skinny_jeans 0.2.2

data/.gitignore

@@ -0,0 +1,5 @@
+*.db
+*.gz
+*.log
+pkg
+.DS*

data/README.rdoc

@@ -0,0 +1,37 @@
+= SKINNY JEANS LOG PARSING WITH RUBY & SQLITE FOR HIPSTERS
+http://img696.imageshack.us/img696/75/skinnys3.jpg
+
+== EXAMPLE
+
+*  your log file has lines that look like
+
+  0.0.0.0 - - [01/Oct/2010:00:00:00 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [01/Oct/2010:00:00:01 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [01/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [02/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+
+*  then you get 2 SQL rows that looks like:
+  2010-10-01, my-first-post, 3
+  2010-10-02, my-first-post, 1
+*  note the date columns truncate timestamp, so the days are in whatever timezone your log file reports in
+
+
+== WHY?
+*  so you can query a database by date and path and get pageviews and have that data stored CHEAP
+*  because i couldn't find anything simpler and Google Analytics is limited to 10,000 API requests per day
+
+
+== USAGE
+  sj = SkinnyJeans::execute(logfile_path = "access.log", sqlite_skinny_jeans = "sqlite_skinny_jeans.db", path_regexp = /\s\/posts\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+  sj.pageview.where("date = '2010-10-01' and path = 'my-first-post'")
+  => #<SkinnyJeans::Pageview id: 1, date: "2010-10-01", path: "my-first-post", pageview_count: 3>
+1.  NOTE: for now *you have to monkey patch the SkinnyJeans#parse_string_as_date*
+2.  Parse oldest logs first, then run regularly against your main log, let logrotate handle the rest (skinny_jeans remembers where it left off)
+3.  ASSUMES reading log files in ascending order, keeps track of last line read so you could put it on a scheduler or cron job
+4.  access the 2 activerecord classes, sj.pageview (returns Pageview class), and sj.update
+5.  enjoy the skinny jeans
+
+
+== PERFORMANCE
+*  it parses 100,000 lines in < 2.5 seconds
+*  persists 1,000 requests with 2 compound indexes in 15 seconds, or 10 seconds with home_run c extension

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rake'
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "skinny_jeans"
+    s.summary = "Fast webserver log parser for persisting daily pageviews per path to sqlite"
+    s.email = "jonathan.otto@gmail.com"
+    s.homepage = "http://github.com/jotto/skinny_jeans"
+    s.authors = ["Jonathan Otto"]
+    s.add_dependency 'sqlite3-ruby', '>= 1.2.4'
+    s.add_dependency 'activerecord', '>= 2.3.8'
+  end
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/lib/skinny_jeans.rb

@@ -0,0 +1,157 @@
+require 'time'
+require 'benchmark'
+require 'rubygems'
+require 'sqlite3'
+require 'active_record'
+require 'zlib'
+# require 'home_run'
+
+class SkinnyJeans
+
+  def self.execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+    self.new(logfile_path, sqlite_db_path, path_regexp, date_regexp).execute
+  end
+
+  attr_accessor :hash_of_dates, :last_pageview_at
+
+  def initialize(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+    @logfile_path, @sqlite_db_path, @path_regexp, @date_regexp = [logfile_path, sqlite_db_path, path_regexp, date_regexp]
+    @is_gzipped = !logfile_path.to_s[/gz/].nil?
+    prepare_db
+    @hash_of_dates = {}
+    @last_datetime = nil
+  end
+
+  def prepare_db
+    # create database if necessary
+    SQLite3::Database.new(@sqlite_db_path)
+    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => @sqlite_db_path)
+    # create tables if necessary
+    if !Pageview.table_exists?
+      ActiveRecord::Base.connection.create_table(:pageviews) do |t|
+        t.column :date, :date
+        t.column :path, :string
+        t.column :pageview_count, :integer
+      end
+      # flow tight like skinny jeans with these compound indexes
+      ActiveRecord::Base.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
+      ActiveRecord::Base.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
+    end
+    if !Update.table_exists?
+      ActiveRecord::Base.connection.create_table(:updates) do |t|
+        t.column :last_pageview_at, :timestamp
+        t.column :lines_parsed, :integer
+        t.column :last_line_parsed, :string
+      end
+    end
+  end
+
+  def execute
+
+    lines_parsed = 0
+    last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
+    # last_update = Update.order("id DESC").limit(1).first
+    last_update = Update.find(:first, :order => "id DESC", :limit => 1)
+
+    # see if the last_line_parsed parsed exists in the current log file
+    # if it doesnt exist, we'll simply read anything with a timestamp greater than last_pageview_at
+    if last_update
+      last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
+      file_reader do |line, lineno|
+        if line == last_line_parsed
+          lineno_of_last_line_parsed = lineno
+          break
+        end
+      end
+      puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
+    end
+
+    realtime = Benchmark.realtime do
+      date_path_pairs_array = []
+      lineno = -1
+
+      file_reader do |line, index|
+        lineno += 1
+        next if lineno_of_last_line_parsed && lineno < lineno_of_last_line_parsed
+
+        path_match = line[@path_regexp, 1]
+        next if path_match.nil?
+        date_match = line[@date_regexp, 1]
+        next if date_match.nil?
+        time_object = parse_string_as_date(date_match)
+
+        next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && time_object < last_pageview_at
+
+        insert_or_increment([time_object,path_match])
+        last_line_parsed = line
+        lines_parsed += 1
+      end
+    end
+
+    puts "completed parsing in #{realtime}"
+
+    persisted = 0
+    realtime = Benchmark.realtime do
+      hash_of_dates.each do |date, hash_of_paths|
+        hash_of_paths.keys.each do |path|
+          pv = Pageview.find_or_create_by_date_and_path(date, path)
+          pv.pageview_count ||= 0
+          pv.pageview_count += hash_of_paths[path]
+          pv.save!
+          persisted += 1
+        end
+      end
+    end
+    
+    puts "completed persistence in #{realtime}"
+
+    Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed})
+
+    puts "total records in DB: #{Pageview.count}\nlines parsed this round: #{lines_parsed}\nlines persisted this round:#{persisted}\ntotal SkinnyJeans executions since inception: #{Update.count}"
+
+    return self
+
+  end
+
+  def file_reader
+    if @is_gzipped
+      lineno = 0
+      Zlib::GzipReader.open(@logfile_path){|line|yield([line.read,lineno]);lineno+=1}
+    else
+      File.new(@logfile_path, "r").each_with_index{|line, lineno| yield([line,lineno])}
+    end
+  end
+
+  def pageview;get_ar_class(Pageview);end
+  def update;get_ar_class(Update);end
+
+  def get_ar_class(klass)
+    begin;return(klass);rescue(ActiveRecord::ConnectionNotEstablished);prepare_db;end
+  end
+
+  private
+
+  # return a ruby Time object
+  def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
+    day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
+    Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
+  end
+
+  def insert_or_increment(date_path_pair)
+    datetime, path = date_path_pair
+    date = datetime.strftime(("%Y-%m-%d"))
+    hash_of_dates[date] ||= {}
+    hash_of_dates[date][path] ||= 0
+    hash_of_dates[date][path] += 1
+    @last_pageview_at = datetime
+  end
+
+  class Pageview < ActiveRecord::Base
+  end
+  class Update < ActiveRecord::Base
+  end
+
+
+end
+
+# SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"

data/skinny_jeans.gemspec

@@ -0,0 +1,47 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{skinny_jeans}
+  s.version = "0.2.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jonathan Otto"]
+  s.date = %q{2010-10-04}
+  s.email = %q{jonathan.otto@gmail.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/skinny_jeans.rb",
+     "skinny_jeans.gemspec"
+  ]
+  s.homepage = %q{http://github.com/jotto/skinny_jeans}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Fast webserver log parser for persisting daily pageviews per path to sqlite}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 2.3.8"])
+    else
+      s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+      s.add_dependency(%q<activerecord>, [">= 2.3.8"])
+    end
+  else
+    s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+    s.add_dependency(%q<activerecord>, [">= 2.3.8"])
+  end
+end
+

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: skinny_jeans
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 2
+  version: 0.2.2
+platform: ruby
+authors: 
+- Jonathan Otto
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-04 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 2
+        - 4
+        version: 1.2.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 2
+        - 3
+        - 8
+        version: 2.3.8
+  type: :runtime
+  version_requirements: *id002
+description: 
+email: jonathan.otto@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .gitignore
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/skinny_jeans.rb
+- skinny_jeans.gemspec
+has_rdoc: true
+homepage: http://github.com/jotto/skinny_jeans
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Fast webserver log parser for persisting daily pageviews per path to sqlite
+test_files: []
+