skinny-jeans 0.1.0 → 0.2.0

data/README.rdoc

@@ -1,7 +1,7 @@
 = SKINNY JEANS LOG PARSING WITH RUBY & SQLITE FOR HIPSTERS
 http://img696.imageshack.us/img696/75/skinnys3.jpg
 
-== example
+== EXAMPLE
 
 *  your log file has lines that look like
 
@@ -10,9 +10,10 @@ http://img696.imageshack.us/img696/75/skinnys3.jpg
   0.0.0.0 - - [01/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
   0.0.0.0 - - [02/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
 
-*  then you get 1 SQL row that looks like:
+*  then you get 2 SQL rows that looks like:
   2010-10-01, my-first-post, 3
   2010-10-02, my-first-post, 1
+*  note the date columns truncate timestamp, so the days are in whatever timezone your log file reports in
 
 
 == WHY?
@@ -20,18 +21,17 @@ http://img696.imageshack.us/img696/75/skinnys3.jpg
 *  because i couldn't find anything simpler and Google Analytics is limited to 10,000 API requests per day
 
 
-== Usage
-  SkinnyJeans::execute(logfile_path = "access.log", sqlite_skinny_jeans = "sqlite_skinny_jeans.db", path_regexp = /\s\/posts\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
-1.  Parse oldest logs first, then run regularly against your main log, let logrotate handle the rest (skinny_jeans remembers where it left off)
-2.  enjoy the skinny jeans
+== USAGE
+  sj = SkinnyJeans::execute(logfile_path = "access.log", sqlite_skinny_jeans = "sqlite_skinny_jeans.db", path_regexp = /\s\/posts\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+  sj.pageview.where("date = '2010-10-01' and path = 'my-first-post'")
+  => #<SkinnyJeans::Pageview id: 1, date: "2010-10-01", path: "my-first-post", pageview_count: 3>
+1.  NOTE: for now *you have to monkey patch the SkinnyJeans#parse_string_as_date*
+2.  Parse oldest logs first, then run regularly against your main log, let logrotate handle the rest (skinny_jeans remembers where it left off)
+3.  ASSUMES reading log files in ascending order, keeps track of last line read so you could put it on a scheduler or cron job
+4.  access the 2 activerecord classes, sj.pageview (returns Pageview class), and sj.update
+5.  enjoy the skinny jeans
 
 
-== WHAT IT DO
+== PERFORMANCE
 *  it parses 100,000 lines in < 2.5 seconds
-*  persists 1,000 requests with 2 compound indexes in 10 seconds (using home_run c date parser, 15 seconds without home_run)
-
-*  parse a webserver's log file to aggregate paths by DAY with pageview counts
-
-*  creates sqlite database with columns: date, path, pageview_count
-
-*  ASSUMES reading log files in ascending order, keeps track of last line read so you could put it on a scheduler or cron job
+*  persists 1,000 requests with 2 compound indexes in 15 seconds, or 10 seconds with home_run c extension

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.2.0

data/lib/skinny_jeans.rb

@@ -3,115 +3,137 @@ require 'benchmark'
 require 'rubygems'
 require 'sqlite3'
 require 'active_record'
+require 'zlib'
 # require 'home_run'
 
-class Pageview < ActiveRecord::Base
-end
-class Update < ActiveRecord::Base
-end
-
 class SkinnyJeans
-  class << self
-
-    def prepare_db(sqlite_db_path)
-      # create database if necessary
-      SQLite3::Database.new(sqlite_db_path)
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => sqlite_db_path)
-      # create tables if necessary
-      if !Pageview.table_exists?
-        ActiveRecord::Base.connection.create_table(:pageviews) do |t|
-          t.column :date, :date
-          t.column :path, :string
-          t.column :pageview_count, :integer
-        end
-        # flow tight like skinny jeans with these compound indexes
-        ActiveRecord::Base.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
-        ActiveRecord::Base.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
+
+  def self.execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+    self.new(logfile_path, sqlite_db_path, path_regexp, date_regexp).execute
+  end
+
+  attr_accessor :hash_of_dates, :last_pageview_at
+
+  def initialize(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+    @logfile_path, @sqlite_db_path, @path_regexp, @date_regexp = [logfile_path, sqlite_db_path, path_regexp, date_regexp]
+    @is_gzipped = !logfile_path.to_s[/gz/].nil?
+    prepare_db
+    @hash_of_dates = {}
+    @last_datetime = nil
+  end
+
+  def prepare_db
+    # create database if necessary
+    SQLite3::Database.new(@sqlite_db_path)
+    ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => @sqlite_db_path)
+    # create tables if necessary
+    if !Pageview.table_exists?
+      ActiveRecord::Base.connection.create_table(:pageviews) do |t|
+        t.column :date, :date
+        t.column :path, :string
+        t.column :pageview_count, :integer
       end
-      if !Update.table_exists?
-        ActiveRecord::Base.connection.create_table(:updates) do |t|
-          t.column :last_pageview_at, :timestamp
-          t.column :lines_parsed, :integer
-          t.column :last_line_parsed, :string
-        end
+      # flow tight like skinny jeans with these compound indexes
+      ActiveRecord::Base.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
+      ActiveRecord::Base.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
+    end
+    if !Update.table_exists?
+      ActiveRecord::Base.connection.create_table(:updates) do |t|
+        t.column :last_pageview_at, :timestamp
+        t.column :lines_parsed, :integer
+        t.column :last_line_parsed, :string
       end
     end
+  end
+
+  def execute
+
+    lines_parsed = 0
+    last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
+    last_update = Update.order("id DESC").limit(1).first
 
-    def execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
-
-      prepare_db(sqlite_db_path)
-      skinny_jean = self.new
-      lines_parsed = 0
-      last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
-      last_update = Update.order("id DESC").limit(1).first
-
-      if last_update
-        last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
-        File.new(logfile_path, "r").each_with_index do |line, lineno|
-          if line == last_line_parsed
-            lineno_of_last_line_parsed = lineno
-            break
-          end
+    # see if the last_line_parsed parsed exists in the current log file
+    # if it doesnt exist, we'll simply read anything with a timestamp greater than last_pageview_at
+    if last_update
+      last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
+      file_reader do |line, lineno|
+        if line == last_line_parsed
+          lineno_of_last_line_parsed = lineno
+          break
         end
-        puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
       end
+      puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
+    end
 
-      realtime = Benchmark.realtime do
-        date_path_pairs_array = []
-
-        File.new(logfile_path, "r").each_with_index do |line, lineno|
-          next if lineno_of_last_line_parsed && lineno < lineno_of_last_line_parsed
+    realtime = Benchmark.realtime do
+      date_path_pairs_array = []
+      lineno = -1
 
-          path_match = line[path_regexp, 1]
-          next if path_match.nil?
-          date_match = line[date_regexp, 1]
-          next if date_match.nil?
-          time_object = parse_string_as_date(date_match)
+      file_reader do |line, index|
+        lineno += 1
+        next if lineno_of_last_line_parsed && lineno < lineno_of_last_line_parsed
 
-          next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && time_object < last_pageview_at
+        path_match = line[@path_regexp, 1]
+        next if path_match.nil?
+        date_match = line[@date_regexp, 1]
+        next if date_match.nil?
+        time_object = parse_string_as_date(date_match)
 
-          skinny_jean.insert_or_increment([time_object,path_match])
-          last_line_parsed = line
-          lines_parsed += 1
-        end
+        next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && time_object < last_pageview_at
 
-        skinny_jean.hash_of_dates
+        insert_or_increment([time_object,path_match])
+        last_line_parsed = line
+        lines_parsed += 1
       end
+    end
 
-      puts "completed parsing in #{realtime}"
-
-      realtime = Benchmark.realtime do
-        skinny_jean.hash_of_dates.each do |date, hash_of_paths|
-          hash_of_paths.keys.each do |path|
-            pv = Pageview.find_or_create_by_date_and_path(date, path)
-            pv.pageview_count ||= 0
-            pv.pageview_count += hash_of_paths[path]
-            pv.save!
-          end
+    puts "completed parsing in #{realtime}"
+
+    persisted = 0
+    realtime = Benchmark.realtime do
+      hash_of_dates.each do |date, hash_of_paths|
+        hash_of_paths.keys.each do |path|
+          pv = Pageview.find_or_create_by_date_and_path(date, path)
+          pv.pageview_count ||= 0
+          pv.pageview_count += hash_of_paths[path]
+          pv.save!
+          persisted += 1
         end
       end
-      
-      puts "completed persistence in #{realtime}"
+    end
+    
+    puts "completed persistence in #{realtime}"
 
-      Update.create!({:last_pageview_at => skinny_jean.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed})
+    Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed})
 
-      puts "total records in DB: #{Pageview.count}\nlines parsed this round: #{lines_parsed}\ntotal SkinnyJeans executions since inception: #{Update.count}"
+    puts "total records in DB: #{Pageview.count}\nlines parsed this round: #{lines_parsed}\nlines persisted this round:#{persisted}\ntotal SkinnyJeans executions since inception: #{Update.count}"
 
-    end
+    return self
+
+  end
 
-    # return a ruby Time object
-    def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
-      day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
-      Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
+  def file_reader
+    if @is_gzipped
+      lineno = 0
+      Zlib::GzipReader.open(@logfile_path){|line|yield([line.read,lineno]);lineno+=1}
+    else
+      File.new(@logfile_path, "r").each_with_index{|line, lineno| yield([line,lineno])}
     end
+  end
+
+  def pageview;get_ar_class(Pageview);end
+  def update;get_ar_class(Update);end
 
+  def get_ar_class(klass)
+    begin;return(klass);rescue(ActiveRecord::ConnectionNotEstablished);prepare_db;end
   end
 
-  attr_accessor :hash_of_dates, :last_pageview_at
+  private
 
-  def initialize
-    @hash_of_dates = {}
-    @last_datetime = nil
+  # return a ruby Time object
+  def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
+    day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
+    Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
   end
 
   def insert_or_increment(date_path_pair)
@@ -123,6 +145,12 @@ class SkinnyJeans
     @last_pageview_at = datetime
   end
 
+  class Pageview < ActiveRecord::Base
+  end
+  class Update < ActiveRecord::Base
+  end
+
+
 end
 
 # SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"

data/skinny-jeans.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{skinny-jeans}
-  s.version = "0.1.0"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jonathan Otto"]
-  s.date = %q{2010-10-03}
+  s.date = %q{2010-10-04}
   s.email = %q{jonathan.otto@gmail.com}
   s.extra_rdoc_files = [
     "README.rdoc"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: skinny-jeans
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Jonathan Otto
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-03 00:00:00 -05:00
+date: 2010-10-04 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency