skinny-jeans 0.1.0

data/.gitignore

@@ -0,0 +1,3 @@
+*.db
+*.log
+pkg

data/README.rdoc

@@ -0,0 +1,37 @@
+= SKINNY JEANS LOG PARSING WITH RUBY & SQLITE FOR HIPSTERS
+http://img696.imageshack.us/img696/75/skinnys3.jpg
+
+== example
+
+*  your log file has lines that look like
+
+  0.0.0.0 - - [01/Oct/2010:00:00:00 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [01/Oct/2010:00:00:01 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [01/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [02/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+
+*  then you get 1 SQL row that looks like:
+  2010-10-01, my-first-post, 3
+  2010-10-02, my-first-post, 1
+
+
+== WHY?
+*  so you can query a database by date and path and get pageviews and have that data stored CHEAP
+*  because i couldn't find anything simpler and Google Analytics is limited to 10,000 API requests per day
+
+
+== Usage
+  SkinnyJeans::execute(logfile_path = "access.log", sqlite_skinny_jeans = "sqlite_skinny_jeans.db", path_regexp = /\s\/posts\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+1.  Parse oldest logs first, then run regularly against your main log, let logrotate handle the rest (skinny_jeans remembers where it left off)
+2.  enjoy the skinny jeans
+
+
+== WHAT IT DO
+*  it parses 100,000 lines in < 2.5 seconds
+*  persists 1,000 requests with 2 compound indexes in 10 seconds (using home_run c date parser, 15 seconds without home_run)
+
+*  parse a webserver's log file to aggregate paths by DAY with pageview counts
+
+*  creates sqlite database with columns: date, path, pageview_count
+
+*  ASSUMES reading log files in ascending order, keeps track of last line read so you could put it on a scheduler or cron job

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rake'
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "skinny-jeans"
+    s.summary = "Fast webserver log parser for persisting daily pageviews per path to sqlite"
+    s.email = "jonathan.otto@gmail.com"
+    s.homepage = "http://github.com/jotto/skinny-jeans"
+    s.authors = ["Jonathan Otto"]
+    s.add_dependency 'sqlite3-ruby', '>= 1.2.4'
+    s.add_dependency 'activerecord', '>= 3.0.0'
+  end
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/skinny_jeans.rb

@@ -0,0 +1,128 @@
+require 'time'
+require 'benchmark'
+require 'rubygems'
+require 'sqlite3'
+require 'active_record'
+# require 'home_run'
+
+class Pageview < ActiveRecord::Base
+end
+class Update < ActiveRecord::Base
+end
+
+class SkinnyJeans
+  class << self
+
+    def prepare_db(sqlite_db_path)
+      # create database if necessary
+      SQLite3::Database.new(sqlite_db_path)
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => sqlite_db_path)
+      # create tables if necessary
+      if !Pageview.table_exists?
+        ActiveRecord::Base.connection.create_table(:pageviews) do |t|
+          t.column :date, :date
+          t.column :path, :string
+          t.column :pageview_count, :integer
+        end
+        # flow tight like skinny jeans with these compound indexes
+        ActiveRecord::Base.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
+        ActiveRecord::Base.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
+      end
+      if !Update.table_exists?
+        ActiveRecord::Base.connection.create_table(:updates) do |t|
+          t.column :last_pageview_at, :timestamp
+          t.column :lines_parsed, :integer
+          t.column :last_line_parsed, :string
+        end
+      end
+    end
+
+    def execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+
+      prepare_db(sqlite_db_path)
+      skinny_jean = self.new
+      lines_parsed = 0
+      last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
+      last_update = Update.order("id DESC").limit(1).first
+
+      if last_update
+        last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
+        File.new(logfile_path, "r").each_with_index do |line, lineno|
+          if line == last_line_parsed
+            lineno_of_last_line_parsed = lineno
+            break
+          end
+        end
+        puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
+      end
+
+      realtime = Benchmark.realtime do
+        date_path_pairs_array = []
+
+        File.new(logfile_path, "r").each_with_index do |line, lineno|
+          next if lineno_of_last_line_parsed && lineno < lineno_of_last_line_parsed
+
+          path_match = line[path_regexp, 1]
+          next if path_match.nil?
+          date_match = line[date_regexp, 1]
+          next if date_match.nil?
+          time_object = parse_string_as_date(date_match)
+
+          next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && time_object < last_pageview_at
+
+          skinny_jean.insert_or_increment([time_object,path_match])
+          last_line_parsed = line
+          lines_parsed += 1
+        end
+
+        skinny_jean.hash_of_dates
+      end
+
+      puts "completed parsing in #{realtime}"
+
+      realtime = Benchmark.realtime do
+        skinny_jean.hash_of_dates.each do |date, hash_of_paths|
+          hash_of_paths.keys.each do |path|
+            pv = Pageview.find_or_create_by_date_and_path(date, path)
+            pv.pageview_count ||= 0
+            pv.pageview_count += hash_of_paths[path]
+            pv.save!
+          end
+        end
+      end
+      
+      puts "completed persistence in #{realtime}"
+
+      Update.create!({:last_pageview_at => skinny_jean.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed})
+
+      puts "total records in DB: #{Pageview.count}\nlines parsed this round: #{lines_parsed}\ntotal SkinnyJeans executions since inception: #{Update.count}"
+
+    end
+
+    # return a ruby Time object
+    def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
+      day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
+      Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
+    end
+
+  end
+
+  attr_accessor :hash_of_dates, :last_pageview_at
+
+  def initialize
+    @hash_of_dates = {}
+    @last_datetime = nil
+  end
+
+  def insert_or_increment(date_path_pair)
+    datetime, path = date_path_pair
+    date = datetime.strftime(("%Y-%m-%d"))
+    hash_of_dates[date] ||= {}
+    hash_of_dates[date][path] ||= 0
+    hash_of_dates[date][path] += 1
+    @last_pageview_at = datetime
+  end
+
+end
+
+# SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"

data/skinny-jeans.gemspec

@@ -0,0 +1,47 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{skinny-jeans}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jonathan Otto"]
+  s.date = %q{2010-10-03}
+  s.email = %q{jonathan.otto@gmail.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/skinny_jeans.rb",
+     "skinny-jeans.gemspec"
+  ]
+  s.homepage = %q{http://github.com/jotto/skinny-jeans}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Fast webserver log parser for persisting daily pageviews per path to sqlite}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
+    else
+      s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+      s.add_dependency(%q<activerecord>, [">= 3.0.0"])
+    end
+  else
+    s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
+    s.add_dependency(%q<activerecord>, [">= 3.0.0"])
+  end
+end
+

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: skinny-jeans
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Jonathan Otto
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-03 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3-ruby
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 2
+        - 4
+        version: 1.2.4
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 3
+        - 0
+        - 0
+        version: 3.0.0
+  type: :runtime
+  version_requirements: *id002
+description: 
+email: jonathan.otto@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .gitignore
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/skinny_jeans.rb
+- skinny-jeans.gemspec
+has_rdoc: true
+homepage: http://github.com/jotto/skinny-jeans
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Fast webserver log parser for persisting daily pageviews per path to sqlite
+test_files: []
+