skinny_jeans 0.4.0 → 0.5.0

data/README.rdoc

@@ -9,10 +9,14 @@ http://img696.imageshack.us/img696/75/skinnys3.jpg
   0.0.0.0 - - [01/Oct/2010:00:00:01 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
   0.0.0.0 - - [01/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
   0.0.0.0 - - [02/Oct/2010:00:00:03 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" "-"
+  0.0.0.0 - - [02/Oct/2010:00:00:04 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/?q=some+stuff)" "-"
+  0.0.0.0 - - [02/Oct/2010:00:00:05 -0700] "GET /posts/my-first-post HTTP/1.1" 200 1337 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://search.yahoo.com/?p=some+stuff)" "-"
 
-*  then you get 2 SQL rows that look like:
+*  then you get 2 SQL rows in the pageviews table that look like:
   2010-10-01, my-first-post, 3
   2010-10-02, my-first-post, 1
+*  and 1 SQL row in the pageview_keywords table that look like: (search referrals ONLY, but... WITH keyword tracking)
+  2010-10-02, "my-first-post", 2, "some stuff"
 *  note the date columns truncate timestamp, so the days are in whatever timezone your log file reports in
 
 
@@ -39,4 +43,7 @@ http://img696.imageshack.us/img696/75/skinnys3.jpg
 
 == NOTES
 *  supports gzipped files
-*  creates a temp copy of the log file before parsing
+*  creates a temp copy of the log file before parsing
+*  currently requires each line to be unique
+ *  this could be a problem if a single client manages to hit the same page more than once in less than 1 second.
+ *  this is only a problem if the last line parsed is one of the pages that was hit by the same client more than once in less than one second

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/lib/skinny_jeans.rb

@@ -5,6 +5,9 @@ require 'sqlite3'
 require 'active_record'
 require 'zlib'
 require 'fileutils'
+require 'uri'
+require 'cgi'
+require 'skinny_jeans_string_parser'
 # require 'home_run'
 
 class SkinnyJeans
@@ -13,13 +16,14 @@ class SkinnyJeans
     self.new(logfile_path, sqlite_db_path, path_regexp, date_regexp).execute
   end
 
-  attr_accessor :hash_of_dates, :last_pageview_at
+  attr_accessor :hash_of_dates, :hash_of_dates_for_keywords, :last_pageview_at
 
   def initialize(logfile_path, sqlite_db_path, path_regexp, date_regexp)
     @logfile_path, @sqlite_db_path, @path_regexp, @date_regexp = [logfile_path, sqlite_db_path, path_regexp, date_regexp]
     @is_gzipped = !logfile_path.to_s[/gz/].nil?
     prepare_db
     @hash_of_dates = {}
+    @hash_of_dates_for_keywords = {}
     @last_datetime = nil
   end
 
@@ -49,6 +53,19 @@ class SkinnyJeans
         t.column :last_line_parsed, :string
       end
     end
+
+    # addition from 2010-12-06 to track search traffic specifically
+    if !PageviewKeyword.table_exists?
+      SkinnyJeanDb.connection.create_table(:pageview_keywords) do |t|
+        t.column :date, :date
+        t.column :path, :string
+        t.column :pageview_count, :integer
+        t.column :keyword, :string
+      end
+      SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :path, :keyword], :name => "date_path_keyword_index")
+      # SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :pageview_count], :name => "date_pageview_count_index")
+    end
+
   end
 
   def execute
@@ -63,7 +80,7 @@ class SkinnyJeans
     if last_update
       last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
       file_reader do |line, lineno|
-        if line == last_line_parsed
+        if line.to_s[0..254] == last_line_parsed.to_s[0..254]
           lineno_of_last_line_parsed = lineno
           break
         end
@@ -77,18 +94,19 @@ class SkinnyJeans
 
       file_reader do |line, index|
         lineno += 1
-        next if lineno_of_last_line_parsed && lineno < lineno_of_last_line_parsed
+        next if lineno_of_last_line_parsed && lineno <= lineno_of_last_line_parsed
 
         path_match = line[@path_regexp, 1]
         next if path_match.nil?
         date_match = line[@date_regexp, 1]
         next if date_match.nil?
-        time_object = parse_string_as_date(date_match)
+        datetime_obj = parse_string_as_date(date_match)
 
-        next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && time_object < last_pageview_at
+        next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && datetime_obj < last_pageview_at
 
-        insert_or_increment([time_object,path_match])
-        last_line_parsed = line
+        insert_or_increment(datetime_obj, path_match, SkinnyJeansStringParser.extract_search_query(line))
+        @last_pageview_at = datetime_obj
+        last_line_parsed = line.to_s[0..254] # only 255 characters because we store it in the database
         lines_parsed += 1
       end
     end
@@ -96,7 +114,9 @@ class SkinnyJeans
     puts "completed parsing in #{realtime}"
 
     persisted = 0
+    persisted_pageview_keywords = 0
     realtime = Benchmark.realtime do
+
       hash_of_dates.each do |date, hash_of_paths|
         hash_of_paths.keys.each do |path|
           pv = Pageview.find_or_create_by_date_and_path(date, path)
@@ -106,13 +126,30 @@ class SkinnyJeans
           persisted += 1
         end
       end
+
+      hash_of_dates_for_keywords.each do |date, hash_of_paths|
+        hash_of_paths.keys.each do |path|
+          hash_of_paths[path].keys.each do |keyword|
+            pvk = PageviewKeyword.find_or_create_by_date_and_path_and_keyword(date, path, keyword)
+            pvk.keyword = keyword.to_s[0..254]
+            pvk.pageview_count ||= 0
+            pvk.pageview_count += hash_of_paths[path][keyword]
+            pvk.save!
+            persisted_pageview_keywords += 1
+          end
+        end
+      end
+
     end
-    
+
     puts "completed persistence in #{realtime}"
 
-    Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed})
+    Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed.to_s[0..254]})
 
-    puts "total records in DB: #{Pageview.count}\nlines parsed this round: #{lines_parsed}\nlines persisted this round:#{persisted}\ntotal SkinnyJeans executions since inception: #{Update.count}"
+    puts("total records in DB: #{Pageview.count}
+lines parsed this round: #{lines_parsed}
+lines persisted this round:#{persisted}
+total SkinnyJeans executions since inception: #{Update.count}")
 
     return self
 
@@ -137,6 +174,7 @@ class SkinnyJeans
 
   def pageview;get_ar_class(Pageview);end
   def update;get_ar_class(Update);end
+  def pageview_keyword;get_ar_class(PageviewKeyword);end
 
   def get_ar_class(klass)
     begin;return(klass);rescue(ActiveRecord::ConnectionNotEstablished);prepare_db;end
@@ -150,21 +188,32 @@ class SkinnyJeans
     Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
   end
 
-  def insert_or_increment(date_path_pair)
-    datetime, path = date_path_pair
-    date = datetime.strftime(("%Y-%m-%d"))
-    hash_of_dates[date] ||= {}
-    hash_of_dates[date][path] ||= 0
-    hash_of_dates[date][path] += 1
-    @last_pageview_at = datetime
+  def insert_or_increment(_datetime_obj, _path, _search_keyword = nil)
+
+    date_string = _datetime_obj.strftime(("%Y-%m-%d"))
+
+    # data for all pageviews
+    hash_of_dates[date_string] ||= {}
+    hash_of_dates[date_string][_path] ||= 0
+    hash_of_dates[date_string][_path] += 1
+
+    return if _search_keyword.nil?
+
+    # data for just pageviews coming from search
+    hash_of_dates_for_keywords[date_string] ||= {}
+    hash_of_dates_for_keywords[date_string][_path] ||= {}
+    hash_of_dates_for_keywords[date_string][_path][_search_keyword] ||= 0
+    hash_of_dates_for_keywords[date_string][_path][_search_keyword] += 1
+
   end
 
   class Pageview < SkinnyJeanDb
   end
+  class PageviewKeyword < SkinnyJeanDb
+  end
   class Update < SkinnyJeanDb
   end
 
-
 end
 
 # SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"

data/lib/skinny_jeans_string_parser.rb

@@ -0,0 +1,59 @@
+# example
+# SkinnyJeansStringParser.extract_search_query("http://search.aol.com/aol/search?enabled_terms=&s_it=comsearch50&q=cool+stuff")
+# => "cool stuff"
+
+class SkinnyJeansStringParser
+
+  def self.extract_search_query(_url)
+    self.new(_url).get_search_keyword
+  end
+
+  attr_accessor :string_value
+  def initialize(string_value)
+    @string_value = string_value
+  end
+
+  # iterate through any URLs we find in a string and return a search query or nil
+  def get_search_keyword
+    !all_urls.nil? ? all_urls.collect { |_url| extract_search_query_from_url(_url) }[0] : nil
+  end
+
+  # pre: some referring URL from google, yahoo, AOL, bing, ask
+  # post: whatever the search query was, ASCII or GTFO
+  def extract_search_query_from_url(url)
+    val = nil
+    case url
+    when /google\.com/
+      val=return_param_from_url(url, "q")
+    when /search\.yahoo\.com/
+      val=return_param_from_url(url, "p")
+    when /search\.aol\.com/
+      val=return_param_from_url(url, "q")
+    when /ask\.com/
+      val=return_param_from_url(url, "q")
+    when /bing\.com/
+      val=return_param_from_url(url, "q")
+    end
+    # whitelist of acceptable characters
+    val = val.present? && val.gsub(/[^0-9A-Za-z\s"'!@#\$%\^&\*\(\)\?\<\>\[\]:;,\.+-_=]/, '') != val ? nil : val
+    return val
+  end
+
+  # pre: like http://example.org?q=cool&fun=no, "fun"
+  # post: "no"
+  def return_param_from_url(url, param_name)
+    _uri = URI.parse(url)
+    if _uri.query.present?
+      _cgi = CGI.parse(_uri.query)
+      return _cgi[param_name].to_s.strip if _cgi[param_name]
+    end
+    return nil
+  end
+
+  # find all URLs in a string that are at beginning or end of string or are tokenized by spaces
+  def all_urls
+    @all_urls ||= string_value.split(/\s+/).reject { |_string| !_string.match(/^['"]?https?:['"]?/) }.collect { |url| url.gsub(/["']/,'') }
+    @all_urls.empty? ? nil : @all_urls
+  end
+
+end

data/skinny_jeans.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{skinny_jeans}
-  s.version = "0.4.0"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jonathan Otto"]
-  s.date = %q{2010-10-06}
+  s.date = %q{2010-12-06}
   s.email = %q{jonathan.otto@gmail.com}
   s.extra_rdoc_files = [
     "README.rdoc",
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
      "TODO",
      "VERSION",
      "lib/skinny_jeans.rb",
+     "lib/skinny_jeans_string_parser.rb",
      "skinny_jeans.gemspec"
   ]
   s.homepage = %q{http://github.com/jotto/skinny_jeans}
@@ -29,6 +30,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Fast webserver log parser for persisting daily pageviews per path to sqlite}
+  s.test_files = [
+    "test/skinny_jeans_string_parser_test.rb",
+     "test/skinny_jeans_test.rb"
+  ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION

data/test/skinny_jeans_string_parser_test.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/../lib/skinny_jeans'
+require 'test/unit'
+require 'pp'
+require 'fileutils'
+class SkinnyJeansStringParserTest < Test::Unit::TestCase
+
+
+  def test_can_get_all_urls_from_string
+    _string=<<-EOF
+    98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
+    EOF
+    sjsp = SkinnyJeansStringParser.new(_string)
+    assert_equal 1, sjsp.all_urls.size
+    assert_equal "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA",
+      sjsp.all_urls.first
+  end
+
+  def test_return_nil_of_no_urls
+    _string=<<-EOF
+    207.46.12.204 - - [01/Dec/2010:11:48:00 -0800] "GET /deals/skullcandy-inkd-earbuds HTTP/1.1" 200 5732 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;  SLCC1;  .NET CLR 1.1.4322;  .NET CLR 2.0.40607;  .NET CLR 3.0.04506.648)" "-"
+    EOF
+    sjsp = SkinnyJeansStringParser.new(_string)
+    assert_nil sjsp.all_urls
+
+    assert_nil sjsp.get_search_keyword
+  end
+
+  def test_can_get_keyword_from_url
+    _string=<<-EOF
+    98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
+    EOF
+    sjsp = SkinnyJeansStringParser.new(_string)
+    assert_equal "best deals for the 4th generation iPod touch 32 gb", sjsp.get_search_keyword
+  end
+
+end

data/test/skinny_jeans_test.rb

@@ -0,0 +1,53 @@
+require File.dirname(__FILE__) + '/../lib/skinny_jeans'
+require 'test/unit'
+require 'pp'
+require 'fileutils'
+class SkinnyJeansTest < Test::Unit::TestCase
+
+
+  def test_parse_pick_up_where_left_off
+    db_path = "./skinny_jeans_test.db"
+    FileUtils.rm(db_path) if File.exists?(db_path)
+    sj=SkinnyJeans.new(logfile_path = "small_access_log.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    sj.execute
+    assert_equal 20, sj.pageview.count
+    assert_equal 2, sj.pageview.find_by_path("flip-video").pageview_count
+    assert_equal 2, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
+
+# puts "
+# 
+# -----
+# 
+# "
+    # the 2nd file is the same, but with 2 additional lines for flip-video and apple-ipod-touch
+    sj=SkinnyJeans.new(logfile_path = "small_access_log_part_2.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    sj.execute
+    assert_equal 3, sj.pageview.find_by_path("flip-video").pageview_count
+    assert_equal 2, sj.pageview.find_by_path("apple-ipod-touch").pageview_count
+
+    # the last line of small_access_log_part_2 is over 1000 chars, make sure we can handle it properly
+    assert_equal 255, sj.update.last.last_line_parsed.size
+    _last_line=<<-EOF
+    82.31.245.117 - - [01/Dec/2010:11:52:29 -0800] "GET /deals/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control HTTP/1.1" 200 6244 "http://www.google.co.uk/imgres?imgurl=http://dealzon.com/pictures/deals/6656/large/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control.jpg%3F1264075353&imgrefurl=http://dealzon.com/deals/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control&usg=__Y-Tkbg0EefRA5uKrR0b2gs7HqSo=&h=300&w=287&sz=11&hl=en&start=49&zoom=1&tbnid=18tsbSh1DQmf2M:&tbnh=142&tbnw=133&prev=/images%3Fq%3Dblack%2Bradiator%2Bheater%26um%3D1%26hl%3Den%26biw%3D1020%26bih%3D624%26tbs%3Disch:10%2C1872&um=1&itbs=1&iact=hc&vpx=757&vpy=67&dur=140&hovh=230&hovw=220&tx=158&ty=118&ei=Tab2TMWkCMqL4AauyI2SBw&oei=Q6b2TLPJDoW3hQfDlri5BQ&esq=4&page=4&ndsp=15&ved=1t:429,r:14,s:49&biw=1020&bih=624" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Sky Broadband; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; InfoPath.3)" "-"
+    EOF
+    assert_equal _last_line.strip![0..254], sj.update.last.last_line_parsed
+    assert_equal 3, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
+
+# puts "
+# 
+# -----
+# 
+# "
+
+    # the 3rd has 1 additional line so we can ensure we can leave off on a line over 255 characters
+    sj=SkinnyJeans.new(logfile_path = "small_access_log_part_3.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    sj.execute
+    assert_equal 3, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
+    assert_equal 3, sj.pageview.find_by_path("apple-ipod-touch").pageview_count
+    assert_equal 3, sj.pageview_keyword.find_by_path_and_keyword("apple-ipod-touch", "best deals for the 4th generation iPod touch 32 gb").pageview_count
+    # PP.pp sj.pageview_keyword.all
+  end
+
+
+
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: skinny_jeans
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
+  - 5
   - 0
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Jonathan Otto
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-06 00:00:00 -05:00
+date: 2010-12-06 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,7 +66,10 @@ files:
 - TODO
 - VERSION
 - lib/skinny_jeans.rb
+- lib/skinny_jeans_string_parser.rb
 - skinny_jeans.gemspec
+- test/skinny_jeans_string_parser_test.rb
+- test/skinny_jeans_test.rb
 has_rdoc: true
 homepage: http://github.com/jotto/skinny_jeans
 licenses: []
@@ -101,5 +104,6 @@ rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Fast webserver log parser for persisting daily pageviews per path to sqlite
-test_files: []
-
+test_files: 
+- test/skinny_jeans_string_parser_test.rb
+- test/skinny_jeans_test.rb