skinny_jeans 0.5.2 → 0.6.0

data/VERSION

@@ -1 +1 @@
-0.5.2
+0.6.0

data/lib/skinny_jeans/log_parser.rb

@@ -0,0 +1,164 @@
+module SkinnyJeans
+
+  class LogParser
+
+    def self.execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+      self.new(logfile_path, sqlite_db_path, path_regexp, date_regexp).execute
+    end
+
+    attr_accessor :hash_of_dates, :hash_of_dates_for_keywords, :last_pageview_at
+
+    def initialize(logfile_path, sqlite_db_path, path_regexp, date_regexp)
+      @logfile_path, @sqlite_db_path, @path_regexp, @date_regexp = [logfile_path, sqlite_db_path, path_regexp, date_regexp]
+      @is_gzipped = !logfile_path.to_s[/gz/].nil?
+      SkinnyJeans::prepare_db(@sqlite_db_path)
+      @hash_of_dates = {}
+      @hash_of_dates_for_keywords = {}
+      @last_datetime = nil
+    end
+
+
+    def execute
+
+      lines_parsed = 0
+      last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
+      # last_update = Update.order("id DESC").limit(1).first
+      last_update = Update.find(:first, :order => "id DESC", :limit => 1)
+
+      # see if the last_line_parsed parsed exists in the current log file
+      # if it doesnt exist, we'll simply read anything with a timestamp greater than last_pageview_at
+      if last_update
+        last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
+        file_reader do |line, lineno|
+          if line.to_s[0..254] == last_line_parsed.to_s[0..254]
+            lineno_of_last_line_parsed = lineno
+            break
+          end
+        end
+        puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
+      end
+
+      realtime = Benchmark.realtime do
+        date_path_pairs_array = []
+        lineno = -1
+
+        file_reader do |line, index|
+          lineno += 1
+          next if lineno_of_last_line_parsed && lineno <= lineno_of_last_line_parsed
+
+          path_match = line[@path_regexp, 1]
+          next if path_match.nil?
+          date_match = line[@date_regexp, 1]
+          next if date_match.nil?
+          datetime_obj = parse_string_as_date(date_match)
+
+          next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && datetime_obj < last_pageview_at
+
+          insert_or_increment(datetime_obj, path_match, SkinnyJeans::StringParser.extract_search_query(line))
+          @last_pageview_at = datetime_obj
+          last_line_parsed = line.to_s[0..254] # only 255 characters because we store it in the database
+          lines_parsed += 1
+        end
+      end
+
+      puts "completed parsing in #{realtime}"
+
+      persisted = 0
+      persisted_pageview_keywords = 0
+      realtime = Benchmark.realtime do
+
+        hash_of_dates.each do |date, hash_of_paths|
+          hash_of_paths.keys.each do |path|
+            pv = Pageview.find_or_create_by_date_and_path(date, path)
+            pv.pageview_count ||= 0
+            pv.pageview_count += hash_of_paths[path]
+            pv.save!
+            persisted += 1
+          end
+        end
+
+        hash_of_dates_for_keywords.each do |date, hash_of_paths|
+          hash_of_paths.keys.each do |path|
+            hash_of_paths[path].keys.each do |keyword|
+              pvk = PageviewKeyword.find_or_create_by_date_and_path_and_keyword(date, path, keyword)
+              pvk.keyword = keyword.to_s[0..254]
+              pvk.pageview_count ||= 0
+              pvk.pageview_count += hash_of_paths[path][keyword]
+              pvk.save!
+              persisted_pageview_keywords += 1
+            end
+          end
+        end
+
+      end
+
+      puts "completed persistence in #{realtime}"
+
+      Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed.to_s[0..254]})
+
+      puts("total records in DB: #{Pageview.count}
+  lines parsed this round: #{lines_parsed}
+  lines persisted this round:#{persisted}
+  total SkinnyJeans executions since inception: #{Update.count}")
+
+      return self
+
+    end
+
+    # copies the log file, reads it, then removes it
+    def file_reader
+
+      temp_file_path = "#{@logfile_path}.copy"
+      temp_file = FileUtils.cp(@logfile_path, temp_file_path)
+
+      if @is_gzipped
+        lineno = 0
+        Zlib::GzipReader.new(File.new(temp_file_path, "r")).each_line{|line|yield([line,lineno]);lineno+=1}
+        # Zlib::GzipReader.open(@logfile_path).each_line{|line|yield([line,lineno]);lineno+=1}
+      else
+        File.new(temp_file_path, "r").each_with_index{|line, lineno| yield([line,lineno])}
+      end
+
+      FileUtils.rm_f(temp_file_path)
+    end
+
+    def pageview;get_ar_class(Pageview);end
+    def update;get_ar_class(Update);end
+    def pageview_keyword;get_ar_class(PageviewKeyword);end
+
+    def get_ar_class(klass)
+      begin;return(klass);rescue(ActiveRecord::ConnectionNotEstablished);SkinnyJeans::prepare_db(@sqlite_db_path);end
+    end
+
+    private
+
+    # return a ruby Time object
+    def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
+      day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
+      Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
+    end
+
+    def insert_or_increment(_datetime_obj, _path, _search_keyword = nil)
+
+      date_string = _datetime_obj.strftime(("%Y-%m-%d"))
+
+      # data for all pageviews
+      hash_of_dates[date_string] ||= {}
+      hash_of_dates[date_string][_path] ||= 0
+      hash_of_dates[date_string][_path] += 1
+
+      return if _search_keyword.nil?
+
+      # data for just pageviews coming from search
+      hash_of_dates_for_keywords[date_string] ||= {}
+      hash_of_dates_for_keywords[date_string][_path] ||= {}
+      hash_of_dates_for_keywords[date_string][_path][_search_keyword] ||= 0
+      hash_of_dates_for_keywords[date_string][_path][_search_keyword] += 1
+
+    end
+
+
+
+  end
+
+end

data/lib/skinny_jeans/string_parser.rb

@@ -0,0 +1,72 @@
+# example
+# SkinnyJeans::StringParser.extract_search_query("http://search.aol.com/aol/search?enabled_terms=&s_it=comsearch50&q=cool+stuff")
+# => "cool stuff"
+
+module SkinnyJeans
+  class StringParser
+
+    class << self
+      def extract_search_query(_url)
+        self.new(_url).get_search_keyword
+      end
+
+      # pre: some referring URL from google, yahoo, AOL, bing, ask
+      # post: whatever the search query was, ASCII or GTFO
+      def extract_search_query_from_valid_url(url)
+        val = nil
+        case url
+        when /google\.com/
+          val=return_param_from_valid_url_or_path(url,"q")
+        when /search\.yahoo\.com/
+          val=return_param_from_valid_url_or_path(url,"p")
+        when /search\.aol\.com/
+          val=return_param_from_valid_url_or_path(url,"q")
+        when /ask\.com/
+          val=return_param_from_valid_url_or_path(url,"q")
+        when /bing\.com/
+          val=return_param_from_valid_url_or_path(url,"q")
+        end
+        # whitelist of acceptable characters
+        val = val.present? && val.gsub(/[^0-9A-Za-z\s"'!@#\$%\^&\*\(\)\?\<\>\[\]:;,\.+-_=]/, '') != val ? nil : val
+        return val
+      end
+
+      # pre: like http://example.org?q=cool&fun=no, "fun"
+      # post: "no"
+      def return_param_from_valid_url_or_path(url_or_path, param_name)
+        _uri = URI.parse(URI.encode(url_or_path))
+        if _uri.query.present?
+          _cgi = CGI.parse(_uri.query)
+          if _cgi[param_name]
+            val = unescape_string(_cgi[param_name].join).strip.downcase
+            return (!val.nil? && val!='' ? val : nil)
+          end
+        end
+        return nil
+      end
+
+      def unescape_string(_string)
+        temp = _string.dup
+        temp = CGI.unescape(temp) while CGI.unescape(temp) != temp
+        temp
+      end
+    end
+
+    attr_accessor :string_value
+    def initialize(string_value)
+      @string_value = string_value
+    end
+
+    # find all URLs in a string that are at beginning or end of string or are tokenized by spaces
+    def all_urls
+      @all_urls ||= string_value.split(/\s+/).reject { |_string| !_string.match(/^['"]?https?:['"]?/) }.collect { |url| url.gsub(/["']/,'') }
+      @all_urls.empty? ? nil : @all_urls
+    end
+
+    # iterate through any URLs we find in a string and return a search query or nil
+    def get_search_keyword
+      !all_urls.nil? ? all_urls.collect { |_url| self.class.extract_search_query_from_valid_url(_url) }[0] : nil
+    end
+
+  end
+end

data/lib/skinny_jeans.rb

@@ -1,219 +1,65 @@
 require 'time'
 require 'benchmark'
-require 'rubygems'
-require 'sqlite3'
-require 'active_record'
 require 'zlib'
 require 'fileutils'
 require 'uri'
 require 'cgi'
-require 'skinny_jeans_string_parser'
+require 'rubygems'
+require 'active_record'
+require 'sqlite3'
+require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/string_parser")
+require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/log_parser")
 # require 'home_run'
 
-class SkinnyJeans
-
-  def self.execute(logfile_path, sqlite_db_path, path_regexp, date_regexp)
-    self.new(logfile_path, sqlite_db_path, path_regexp, date_regexp).execute
-  end
-
-  attr_accessor :hash_of_dates, :hash_of_dates_for_keywords, :last_pageview_at
-
-  def initialize(logfile_path, sqlite_db_path, path_regexp, date_regexp)
-    @logfile_path, @sqlite_db_path, @path_regexp, @date_regexp = [logfile_path, sqlite_db_path, path_regexp, date_regexp]
-    @is_gzipped = !logfile_path.to_s[/gz/].nil?
-    prepare_db
-    @hash_of_dates = {}
-    @hash_of_dates_for_keywords = {}
-    @last_datetime = nil
-  end
 
+# SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"
+module SkinnyJeans
   class SkinnyJeanDb < ActiveRecord::Base
     self.abstract_class = true
   end
-
-  def prepare_db
-    # create database if necessary
-    SQLite3::Database.new(@sqlite_db_path)
-    SkinnyJeanDb.establish_connection(:adapter => 'sqlite3', :database => @sqlite_db_path)
-    # create tables if necessary
-    if !Pageview.table_exists?
-      SkinnyJeanDb.connection.create_table(:pageviews) do |t|
-        t.column :date, :date
-        t.column :path, :string
-        t.column :pageview_count, :integer
-      end
-      # flow tight like skinny jeans with these compound indexes
-      SkinnyJeanDb.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
-      SkinnyJeanDb.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
-    end
-    if !Update.table_exists?
-      SkinnyJeanDb.connection.create_table(:updates) do |t|
-        t.column :last_pageview_at, :timestamp
-        t.column :lines_parsed, :integer
-        t.column :last_line_parsed, :string
-      end
-    end
-
-    # addition from 2010-12-06 to track search traffic specifically
-    if !PageviewKeyword.table_exists?
-      SkinnyJeanDb.connection.create_table(:pageview_keywords) do |t|
-        t.column :date, :date
-        t.column :path, :string
-        t.column :pageview_count, :integer
-        t.column :keyword, :string
-      end
-      SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :path, :keyword], :name => "date_path_keyword_index")
-      # SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :pageview_count], :name => "date_pageview_count_index")
-    end
-
-  end
-
-  def execute
-
-    lines_parsed = 0
-    last_line_parsed, last_pageview_at, lineno_of_last_line_parsed = [nil,nil,nil]
-    # last_update = Update.order("id DESC").limit(1).first
-    last_update = Update.find(:first, :order => "id DESC", :limit => 1)
-
-    # see if the last_line_parsed parsed exists in the current log file
-    # if it doesnt exist, we'll simply read anything with a timestamp greater than last_pageview_at
-    if last_update
-      last_pageview_at, last_line_parsed = last_update.last_pageview_at, last_update.last_line_parsed
-      file_reader do |line, lineno|
-        if line.to_s[0..254] == last_line_parsed.to_s[0..254]
-          lineno_of_last_line_parsed = lineno
-          break
+  class Pageview < SkinnyJeanDb;end
+  class PageviewKeyword < SkinnyJeanDb;end
+  class Update < SkinnyJeanDb;end
+
+  class << self
+    def prepare_db(sqlite_db_path)
+      # create database if necessary
+      SQLite3::Database.new(sqlite_db_path)
+      SkinnyJeanDb.establish_connection(:adapter => 'sqlite3', :database => sqlite_db_path)
+      # create tables if necessary
+      if !Pageview.table_exists?
+        SkinnyJeanDb.connection.create_table(:pageviews) do |t|
+          t.column :date, :date
+          t.column :path, :string
+          t.column :pageview_count, :integer
         end
+        # flow tight like skinny jeans with these compound indexes
+        SkinnyJeanDb.connection.add_index(:pageviews, [:date, :path], :name => "date_path_index")
+        SkinnyJeanDb.connection.add_index(:pageviews, [:date, :pageview_count], :name => "date_pageview_count_index")
       end
-      puts "last line parsed was\n#{last_line_parsed}\nat lineno #{lineno_of_last_line_parsed}"
-    end
-
-    realtime = Benchmark.realtime do
-      date_path_pairs_array = []
-      lineno = -1
-
-      file_reader do |line, index|
-        lineno += 1
-        next if lineno_of_last_line_parsed && lineno <= lineno_of_last_line_parsed
-
-        path_match = line[@path_regexp, 1]
-        next if path_match.nil?
-        date_match = line[@date_regexp, 1]
-        next if date_match.nil?
-        datetime_obj = parse_string_as_date(date_match)
-
-        next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && datetime_obj < last_pageview_at
-
-        insert_or_increment(datetime_obj, path_match, SkinnyJeansStringParser.extract_search_query(line))
-        @last_pageview_at = datetime_obj
-        last_line_parsed = line.to_s[0..254] # only 255 characters because we store it in the database
-        lines_parsed += 1
-      end
-    end
-
-    puts "completed parsing in #{realtime}"
-
-    persisted = 0
-    persisted_pageview_keywords = 0
-    realtime = Benchmark.realtime do
-
-      hash_of_dates.each do |date, hash_of_paths|
-        hash_of_paths.keys.each do |path|
-          pv = Pageview.find_or_create_by_date_and_path(date, path)
-          pv.pageview_count ||= 0
-          pv.pageview_count += hash_of_paths[path]
-          pv.save!
-          persisted += 1
+      if !Update.table_exists?
+        SkinnyJeanDb.connection.create_table(:updates) do |t|
+          t.column :last_pageview_at, :timestamp
+          t.column :lines_parsed, :integer
+          t.column :last_line_parsed, :string
         end
       end
 
-      hash_of_dates_for_keywords.each do |date, hash_of_paths|
-        hash_of_paths.keys.each do |path|
-          hash_of_paths[path].keys.each do |keyword|
-            pvk = PageviewKeyword.find_or_create_by_date_and_path_and_keyword(date, path, keyword)
-            pvk.keyword = keyword.to_s[0..254]
-            pvk.pageview_count ||= 0
-            pvk.pageview_count += hash_of_paths[path][keyword]
-            pvk.save!
-            persisted_pageview_keywords += 1
-          end
+      # addition from 2010-12-06 to track search traffic specifically
+      if !PageviewKeyword.table_exists?
+        SkinnyJeanDb.connection.create_table(:pageview_keywords) do |t|
+          t.column :date, :date
+          t.column :path, :string
+          t.column :pageview_count, :integer
+          t.column :keyword, :string
         end
+        SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :path, :keyword], :name => "date_path_keyword_index")
+        # SkinnyJeanDb.connection.add_index(:pageview_keywords, [:date, :pageview_count], :name => "date_pageview_count_index")
       end
 
     end
 
-    puts "completed persistence in #{realtime}"
-
-    Update.create!({:last_pageview_at => self.last_pageview_at, :lines_parsed => lines_parsed, :last_line_parsed => last_line_parsed.to_s[0..254]})
-
-    puts("total records in DB: #{Pageview.count}
-lines parsed this round: #{lines_parsed}
-lines persisted this round:#{persisted}
-total SkinnyJeans executions since inception: #{Update.count}")
-
-    return self
-
-  end
-
-  # copies the log file, reads it, then removes it
-  def file_reader
-
-    temp_file_path = "#{@logfile_path}.copy"
-    temp_file = FileUtils.cp(@logfile_path, temp_file_path)
-
-    if @is_gzipped
-      lineno = 0
-      Zlib::GzipReader.new(File.new(temp_file_path, "r")).each_line{|line|yield([line,lineno]);lineno+=1}
-      # Zlib::GzipReader.open(@logfile_path).each_line{|line|yield([line,lineno]);lineno+=1}
-    else
-      File.new(temp_file_path, "r").each_with_index{|line, lineno| yield([line,lineno])}
-    end
-
-    FileUtils.rm_f(temp_file_path)
-  end
-
-  def pageview;get_ar_class(Pageview);end
-  def update;get_ar_class(Update);end
-  def pageview_keyword;get_ar_class(PageviewKeyword);end
-
-  def get_ar_class(klass)
-    begin;return(klass);rescue(ActiveRecord::ConnectionNotEstablished);prepare_db;end
-  end
-
-  private
-
-  # return a ruby Time object
-  def parse_string_as_date(date_string = "02/Oct/2010:11:17:44 -0700")
-    day,month,year,hour,minute,seconds,zone = date_string.scan(/(\d{1,2})\/(\w{3,5})\/(\d{4}):(\d\d):(\d\d):(\d\d)\s(-?\d{3,4})/).flatten
-    Time.parse("#{year}-#{month}-#{day} #{hour}:#{minute}:#{seconds} #{zone}")
-  end
-
-  def insert_or_increment(_datetime_obj, _path, _search_keyword = nil)
-
-    date_string = _datetime_obj.strftime(("%Y-%m-%d"))
-
-    # data for all pageviews
-    hash_of_dates[date_string] ||= {}
-    hash_of_dates[date_string][_path] ||= 0
-    hash_of_dates[date_string][_path] += 1
-
-    return if _search_keyword.nil?
-
-    # data for just pageviews coming from search
-    hash_of_dates_for_keywords[date_string] ||= {}
-    hash_of_dates_for_keywords[date_string][_path] ||= {}
-    hash_of_dates_for_keywords[date_string][_path][_search_keyword] ||= 0
-    hash_of_dates_for_keywords[date_string][_path][_search_keyword] += 1
-
-  end
-
-  class Pageview < SkinnyJeanDb
-  end
-  class PageviewKeyword < SkinnyJeanDb
-  end
-  class Update < SkinnyJeanDb
   end
 
 end
 
-# SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"

data/skinny_jeans.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{skinny_jeans}
-  s.version = "0.5.2"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jonathan Otto"]
-  s.date = %q{2010-12-08}
+  s.date = %q{2011-01-05}
   s.email = %q{jonathan.otto@gmail.com}
   s.extra_rdoc_files = [
     "README.rdoc",
@@ -22,7 +22,8 @@ Gem::Specification.new do |s|
      "TODO",
      "VERSION",
      "lib/skinny_jeans.rb",
-     "lib/skinny_jeans_string_parser.rb",
+     "lib/skinny_jeans/log_parser.rb",
+     "lib/skinny_jeans/string_parser.rb",
      "skinny_jeans.gemspec"
   ]
   s.homepage = %q{http://github.com/jotto/skinny_jeans}

data/test/skinny_jeans_string_parser_test.rb

@@ -1,15 +1,16 @@
-require File.dirname(__FILE__) + '/../lib/skinny_jeans'
+# require File.dirname(__FILE__) + '/../lib/skinny_jeans'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/skinny_jeans")
 require 'test/unit'
 require 'pp'
-require 'fileutils'
-class SkinnyJeansStringParserTest < Test::Unit::TestCase
+
+class SkinnyJeans::StringParserTest < Test::Unit::TestCase
 
 
   def test_can_get_all_urls_from_string
     _string=<<-EOF
     98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
     EOF
-    sjsp = SkinnyJeansStringParser.new(_string)
+    sjsp = SkinnyJeans::StringParser.new(_string)
     assert_equal 1, sjsp.all_urls.size
     assert_equal "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA",
       sjsp.all_urls.first
@@ -19,7 +20,7 @@ class SkinnyJeansStringParserTest < Test::Unit::TestCase
     _string=<<-EOF
     http://www.bing.com/search?q=XPS+17+coupon&src={referrer:source?}
     EOF
-    sjsp = SkinnyJeansStringParser.new(_string)
+    sjsp = SkinnyJeans::StringParser.new(_string)
     assert_equal "xps 17 coupon", sjsp.get_search_keyword
   end
 
@@ -27,7 +28,7 @@ class SkinnyJeansStringParserTest < Test::Unit::TestCase
     _string=<<-EOF
     207.46.12.204 - - [01/Dec/2010:11:48:00 -0800] "GET /deals/skullcandy-inkd-earbuds HTTP/1.1" 200 5732 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2;  SLCC1;  .NET CLR 1.1.4322;  .NET CLR 2.0.40607;  .NET CLR 3.0.04506.648)" "-"
     EOF
-    sjsp = SkinnyJeansStringParser.new(_string)
+    sjsp = SkinnyJeans::StringParser.new(_string)
     assert_nil sjsp.all_urls
 
     assert_nil sjsp.get_search_keyword
@@ -37,8 +38,27 @@ class SkinnyJeansStringParserTest < Test::Unit::TestCase
     _string=<<-EOF
     98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
     EOF
-    sjsp = SkinnyJeansStringParser.new(_string)
+    sjsp = SkinnyJeans::StringParser.new(_string)
     assert_equal "best deals for the 4th generation ipod touch 32 gb", sjsp.get_search_keyword
   end
 
+  def test_return_param_from_url
+    assert_equal "nowai", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","drc")
+    assert_equal "iDesign Tower Stereo System".downcase, SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://us.yhs.search.yahoo.com/if?partnerid=yhs-if-freecause&fr=yhs-if-freecause&ei=UTF-8&YST_b=21&tid=61613&uid=47727219&p=iDesign Tower Stereo System","p")
+    assert_equal "yes", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","ok")
+    assert_equal "yes", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://dealzon.com/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","ok")
+    assert_equal "nowai", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://dealzon.com/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","drc")
+    
+    
+    assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.yahoo.com/search;_ylt=AqnIgbSoqn0rhe69ABoUdv.bvZx4?p=hp+coupon&toggle=1&cop=mss&ei=UTF-8&fr=yfp-t-701","p")
+    assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.yahoo.com/search;_ylt=A0oG7h6HJPtMlGIBsFyl87UF;_ylc=X1MDMjE0MjQ3ODk0OARfcgMyBGZyA3NmcARuX2dwcwMxMARvcmlnaW4Dc3ljBHF1ZXJ5A2hwIGNvdXBvbgRzYW8DMQ--?p=hp+coupon&fr=sfp&fr2=&iscqry=","p")
+    assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.ask.com/web?q=hp+coupon&search=&qsrc=0&o=0&l=dir","q")
+    assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.aol.com/aol/search?enabled_terms=&s_it=comsearch50&q=hp+coupon","q")
+    assert_equal "hp coupns", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.bing.com/search?q=hp+coupns&go=&form=QBLH&qs=n&sk=&sc=8-6","q")
+    assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&ved=0CCEQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon&ei=1Pz7TKb5AoP7lwfEoeSgBQ&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=Vs7s9a1z2Elm23NVffMJ8A","q")
+    assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&sqi=2&ved=0CCAQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon&ei=xyT7TO7kEYGKlwfT9tSPDA&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=7pnUG3YTE8sHONsIqLl1sg","q")
+    
+    assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&sqi=2&ved=0CCAQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon%20&ei=xyT7TO7kEYGKlwfT9tSPDA&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=7pnUG3YTE8sHONsIqLl1sg","q")
+  end
+
 end

data/test/skinny_jeans_test.rb

@@ -1,14 +1,17 @@
-require File.dirname(__FILE__) + '/../lib/skinny_jeans'
+
+require File.expand_path(File.dirname(__FILE__) + "/../lib/skinny_jeans")
 require 'test/unit'
 require 'pp'
-require 'fileutils'
+
 class SkinnyJeansTest < Test::Unit::TestCase
 
 
   def test_parse_pick_up_where_left_off
-    db_path = "./skinny_jeans_test.db"
+    db_path = File.expand_path(File.dirname(__FILE__) + "/skinny_jeans_test.db")
+    # db_path = "./skinny_jeans_test.db"
     FileUtils.rm(db_path) if File.exists?(db_path)
-    sj=SkinnyJeans.new(logfile_path = "small_access_log.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    _logfile_path = File.expand_path(File.dirname(__FILE__) + "/small_access_log.log")
+    sj=SkinnyJeans::LogParser.new(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
     sj.execute
     assert_equal 20, sj.pageview.count
     assert_equal 2, sj.pageview.find_by_path("flip-video").pageview_count
@@ -20,7 +23,8 @@ class SkinnyJeansTest < Test::Unit::TestCase
 # 
 # "
     # the 2nd file is the same, but with 2 additional lines for flip-video and apple-ipod-touch
-    sj=SkinnyJeans.new(logfile_path = "small_access_log_part_2.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    _logfile_path_2 = File.expand_path(File.dirname(__FILE__) + "/small_access_log_part_2.log")
+    sj=SkinnyJeans::LogParser.new(_logfile_path_2, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
     sj.execute
     assert_equal 3, sj.pageview.find_by_path("flip-video").pageview_count
     assert_equal 2, sj.pageview.find_by_path("apple-ipod-touch").pageview_count
@@ -39,8 +43,9 @@ class SkinnyJeansTest < Test::Unit::TestCase
 # 
 # "
 
+    _logfile_path_3 = File.expand_path(File.dirname(__FILE__) + "/small_access_log_part_3.log")
     # the 3rd has 1 additional line so we can ensure we can leave off on a line over 255 characters
-    sj=SkinnyJeans.new(logfile_path = "small_access_log_part_3.log", sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+    sj=SkinnyJeans::LogParser.new(_logfile_path_3, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
     sj.execute
     assert_equal 3, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
     assert_equal 3, sj.pageview.find_by_path("apple-ipod-touch").pageview_count

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: skinny_jeans
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 7
   prerelease: false
   segments: 
   - 0
-  - 5
-  - 2
-  version: 0.5.2
+  - 6
+  - 0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Jonathan Otto
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-08 00:00:00 -06:00
+date: 2011-01-05 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,7 +66,8 @@ files:
 - TODO
 - VERSION
 - lib/skinny_jeans.rb
-- lib/skinny_jeans_string_parser.rb
+- lib/skinny_jeans/log_parser.rb
+- lib/skinny_jeans/string_parser.rb
 - skinny_jeans.gemspec
 - test/skinny_jeans_string_parser_test.rb
 - test/skinny_jeans_test.rb

data/lib/skinny_jeans_string_parser.rb

@@ -1,70 +0,0 @@
-# example
-# SkinnyJeansStringParser.extract_search_query("http://search.aol.com/aol/search?enabled_terms=&s_it=comsearch50&q=cool+stuff")
-# => "cool stuff"
-
-class SkinnyJeansStringParser
-
-  def self.extract_search_query(_url)
-    self.new(_url).get_search_keyword
-  end
-
-  attr_accessor :string_value
-  def initialize(string_value)
-    @string_value = string_value
-  end
-
-  # iterate through any URLs we find in a string and return a search query or nil
-  def get_search_keyword
-    !all_urls.nil? ? all_urls.collect { |_url| extract_search_query_from_url(_url) }[0] : nil
-  end
-
-  # pre: some referring URL from google, yahoo, AOL, bing, ask
-  # post: whatever the search query was, ASCII or GTFO
-  def extract_search_query_from_url(url)
-    val = nil
-    case url
-    when /google\.com/
-      val=return_param_from_url(url, "q")
-    when /search\.yahoo\.com/
-      val=return_param_from_url(url, "p")
-    when /search\.aol\.com/
-      val=return_param_from_url(url, "q")
-    when /ask\.com/
-      val=return_param_from_url(url, "q")
-    when /bing\.com/
-      val=return_param_from_url(url, "q")
-    end
-    # whitelist of acceptable characters
-    val = val.present? && val.gsub(/[^0-9A-Za-z\s"'!@#\$%\^&\*\(\)\?\<\>\[\]:;,\.+-_=]/, '') != val ? nil : val
-    return val
-  end
-
-  # pre: like http://example.org?q=cool&fun=no, "fun"
-  # post: "no"
-  def return_param_from_url(url, param_name)
-    _uri = URI.parse(URI.encode(url))
-    if _uri.query.present?
-      _cgi = CGI.parse(_uri.query)
-      if _cgi[param_name]
-        val = unescape_string(_cgi[param_name].to_s).strip.downcase
-        return (!val.nil? && val!='' ? val : nil)
-      end
-    end
-    return nil
-  end
-
-  # find all URLs in a string that are at beginning or end of string or are tokenized by spaces
-  def all_urls
-    @all_urls ||= string_value.split(/\s+/).reject { |_string| !_string.match(/^['"]?https?:['"]?/) }.collect { |url| url.gsub(/["']/,'') }
-    @all_urls.empty? ? nil : @all_urls
-  end
-
-  private
-  def unescape_string(_string)
-    temp = _string.dup
-    temp = CGI.unescape(temp) while CGI.unescape(temp) != temp
-    temp
-  end
-
-
-end