skinny_jeans 0.8.1 → 0.9.0

data/README.rdoc

@@ -1,4 +1,4 @@
-= SKINNY JEANS LOG PARSING WITH RUBY & SQLITE FOR HIPSTERS
+= SKINNY JEANS FAST LOG PARSING WITH RUBY & SQLITE FOR HIPSTERS
 http://img696.imageshack.us/img696/75/skinnys3.jpg
 
 == EXAMPLE
@@ -30,24 +30,22 @@ http://img696.imageshack.us/img696/75/skinnys3.jpg
   sj = SkinnyJeans::LogParser::execute(logfile_path = "access.log", sqlite_skinny_jeans = "sqlite_skinny_jeans.db", path_regexp = /\s\/posts\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
   sj.pageview.where("date = '2010-10-01' and path = 'my-first-post'")
   => #<SkinnyJeans::Pageview id: 1, date: "2010-10-01", path: "my-first-post", pageview_count: 3>
-1.  NOTE: for now **you have to monkey patch the SkinnyJeans#parse_string_as_date**
+1.  NOTE: for now **you may have to monkey patch the SkinnyJeans#parse_string_as_date**
 2.  Parse oldest logs first, then run regularly against your main log, let logrotate handle the rest (skinny_jeans remembers where it left off)
 3.  ASSUMES reading log files in ascending order, keeps track of last line read so you could put it on a scheduler or cron job
-4.  access the 2 activerecord classes, sj.pageview (returns Pageview class), and sj.update
+4.  access the 2 activerecord classes, sj.pageview (returns Pageview class), and sj.update happily within your own app
 5.  enjoy the skinny jeans
 
 
-== PERFORMANCE (without organic search tracking)
-*  it parses 100,000 lines in < 2.5 seconds
-*  persists 1,000 requests with 2 compound indexes in 15 seconds, or 10 seconds with home_run c extension
-*  25,000 rows == 4 megabyte sqlite database
+== PERFORMANCE
+*  it parses 300,000 lines in < 9 seconds and
+*  persists 6,000 requests (in 2 different tables, one of the tables is for search engine referrals) with 2 compound indexes in 18 seconds
 
 == NOTES
 *  supports gzipped files
 *  creates a temp copy of the log file before parsing
 *  currently requires each line to be unique
-  *  this could be a problem if a single client manages to hit the same page more than once in less than 1 second.
-  *  this is only a problem if the last line parsed is one of the pages that was hit by the same client more than once in less than one second
+*  line uniqueness is potential problem if a single client manages to hit the same page more than once in less than 1 second AND it's the last line of a log file
 
 == LICENSE
 

data/Rakefile

@@ -8,9 +8,10 @@ begin
     s.email = "jonathan.otto@gmail.com"
     s.homepage = "http://github.com/jotto/skinny_jeans"
     s.authors = ["Jonathan Otto"]
-    s.add_dependency 'sqlite3-ruby', '>= 1.2.4'
-    s.add_dependency 'activerecord', '>= 2.3.8'
+    s.add_dependency 'sqlite3-ruby', '>= 1.3.3'
+    s.add_dependency 'activerecord', '>= 3.0.0'
     s.add_dependency 'spinner', '>= 1.0.0'
+    s.add_dependency 'home_run', '>= 1.0.1'
   end
 rescue LoadError
   puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"

data/VERSION

@@ -1 +1 @@
-0.8.1
+0.9.0

data/lib/skinny_jeans/string_parser.rb

@@ -29,7 +29,7 @@ module SkinnyJeans
           val=return_param_from_valid_url_or_path(url,"q")
         end
         # whitelist of acceptable characters
-        val = val.present? && val.gsub(/[^0-9A-Za-z\s"'!@#\$%\^&\*\(\)\?\<\>\[\]:;,\.+-_=]/, '') != val ? nil : val
+        val = !!val && val.gsub(/[^0-9A-Za-z\s"'!@#\$%\^&\*\(\)\?\<\>\[\]:;,\.+-_=]/, '') != val ? nil : val
         return val
       end
 
@@ -40,18 +40,13 @@ module SkinnyJeans
         if _uri.query.present?
           _cgi = CGI.parse(_uri.query)
           if _cgi[param_name]
-            val = unescape_string(_cgi[param_name].join).strip.downcase
+            val = URI.decode(_cgi[param_name].join).strip.downcase
             return (!val.nil? && val!='' ? val : nil)
           end
         end
         return nil
       end
 
-      def unescape_string(_string)
-        temp = _string.dup
-        temp = CGI.unescape(temp) while CGI.unescape(temp) != temp
-        temp
-      end
     end
 
     attr_accessor :string_value

data/lib/skinny_jeans.rb

@@ -8,9 +8,16 @@ require 'rubygems'
 require 'active_record'
 require 'sqlite3'
 require 'spinner'
+
+# faster URI decoding (neglible savings)
+# leaving comment for reference
+# require 'escape_utils'
+
 require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/string_parser")
 require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/log_parser")
-# require 'home_run'
+
+# faster date parsing (about a 17% speed boost)
+require 'home_run'
 
 
 # SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"
@@ -64,3 +71,25 @@ module SkinnyJeans
 
 end
 
+# class CGI
+#   # @@accept_charset="UTF-8" unless defined?(@@accept_charset)
+#   # # URL-encode a string.
+#   # #   url_encoded_string = CGI::escape("'Stop!' said Fred")
+#   # #      # => "%27Stop%21%27+said+Fred"
+#   # def CGI::escape(string)
+#   #   string.gsub(/([^ a-zA-Z0-9_.-]+)/) do
+#   #     '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
+#   #   end.tr(' ', '+')
+#   # end
+# 
+# 
+#   # URL-decode a string with encoding(optional).
+#   #   string = CGI::unescape("%27Stop%21%27+said+Fred")
+#   #      # => "'Stop!' said Fred"
+#   def CGI::unescape(string,encoding=@@accept_charset)
+#     str=string.tr('+', ' ').force_encoding(Encoding::ASCII_8BIT).gsub(/((?:%[0-9a-fA-F]{2})+)/u) do
+#       [$1.delete('%')].pack('H*')
+#     end.force_encoding(encoding)
+#     str.valid_encoding? ? str : str.force_encoding(string.encoding)
+#   end
+# end

data/skinny_jeans.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{skinny_jeans}
-  s.version = "0.8.1"
+  s.version = "0.9.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jonathan Otto"]
@@ -40,18 +40,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
-      s.add_runtime_dependency(%q<activerecord>, [">= 2.3.8"])
+      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
       s.add_runtime_dependency(%q<spinner>, [">= 1.0.0"])
+      s.add_runtime_dependency(%q<home_run>, [">= 1.0.1"])
     else
-      s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
-      s.add_dependency(%q<activerecord>, [">= 2.3.8"])
+      s.add_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
+      s.add_dependency(%q<activerecord>, [">= 3.0.0"])
       s.add_dependency(%q<spinner>, [">= 1.0.0"])
+      s.add_dependency(%q<home_run>, [">= 1.0.1"])
     end
   else
-    s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
-    s.add_dependency(%q<activerecord>, [">= 2.3.8"])
+    s.add_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
+    s.add_dependency(%q<activerecord>, [">= 3.0.0"])
     s.add_dependency(%q<spinner>, [">= 1.0.0"])
+    s.add_dependency(%q<home_run>, [">= 1.0.1"])
   end
 end
 

data/test/skinny_jeans_string_parser_test.rb

@@ -65,6 +65,7 @@ class SkinnyJeans::StringParserTest < Test::Unit::TestCase
     
     assert_equal "\"dealzon\"", SkinnyJeans::StringParser.extract_search_query("http://www.google.com/search?client=safari&rls=en&q=%22dealzon%22&ie=UTF-8&oe=UTF-8")
     assert_equal "\'dealzon\'", SkinnyJeans::StringParser.extract_search_query("http://www.google.com/search?client=safari&rls=en&q='dealzon'&ie=UTF-8&oe=UTF-8")
+    assert_equal "legos on sale %80 off", SkinnyJeans::StringParser::extract_search_query("http://www.google.com/url?sa=t&source=web&cd=8&ved=0CF0QFjAH&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Flego-mindstorms-nxt-robotics-micro-controller-brick-black&rct=j&q=legos%20on%20sale%20%2580%20off&ei=klp9TdyXEPC10QHMpujiAw&usg=AFQjCNFksmyHsu8gLq436-GpHmeu-OAcWQ")
   end
 
 end

data/test/skinny_jeans_test.rb

@@ -9,6 +9,7 @@ class SkinnyJeansTest < Test::Unit::TestCase
   def test_will_work
     db_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/skinny_jeans_deal_show.db")
     FileUtils.rm(db_path) if File.exists?(db_path)
+    # broken_access_big.log is 337,925 lines
     _logfile_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/broken_access_big.log")
     puts db_path.inspect
     sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 8
-  - 1
-  version: 0.8.1
+  - 9
+  - 0
+  version: 0.9.0
 platform: ruby
 authors: 
 - Jonathan Otto
@@ -27,9 +27,9 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 1
-        - 2
-        - 4
-        version: 1.2.4
+        - 3
+        - 3
+        version: 1.3.3
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -41,10 +41,10 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version 
         segments: 
-        - 2
         - 3
-        - 8
-        version: 2.3.8
+        - 0
+        - 0
+        version: 3.0.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -62,6 +62,21 @@ dependencies:
         version: 1.0.0
   type: :runtime
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: home_run
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 1
+        version: 1.0.1
+  type: :runtime
+  version_requirements: *id004
 description: Fast webserver log parser for persisting daily pageviews per path to sqlite
 email: jonathan.otto@gmail.com
 executables: []