skinny_jeans 0.6.4 → 0.6.5

data/VERSION

@@ -1 +1 @@
-0.6.4
+0.6.5

data/lib/skinny_jeans/log_parser.rb

@@ -46,12 +46,21 @@ module SkinnyJeans
           lineno += 1
           next if lineno_of_last_line_parsed && lineno <= lineno_of_last_line_parsed
 
-          path_match = line[@path_regexp, 1]
-          next if path_match.nil?
-          date_match = line[@date_regexp, 1]
-          next if date_match.nil?
-          datetime_obj = parse_string_as_date(date_match)
-
+          begin
+            path_match = line[@path_regexp, 1]
+            next if path_match.nil?
+            date_match = line[@date_regexp, 1]
+            next if date_match.nil?
+            datetime_obj = parse_string_as_date(date_match)
+          rescue ArgumentError => e
+            if e.message.match(/invalid byte sequence in UTF-8/)
+              puts "failed to parse the following line because of #{e.class.name}: #{e.message}"
+              puts line
+              next
+            else
+              raise(e)
+            end
+          end
           next if lineno_of_last_line_parsed.nil? && !last_pageview_at.nil? && datetime_obj < last_pageview_at
 
           insert_or_increment(datetime_obj, path_match, SkinnyJeans::StringParser.extract_search_query(line))
@@ -68,7 +77,9 @@ module SkinnyJeans
       realtime = Benchmark.realtime do
 
         hash_of_dates.each do |date, hash_of_paths|
-          hash_of_paths.keys.each do |path|
+          puts "on date #{date.inspect} with #{hash_of_paths.keys.size} keys"
+          hash_of_paths.keys.each_with_index do |path, index|
+            # puts "path is #{path}, #{index.to_f/hash_of_paths.keys.size.to_f}"
             pv = Pageview.find_or_create_by_date_and_path(date, path)
             pv.pageview_count ||= 0
             pv.pageview_count += hash_of_paths[path]
@@ -76,7 +87,7 @@ module SkinnyJeans
             persisted += 1
           end
         end
-
+# puts "now doing osk"
         hash_of_dates_for_keywords.each do |date, hash_of_paths|
           hash_of_paths.keys.each do |path|
             hash_of_paths[path].keys.each do |keyword|

data/test/skinny_jeans_string_parser_test.rb

@@ -5,7 +5,6 @@ require 'pp'
 
 class SkinnyJeans::StringParserTest < Test::Unit::TestCase
 
-
   def test_can_get_all_urls_from_string
     _string=<<-EOF
     98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"

data/test/skinny_jeans_test.rb

@@ -6,6 +6,13 @@ require 'pp'
 class SkinnyJeansTest < Test::Unit::TestCase
 
 
+  def test_will_work
+    db_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/skinny_jeans_deal_show.db")
+    _logfile_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/broken_access_big.log")
+    puts db_path.inspect
+    sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
+  end
+
   def test_can_parse_default_nginx_log_format
     db_path = File.expand_path(File.dirname(__FILE__) + "/skinny_jeans_test.db")
     FileUtils.rm(db_path) if File.exists?(db_path)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 6
-  - 4
-  version: 0.6.4
+  - 5
+  version: 0.6.5
 platform: ruby
 authors: 
 - Jonathan Otto
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-13 00:00:00 -06:00
+date: 2011-03-15 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency