skinny_jeans 0.9.2 → 0.9.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +1 -1
- data/VERSION +1 -1
- data/lib/skinny_jeans.rb +1 -1
- data/skinny_jeans.gemspec +8 -16
- metadata +43 -65
- data/test/skinny_jeans_string_parser_test.rb +0 -74
- data/test/skinny_jeans_test.rb +0 -82
data/Rakefile
CHANGED
@@ -11,7 +11,7 @@ begin
|
|
11
11
|
s.add_dependency 'sqlite3-ruby', '>= 1.3.3'
|
12
12
|
s.add_dependency 'activerecord', '>= 3.0.0'
|
13
13
|
s.add_dependency 'spinner', '>= 1.0.0'
|
14
|
-
s.add_dependency 'home_run', '>= 1.0.1'
|
14
|
+
# s.add_dependency 'home_run', '>= 1.0.1'
|
15
15
|
end
|
16
16
|
rescue LoadError
|
17
17
|
puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.9.
|
1
|
+
0.9.3
|
data/lib/skinny_jeans.rb
CHANGED
@@ -17,7 +17,7 @@ require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/string_parser")
|
|
17
17
|
require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/log_parser")
|
18
18
|
|
19
19
|
# faster date parsing (about a 17% speed boost)
|
20
|
-
require 'home_run'
|
20
|
+
# require 'home_run'
|
21
21
|
|
22
22
|
|
23
23
|
# SkinnyJeans::execute(ARGV.first) if "#{$0}".gsub(/.*\//,"") == "skinny_jeans.rb"
|
data/skinny_jeans.gemspec
CHANGED
@@ -4,14 +4,14 @@
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
|
-
s.name =
|
8
|
-
s.version = "0.9.
|
7
|
+
s.name = "skinny_jeans"
|
8
|
+
s.version = "0.9.3"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jonathan Otto"]
|
12
|
-
s.date =
|
13
|
-
s.description =
|
14
|
-
s.email =
|
12
|
+
s.date = "2011-10-26"
|
13
|
+
s.description = "Fast webserver log parser for persisting daily pageviews per path to sqlite"
|
14
|
+
s.email = "jonathan.otto@gmail.com"
|
15
15
|
s.extra_rdoc_files = [
|
16
16
|
"README.rdoc",
|
17
17
|
"TODO"
|
@@ -26,35 +26,27 @@ Gem::Specification.new do |s|
|
|
26
26
|
"lib/skinny_jeans/string_parser.rb",
|
27
27
|
"skinny_jeans.gemspec"
|
28
28
|
]
|
29
|
-
s.homepage =
|
29
|
+
s.homepage = "http://github.com/jotto/skinny_jeans"
|
30
30
|
s.require_paths = ["lib"]
|
31
|
-
s.rubygems_version =
|
32
|
-
s.summary =
|
33
|
-
s.test_files = [
|
34
|
-
"test/skinny_jeans_string_parser_test.rb",
|
35
|
-
"test/skinny_jeans_test.rb"
|
36
|
-
]
|
31
|
+
s.rubygems_version = "1.8.10"
|
32
|
+
s.summary = "Fast webserver log parser for persisting daily pageviews per path to sqlite"
|
37
33
|
|
38
34
|
if s.respond_to? :specification_version then
|
39
|
-
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
40
35
|
s.specification_version = 3
|
41
36
|
|
42
37
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
43
38
|
s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
|
44
39
|
s.add_runtime_dependency(%q<activerecord>, [">= 3.0.0"])
|
45
40
|
s.add_runtime_dependency(%q<spinner>, [">= 1.0.0"])
|
46
|
-
s.add_runtime_dependency(%q<home_run>, [">= 1.0.1"])
|
47
41
|
else
|
48
42
|
s.add_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
|
49
43
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
50
44
|
s.add_dependency(%q<spinner>, [">= 1.0.0"])
|
51
|
-
s.add_dependency(%q<home_run>, [">= 1.0.1"])
|
52
45
|
end
|
53
46
|
else
|
54
47
|
s.add_dependency(%q<sqlite3-ruby>, [">= 1.3.3"])
|
55
48
|
s.add_dependency(%q<activerecord>, [">= 3.0.0"])
|
56
49
|
s.add_dependency(%q<spinner>, [">= 1.0.0"])
|
57
|
-
s.add_dependency(%q<home_run>, [">= 1.0.1"])
|
58
50
|
end
|
59
51
|
end
|
60
52
|
|
metadata
CHANGED
@@ -1,72 +1,58 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: skinny_jeans
|
3
|
-
version: !ruby/object:Gem::Version
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.9.3
|
4
5
|
prerelease:
|
5
|
-
version: 0.9.2
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Jonathan Otto
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
dependencies:
|
16
|
-
- !ruby/object:Gem::Dependency
|
12
|
+
date: 2011-10-26 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
17
15
|
name: sqlite3-ruby
|
18
|
-
|
19
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
16
|
+
requirement: &70237257142100 !ruby/object:Gem::Requirement
|
20
17
|
none: false
|
21
|
-
requirements:
|
22
|
-
- -
|
23
|
-
- !ruby/object:Gem::Version
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
24
21
|
version: 1.3.3
|
25
22
|
type: :runtime
|
26
|
-
version_requirements: *id001
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: activerecord
|
29
23
|
prerelease: false
|
30
|
-
|
24
|
+
version_requirements: *70237257142100
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: activerecord
|
27
|
+
requirement: &70237257140200 !ruby/object:Gem::Requirement
|
31
28
|
none: false
|
32
|
-
requirements:
|
33
|
-
- -
|
34
|
-
- !ruby/object:Gem::Version
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
35
32
|
version: 3.0.0
|
36
33
|
type: :runtime
|
37
|
-
version_requirements: *id002
|
38
|
-
- !ruby/object:Gem::Dependency
|
39
|
-
name: spinner
|
40
34
|
prerelease: false
|
41
|
-
|
35
|
+
version_requirements: *70237257140200
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: spinner
|
38
|
+
requirement: &70237257139080 !ruby/object:Gem::Requirement
|
42
39
|
none: false
|
43
|
-
requirements:
|
44
|
-
- -
|
45
|
-
- !ruby/object:Gem::Version
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
46
43
|
version: 1.0.0
|
47
44
|
type: :runtime
|
48
|
-
version_requirements: *id003
|
49
|
-
- !ruby/object:Gem::Dependency
|
50
|
-
name: home_run
|
51
45
|
prerelease: false
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
- - ">="
|
56
|
-
- !ruby/object:Gem::Version
|
57
|
-
version: 1.0.1
|
58
|
-
type: :runtime
|
59
|
-
version_requirements: *id004
|
60
|
-
description: Fast webserver log parser for persisting daily pageviews per path to sqlite
|
46
|
+
version_requirements: *70237257139080
|
47
|
+
description: Fast webserver log parser for persisting daily pageviews per path to
|
48
|
+
sqlite
|
61
49
|
email: jonathan.otto@gmail.com
|
62
50
|
executables: []
|
63
|
-
|
64
51
|
extensions: []
|
65
|
-
|
66
|
-
extra_rdoc_files:
|
52
|
+
extra_rdoc_files:
|
67
53
|
- README.rdoc
|
68
54
|
- TODO
|
69
|
-
files:
|
55
|
+
files:
|
70
56
|
- README.rdoc
|
71
57
|
- Rakefile
|
72
58
|
- TODO
|
@@ -75,36 +61,28 @@ files:
|
|
75
61
|
- lib/skinny_jeans/log_parser.rb
|
76
62
|
- lib/skinny_jeans/string_parser.rb
|
77
63
|
- skinny_jeans.gemspec
|
78
|
-
- test/skinny_jeans_string_parser_test.rb
|
79
|
-
- test/skinny_jeans_test.rb
|
80
|
-
has_rdoc: true
|
81
64
|
homepage: http://github.com/jotto/skinny_jeans
|
82
65
|
licenses: []
|
83
|
-
|
84
66
|
post_install_message:
|
85
67
|
rdoc_options: []
|
86
|
-
|
87
|
-
require_paths:
|
68
|
+
require_paths:
|
88
69
|
- lib
|
89
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
70
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
90
71
|
none: false
|
91
|
-
requirements:
|
92
|
-
- -
|
93
|
-
- !ruby/object:Gem::Version
|
94
|
-
version:
|
95
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ! '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
96
77
|
none: false
|
97
|
-
requirements:
|
98
|
-
- -
|
99
|
-
- !ruby/object:Gem::Version
|
100
|
-
version:
|
78
|
+
requirements:
|
79
|
+
- - ! '>='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
101
82
|
requirements: []
|
102
|
-
|
103
83
|
rubyforge_project:
|
104
|
-
rubygems_version: 1.
|
84
|
+
rubygems_version: 1.8.10
|
105
85
|
signing_key:
|
106
86
|
specification_version: 3
|
107
87
|
summary: Fast webserver log parser for persisting daily pageviews per path to sqlite
|
108
|
-
test_files:
|
109
|
-
- test/skinny_jeans_string_parser_test.rb
|
110
|
-
- test/skinny_jeans_test.rb
|
88
|
+
test_files: []
|
@@ -1,74 +0,0 @@
|
|
1
|
-
# require File.dirname(__FILE__) + '/../lib/skinny_jeans'
|
2
|
-
require File.expand_path(File.dirname(__FILE__) + "/../lib/skinny_jeans")
|
3
|
-
require 'test/unit'
|
4
|
-
require 'pp'
|
5
|
-
|
6
|
-
class SkinnyJeans::StringParserTest < Test::Unit::TestCase
|
7
|
-
|
8
|
-
def test_can_get_all_urls_from_string
|
9
|
-
_string=<<-EOF
|
10
|
-
98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
|
11
|
-
EOF
|
12
|
-
sjsp = SkinnyJeans::StringParser.new(_string)
|
13
|
-
assert_equal 1, sjsp.all_urls.size
|
14
|
-
assert_equal "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA",
|
15
|
-
sjsp.all_urls.first
|
16
|
-
end
|
17
|
-
|
18
|
-
def test_can_parse_crappy_url
|
19
|
-
_string=<<-EOF
|
20
|
-
http://www.bing.com/search?q=XPS+17+coupon&src={referrer:source?}
|
21
|
-
EOF
|
22
|
-
sjsp = SkinnyJeans::StringParser.new(_string)
|
23
|
-
assert_equal "xps 17 coupon", sjsp.get_search_keyword
|
24
|
-
end
|
25
|
-
|
26
|
-
def test_return_nil_of_no_urls
|
27
|
-
_string=<<-EOF
|
28
|
-
207.46.12.204 - - [01/Dec/2010:11:48:00 -0800] "GET /deals/skullcandy-inkd-earbuds HTTP/1.1" 200 5732 "-" "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.2; SLCC1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 3.0.04506.648)" "-"
|
29
|
-
EOF
|
30
|
-
sjsp = SkinnyJeans::StringParser.new(_string)
|
31
|
-
assert_nil sjsp.all_urls
|
32
|
-
|
33
|
-
assert_nil sjsp.get_search_keyword
|
34
|
-
end
|
35
|
-
|
36
|
-
def test_can_get_keyword_from_url
|
37
|
-
_string=<<-EOF
|
38
|
-
98.244.200.209 - - [01/Dec/2010:11:51:26 -0800] "GET /deals/apple-ipod-touch HTTP/1.1" 200 11448 "http://www.google.com/m/search?oe=UTF-8&client=safari&hl=en&q=best+deals+for+the+4th+generation+iPod+touch+32+gb&gws_link_params=spell:1&ei=aqb2TJDBLqGutgfp862NAg&ved=0CBEQBSgA" "Mozilla/5.0 (iPod; U; CPU iPhone OS 4_1 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8B117 Safari/6531.22.7" "-"
|
39
|
-
EOF
|
40
|
-
sjsp = SkinnyJeans::StringParser.new(_string)
|
41
|
-
assert_equal "best deals for the 4th generation ipod touch 32 gb", sjsp.get_search_keyword
|
42
|
-
end
|
43
|
-
|
44
|
-
def test_return_param_from_url
|
45
|
-
assert_equal "nowai", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","drc")
|
46
|
-
assert_equal "iDesign Tower Stereo System".downcase, SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://us.yhs.search.yahoo.com/if?partnerid=yhs-if-freecause&fr=yhs-if-freecause&ei=UTF-8&YST_b=21&tid=61613&uid=47727219&p=iDesign Tower Stereo System","p")
|
47
|
-
assert_equal "yes", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","ok")
|
48
|
-
assert_equal "yes", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://dealzon.com/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","ok")
|
49
|
-
assert_equal "nowai", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://dealzon.com/deals/hp-dv6-laptop?ok=yes&yea=true&drc=nowai","drc")
|
50
|
-
|
51
|
-
|
52
|
-
assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.yahoo.com/search;_ylt=AqnIgbSoqn0rhe69ABoUdv.bvZx4?p=hp+coupon&toggle=1&cop=mss&ei=UTF-8&fr=yfp-t-701","p")
|
53
|
-
assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.yahoo.com/search;_ylt=A0oG7h6HJPtMlGIBsFyl87UF;_ylc=X1MDMjE0MjQ3ODk0OARfcgMyBGZyA3NmcARuX2dwcwMxMARvcmlnaW4Dc3ljBHF1ZXJ5A2hwIGNvdXBvbgRzYW8DMQ--?p=hp+coupon&fr=sfp&fr2=&iscqry=","p")
|
54
|
-
assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.ask.com/web?q=hp+coupon&search=&qsrc=0&o=0&l=dir","q")
|
55
|
-
assert_equal "hp coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://search.aol.com/aol/search?enabled_terms=&s_it=comsearch50&q=hp+coupon","q")
|
56
|
-
assert_equal "hp coupns", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.bing.com/search?q=hp+coupns&go=&form=QBLH&qs=n&sk=&sc=8-6","q")
|
57
|
-
assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&ved=0CCEQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon&ei=1Pz7TKb5AoP7lwfEoeSgBQ&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=Vs7s9a1z2Elm23NVffMJ8A","q")
|
58
|
-
assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&sqi=2&ved=0CCAQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon&ei=xyT7TO7kEYGKlwfT9tSPDA&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=7pnUG3YTE8sHONsIqLl1sg","q")
|
59
|
-
|
60
|
-
assert_equal "dv6 coupon", SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com/url?sa=t&source=web&cd=1&sqi=2&ved=0CCAQFjAA&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Fhp-dv6-laptop&rct=j&q=dv6%20coupon%20&ei=xyT7TO7kEYGKlwfT9tSPDA&usg=AFQjCNGFVL4PvZ59hbxkCFVdmHMayEe3UQ&sig2=7pnUG3YTE8sHONsIqLl1sg","q")
|
61
|
-
|
62
|
-
# this search term has chinese characters in it, as of 2011-04-13, we are just returning nil for them
|
63
|
-
assert_equal nil, SkinnyJeans::StringParser.return_param_from_valid_url_or_path("http://www.google.com.hk/search?q=vostro+3400+3450+%C7%F8%B1%F0&client=aff-9991&ie=GB2312&oe=utf8&hl=zh-CN&newwindow=1&channel=searchbutton","q")
|
64
|
-
|
65
|
-
end
|
66
|
-
|
67
|
-
def test_extract_search_query
|
68
|
-
|
69
|
-
assert_equal "\"dealzon\"", SkinnyJeans::StringParser.extract_search_query("http://www.google.com/search?client=safari&rls=en&q=%22dealzon%22&ie=UTF-8&oe=UTF-8")
|
70
|
-
assert_equal "\'dealzon\'", SkinnyJeans::StringParser.extract_search_query("http://www.google.com/search?client=safari&rls=en&q='dealzon'&ie=UTF-8&oe=UTF-8")
|
71
|
-
assert_equal "legos on sale %80 off", SkinnyJeans::StringParser::extract_search_query("http://www.google.com/url?sa=t&source=web&cd=8&ved=0CF0QFjAH&url=http%3A%2F%2Fdealzon.com%2Fdeals%2Flego-mindstorms-nxt-robotics-micro-controller-brick-black&rct=j&q=legos%20on%20sale%20%2580%20off&ei=klp9TdyXEPC10QHMpujiAw&usg=AFQjCNFksmyHsu8gLq436-GpHmeu-OAcWQ")
|
72
|
-
end
|
73
|
-
|
74
|
-
end
|
data/test/skinny_jeans_test.rb
DELETED
@@ -1,82 +0,0 @@
|
|
1
|
-
|
2
|
-
require File.expand_path(File.dirname(__FILE__) + "/../lib/skinny_jeans")
|
3
|
-
require 'test/unit'
|
4
|
-
require 'pp'
|
5
|
-
|
6
|
-
class SkinnyJeansTest < Test::Unit::TestCase
|
7
|
-
|
8
|
-
|
9
|
-
def test_will_work
|
10
|
-
db_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/skinny_jeans_deal_show.db")
|
11
|
-
FileUtils.rm(db_path) if File.exists?(db_path)
|
12
|
-
# broken_access_big.log is 337,925 lines
|
13
|
-
# _logfile_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/broken_access_big.log")
|
14
|
-
_logfile_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/another_broken_access_april_12.log")
|
15
|
-
puts db_path.inspect
|
16
|
-
sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
17
|
-
end
|
18
|
-
|
19
|
-
def test_can_parse_default_nginx_log_format
|
20
|
-
db_path = File.expand_path(File.dirname(__FILE__) + "/skinny_jeans_test.db")
|
21
|
-
FileUtils.rm(db_path) if File.exists?(db_path)
|
22
|
-
_logfile_path = File.expand_path(File.dirname(__FILE__) + "/small_access_log_default_params.log")
|
23
|
-
sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
24
|
-
assert_equal 4, sj.pageview.count
|
25
|
-
assert_equal 4, sj.pageview.find_all_by_date("2011-02-13").count
|
26
|
-
assert_equal 1, sj.pageview.find_by_path("tron-evolution-for-xbox-360").pageview_count
|
27
|
-
assert_equal 1, sj.pageview.find_by_path("samsung-pn50c590-50-inch-plasma-hdtv").pageview_count
|
28
|
-
assert_equal 1, sj.pageview.find_by_path("cheap-ipad-deals").pageview_count
|
29
|
-
assert_equal 1, sj.pageview.find_by_path("buffalo-drivestation-axis-2tb-external-hard-drive").pageview_count
|
30
|
-
end
|
31
|
-
|
32
|
-
|
33
|
-
def test_parse_pick_up_where_left_off
|
34
|
-
db_path = File.expand_path(File.dirname(__FILE__) + "/skinny_jeans_test.db")
|
35
|
-
# db_path = "./skinny_jeans_test.db"
|
36
|
-
FileUtils.rm(db_path) if File.exists?(db_path)
|
37
|
-
_logfile_path = File.expand_path(File.dirname(__FILE__) + "/small_access_log.log")
|
38
|
-
sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
39
|
-
# sj.execute
|
40
|
-
assert_equal 20, sj.pageview.count
|
41
|
-
assert_equal 2, sj.pageview.find_by_path("flip-video").pageview_count
|
42
|
-
assert_equal 2, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
|
43
|
-
|
44
|
-
# puts "
|
45
|
-
#
|
46
|
-
# -----
|
47
|
-
#
|
48
|
-
# "
|
49
|
-
# the 2nd file is the same, but with 2 additional lines for flip-video and apple-ipod-touch
|
50
|
-
_logfile_path_2 = File.expand_path(File.dirname(__FILE__) + "/small_access_log_part_2.log")
|
51
|
-
sj=SkinnyJeans::LogParser.new(_logfile_path_2, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
52
|
-
sj.execute
|
53
|
-
assert_equal 3, sj.pageview.find_by_path("flip-video").pageview_count
|
54
|
-
assert_equal 2, sj.pageview.find_by_path("apple-ipod-touch").pageview_count
|
55
|
-
|
56
|
-
# the last line of small_access_log_part_2 is over 1000 chars, make sure we can handle it properly
|
57
|
-
assert_equal 255, sj.update.last.last_line_parsed.size
|
58
|
-
_last_line=<<-EOF
|
59
|
-
82.31.245.117 - - [01/Dec/2010:11:52:29 -0800] "GET /deals/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control HTTP/1.1" 200 6244 "http://www.google.co.uk/imgres?imgurl=http://dealzon.com/pictures/deals/6656/large/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control.jpg%3F1264075353&imgrefurl=http://dealzon.com/deals/delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control&usg=__Y-Tkbg0EefRA5uKrR0b2gs7HqSo=&h=300&w=287&sz=11&hl=en&start=49&zoom=1&tbnid=18tsbSh1DQmf2M:&tbnh=142&tbnw=133&prev=/images%3Fq%3Dblack%2Bradiator%2Bheater%26um%3D1%26hl%3Den%26biw%3D1020%26bih%3D624%26tbs%3Disch:10%2C1872&um=1&itbs=1&iact=hc&vpx=757&vpy=67&dur=140&hovh=230&hovw=220&tx=158&ty=118&ei=Tab2TMWkCMqL4AauyI2SBw&oei=Q6b2TLPJDoW3hQfDlri5BQ&esq=4&page=4&ndsp=15&ved=1t:429,r:14,s:49&biw=1020&bih=624" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; Sky Broadband; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; InfoPath.3)" "-"
|
60
|
-
EOF
|
61
|
-
assert_equal _last_line.strip![0..254], sj.update.last.last_line_parsed
|
62
|
-
assert_equal 3, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
|
63
|
-
|
64
|
-
# puts "
|
65
|
-
#
|
66
|
-
# -----
|
67
|
-
#
|
68
|
-
# "
|
69
|
-
|
70
|
-
_logfile_path_3 = File.expand_path(File.dirname(__FILE__) + "/small_access_log_part_3.log")
|
71
|
-
# the 3rd has 1 additional line so we can ensure we can leave off on a line over 255 characters
|
72
|
-
sj=SkinnyJeans::LogParser.new(_logfile_path_3, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
73
|
-
sj.execute
|
74
|
-
assert_equal 3, sj.pageview.find_by_path("delonghi-hhp1500-safeheat-mica-panel-radiator-heater-with-thermostat-control").pageview_count
|
75
|
-
assert_equal 3, sj.pageview.find_by_path("apple-ipod-touch").pageview_count
|
76
|
-
assert_equal 3, sj.pageview_keyword.find_by_path_and_keyword("apple-ipod-touch", "best deals for the 4th generation ipod touch 32 gb").pageview_count
|
77
|
-
# PP.pp sj.pageview_keyword.all
|
78
|
-
end
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
end
|