skinny_jeans 0.6.5 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +5 -0
- data/Rakefile +1 -0
- data/VERSION +1 -1
- data/lib/skinny_jeans/log_parser.rb +26 -17
- data/lib/skinny_jeans.rb +1 -0
- data/skinny_jeans.gemspec +5 -2
- data/test/skinny_jeans_test.rb +4 -1
- metadata +28 -6
data/Rakefile
CHANGED
@@ -10,6 +10,7 @@ begin
|
|
10
10
|
s.authors = ["Jonathan Otto"]
|
11
11
|
s.add_dependency 'sqlite3-ruby', '>= 1.2.4'
|
12
12
|
s.add_dependency 'activerecord', '>= 2.3.8'
|
13
|
+
s.add_dependency 'spinner', '>= 1.0.0'
|
13
14
|
end
|
14
15
|
rescue LoadError
|
15
16
|
puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install jeweler"
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.7.0
|
@@ -77,28 +77,37 @@ module SkinnyJeans
|
|
77
77
|
realtime = Benchmark.realtime do
|
78
78
|
|
79
79
|
hash_of_dates.each do |date, hash_of_paths|
|
80
|
-
|
81
|
-
hash_of_paths.keys.
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
80
|
+
|
81
|
+
Spinner::with_spinner(:count=>hash_of_paths.keys.size, :message=>"Inserting rows into database for pageviews #{date}...") do |spin|
|
82
|
+
hash_of_paths.keys.each_with_index do |path, index|
|
83
|
+
# puts "path is #{path}, #{index.to_f/hash_of_paths.keys.size.to_f}"
|
84
|
+
pv = Pageview.find_or_create_by_date_and_path(date, path)
|
85
|
+
pv.pageview_count ||= 0
|
86
|
+
pv.pageview_count += hash_of_paths[path]
|
87
|
+
pv.save!
|
88
|
+
persisted += 1
|
89
|
+
spin.call
|
90
|
+
end
|
88
91
|
end
|
92
|
+
puts "completed pageviews date #{date.inspect} with #{hash_of_paths.keys.size} keys"
|
93
|
+
|
89
94
|
end
|
90
|
-
|
95
|
+
|
91
96
|
hash_of_dates_for_keywords.each do |date, hash_of_paths|
|
92
|
-
hash_of_paths.keys.
|
93
|
-
hash_of_paths
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
97
|
+
Spinner::with_spinner(:count=>hash_of_paths.keys.size, :message=>"Inserting rows into database for pageview_keywords #{date}...") do |spin|
|
98
|
+
hash_of_paths.keys.each do |path|
|
99
|
+
hash_of_paths[path].keys.each do |keyword|
|
100
|
+
pvk = PageviewKeyword.find_or_create_by_date_and_path_and_keyword(date, path, keyword)
|
101
|
+
pvk.keyword = keyword.to_s[0..254]
|
102
|
+
pvk.pageview_count ||= 0
|
103
|
+
pvk.pageview_count += hash_of_paths[path][keyword]
|
104
|
+
pvk.save!
|
105
|
+
persisted_pageview_keywords += 1
|
106
|
+
end
|
107
|
+
spin.call
|
100
108
|
end
|
101
109
|
end
|
110
|
+
puts "completed pageview_keywords date #{date.inspect} with #{hash_of_paths.keys.size} keys"
|
102
111
|
end
|
103
112
|
|
104
113
|
end
|
data/lib/skinny_jeans.rb
CHANGED
@@ -7,6 +7,7 @@ require 'cgi'
|
|
7
7
|
require 'rubygems'
|
8
8
|
require 'active_record'
|
9
9
|
require 'sqlite3'
|
10
|
+
require 'spinner'
|
10
11
|
require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/string_parser")
|
11
12
|
require File.expand_path(File.dirname(__FILE__) + "/skinny_jeans/log_parser")
|
12
13
|
# require 'home_run'
|
data/skinny_jeans.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{skinny_jeans}
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.7.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jonathan Otto"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-03-16}
|
13
13
|
s.description = %q{Fast webserver log parser for persisting daily pageviews per path to sqlite}
|
14
14
|
s.email = %q{jonathan.otto@gmail.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -44,13 +44,16 @@ Gem::Specification.new do |s|
|
|
44
44
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
45
45
|
s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
|
46
46
|
s.add_runtime_dependency(%q<activerecord>, [">= 2.3.8"])
|
47
|
+
s.add_runtime_dependency(%q<spinner>, [">= 1.0.0"])
|
47
48
|
else
|
48
49
|
s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
|
49
50
|
s.add_dependency(%q<activerecord>, [">= 2.3.8"])
|
51
|
+
s.add_dependency(%q<spinner>, [">= 1.0.0"])
|
50
52
|
end
|
51
53
|
else
|
52
54
|
s.add_dependency(%q<sqlite3-ruby>, [">= 1.2.4"])
|
53
55
|
s.add_dependency(%q<activerecord>, [">= 2.3.8"])
|
56
|
+
s.add_dependency(%q<spinner>, [">= 1.0.0"])
|
54
57
|
end
|
55
58
|
end
|
56
59
|
|
data/test/skinny_jeans_test.rb
CHANGED
@@ -8,7 +8,10 @@ class SkinnyJeansTest < Test::Unit::TestCase
|
|
8
8
|
|
9
9
|
def test_will_work
|
10
10
|
db_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/skinny_jeans_deal_show.db")
|
11
|
-
|
11
|
+
FileUtils.rm(db_path) if File.exists?(db_path)
|
12
|
+
# _logfile_path = File.expand_path(File.dirname(__FILE__) + "/more_sample_data/broken_access_big.log")
|
13
|
+
_logfile_path = File.expand_path("/Users/jonathanotto/dznbak/access.log.1")
|
14
|
+
|
12
15
|
puts db_path.inspect
|
13
16
|
sj=SkinnyJeans::LogParser.execute(_logfile_path, sqlite_skinny_jeans = db_path, path_regexp = /\s\/deals\/(.*)\sHTTP/, date_regexp = /\[(\d.*\d)\]/)
|
14
17
|
end
|
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: skinny_jeans
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
+
hash: 3
|
4
5
|
prerelease: false
|
5
6
|
segments:
|
6
7
|
- 0
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 0.
|
8
|
+
- 7
|
9
|
+
- 0
|
10
|
+
version: 0.7.0
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Jonathan Otto
|
@@ -14,7 +15,7 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date: 2011-03-
|
18
|
+
date: 2011-03-16 00:00:00 -05:00
|
18
19
|
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
@@ -25,6 +26,7 @@ dependencies:
|
|
25
26
|
requirements:
|
26
27
|
- - ">="
|
27
28
|
- !ruby/object:Gem::Version
|
29
|
+
hash: 23
|
28
30
|
segments:
|
29
31
|
- 1
|
30
32
|
- 2
|
@@ -40,6 +42,7 @@ dependencies:
|
|
40
42
|
requirements:
|
41
43
|
- - ">="
|
42
44
|
- !ruby/object:Gem::Version
|
45
|
+
hash: 19
|
43
46
|
segments:
|
44
47
|
- 2
|
45
48
|
- 3
|
@@ -47,6 +50,22 @@ dependencies:
|
|
47
50
|
version: 2.3.8
|
48
51
|
type: :runtime
|
49
52
|
version_requirements: *id002
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: spinner
|
55
|
+
prerelease: false
|
56
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
hash: 23
|
62
|
+
segments:
|
63
|
+
- 1
|
64
|
+
- 0
|
65
|
+
- 0
|
66
|
+
version: 1.0.0
|
67
|
+
type: :runtime
|
68
|
+
version_requirements: *id003
|
50
69
|
description: Fast webserver log parser for persisting daily pageviews per path to sqlite
|
51
70
|
email: jonathan.otto@gmail.com
|
52
71
|
executables: []
|
@@ -57,6 +76,7 @@ extra_rdoc_files:
|
|
57
76
|
- README.rdoc
|
58
77
|
- TODO
|
59
78
|
files:
|
79
|
+
- .gitignore
|
60
80
|
- README.rdoc
|
61
81
|
- Rakefile
|
62
82
|
- TODO
|
@@ -72,8 +92,8 @@ homepage: http://github.com/jotto/skinny_jeans
|
|
72
92
|
licenses: []
|
73
93
|
|
74
94
|
post_install_message:
|
75
|
-
rdoc_options:
|
76
|
-
|
95
|
+
rdoc_options:
|
96
|
+
- --charset=UTF-8
|
77
97
|
require_paths:
|
78
98
|
- lib
|
79
99
|
required_ruby_version: !ruby/object:Gem::Requirement
|
@@ -81,6 +101,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
81
101
|
requirements:
|
82
102
|
- - ">="
|
83
103
|
- !ruby/object:Gem::Version
|
104
|
+
hash: 3
|
84
105
|
segments:
|
85
106
|
- 0
|
86
107
|
version: "0"
|
@@ -89,6 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
110
|
requirements:
|
90
111
|
- - ">="
|
91
112
|
- !ruby/object:Gem::Version
|
113
|
+
hash: 3
|
92
114
|
segments:
|
93
115
|
- 0
|
94
116
|
version: "0"
|