croque 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +19 -0
- data/lib/croque.rb +13 -2
- data/lib/croque/aggregator.rb +22 -6
- data/lib/croque/monsieur.rb +17 -2
- data/lib/croque/version.rb +1 -1
- data/log/development.log +4 -0
- data/log/test.log +4 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c4395e1d7c4c85af579598fbcffdb7620700a185
|
4
|
+
data.tar.gz: 0a87659ee43c0475e137a6623065b8ead0338e0a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f8a3b409feb1f1e6d54521681d541bf1cf8ec5b088122c6add9a08f6fac57552aaed517b1964cce7427cdb01e110d4b299d17f75dfc9c8c94ef02bfbdfba0e0f
|
7
|
+
data.tar.gz: 4e4c5253a5a0638c82bf2eb3db7ca468ee59cec4c973aec544b2678e265fee9440016fbbcc44b38fa055a6b6c2095bfeadd137b5c14510e68c5bae9597fcdb14
|
data/README.md
CHANGED
@@ -91,6 +91,25 @@ monsieur.query
|
|
91
91
|
=> "tomato=delicious&kyouha=hare"
|
92
92
|
```
|
93
93
|
|
94
|
+
paginate
|
95
|
+
|
96
|
+
```ruby
|
97
|
+
Croque.ranking(Date.yesterday. page: 1, per: 50)
|
98
|
+
```
|
99
|
+
|
100
|
+
all dates whose the ranking exist
|
101
|
+
|
102
|
+
```ruby
|
103
|
+
Croque.all
|
104
|
+
=> [Sat, 21 Oct 2017]
|
105
|
+
```
|
106
|
+
|
107
|
+
total count of the ranking
|
108
|
+
|
109
|
+
```ruby
|
110
|
+
Croque.total_count(Date.yesterday)
|
111
|
+
=> 1
|
112
|
+
```
|
94
113
|
|
95
114
|
## Development
|
96
115
|
|
data/lib/croque.rb
CHANGED
@@ -41,9 +41,20 @@ module Croque
|
|
41
41
|
Croque::Aggregator.aggregate(date)
|
42
42
|
end
|
43
43
|
|
44
|
-
def
|
44
|
+
def all
|
45
|
+
# Get Aggregated List
|
46
|
+
# return date list as Array
|
47
|
+
Croque::Aggregator.all
|
48
|
+
end
|
49
|
+
|
50
|
+
def ranking(date, page: nil, per: nil)
|
45
51
|
# Get ranking as Sorted Array
|
46
|
-
|
52
|
+
# limit = 0 => all lines
|
53
|
+
Croque::Monsieur.get_list(date, page, per)
|
54
|
+
end
|
55
|
+
|
56
|
+
def total_count(date)
|
57
|
+
Croque::Monsieur.total_count(date)
|
47
58
|
end
|
48
59
|
end
|
49
60
|
end
|
data/lib/croque/aggregator.rb
CHANGED
@@ -24,7 +24,9 @@ module Croque
|
|
24
24
|
while (k-1)*linage < line_count.to_i
|
25
25
|
fragment = `head -n #{k*1000} #{file} | tail -n #{linage}`
|
26
26
|
fragment_lines = fragment.lines
|
27
|
-
lines += fragment_lines.select
|
27
|
+
lines += fragment_lines.select do |line|
|
28
|
+
line.match(date_matcher(date))
|
29
|
+
end
|
28
30
|
k += 1
|
29
31
|
end
|
30
32
|
# extract the matched line (Date)
|
@@ -62,6 +64,16 @@ module Croque
|
|
62
64
|
store_csv(ranking_path(date), data)
|
63
65
|
end
|
64
66
|
|
67
|
+
def all
|
68
|
+
paths = Dir.glob(store_path + '*')
|
69
|
+
paths = paths.select do |path|
|
70
|
+
path.match(/\d{4}\-\d{2}\-\d{2}/)
|
71
|
+
end
|
72
|
+
paths.map do |path|
|
73
|
+
Date.parse(File.basename(path))
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
65
77
|
private
|
66
78
|
def log_files
|
67
79
|
Dir::glob(dir_path + '*').select do |path|
|
@@ -73,8 +85,12 @@ module Croque
|
|
73
85
|
Croque.config.log_dir_path
|
74
86
|
end
|
75
87
|
|
88
|
+
def store_path
|
89
|
+
Croque.config.store_path
|
90
|
+
end
|
91
|
+
|
76
92
|
def ranking_path(date)
|
77
|
-
|
93
|
+
store_path.join("#{date}", "ranking.csv")
|
78
94
|
end
|
79
95
|
|
80
96
|
def log_file_matcher
|
@@ -82,7 +98,7 @@ module Croque
|
|
82
98
|
end
|
83
99
|
|
84
100
|
def remove_files(date)
|
85
|
-
path =
|
101
|
+
path = store_path.join("#{date}")
|
86
102
|
if Dir.exist?(path)
|
87
103
|
FileUtils.remove_dir(path)
|
88
104
|
end
|
@@ -92,7 +108,7 @@ module Croque
|
|
92
108
|
# matcher
|
93
109
|
matcher = convert_matcher(matcher: Croque.config.matcher)
|
94
110
|
# head
|
95
|
-
head_lines = `head -n
|
111
|
+
head_lines = `head -n 100 #{file}`
|
96
112
|
# get lines as Array
|
97
113
|
head_lines = head_lines.lines
|
98
114
|
head_line = head_lines.select do |line|
|
@@ -100,7 +116,7 @@ module Croque
|
|
100
116
|
end.first
|
101
117
|
head_date = get_date_from_line(head_line)
|
102
118
|
# tail
|
103
|
-
tail_lines = `tail -n
|
119
|
+
tail_lines = `tail -n 100 #{file}`
|
104
120
|
# get lines as Array
|
105
121
|
tail_lines = tail_lines.lines
|
106
122
|
tail_line = tail_lines.select do |line|
|
@@ -190,7 +206,7 @@ module Croque
|
|
190
206
|
end
|
191
207
|
|
192
208
|
def csv_path(date, hour)
|
193
|
-
|
209
|
+
store_path.join("#{date}", "#{hour}.csv")
|
194
210
|
end
|
195
211
|
|
196
212
|
def csv_option
|
data/lib/croque/monsieur.rb
CHANGED
@@ -46,16 +46,31 @@ module Croque
|
|
46
46
|
end
|
47
47
|
|
48
48
|
class << self
|
49
|
-
def get_list(date,
|
49
|
+
def get_list(date, page, per)
|
50
|
+
# maybe String
|
51
|
+
page_num = (page || 1).to_i
|
52
|
+
per_num = (per || total_count(date)).to_i
|
53
|
+
# get csv data
|
50
54
|
csv_data = File.open(ranking_path(date), "r").read.gsub(/\r/, "")
|
51
55
|
csv = CSV.new(csv_data)
|
52
56
|
# Sorted lines as ranking
|
53
|
-
|
57
|
+
start = ((page_num-1)*per_num)
|
58
|
+
# csv to Array
|
59
|
+
lines = csv.to_a
|
60
|
+
# start..end
|
61
|
+
lines = lines.slice(start, per_num) || []
|
62
|
+
# generate this class instance
|
63
|
+
lines.map do |line|
|
54
64
|
# line = [date, hour, uuid, processing_time (ms)]
|
55
65
|
self.new(*line)
|
56
66
|
end
|
57
67
|
end
|
58
68
|
|
69
|
+
def total_count(date)
|
70
|
+
wc_result = `wc -l #{ranking_path(date)}`
|
71
|
+
wc_result.match(/\d+/)[0].try(:to_i)
|
72
|
+
end
|
73
|
+
|
59
74
|
private
|
60
75
|
def ranking_path(date)
|
61
76
|
Croque.config.store_path.join("#{date}", "ranking.csv")
|
data/lib/croque/version.rb
CHANGED
data/log/development.log
CHANGED
@@ -1,4 +1,8 @@
|
|
1
1
|
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T06:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
3
|
+
I, [2017-10-21T06:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T06:55:28.898918 #20032] INFO -- : Completed 200 OK in 1111ms (Views: 1111.0ms)
|
5
|
+
|
2
6
|
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
7
|
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
8
|
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
data/log/test.log
CHANGED
@@ -1,4 +1,8 @@
|
|
1
1
|
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T06:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
3
|
+
I, [2017-10-21T06:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T06:55:28.898918 #20032] INFO -- : Completed 200 OK in 1111ms (Views: 1111.0ms)
|
5
|
+
|
2
6
|
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
7
|
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
8
|
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: croque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takuya Okuhara
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|