croque 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +77 -17
- data/bin/console +2 -4
- data/croque.gemspec +3 -1
- data/lib/croque.rb +44 -1
- data/lib/croque/aggregator.rb +334 -0
- data/lib/croque/monsieur.rb +80 -0
- data/lib/croque/version.rb +1 -1
- data/log/development.log +27 -0
- data/log/test.log +27 -0
- metadata +22 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3ab113d4a608bb3845958d8473d9f16bb907215b
|
4
|
+
data.tar.gz: a13e14c6925aadf4ffaa87bde6debfcb2dbd8641
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8b8ed926eed351b8a48f88896ec7f444535db5b1b486684ac3cf78825fa0b9c93ad3bc0352ab620a728542c61d57505068f48c73cb3b52744543389599696941
|
7
|
+
data.tar.gz: d9aef0bee5b2e9603985098f9cfbd01e48b3a13648f9c024b0fe374c7591e14423eb56d22074a5d318e74e85429c485fe0f8e5c9abc4ea8e0acd301f39681dce
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,43 +1,103 @@
|
|
1
1
|
# Croque
|
2
2
|
|
3
|
-
|
3
|
+
Croque is a simple aggregator of log. It will be useful for notifications of slow request.
|
4
4
|
|
5
|
-
|
5
|
+
By the way, **Croque** Monsieur is a baked or fried boiled ham and cheese sandwich. The dish originated in French cafés and bars as a quick snack.
|
6
|
+
|
7
|
+
<img src="https://user-images.githubusercontent.com/4189626/31853769-560ed0b6-b6c9-11e7-8166-8351a0eecc8e.jpg" width="200px">
|
6
8
|
|
7
9
|
## Installation
|
8
10
|
|
11
|
+
This gem is developed as a plugin for rails gem.
|
12
|
+
|
9
13
|
Add this line to your application's Gemfile:
|
10
14
|
|
11
15
|
```ruby
|
12
16
|
gem 'croque'
|
13
17
|
```
|
14
18
|
|
15
|
-
|
16
|
-
|
17
|
-
$ bundle
|
19
|
+
## Configuration
|
18
20
|
|
19
|
-
|
21
|
+
Croque's default configurations.
|
20
22
|
|
21
|
-
|
23
|
+
```ruby
|
24
|
+
Croque.configure do |config|
|
25
|
+
config.root_path = Pathname.new(Rails.root || Dir.pwd)
|
26
|
+
config.log_dir_path = config.root_path.join('log')
|
27
|
+
config.store_path = config.root_path.join('tmp', 'croque', Rails.env)
|
28
|
+
config.log_file_matcher = /#{Rails.env}.log/
|
29
|
+
config.hour_matcher = /dateThour/
|
30
|
+
config.severity_matcher = /severity/
|
31
|
+
config.matcher = /\[#{config.hour_matcher.source}:\d{2}:\d{2}\.\d+ #{config.severity_matcher.source}\]/
|
32
|
+
config.start_matcher = /\-\- : Started/
|
33
|
+
config.end_matcher = /\-\- : Completed/
|
34
|
+
config.lower_time = 1000 # ms
|
35
|
+
end
|
36
|
+
```
|
22
37
|
|
23
38
|
## Usage
|
24
39
|
|
25
|
-
|
40
|
+
Croque treats the date as a unit.
|
26
41
|
|
27
|
-
|
42
|
+
First, do aggregate.
|
28
43
|
|
29
|
-
|
44
|
+
```ruby
|
45
|
+
Croque.aggregate(Date.yesterday)
|
46
|
+
```
|
30
47
|
|
31
|
-
|
48
|
+
Then, csv files will be output to the directory pointed by store_path.
|
32
49
|
|
33
|
-
|
50
|
+
Next, get ranking as Array.
|
34
51
|
|
35
|
-
|
52
|
+
```ruby
|
53
|
+
ranking_list = Croque.ranking(Date.yesterday)
|
54
|
+
=> [
|
55
|
+
#<Croque::Monsieur:0x00007fd727979ce8 @date=Sat, 21 Oct 2017, @hour=12, @id="3441444b-a6d4-460f-a37d-821e699d7a63", @time="1200.0">,
|
56
|
+
#<Croque::Monsieur:0x00007fd727929400 @date=Sat, 21 Oct 2017, @hour=8, @id="becb857d-31f2-47ee-9029-e034e07c7f06", @time="812.0">,
|
57
|
+
#<Croque::Monsieur:0x00007fd727929400 @date=Sat, 21 Oct 2017, @hour=23, @id="c29c7e0d-a56d-468e-8ab0-636e09b44996", @time="564.0">
|
58
|
+
]
|
59
|
+
|
60
|
+
# monsieur is a ranking object
|
61
|
+
monsieur = ranking_list[0]
|
62
|
+
=> #<Croque::Monsieur:0x00007fd727979ce8 @date=Sat, 21 Oct 2017, @hour=12, @id="3441444b-a6d4-460f-a37d-821e699d7a63", @time="1200.0">
|
63
|
+
|
64
|
+
monsieur.body
|
65
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
66
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
67
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
68
|
+
=> [
|
69
|
+
"I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET \"/demo?tomato=delicious&kyouha=hare\" for 127.0.0.1 at 2017-10-21 12:55:30 +0900",
|
70
|
+
"I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML",
|
71
|
+
"I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)"
|
72
|
+
]
|
73
|
+
|
74
|
+
monsieur.views_time
|
75
|
+
=> 199.9 # ms
|
76
|
+
|
77
|
+
monsieur.active_record_time
|
78
|
+
=> 1000.1 # ms
|
79
|
+
|
80
|
+
monsieur.processing_time
|
81
|
+
=> 1200.0 # ms
|
82
|
+
|
83
|
+
monsieur.full_path
|
84
|
+
=> "/demo?tomato=delicious&kyouha=hare"
|
85
|
+
|
86
|
+
monsieur.path_info
|
87
|
+
=> "/demo"
|
88
|
+
|
89
|
+
monsieur.query
|
90
|
+
=> "tomato=delicious&kyouha=hare"
|
91
|
+
```
|
36
92
|
|
37
|
-
## License
|
38
93
|
|
39
|
-
|
94
|
+
## Development
|
40
95
|
|
41
|
-
|
96
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
97
|
+
|
98
|
+
## Contributing
|
99
|
+
|
100
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/croque. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
42
101
|
|
43
|
-
|
102
|
+
## Copyright
|
103
|
+
Copyright (c) 2017 Takuya Okuhara. Licensed under the [MIT License](http://opensource.org/licenses/MIT).
|
data/bin/console
CHANGED
@@ -7,8 +7,6 @@ require "croque"
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
9
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
10
|
|
13
|
-
require "
|
14
|
-
|
11
|
+
require "pry"
|
12
|
+
Pry.start
|
data/croque.gemspec
CHANGED
@@ -20,7 +20,9 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.
|
23
|
+
spec.add_dependency "rails", ">= 4"
|
24
|
+
|
25
|
+
spec.add_development_dependency "pry"
|
24
26
|
spec.add_development_dependency "rake"
|
25
27
|
spec.add_development_dependency "rspec"
|
26
28
|
end
|
data/lib/croque.rb
CHANGED
@@ -1,5 +1,48 @@
|
|
1
1
|
require "croque/version"
|
2
|
+
require "croque/aggregator"
|
3
|
+
require "croque/monsieur"
|
4
|
+
require 'rails'
|
5
|
+
require 'active_support'
|
6
|
+
require 'active_support/configurable'
|
7
|
+
require 'csv'
|
2
8
|
|
3
9
|
module Croque
|
4
|
-
|
10
|
+
def self.configure(&block)
|
11
|
+
yield @config ||= Croque::Configuration.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.config
|
15
|
+
@config
|
16
|
+
end
|
17
|
+
|
18
|
+
class Configuration
|
19
|
+
include ActiveSupport::Configurable
|
20
|
+
config_accessor :root_path, :log_dir_path, :store_path,
|
21
|
+
:log_file_matcher, :hour_matcher, :matcher, :severity_matcher,
|
22
|
+
:start_matcher, :end_matcher, :lower_time
|
23
|
+
end
|
24
|
+
|
25
|
+
configure do |config|
|
26
|
+
config.root_path = Pathname.new(Rails.root || Dir.pwd)
|
27
|
+
config.log_dir_path = config.root_path.join('log')
|
28
|
+
config.store_path = config.root_path.join('tmp', 'croque', Rails.env)
|
29
|
+
config.log_file_matcher = /#{Rails.env}.log/
|
30
|
+
config.hour_matcher = /dateThour/
|
31
|
+
config.severity_matcher = /severity/
|
32
|
+
config.matcher = /\[#{config.hour_matcher.source}:\d{2}:\d{2}\.\d+ #{config.severity_matcher.source}\]/
|
33
|
+
config.start_matcher = /\-\- : Started/
|
34
|
+
config.end_matcher = /\-\- : Completed/
|
35
|
+
config.lower_time = 1000 # ms
|
36
|
+
end
|
37
|
+
|
38
|
+
class << self
|
39
|
+
def aggregate(date)
|
40
|
+
Croque::Aggregator.aggregate(date)
|
41
|
+
end
|
42
|
+
|
43
|
+
def ranking(date, limit: 0)
|
44
|
+
# Get ranking as Sorted Array
|
45
|
+
Croque::Monsieur.get_list(date, limit)
|
46
|
+
end
|
47
|
+
end
|
5
48
|
end
|
@@ -0,0 +1,334 @@
|
|
1
|
+
module Croque
|
2
|
+
module Aggregator
|
3
|
+
class << self
|
4
|
+
def aggregate(date)
|
5
|
+
# remove files
|
6
|
+
remove_files(date)
|
7
|
+
# aggregate per hour
|
8
|
+
aggregate_per_hour(date)
|
9
|
+
# generate_ranking
|
10
|
+
generate_ranking(date)
|
11
|
+
end
|
12
|
+
|
13
|
+
def aggregate_per_hour(date)
|
14
|
+
# scan each file
|
15
|
+
log_files.each do |file|
|
16
|
+
# check skippable
|
17
|
+
next if skippable?(date, file)
|
18
|
+
# all lines
|
19
|
+
linage = 1000
|
20
|
+
wc_result = `wc -l #{file}`
|
21
|
+
line_count = wc_result.match(/\d+/)[0]
|
22
|
+
k = 1
|
23
|
+
lines = []
|
24
|
+
while (k-1)*linage < line_count.to_i
|
25
|
+
fragment = `head -n #{k*1000} #{file} | tail -n #{linage}`
|
26
|
+
fragment_lines = fragment.lines
|
27
|
+
lines += fragment_lines.select{ |line| line.match(date_matcher(date)) }
|
28
|
+
k += 1
|
29
|
+
end
|
30
|
+
# extract the matched line (Date)
|
31
|
+
lines = lines
|
32
|
+
hours.each do |hour|
|
33
|
+
# craete csv file
|
34
|
+
create_csv(date, hour, lines)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def generate_ranking(date)
|
40
|
+
array = []
|
41
|
+
hours.each do |hour|
|
42
|
+
# csv data
|
43
|
+
path = csv_path(date, hour)
|
44
|
+
# next if no file
|
45
|
+
next unless File.exist?(path)
|
46
|
+
csv_data = File.open(path, "r").read.gsub(/\r/, "")
|
47
|
+
csv = CSV.new(csv_data)
|
48
|
+
csv.to_a.each do |line|
|
49
|
+
uuid = line[0]
|
50
|
+
processing_time = line[1].to_f
|
51
|
+
# next if processing_time < config.lower_time
|
52
|
+
next if low?(processing_time)
|
53
|
+
array << [date, hour, uuid, processing_time]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
# Processing Time Desc
|
57
|
+
array = array.sort{ |a, b| b[3] <=> a[3] }
|
58
|
+
# Generate CSV
|
59
|
+
data = CSV.generate("", csv_option) do |csv|
|
60
|
+
array.each{ |line| csv << line }
|
61
|
+
end
|
62
|
+
store_csv(ranking_path(date), data)
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
def log_files
|
67
|
+
Dir::glob(dir_path + '*').select do |path|
|
68
|
+
path.match(log_file_matcher)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def dir_path
|
73
|
+
Croque.config.log_dir_path
|
74
|
+
end
|
75
|
+
|
76
|
+
def ranking_path(date)
|
77
|
+
Croque.config.store_path.join("#{date}", "ranking.csv")
|
78
|
+
end
|
79
|
+
|
80
|
+
def log_file_matcher
|
81
|
+
Croque.config.log_file_matcher
|
82
|
+
end
|
83
|
+
|
84
|
+
def remove_files(date)
|
85
|
+
path = Croque.config.store_path.join("#{date}")
|
86
|
+
if Dir.exist?(path)
|
87
|
+
FileUtils.remove_dir(path)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def skippable?(date, file)
|
92
|
+
# matcher
|
93
|
+
matcher = convert_matcher(matcher: Croque.config.matcher)
|
94
|
+
# head
|
95
|
+
head_lines = `head -n 10 #{file}`
|
96
|
+
# get lines as Array
|
97
|
+
head_lines = head_lines.lines
|
98
|
+
head_line = head_lines.select do |line|
|
99
|
+
line.match(matcher)
|
100
|
+
end.first
|
101
|
+
head_date = get_date_from_line(head_line)
|
102
|
+
# tail
|
103
|
+
tail_lines = `tail -n 10 #{file}`
|
104
|
+
# get lines as Array
|
105
|
+
tail_lines = tail_lines.lines
|
106
|
+
tail_line = tail_lines.select do |line|
|
107
|
+
line.match(matcher)
|
108
|
+
end.last
|
109
|
+
tail_date = get_date_from_line(tail_line)
|
110
|
+
# include date during range
|
111
|
+
return !(head_date && tail_date && (head_date..tail_date).include?(date))
|
112
|
+
end
|
113
|
+
|
114
|
+
def get_date_from_line(line)
|
115
|
+
if line.present?
|
116
|
+
match = line.match(/\d{4}\-\d{2}\-\d{2}/)
|
117
|
+
if match
|
118
|
+
begin
|
119
|
+
Date.parse(match[0])
|
120
|
+
rescue
|
121
|
+
nil
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def date_matcher(date)
|
128
|
+
convert_matcher(
|
129
|
+
matcher: Croque.config.matcher,
|
130
|
+
date: date
|
131
|
+
)
|
132
|
+
end
|
133
|
+
|
134
|
+
def hour_matcher(hour)
|
135
|
+
convert_matcher(
|
136
|
+
matcher: Croque.config.hour_matcher,
|
137
|
+
hour: hour
|
138
|
+
)
|
139
|
+
end
|
140
|
+
|
141
|
+
def severity_matcher(date, severity)
|
142
|
+
convert_matcher(
|
143
|
+
matcher: Croque.config.matcher,
|
144
|
+
severity: severity
|
145
|
+
)
|
146
|
+
end
|
147
|
+
|
148
|
+
def start_matcher
|
149
|
+
Croque.config.start_matcher
|
150
|
+
end
|
151
|
+
|
152
|
+
def end_matcher
|
153
|
+
Croque.config.end_matcher
|
154
|
+
end
|
155
|
+
|
156
|
+
def convert_matcher(matcher:, date: nil, hour: nil, severity: nil)
|
157
|
+
# Regexp => String
|
158
|
+
matcher = matcher.source
|
159
|
+
# date => XXXX-XX-XX
|
160
|
+
date = if date
|
161
|
+
date.to_s
|
162
|
+
else
|
163
|
+
"\\d{4}-\\d{2}-\\d{2}"
|
164
|
+
end.gsub(/\-/, "\\-")
|
165
|
+
# hour = format("%02d", hour)
|
166
|
+
hour = if hour
|
167
|
+
format("%02d", hour)
|
168
|
+
else
|
169
|
+
"\\d{2}"
|
170
|
+
end
|
171
|
+
severity = if severity
|
172
|
+
severity
|
173
|
+
else
|
174
|
+
"#\\d+"
|
175
|
+
end
|
176
|
+
# replace particular string
|
177
|
+
matcher = matcher.gsub(/severity/, severity)
|
178
|
+
matcher = matcher.gsub(/hour/, hour)
|
179
|
+
matcher = matcher.gsub(/date/, date)
|
180
|
+
# String => Regexp
|
181
|
+
Regexp.new(matcher)
|
182
|
+
end
|
183
|
+
|
184
|
+
def hours
|
185
|
+
(0..23).to_a
|
186
|
+
end
|
187
|
+
|
188
|
+
def csv_path(date, hour)
|
189
|
+
Croque.config.store_path.join("#{date}", "#{hour}.csv")
|
190
|
+
end
|
191
|
+
|
192
|
+
def csv_option
|
193
|
+
{
|
194
|
+
row_sep: "\r\n",
|
195
|
+
headers: false,
|
196
|
+
write_headers: true,
|
197
|
+
force_quotes: true
|
198
|
+
}
|
199
|
+
end
|
200
|
+
|
201
|
+
def headers
|
202
|
+
[
|
203
|
+
"Line ID", # 0
|
204
|
+
"Processing Time (ms)", # 1
|
205
|
+
"Views Time (ms)", # 2
|
206
|
+
"ActiveRecord Time (ms)", # 3
|
207
|
+
"Full Path", # 4
|
208
|
+
"Path Info", # 5
|
209
|
+
"Params", # 6
|
210
|
+
"Body" # 7
|
211
|
+
]
|
212
|
+
end
|
213
|
+
|
214
|
+
def create_csv(date, hour, lines)
|
215
|
+
# extract the matched line (Hour)
|
216
|
+
path = csv_path(date, hour)
|
217
|
+
lines_per_hour = lines.select{ |line| line.match(hour_matcher(hour)) }
|
218
|
+
# get start line of request
|
219
|
+
start_indexes = get_start_indexes(lines_per_hour)
|
220
|
+
data = CSV.generate("", csv_option) do |csv|
|
221
|
+
start_indexes.each do |start_index|
|
222
|
+
values = []
|
223
|
+
start_line = lines_per_hour[start_index]
|
224
|
+
severity = get_severity(start_line)
|
225
|
+
end_index = get_end_index(date, severity, start_index, lines_per_hour)
|
226
|
+
if end_index
|
227
|
+
# Line ID
|
228
|
+
values << SecureRandom.uuid
|
229
|
+
# get End Line
|
230
|
+
end_line = lines_per_hour[end_index]
|
231
|
+
# Processing Time
|
232
|
+
values << get_processing_time(end_line)
|
233
|
+
# Views Time
|
234
|
+
values << get_views_time(end_line)
|
235
|
+
# ActiveRecord Time
|
236
|
+
values << get_active_record_time(end_line)
|
237
|
+
# Full path
|
238
|
+
full_path = get_full_path(start_line)
|
239
|
+
values << full_path
|
240
|
+
# Path Info
|
241
|
+
values << get_path_info(full_path)
|
242
|
+
# Params
|
243
|
+
values << get_params(full_path)
|
244
|
+
# Body
|
245
|
+
lines_per_severity = get_lines_per_severity(date, start_index, end_index, severity, lines_per_hour)
|
246
|
+
values << lines_per_severity.join("\t")
|
247
|
+
# values to CSV
|
248
|
+
csv << values
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
store_csv(path, data)
|
253
|
+
end
|
254
|
+
|
255
|
+
def store_csv(path, data)
|
256
|
+
# make dirctroy
|
257
|
+
unless Dir.exist?(File.dirname(path))
|
258
|
+
FileUtils.mkdir_p(File.dirname(path))
|
259
|
+
end
|
260
|
+
File.open(path, 'a') do |f|
|
261
|
+
f.write data
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def get_start_indexes(lines)
|
266
|
+
# map index only matched line
|
267
|
+
lines.map.with_index do |line, index|
|
268
|
+
index if line.match(start_matcher)
|
269
|
+
end.compact
|
270
|
+
end
|
271
|
+
|
272
|
+
def get_severity(line)
|
273
|
+
line.match(/#\d+/)[0]
|
274
|
+
end
|
275
|
+
|
276
|
+
def get_end_index(date, severity, start_index, lines)
|
277
|
+
# end line = first of matched lines
|
278
|
+
lines.map.with_index do |line, index|
|
279
|
+
index if start_index < index && line.match(end_matcher) &&
|
280
|
+
line.match(severity_matcher(date, severity))
|
281
|
+
end.compact.first
|
282
|
+
end
|
283
|
+
|
284
|
+
def get_lines_per_severity(date, start_index, end_index, severity, lines)
|
285
|
+
lines[start_index..end_index].select do |line|
|
286
|
+
line.match(severity_matcher(date, severity))
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
def get_processing_time(line)
|
291
|
+
match = line.match(/([1-9]\d*|0)(\.\d+)?ms/)
|
292
|
+
if match
|
293
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
294
|
+
else
|
295
|
+
0
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
def get_views_time(line)
|
300
|
+
match = line.match(/Views: ([1-9]\d*|0)(\.\d+)?ms/)
|
301
|
+
if match
|
302
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
303
|
+
else
|
304
|
+
0
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
def get_active_record_time(line)
|
309
|
+
match = line.match(/ActiveRecord: ([1-9]\d*|0)(\.\d+)?ms/)
|
310
|
+
if match
|
311
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
312
|
+
else
|
313
|
+
0
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
def get_full_path(line)
|
318
|
+
line.match(/\".*\"/)[0].gsub(/\"/, '')
|
319
|
+
end
|
320
|
+
|
321
|
+
def get_path_info(full_path)
|
322
|
+
URI.parse("http://example.com#{full_path}").path
|
323
|
+
end
|
324
|
+
|
325
|
+
def get_params(full_path)
|
326
|
+
URI.parse("http://example.com#{full_path}").query
|
327
|
+
end
|
328
|
+
|
329
|
+
def low?(time)
|
330
|
+
time < Croque.config.lower_time
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Croque
|
2
|
+
class Monsieur
|
3
|
+
|
4
|
+
attr_accessor :date, :hour, :id, :time, :line
|
5
|
+
|
6
|
+
def initialize(date, hour, id, time)
|
7
|
+
self.date = Date.parse(date)
|
8
|
+
self.hour = hour.to_i
|
9
|
+
self.id = id
|
10
|
+
self.time = time
|
11
|
+
end
|
12
|
+
|
13
|
+
def body
|
14
|
+
# return Array
|
15
|
+
lines = get_line[7].split("\t")
|
16
|
+
lines = lines.map{ |line| line.strip }
|
17
|
+
lines.each do |line|
|
18
|
+
print "#{line}\n"
|
19
|
+
end
|
20
|
+
lines = lines.map{ |line| line.gsub(/\e\[\d+m/, '') }
|
21
|
+
lines
|
22
|
+
end
|
23
|
+
|
24
|
+
def views_time
|
25
|
+
get_line[2].to_f
|
26
|
+
end
|
27
|
+
|
28
|
+
def active_record_time
|
29
|
+
get_line[3].to_f
|
30
|
+
end
|
31
|
+
|
32
|
+
def processing_time
|
33
|
+
self.time.to_f
|
34
|
+
end
|
35
|
+
|
36
|
+
def full_path
|
37
|
+
URI.unescape(get_line[4])
|
38
|
+
end
|
39
|
+
|
40
|
+
def path_info
|
41
|
+
get_line[5]
|
42
|
+
end
|
43
|
+
|
44
|
+
def query
|
45
|
+
URI.unescape(get_line[6])
|
46
|
+
end
|
47
|
+
|
48
|
+
class << self
|
49
|
+
def get_list(date, limit)
|
50
|
+
csv_data = File.open(ranking_path(date), "r").read.gsub(/\r/, "")
|
51
|
+
csv = CSV.new(csv_data)
|
52
|
+
# Sorted lines as ranking
|
53
|
+
csv.to_a[0..(limit-1)].map do |line|
|
54
|
+
# line = [date, hour, uuid, processing_time (ms)]
|
55
|
+
self.new(*line)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
def ranking_path(date)
|
61
|
+
Croque.config.store_path.join("#{date}", "ranking.csv")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
def get_line
|
67
|
+
self.line ||= if File.exist?(csv_path)
|
68
|
+
csv_data = File.open(csv_path, "r").read.gsub(/\r/, "")
|
69
|
+
csv = CSV.new(csv_data)
|
70
|
+
csv.to_a.find{ |line| line[0] == self.id }
|
71
|
+
else
|
72
|
+
nil
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def csv_path
|
77
|
+
Croque.config.store_path.join("#{self.date}", "#{self.hour}.csv")
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
data/lib/croque/version.rb
CHANGED
data/log/development.log
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
|
+
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
5
|
+
I, [2017-10-21T12:53:05.807962 #11178] INFO -- : Completed 200 OK in 241ms (Views: 8.4ms)
|
6
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
7
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
8
|
+
|
9
|
+
I, [2017-10-22T08:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
10
|
+
I, [2017-10-22T08:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
11
|
+
I, [2017-10-22T08:55:28.898918 #20032] INFO -- : Completed 200 OK in 6ms (Views: 8.0ms)
|
12
|
+
|
13
|
+
I, [2017-10-23T21:55:29.579666 #90113] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:29 +0900
|
14
|
+
I, [2017-10-23T21:55:29.580536 #90113] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
15
|
+
I, [2017-10-23T21:55:29.587382 #90113] INFO -- : Completed 200 OK in 7ms (Views: 7.3ms)
|
16
|
+
|
17
|
+
I, [2017-10-24T05:55:30.531547 #70212] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
18
|
+
I, [2017-10-24T05:55:30.532415 #70212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
19
|
+
I, [2017-10-24T05:55:30.539637 #70212] INFO -- : Completed 200 OK in 7ms (Views: 10.3ms)
|
20
|
+
|
21
|
+
I, [2017-10-25T00:55:30.531547 #92121] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
22
|
+
I, [2017-10-25T00:55:30.532415 #92121] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
23
|
+
I, [2017-10-25T00:55:30.539637 #92121] INFO -- : Completed 200 OK in 7ms (Views: 100.3ms)
|
24
|
+
|
25
|
+
I, [2017-10-26T16:55:30.531547 #89211] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
26
|
+
I, [2017-10-26T16:55:30.532415 #89211] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
27
|
+
I, [2017-10-26T16:55:30.539637 #89211] INFO -- : Completed 200 OK in 7ms (Views: 102.3ms)
|
data/log/test.log
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
|
+
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
5
|
+
I, [2017-10-21T12:53:05.807962 #11178] INFO -- : Completed 200 OK in 241ms (Views: 8.4ms)
|
6
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
7
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
8
|
+
|
9
|
+
I, [2017-10-22T08:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
10
|
+
I, [2017-10-22T08:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
11
|
+
I, [2017-10-22T08:55:28.898918 #20032] INFO -- : Completed 200 OK in 6ms (Views: 8.0ms)
|
12
|
+
|
13
|
+
I, [2017-10-23T21:55:29.579666 #90113] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:29 +0900
|
14
|
+
I, [2017-10-23T21:55:29.580536 #90113] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
15
|
+
I, [2017-10-23T21:55:29.587382 #90113] INFO -- : Completed 200 OK in 7ms (Views: 7.3ms)
|
16
|
+
|
17
|
+
I, [2017-10-24T05:55:30.531547 #70212] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
18
|
+
I, [2017-10-24T05:55:30.532415 #70212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
19
|
+
I, [2017-10-24T05:55:30.539637 #70212] INFO -- : Completed 200 OK in 7ms (Views: 10.3ms)
|
20
|
+
|
21
|
+
I, [2017-10-25T00:55:30.531547 #92121] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
22
|
+
I, [2017-10-25T00:55:30.532415 #92121] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
23
|
+
I, [2017-10-25T00:55:30.539637 #92121] INFO -- : Completed 200 OK in 7ms (Views: 100.3ms)
|
24
|
+
|
25
|
+
I, [2017-10-26T16:55:30.531547 #89211] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
26
|
+
I, [2017-10-26T16:55:30.532415 #89211] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
27
|
+
I, [2017-10-26T16:55:30.539637 #89211] INFO -- : Completed 200 OK in 7ms (Views: 102.3ms)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: croque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takuya Okuhara
|
@@ -14,16 +14,30 @@ dependencies:
|
|
14
14
|
name: rails
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '4'
|
20
|
-
type: :
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pry
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: rake
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -70,7 +84,11 @@ files:
|
|
70
84
|
- bin/setup
|
71
85
|
- croque.gemspec
|
72
86
|
- lib/croque.rb
|
87
|
+
- lib/croque/aggregator.rb
|
88
|
+
- lib/croque/monsieur.rb
|
73
89
|
- lib/croque/version.rb
|
90
|
+
- log/development.log
|
91
|
+
- log/test.log
|
74
92
|
homepage: https://github.com/okutaku0507/croque
|
75
93
|
licenses:
|
76
94
|
- MIT
|