croque 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +77 -17
- data/bin/console +2 -4
- data/croque.gemspec +3 -1
- data/lib/croque.rb +44 -1
- data/lib/croque/aggregator.rb +334 -0
- data/lib/croque/monsieur.rb +80 -0
- data/lib/croque/version.rb +1 -1
- data/log/development.log +27 -0
- data/log/test.log +27 -0
- metadata +22 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3ab113d4a608bb3845958d8473d9f16bb907215b
|
4
|
+
data.tar.gz: a13e14c6925aadf4ffaa87bde6debfcb2dbd8641
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8b8ed926eed351b8a48f88896ec7f444535db5b1b486684ac3cf78825fa0b9c93ad3bc0352ab620a728542c61d57505068f48c73cb3b52744543389599696941
|
7
|
+
data.tar.gz: d9aef0bee5b2e9603985098f9cfbd01e48b3a13648f9c024b0fe374c7591e14423eb56d22074a5d318e74e85429c485fe0f8e5c9abc4ea8e0acd301f39681dce
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,43 +1,103 @@
|
|
1
1
|
# Croque
|
2
2
|
|
3
|
-
|
3
|
+
Croque is a simple aggregator of log. It will be useful for notifications of slow request.
|
4
4
|
|
5
|
-
|
5
|
+
By the way, **Croque** Monsieur is a baked or fried boiled ham and cheese sandwich. The dish originated in French cafés and bars as a quick snack.
|
6
|
+
|
7
|
+
<img src="https://user-images.githubusercontent.com/4189626/31853769-560ed0b6-b6c9-11e7-8166-8351a0eecc8e.jpg" width="200px">
|
6
8
|
|
7
9
|
## Installation
|
8
10
|
|
11
|
+
This gem is developed as a plugin for rails gem.
|
12
|
+
|
9
13
|
Add this line to your application's Gemfile:
|
10
14
|
|
11
15
|
```ruby
|
12
16
|
gem 'croque'
|
13
17
|
```
|
14
18
|
|
15
|
-
|
16
|
-
|
17
|
-
$ bundle
|
19
|
+
## Configuration
|
18
20
|
|
19
|
-
|
21
|
+
Croque's default configurations.
|
20
22
|
|
21
|
-
|
23
|
+
```ruby
|
24
|
+
Croque.configure do |config|
|
25
|
+
config.root_path = Pathname.new(Rails.root || Dir.pwd)
|
26
|
+
config.log_dir_path = config.root_path.join('log')
|
27
|
+
config.store_path = config.root_path.join('tmp', 'croque', Rails.env)
|
28
|
+
config.log_file_matcher = /#{Rails.env}.log/
|
29
|
+
config.hour_matcher = /dateThour/
|
30
|
+
config.severity_matcher = /severity/
|
31
|
+
config.matcher = /\[#{config.hour_matcher.source}:\d{2}:\d{2}\.\d+ #{config.severity_matcher.source}\]/
|
32
|
+
config.start_matcher = /\-\- : Started/
|
33
|
+
config.end_matcher = /\-\- : Completed/
|
34
|
+
config.lower_time = 1000 # ms
|
35
|
+
end
|
36
|
+
```
|
22
37
|
|
23
38
|
## Usage
|
24
39
|
|
25
|
-
|
40
|
+
Croque treats the date as a unit.
|
26
41
|
|
27
|
-
|
42
|
+
First, do aggregate.
|
28
43
|
|
29
|
-
|
44
|
+
```ruby
|
45
|
+
Croque.aggregate(Date.yesterday)
|
46
|
+
```
|
30
47
|
|
31
|
-
|
48
|
+
Then, csv files will be output to the directory pointed by store_path.
|
32
49
|
|
33
|
-
|
50
|
+
Next, get ranking as Array.
|
34
51
|
|
35
|
-
|
52
|
+
```ruby
|
53
|
+
ranking_list = Croque.ranking(Date.yesterday)
|
54
|
+
=> [
|
55
|
+
#<Croque::Monsieur:0x00007fd727979ce8 @date=Sat, 21 Oct 2017, @hour=12, @id="3441444b-a6d4-460f-a37d-821e699d7a63", @time="1200.0">,
|
56
|
+
#<Croque::Monsieur:0x00007fd727929400 @date=Sat, 21 Oct 2017, @hour=8, @id="becb857d-31f2-47ee-9029-e034e07c7f06", @time="812.0">,
|
57
|
+
#<Croque::Monsieur:0x00007fd727929400 @date=Sat, 21 Oct 2017, @hour=23, @id="c29c7e0d-a56d-468e-8ab0-636e09b44996", @time="564.0">
|
58
|
+
]
|
59
|
+
|
60
|
+
# monsieur is a ranking object
|
61
|
+
monsieur = ranking_list[0]
|
62
|
+
=> #<Croque::Monsieur:0x00007fd727979ce8 @date=Sat, 21 Oct 2017, @hour=12, @id="3441444b-a6d4-460f-a37d-821e699d7a63", @time="1200.0">
|
63
|
+
|
64
|
+
monsieur.body
|
65
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
66
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
67
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
68
|
+
=> [
|
69
|
+
"I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET \"/demo?tomato=delicious&kyouha=hare\" for 127.0.0.1 at 2017-10-21 12:55:30 +0900",
|
70
|
+
"I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML",
|
71
|
+
"I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)"
|
72
|
+
]
|
73
|
+
|
74
|
+
monsieur.views_time
|
75
|
+
=> 199.9 # ms
|
76
|
+
|
77
|
+
monsieur.active_record_time
|
78
|
+
=> 1000.1 # ms
|
79
|
+
|
80
|
+
monsieur.processing_time
|
81
|
+
=> 1200.0 # ms
|
82
|
+
|
83
|
+
monsieur.full_path
|
84
|
+
=> "/demo?tomato=delicious&kyouha=hare"
|
85
|
+
|
86
|
+
monsieur.path_info
|
87
|
+
=> "/demo"
|
88
|
+
|
89
|
+
monsieur.query
|
90
|
+
=> "tomato=delicious&kyouha=hare"
|
91
|
+
```
|
36
92
|
|
37
|
-
## License
|
38
93
|
|
39
|
-
|
94
|
+
## Development
|
40
95
|
|
41
|
-
|
96
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
97
|
+
|
98
|
+
## Contributing
|
99
|
+
|
100
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/croque. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
42
101
|
|
43
|
-
|
102
|
+
## Copyright
|
103
|
+
Copyright (c) 2017 Takuya Okuhara. Licensed under the [MIT License](http://opensource.org/licenses/MIT).
|
data/bin/console
CHANGED
@@ -7,8 +7,6 @@ require "croque"
|
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
8
8
|
|
9
9
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
10
|
|
13
|
-
require "
|
14
|
-
|
11
|
+
require "pry"
|
12
|
+
Pry.start
|
data/croque.gemspec
CHANGED
@@ -20,7 +20,9 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
21
|
spec.require_paths = ["lib"]
|
22
22
|
|
23
|
-
spec.
|
23
|
+
spec.add_dependency "rails", ">= 4"
|
24
|
+
|
25
|
+
spec.add_development_dependency "pry"
|
24
26
|
spec.add_development_dependency "rake"
|
25
27
|
spec.add_development_dependency "rspec"
|
26
28
|
end
|
data/lib/croque.rb
CHANGED
@@ -1,5 +1,48 @@
|
|
1
1
|
require "croque/version"
|
2
|
+
require "croque/aggregator"
|
3
|
+
require "croque/monsieur"
|
4
|
+
require 'rails'
|
5
|
+
require 'active_support'
|
6
|
+
require 'active_support/configurable'
|
7
|
+
require 'csv'
|
2
8
|
|
3
9
|
module Croque
|
4
|
-
|
10
|
+
def self.configure(&block)
|
11
|
+
yield @config ||= Croque::Configuration.new
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.config
|
15
|
+
@config
|
16
|
+
end
|
17
|
+
|
18
|
+
class Configuration
|
19
|
+
include ActiveSupport::Configurable
|
20
|
+
config_accessor :root_path, :log_dir_path, :store_path,
|
21
|
+
:log_file_matcher, :hour_matcher, :matcher, :severity_matcher,
|
22
|
+
:start_matcher, :end_matcher, :lower_time
|
23
|
+
end
|
24
|
+
|
25
|
+
configure do |config|
|
26
|
+
config.root_path = Pathname.new(Rails.root || Dir.pwd)
|
27
|
+
config.log_dir_path = config.root_path.join('log')
|
28
|
+
config.store_path = config.root_path.join('tmp', 'croque', Rails.env)
|
29
|
+
config.log_file_matcher = /#{Rails.env}.log/
|
30
|
+
config.hour_matcher = /dateThour/
|
31
|
+
config.severity_matcher = /severity/
|
32
|
+
config.matcher = /\[#{config.hour_matcher.source}:\d{2}:\d{2}\.\d+ #{config.severity_matcher.source}\]/
|
33
|
+
config.start_matcher = /\-\- : Started/
|
34
|
+
config.end_matcher = /\-\- : Completed/
|
35
|
+
config.lower_time = 1000 # ms
|
36
|
+
end
|
37
|
+
|
38
|
+
class << self
|
39
|
+
def aggregate(date)
|
40
|
+
Croque::Aggregator.aggregate(date)
|
41
|
+
end
|
42
|
+
|
43
|
+
def ranking(date, limit: 0)
|
44
|
+
# Get ranking as Sorted Array
|
45
|
+
Croque::Monsieur.get_list(date, limit)
|
46
|
+
end
|
47
|
+
end
|
5
48
|
end
|
@@ -0,0 +1,334 @@
|
|
1
|
+
module Croque
|
2
|
+
module Aggregator
|
3
|
+
class << self
|
4
|
+
def aggregate(date)
|
5
|
+
# remove files
|
6
|
+
remove_files(date)
|
7
|
+
# aggregate per hour
|
8
|
+
aggregate_per_hour(date)
|
9
|
+
# generate_ranking
|
10
|
+
generate_ranking(date)
|
11
|
+
end
|
12
|
+
|
13
|
+
def aggregate_per_hour(date)
|
14
|
+
# scan each file
|
15
|
+
log_files.each do |file|
|
16
|
+
# check skippable
|
17
|
+
next if skippable?(date, file)
|
18
|
+
# all lines
|
19
|
+
linage = 1000
|
20
|
+
wc_result = `wc -l #{file}`
|
21
|
+
line_count = wc_result.match(/\d+/)[0]
|
22
|
+
k = 1
|
23
|
+
lines = []
|
24
|
+
while (k-1)*linage < line_count.to_i
|
25
|
+
fragment = `head -n #{k*1000} #{file} | tail -n #{linage}`
|
26
|
+
fragment_lines = fragment.lines
|
27
|
+
lines += fragment_lines.select{ |line| line.match(date_matcher(date)) }
|
28
|
+
k += 1
|
29
|
+
end
|
30
|
+
# extract the matched line (Date)
|
31
|
+
lines = lines
|
32
|
+
hours.each do |hour|
|
33
|
+
# craete csv file
|
34
|
+
create_csv(date, hour, lines)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def generate_ranking(date)
|
40
|
+
array = []
|
41
|
+
hours.each do |hour|
|
42
|
+
# csv data
|
43
|
+
path = csv_path(date, hour)
|
44
|
+
# next if no file
|
45
|
+
next unless File.exist?(path)
|
46
|
+
csv_data = File.open(path, "r").read.gsub(/\r/, "")
|
47
|
+
csv = CSV.new(csv_data)
|
48
|
+
csv.to_a.each do |line|
|
49
|
+
uuid = line[0]
|
50
|
+
processing_time = line[1].to_f
|
51
|
+
# next if processing_time < config.lower_time
|
52
|
+
next if low?(processing_time)
|
53
|
+
array << [date, hour, uuid, processing_time]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
# Processing Time Desc
|
57
|
+
array = array.sort{ |a, b| b[3] <=> a[3] }
|
58
|
+
# Generate CSV
|
59
|
+
data = CSV.generate("", csv_option) do |csv|
|
60
|
+
array.each{ |line| csv << line }
|
61
|
+
end
|
62
|
+
store_csv(ranking_path(date), data)
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
def log_files
|
67
|
+
Dir::glob(dir_path + '*').select do |path|
|
68
|
+
path.match(log_file_matcher)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def dir_path
|
73
|
+
Croque.config.log_dir_path
|
74
|
+
end
|
75
|
+
|
76
|
+
def ranking_path(date)
|
77
|
+
Croque.config.store_path.join("#{date}", "ranking.csv")
|
78
|
+
end
|
79
|
+
|
80
|
+
def log_file_matcher
|
81
|
+
Croque.config.log_file_matcher
|
82
|
+
end
|
83
|
+
|
84
|
+
def remove_files(date)
|
85
|
+
path = Croque.config.store_path.join("#{date}")
|
86
|
+
if Dir.exist?(path)
|
87
|
+
FileUtils.remove_dir(path)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def skippable?(date, file)
|
92
|
+
# matcher
|
93
|
+
matcher = convert_matcher(matcher: Croque.config.matcher)
|
94
|
+
# head
|
95
|
+
head_lines = `head -n 10 #{file}`
|
96
|
+
# get lines as Array
|
97
|
+
head_lines = head_lines.lines
|
98
|
+
head_line = head_lines.select do |line|
|
99
|
+
line.match(matcher)
|
100
|
+
end.first
|
101
|
+
head_date = get_date_from_line(head_line)
|
102
|
+
# tail
|
103
|
+
tail_lines = `tail -n 10 #{file}`
|
104
|
+
# get lines as Array
|
105
|
+
tail_lines = tail_lines.lines
|
106
|
+
tail_line = tail_lines.select do |line|
|
107
|
+
line.match(matcher)
|
108
|
+
end.last
|
109
|
+
tail_date = get_date_from_line(tail_line)
|
110
|
+
# include date during range
|
111
|
+
return !(head_date && tail_date && (head_date..tail_date).include?(date))
|
112
|
+
end
|
113
|
+
|
114
|
+
def get_date_from_line(line)
|
115
|
+
if line.present?
|
116
|
+
match = line.match(/\d{4}\-\d{2}\-\d{2}/)
|
117
|
+
if match
|
118
|
+
begin
|
119
|
+
Date.parse(match[0])
|
120
|
+
rescue
|
121
|
+
nil
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def date_matcher(date)
|
128
|
+
convert_matcher(
|
129
|
+
matcher: Croque.config.matcher,
|
130
|
+
date: date
|
131
|
+
)
|
132
|
+
end
|
133
|
+
|
134
|
+
def hour_matcher(hour)
|
135
|
+
convert_matcher(
|
136
|
+
matcher: Croque.config.hour_matcher,
|
137
|
+
hour: hour
|
138
|
+
)
|
139
|
+
end
|
140
|
+
|
141
|
+
def severity_matcher(date, severity)
|
142
|
+
convert_matcher(
|
143
|
+
matcher: Croque.config.matcher,
|
144
|
+
severity: severity
|
145
|
+
)
|
146
|
+
end
|
147
|
+
|
148
|
+
def start_matcher
|
149
|
+
Croque.config.start_matcher
|
150
|
+
end
|
151
|
+
|
152
|
+
def end_matcher
|
153
|
+
Croque.config.end_matcher
|
154
|
+
end
|
155
|
+
|
156
|
+
def convert_matcher(matcher:, date: nil, hour: nil, severity: nil)
|
157
|
+
# Regexp => String
|
158
|
+
matcher = matcher.source
|
159
|
+
# date => XXXX-XX-XX
|
160
|
+
date = if date
|
161
|
+
date.to_s
|
162
|
+
else
|
163
|
+
"\\d{4}-\\d{2}-\\d{2}"
|
164
|
+
end.gsub(/\-/, "\\-")
|
165
|
+
# hour = format("%02d", hour)
|
166
|
+
hour = if hour
|
167
|
+
format("%02d", hour)
|
168
|
+
else
|
169
|
+
"\\d{2}"
|
170
|
+
end
|
171
|
+
severity = if severity
|
172
|
+
severity
|
173
|
+
else
|
174
|
+
"#\\d+"
|
175
|
+
end
|
176
|
+
# replace particular string
|
177
|
+
matcher = matcher.gsub(/severity/, severity)
|
178
|
+
matcher = matcher.gsub(/hour/, hour)
|
179
|
+
matcher = matcher.gsub(/date/, date)
|
180
|
+
# String => Regexp
|
181
|
+
Regexp.new(matcher)
|
182
|
+
end
|
183
|
+
|
184
|
+
def hours
|
185
|
+
(0..23).to_a
|
186
|
+
end
|
187
|
+
|
188
|
+
def csv_path(date, hour)
|
189
|
+
Croque.config.store_path.join("#{date}", "#{hour}.csv")
|
190
|
+
end
|
191
|
+
|
192
|
+
def csv_option
|
193
|
+
{
|
194
|
+
row_sep: "\r\n",
|
195
|
+
headers: false,
|
196
|
+
write_headers: true,
|
197
|
+
force_quotes: true
|
198
|
+
}
|
199
|
+
end
|
200
|
+
|
201
|
+
def headers
|
202
|
+
[
|
203
|
+
"Line ID", # 0
|
204
|
+
"Processing Time (ms)", # 1
|
205
|
+
"Views Time (ms)", # 2
|
206
|
+
"ActiveRecord Time (ms)", # 3
|
207
|
+
"Full Path", # 4
|
208
|
+
"Path Info", # 5
|
209
|
+
"Params", # 6
|
210
|
+
"Body" # 7
|
211
|
+
]
|
212
|
+
end
|
213
|
+
|
214
|
+
def create_csv(date, hour, lines)
|
215
|
+
# extract the matched line (Hour)
|
216
|
+
path = csv_path(date, hour)
|
217
|
+
lines_per_hour = lines.select{ |line| line.match(hour_matcher(hour)) }
|
218
|
+
# get start line of request
|
219
|
+
start_indexes = get_start_indexes(lines_per_hour)
|
220
|
+
data = CSV.generate("", csv_option) do |csv|
|
221
|
+
start_indexes.each do |start_index|
|
222
|
+
values = []
|
223
|
+
start_line = lines_per_hour[start_index]
|
224
|
+
severity = get_severity(start_line)
|
225
|
+
end_index = get_end_index(date, severity, start_index, lines_per_hour)
|
226
|
+
if end_index
|
227
|
+
# Line ID
|
228
|
+
values << SecureRandom.uuid
|
229
|
+
# get End Line
|
230
|
+
end_line = lines_per_hour[end_index]
|
231
|
+
# Processing Time
|
232
|
+
values << get_processing_time(end_line)
|
233
|
+
# Views Time
|
234
|
+
values << get_views_time(end_line)
|
235
|
+
# ActiveRecord Time
|
236
|
+
values << get_active_record_time(end_line)
|
237
|
+
# Full path
|
238
|
+
full_path = get_full_path(start_line)
|
239
|
+
values << full_path
|
240
|
+
# Path Info
|
241
|
+
values << get_path_info(full_path)
|
242
|
+
# Params
|
243
|
+
values << get_params(full_path)
|
244
|
+
# Body
|
245
|
+
lines_per_severity = get_lines_per_severity(date, start_index, end_index, severity, lines_per_hour)
|
246
|
+
values << lines_per_severity.join("\t")
|
247
|
+
# values to CSV
|
248
|
+
csv << values
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
store_csv(path, data)
|
253
|
+
end
|
254
|
+
|
255
|
+
def store_csv(path, data)
|
256
|
+
# make dirctroy
|
257
|
+
unless Dir.exist?(File.dirname(path))
|
258
|
+
FileUtils.mkdir_p(File.dirname(path))
|
259
|
+
end
|
260
|
+
File.open(path, 'a') do |f|
|
261
|
+
f.write data
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
def get_start_indexes(lines)
|
266
|
+
# map index only matched line
|
267
|
+
lines.map.with_index do |line, index|
|
268
|
+
index if line.match(start_matcher)
|
269
|
+
end.compact
|
270
|
+
end
|
271
|
+
|
272
|
+
def get_severity(line)
|
273
|
+
line.match(/#\d+/)[0]
|
274
|
+
end
|
275
|
+
|
276
|
+
def get_end_index(date, severity, start_index, lines)
|
277
|
+
# end line = first of matched lines
|
278
|
+
lines.map.with_index do |line, index|
|
279
|
+
index if start_index < index && line.match(end_matcher) &&
|
280
|
+
line.match(severity_matcher(date, severity))
|
281
|
+
end.compact.first
|
282
|
+
end
|
283
|
+
|
284
|
+
def get_lines_per_severity(date, start_index, end_index, severity, lines)
|
285
|
+
lines[start_index..end_index].select do |line|
|
286
|
+
line.match(severity_matcher(date, severity))
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
def get_processing_time(line)
|
291
|
+
match = line.match(/([1-9]\d*|0)(\.\d+)?ms/)
|
292
|
+
if match
|
293
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
294
|
+
else
|
295
|
+
0
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
def get_views_time(line)
|
300
|
+
match = line.match(/Views: ([1-9]\d*|0)(\.\d+)?ms/)
|
301
|
+
if match
|
302
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
303
|
+
else
|
304
|
+
0
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
def get_active_record_time(line)
|
309
|
+
match = line.match(/ActiveRecord: ([1-9]\d*|0)(\.\d+)?ms/)
|
310
|
+
if match
|
311
|
+
match[0].match(/([1-9]\d*|0)(\.\d+)?/)[0].to_f.round(1)
|
312
|
+
else
|
313
|
+
0
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
def get_full_path(line)
|
318
|
+
line.match(/\".*\"/)[0].gsub(/\"/, '')
|
319
|
+
end
|
320
|
+
|
321
|
+
def get_path_info(full_path)
|
322
|
+
URI.parse("http://example.com#{full_path}").path
|
323
|
+
end
|
324
|
+
|
325
|
+
def get_params(full_path)
|
326
|
+
URI.parse("http://example.com#{full_path}").query
|
327
|
+
end
|
328
|
+
|
329
|
+
def low?(time)
|
330
|
+
time < Croque.config.lower_time
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Croque
|
2
|
+
class Monsieur
|
3
|
+
|
4
|
+
attr_accessor :date, :hour, :id, :time, :line
|
5
|
+
|
6
|
+
def initialize(date, hour, id, time)
|
7
|
+
self.date = Date.parse(date)
|
8
|
+
self.hour = hour.to_i
|
9
|
+
self.id = id
|
10
|
+
self.time = time
|
11
|
+
end
|
12
|
+
|
13
|
+
def body
|
14
|
+
# return Array
|
15
|
+
lines = get_line[7].split("\t")
|
16
|
+
lines = lines.map{ |line| line.strip }
|
17
|
+
lines.each do |line|
|
18
|
+
print "#{line}\n"
|
19
|
+
end
|
20
|
+
lines = lines.map{ |line| line.gsub(/\e\[\d+m/, '') }
|
21
|
+
lines
|
22
|
+
end
|
23
|
+
|
24
|
+
def views_time
|
25
|
+
get_line[2].to_f
|
26
|
+
end
|
27
|
+
|
28
|
+
def active_record_time
|
29
|
+
get_line[3].to_f
|
30
|
+
end
|
31
|
+
|
32
|
+
def processing_time
|
33
|
+
self.time.to_f
|
34
|
+
end
|
35
|
+
|
36
|
+
def full_path
|
37
|
+
URI.unescape(get_line[4])
|
38
|
+
end
|
39
|
+
|
40
|
+
def path_info
|
41
|
+
get_line[5]
|
42
|
+
end
|
43
|
+
|
44
|
+
def query
|
45
|
+
URI.unescape(get_line[6])
|
46
|
+
end
|
47
|
+
|
48
|
+
class << self
|
49
|
+
def get_list(date, limit)
|
50
|
+
csv_data = File.open(ranking_path(date), "r").read.gsub(/\r/, "")
|
51
|
+
csv = CSV.new(csv_data)
|
52
|
+
# Sorted lines as ranking
|
53
|
+
csv.to_a[0..(limit-1)].map do |line|
|
54
|
+
# line = [date, hour, uuid, processing_time (ms)]
|
55
|
+
self.new(*line)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
def ranking_path(date)
|
61
|
+
Croque.config.store_path.join("#{date}", "ranking.csv")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
def get_line
|
67
|
+
self.line ||= if File.exist?(csv_path)
|
68
|
+
csv_data = File.open(csv_path, "r").read.gsub(/\r/, "")
|
69
|
+
csv = CSV.new(csv_data)
|
70
|
+
csv.to_a.find{ |line| line[0] == self.id }
|
71
|
+
else
|
72
|
+
nil
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def csv_path
|
77
|
+
Croque.config.store_path.join("#{self.date}", "#{self.hour}.csv")
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
data/lib/croque/version.rb
CHANGED
data/log/development.log
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
|
+
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
5
|
+
I, [2017-10-21T12:53:05.807962 #11178] INFO -- : Completed 200 OK in 241ms (Views: 8.4ms)
|
6
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
7
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
8
|
+
|
9
|
+
I, [2017-10-22T08:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
10
|
+
I, [2017-10-22T08:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
11
|
+
I, [2017-10-22T08:55:28.898918 #20032] INFO -- : Completed 200 OK in 6ms (Views: 8.0ms)
|
12
|
+
|
13
|
+
I, [2017-10-23T21:55:29.579666 #90113] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:29 +0900
|
14
|
+
I, [2017-10-23T21:55:29.580536 #90113] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
15
|
+
I, [2017-10-23T21:55:29.587382 #90113] INFO -- : Completed 200 OK in 7ms (Views: 7.3ms)
|
16
|
+
|
17
|
+
I, [2017-10-24T05:55:30.531547 #70212] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
18
|
+
I, [2017-10-24T05:55:30.532415 #70212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
19
|
+
I, [2017-10-24T05:55:30.539637 #70212] INFO -- : Completed 200 OK in 7ms (Views: 10.3ms)
|
20
|
+
|
21
|
+
I, [2017-10-25T00:55:30.531547 #92121] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
22
|
+
I, [2017-10-25T00:55:30.532415 #92121] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
23
|
+
I, [2017-10-25T00:55:30.539637 #92121] INFO -- : Completed 200 OK in 7ms (Views: 100.3ms)
|
24
|
+
|
25
|
+
I, [2017-10-26T16:55:30.531547 #89211] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
26
|
+
I, [2017-10-26T16:55:30.532415 #89211] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
27
|
+
I, [2017-10-26T16:55:30.539637 #89211] INFO -- : Completed 200 OK in 7ms (Views: 102.3ms)
|
data/log/test.log
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# Logfile created on 2017-10-21 12:52:41 +0900 by logger.rb/56815
|
2
|
+
I, [2017-10-21T12:53:04.553805 #11178] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:53:04 +0900
|
3
|
+
I, [2017-10-21T12:53:04.566846 #11178] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
4
|
+
I, [2017-10-21T12:55:04.566846 #22212] INFO -- : Started GET "/demo?tomato=delicious&kyouha=hare" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
5
|
+
I, [2017-10-21T12:53:05.807962 #11178] INFO -- : Completed 200 OK in 241ms (Views: 8.4ms)
|
6
|
+
I, [2017-10-21T12:53:06.566846 #22212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
7
|
+
I, [2017-10-21T12:53:10.807962 #22212] INFO -- : Completed 200 OK in 1200ms (Views: 199.9ms | ActiveRecord: 1000.1ms)
|
8
|
+
|
9
|
+
I, [2017-10-22T08:55:28.313141 #20032] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:28 +0900
|
10
|
+
I, [2017-10-22T08:55:28.112109 #20032] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
11
|
+
I, [2017-10-22T08:55:28.898918 #20032] INFO -- : Completed 200 OK in 6ms (Views: 8.0ms)
|
12
|
+
|
13
|
+
I, [2017-10-23T21:55:29.579666 #90113] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:29 +0900
|
14
|
+
I, [2017-10-23T21:55:29.580536 #90113] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
15
|
+
I, [2017-10-23T21:55:29.587382 #90113] INFO -- : Completed 200 OK in 7ms (Views: 7.3ms)
|
16
|
+
|
17
|
+
I, [2017-10-24T05:55:30.531547 #70212] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
18
|
+
I, [2017-10-24T05:55:30.532415 #70212] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
19
|
+
I, [2017-10-24T05:55:30.539637 #70212] INFO -- : Completed 200 OK in 7ms (Views: 10.3ms)
|
20
|
+
|
21
|
+
I, [2017-10-25T00:55:30.531547 #92121] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
22
|
+
I, [2017-10-25T00:55:30.532415 #92121] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
23
|
+
I, [2017-10-25T00:55:30.539637 #92121] INFO -- : Completed 200 OK in 7ms (Views: 100.3ms)
|
24
|
+
|
25
|
+
I, [2017-10-26T16:55:30.531547 #89211] INFO -- : Started GET "/" for 127.0.0.1 at 2017-10-21 12:55:30 +0900
|
26
|
+
I, [2017-10-26T16:55:30.532415 #89211] INFO -- : Processing by Rails::WelcomeController#index as HTML
|
27
|
+
I, [2017-10-26T16:55:30.539637 #89211] INFO -- : Completed 200 OK in 7ms (Views: 102.3ms)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: croque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takuya Okuhara
|
@@ -14,16 +14,30 @@ dependencies:
|
|
14
14
|
name: rails
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '4'
|
20
|
-
type: :
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pry
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: rake
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -70,7 +84,11 @@ files:
|
|
70
84
|
- bin/setup
|
71
85
|
- croque.gemspec
|
72
86
|
- lib/croque.rb
|
87
|
+
- lib/croque/aggregator.rb
|
88
|
+
- lib/croque/monsieur.rb
|
73
89
|
- lib/croque/version.rb
|
90
|
+
- log/development.log
|
91
|
+
- log/test.log
|
74
92
|
homepage: https://github.com/okutaku0507/croque
|
75
93
|
licenses:
|
76
94
|
- MIT
|