croque 0.5.3 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +1 -0
- data/lib/croque.rb +3 -1
- data/lib/croque/aggregator.rb +17 -3
- data/lib/croque/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43b4e00f40d1bf501a490be5b58db0cca92eb260
|
4
|
+
data.tar.gz: 110503e639572246320bfc33dc994e1dab2aecbf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0710a4e309cad39c2d1cbe5fccd29dc9dca4cbc42300a8f936da0f320e28ba9a71889fe81303e5a605ebfd27acd0b9d950a16bce1be4d021208abb4c47f52dc3
|
7
|
+
data.tar.gz: 8360e2fff8d5a4e190c8e745235b191ce069c21ccb0a48431c0ca60d3e8bb3c5d386c36068a34ed729fcfbf24a1b2757a0a794eb1df108f8044c972e926b7047
|
data/.gitignore
CHANGED
data/README.md
CHANGED
data/lib/croque.rb
CHANGED
@@ -19,7 +19,8 @@ module Croque
|
|
19
19
|
include ActiveSupport::Configurable
|
20
20
|
config_accessor :root_path, :log_dir_path, :store_path,
|
21
21
|
:log_file_matcher, :hour_matcher, :matcher, :severity_matcher,
|
22
|
-
:start_matcher, :end_matcher, :lower_time, :except_path_matcher
|
22
|
+
:start_matcher, :end_matcher, :lower_time, :except_path_matcher,
|
23
|
+
:logger
|
23
24
|
end
|
24
25
|
|
25
26
|
configure do |config|
|
@@ -34,6 +35,7 @@ module Croque
|
|
34
35
|
config.end_matcher = /\-\- : Completed/
|
35
36
|
config.lower_time = 1000 # ms
|
36
37
|
config.except_path_matcher = /\/assets\//
|
38
|
+
config.logger = Logger.new(config.log_dir_path.join("croque.#{Rails.env}.log"))
|
37
39
|
end
|
38
40
|
|
39
41
|
class << self
|
data/lib/croque/aggregator.rb
CHANGED
@@ -12,16 +12,20 @@ module Croque
|
|
12
12
|
|
13
13
|
def aggregate_per_hour(date)
|
14
14
|
# scan each file
|
15
|
+
log("aggregate logs per hour on #{date} start")
|
15
16
|
log_files.each do |file|
|
17
|
+
log("check skippable of #{file}")
|
16
18
|
# check skippable
|
17
19
|
next if skippable?(date, file)
|
20
|
+
log("aggregate logs of #{file}")
|
18
21
|
# all lines
|
19
22
|
linage = 1000
|
20
23
|
wc_result = `wc -l #{file}`
|
21
|
-
line_count = wc_result.match(/\d+/)[0]
|
24
|
+
line_count = wc_result.match(/\d+/)[0].to_i
|
22
25
|
k = 1
|
23
26
|
lines = []
|
24
|
-
while (k-1)*linage < line_count
|
27
|
+
while (k-1)*linage < line_count
|
28
|
+
log("aggregate logs for #{(k-1)*linage}-#{k*linage} in #{line_count} on #{date}")
|
25
29
|
fragment = `head -n #{k*1000} #{file} | tail -n #{linage}`
|
26
30
|
fragment_lines = fragment.lines
|
27
31
|
lines += fragment_lines.select do |line|
|
@@ -33,14 +37,18 @@ module Croque
|
|
33
37
|
lines = lines
|
34
38
|
hours.each do |hour|
|
35
39
|
# craete csv file
|
40
|
+
log("create csv for #{date} #{hour} hour")
|
36
41
|
create_csv(date, hour, lines)
|
37
42
|
end
|
38
43
|
end
|
44
|
+
log("aggregate logs per hour on #{date} end")
|
39
45
|
end
|
40
46
|
|
41
47
|
def generate_ranking(date)
|
48
|
+
log("generate ranking on #{date} start")
|
42
49
|
array = []
|
43
50
|
hours.each do |hour|
|
51
|
+
log("generate array for ranking in #{date} #{hour} hour")
|
44
52
|
# csv data
|
45
53
|
path = csv_path(date, hour)
|
46
54
|
# next if no file
|
@@ -50,18 +58,20 @@ module Croque
|
|
50
58
|
csv.to_a.each do |line|
|
51
59
|
uuid = line[0]
|
52
60
|
processing_time = line[1].to_f
|
53
|
-
# next if processing_time < config.lower_time
|
54
61
|
next if low?(processing_time)
|
55
62
|
array << [date, hour, uuid, processing_time]
|
56
63
|
end
|
57
64
|
end
|
65
|
+
log("sort array for ranking on #{date}")
|
58
66
|
# Processing Time Desc
|
59
67
|
array = array.sort{ |a, b| b[3] <=> a[3] }
|
68
|
+
log("generate ranking csv on #{date}")
|
60
69
|
# Generate CSV
|
61
70
|
data = CSV.generate("", csv_option) do |csv|
|
62
71
|
array.each{ |line| csv << line }
|
63
72
|
end
|
64
73
|
store_csv(ranking_path(date), data)
|
74
|
+
log("generate ranking on #{date} end")
|
65
75
|
end
|
66
76
|
|
67
77
|
def all
|
@@ -349,6 +359,10 @@ module Croque
|
|
349
359
|
def low?(time)
|
350
360
|
time < Croque.config.lower_time
|
351
361
|
end
|
362
|
+
|
363
|
+
def log(message)
|
364
|
+
Croque.config.logger.try(:info, message)
|
365
|
+
end
|
352
366
|
end
|
353
367
|
end
|
354
368
|
end
|
data/lib/croque/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: croque
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takuya Okuhara
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-11-
|
11
|
+
date: 2017-11-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|