fluent-plugin-http-record-modifier 0.0.0 → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-http-record-modifier"
|
7
|
-
spec.version = "0.0.
|
7
|
+
spec.version = "0.0.1"
|
8
8
|
spec.authors = ["Augusto Nishi"]
|
9
9
|
spec.email = ["augusto.nishi@gmail.com"]
|
10
10
|
spec.summary = %q{fluentd filter plugin for modifing record based on a HTTP request}
|
@@ -22,6 +22,9 @@ module Fluent
|
|
22
22
|
config_param :username, :string, :default => ''
|
23
23
|
config_param :password, :string, :default => ''
|
24
24
|
config_param :raise_on_error, :bool, :default => true
|
25
|
+
config_param :cache, :bool, :default => true
|
26
|
+
config_param :expire, :integer, :default => 600 #10 min
|
27
|
+
config_param :renew_time_key, :string, :default => nil
|
25
28
|
|
26
29
|
def configure(conf)
|
27
30
|
super
|
@@ -53,6 +56,8 @@ module Fluent
|
|
53
56
|
@keep_keys = @keep_keys.split(',')
|
54
57
|
end
|
55
58
|
|
59
|
+
@request_cache = Cache.new(@cache, @expire)
|
60
|
+
|
56
61
|
@placeholder_expander = PlaceholderExpander.new({
|
57
62
|
:log => log,
|
58
63
|
:auto_typecast => @auto_typecast,
|
@@ -77,8 +82,12 @@ module Fluent
|
|
77
82
|
es.each do |time, record|
|
78
83
|
last_record = record # for debug log
|
79
84
|
req, uri = create_request(tag, time, record)
|
80
|
-
|
81
|
-
body
|
85
|
+
body = @request_cache.get(uri.to_s)
|
86
|
+
if body.nil?
|
87
|
+
res = send_request(req, uri)
|
88
|
+
body = deserialize_body(res)
|
89
|
+
@request_cache.set(uri.to_s, body)
|
90
|
+
end
|
82
91
|
new_record = reform(time, record, placeholders, body)
|
83
92
|
if @renew_time_key && new_record.has_key?(@renew_time_key)
|
84
93
|
time = new_record[@renew_time_key].to_i
|
@@ -220,6 +229,34 @@ module Fluent
|
|
220
229
|
rev_tag_suffix.reverse!
|
221
230
|
end
|
222
231
|
|
232
|
+
class Cache
|
233
|
+
def initialize(cache, expire)
|
234
|
+
@data = {}
|
235
|
+
@cache = cache
|
236
|
+
@expire = expire
|
237
|
+
end
|
238
|
+
|
239
|
+
def get(key)
|
240
|
+
unless @data.has_key?(key) and @cache
|
241
|
+
return nil
|
242
|
+
end
|
243
|
+
if Time.now.to_i > @data[key]['time'] + @expire
|
244
|
+
@data.delete(key)
|
245
|
+
return nil
|
246
|
+
end
|
247
|
+
return @data[key]['value']
|
248
|
+
end
|
249
|
+
|
250
|
+
def set(key, value)
|
251
|
+
if @cache
|
252
|
+
@data[key] = {
|
253
|
+
'time' => Time.now.to_i,
|
254
|
+
'value' => value
|
255
|
+
}
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
223
260
|
class PlaceholderExpander
|
224
261
|
attr_reader :placeholders, :log
|
225
262
|
|
@@ -230,8 +267,7 @@ module Fluent
|
|
230
267
|
|
231
268
|
def prepare_placeholders(time, record, opts)
|
232
269
|
placeholders = { '${time}' => Time.at(time).to_s }
|
233
|
-
record.each {|key, value| crawl_placeholder(value, placeholders, "#{key}")
|
234
|
-
}
|
270
|
+
record.each {|key, value| crawl_placeholder(value, placeholders, "#{key}")}
|
235
271
|
opts.each do |key, value|
|
236
272
|
if value.kind_of?(Array) # tag_parts, etc
|
237
273
|
size = value.size
|
metadata
CHANGED
@@ -1,55 +1,62 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-http-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
5
6
|
platform: ruby
|
6
7
|
authors:
|
7
8
|
- Augusto Nishi
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2015-
|
12
|
+
date: 2015-08-14 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: bundler
|
15
16
|
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
16
18
|
requirements:
|
17
|
-
- -
|
19
|
+
- - ~>
|
18
20
|
- !ruby/object:Gem::Version
|
19
21
|
version: '1.5'
|
20
22
|
type: :development
|
21
23
|
prerelease: false
|
22
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
23
26
|
requirements:
|
24
|
-
- -
|
27
|
+
- - ~>
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '1.5'
|
27
30
|
- !ruby/object:Gem::Dependency
|
28
31
|
name: rake
|
29
32
|
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
30
34
|
requirements:
|
31
|
-
- -
|
35
|
+
- - ! '>='
|
32
36
|
- !ruby/object:Gem::Version
|
33
37
|
version: '0'
|
34
38
|
type: :development
|
35
39
|
prerelease: false
|
36
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
37
42
|
requirements:
|
38
|
-
- -
|
43
|
+
- - ! '>='
|
39
44
|
- !ruby/object:Gem::Version
|
40
45
|
version: '0'
|
41
46
|
- !ruby/object:Gem::Dependency
|
42
47
|
name: fluentd
|
43
48
|
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
44
50
|
requirements:
|
45
|
-
- -
|
51
|
+
- - ! '>='
|
46
52
|
- !ruby/object:Gem::Version
|
47
53
|
version: '0'
|
48
54
|
type: :development
|
49
55
|
prerelease: false
|
50
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
51
58
|
requirements:
|
52
|
-
- -
|
59
|
+
- - ! '>='
|
53
60
|
- !ruby/object:Gem::Version
|
54
61
|
version: '0'
|
55
62
|
description: fluentd filter plugin for modifing record based on a HTTP request
|
@@ -59,36 +66,36 @@ executables: []
|
|
59
66
|
extensions: []
|
60
67
|
extra_rdoc_files: []
|
61
68
|
files:
|
62
|
-
-
|
69
|
+
- .gitignore
|
63
70
|
- Gemfile
|
64
71
|
- LICENSE.txt
|
65
72
|
- README.md
|
66
73
|
- Rakefile
|
67
74
|
- fluent-plugin-http-record-modifier.gemspec
|
68
|
-
- lib/fluent/plugin/.filter_http_record_modifier.rb.swp
|
69
75
|
- lib/fluent/plugin/filter_http_record_modifier.rb
|
70
76
|
homepage: http://github.com/DEVTecnologia/fluent-plugin-http-record-modifier
|
71
77
|
licenses:
|
72
78
|
- MIT
|
73
|
-
metadata: {}
|
74
79
|
post_install_message:
|
75
80
|
rdoc_options: []
|
76
81
|
require_paths:
|
77
82
|
- lib
|
78
83
|
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
79
85
|
requirements:
|
80
|
-
- -
|
86
|
+
- - ! '>='
|
81
87
|
- !ruby/object:Gem::Version
|
82
88
|
version: '0'
|
83
89
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
84
91
|
requirements:
|
85
|
-
- -
|
92
|
+
- - ! '>='
|
86
93
|
- !ruby/object:Gem::Version
|
87
94
|
version: '0'
|
88
95
|
requirements: []
|
89
96
|
rubyforge_project:
|
90
|
-
rubygems_version:
|
97
|
+
rubygems_version: 1.8.23
|
91
98
|
signing_key:
|
92
|
-
specification_version:
|
99
|
+
specification_version: 3
|
93
100
|
summary: fluentd filter plugin for modifing record based on a HTTP request
|
94
101
|
test_files: []
|
checksums.yaml
DELETED
@@ -1,7 +0,0 @@
|
|
1
|
-
---
|
2
|
-
SHA1:
|
3
|
-
metadata.gz: 2ea15085273d646c9d75d934aa5b2e90eeef89af
|
4
|
-
data.tar.gz: 693781df064fceb076c7d99a8718a89724898aa8
|
5
|
-
SHA512:
|
6
|
-
metadata.gz: d938f24e8a733b6a9161f94e09d066d2eed540d29a5487684c14e226867a92f8fb6f7fa229ab49a09d25a89d178c2379245c65d77ad4e2ecd251d651e4a40122
|
7
|
-
data.tar.gz: 859c43d550ef56cc379bf899c6fbf7e43964103d4369fe85d9eb70e2c87b4e245eb50f8edb9ce0bec993b31600968a06fb13cfcbd66e04f6910ecdeba79cebac
|
Binary file
|