fluent-plugin-norikra 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +8 -1
- data/example/test2.conf +2 -0
- data/fluent-plugin-norikra.gemspec +1 -1
- data/lib/fluent/plugin/in_norikra.rb +5 -0
- data/lib/fluent/plugin/norikra/input.rb +3 -3
- data/lib/fluent/plugin/norikra/output.rb +19 -11
- data/lib/fluent/plugin/out_norikra.rb +7 -0
- data/lib/fluent/plugin/out_norikra_filter.rb +14 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 86ac7050ae6300621a0e0a9602f073668f43b2af
|
4
|
+
data.tar.gz: 2ad530592d6771c3a82f29a77147cd99d11b188d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40a6c94ac9f9b2ee05fedcc97dd3f1a879eb50c815f7a961d5515275399d5fea21bd8d6fe3899a4f7a027bc01fb8bf64a28857fa3a15411b228f2dfda5904f9c
|
7
|
+
data.tar.gz: 85b3f3e7f757b4e97a3bca27a62b604ec55fc5a16dd2f418a91bc47a635f026e8bbb28054f9f3ef01ad45aaf8057a12c5937fd466be8c9c8391b5150abee9187
|
data/README.md
CHANGED
@@ -57,9 +57,15 @@ NorikraOutput plugin opens Norikra's target for newly incoming tags. You can spe
|
|
57
57
|
|
58
58
|
target_map_tag true # fluentd's tag -> norikra's target
|
59
59
|
remove_tag_prefix data
|
60
|
+
|
60
61
|
# other options:
|
61
62
|
# target_map_key KEY_NAME # use specified key's value as target in fluentd event
|
62
63
|
# target_string STRING # use fixed target name specified
|
64
|
+
# drop_error_record true # drop records chunk which includes records to occur ClientError on norikra server
|
65
|
+
# # default: true
|
66
|
+
# # (ex: specified (non-optional) fields missing or invalid value for specified type)
|
67
|
+
# drop_server_error_record true # drop records chunk when any ServerError occurs
|
68
|
+
# # default: false (to retry)
|
63
69
|
|
64
70
|
<default>
|
65
71
|
include * # send all fields values to norikra
|
@@ -179,6 +185,7 @@ Configuration example to receive tags like `event.foo` and send norikra's target
|
|
179
185
|
group count_query_group # or default when omitted
|
180
186
|
expression SELECT count(*) AS cnt FROM ${target}.win:time_batch(1 minute)
|
181
187
|
tag count.min.${target}
|
188
|
+
fetch_interval 10s
|
182
189
|
</query>
|
183
190
|
<query>
|
184
191
|
name count_hour_${target}
|
@@ -234,7 +241,7 @@ Norikra's target (like table name) can be generated from:
|
|
234
241
|
|
235
242
|
# TODO
|
236
243
|
|
237
|
-
* write
|
244
|
+
* write about these topics
|
238
245
|
* error logs for new target, success logs of retry
|
239
246
|
|
240
247
|
# Copyright
|
data/example/test2.conf
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |spec|
|
4
4
|
spec.name = "fluent-plugin-norikra"
|
5
|
-
spec.version = "0.
|
5
|
+
spec.version = "0.2.0"
|
6
6
|
spec.authors = ["TAGOMORI Satoshi"]
|
7
7
|
spec.email = ["tagomoris@gmail.com"]
|
8
8
|
spec.description = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}
|
@@ -72,7 +72,7 @@ module Fluent::NorikraPlugin
|
|
72
72
|
@fetch_queue.sort!
|
73
73
|
end
|
74
74
|
rescue => e
|
75
|
-
|
75
|
+
log.error "unknown log encountered", :error_class => e.class, :message => e.message
|
76
76
|
end
|
77
77
|
|
78
78
|
def fetch_worker
|
@@ -88,7 +88,7 @@ module Fluent::NorikraPlugin
|
|
88
88
|
begin
|
89
89
|
data = req.fetch(client())
|
90
90
|
rescue => e
|
91
|
-
|
91
|
+
log.error "failed to fetch", :norikra => "#{@host}:#{@port}", :method => req.method, :target => req.target, :error => e.class, :message => e.message
|
92
92
|
end
|
93
93
|
|
94
94
|
data.each do |tag, event_array|
|
@@ -96,7 +96,7 @@ module Fluent::NorikraPlugin
|
|
96
96
|
begin
|
97
97
|
Fluent::Engine.emit(tag, time, event)
|
98
98
|
rescue => e
|
99
|
-
|
99
|
+
log.error "failed to emit event from norikra query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message, :tag => tag, :record => event
|
100
100
|
end
|
101
101
|
end
|
102
102
|
end
|
@@ -81,9 +81,9 @@ module Fluent::NorikraPlugin
|
|
81
81
|
targets.each do |t|
|
82
82
|
next if @target_map[t.name]
|
83
83
|
|
84
|
-
|
84
|
+
log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
|
85
85
|
if prepare_target(c, t)
|
86
|
-
|
86
|
+
log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
|
87
87
|
|
88
88
|
if @enable_auto_query
|
89
89
|
raise "bug" unless self.respond_to?(:insert_fetch_queue)
|
@@ -96,7 +96,7 @@ module Fluent::NorikraPlugin
|
|
96
96
|
@target_map[t.name] = t
|
97
97
|
@registered_targets.delete(t.name)
|
98
98
|
else
|
99
|
-
|
99
|
+
log.error "Failed to prepare norikra data for target:#{t.name}"
|
100
100
|
@norikra_started.push(t)
|
101
101
|
end
|
102
102
|
end
|
@@ -105,12 +105,12 @@ module Fluent::NorikraPlugin
|
|
105
105
|
|
106
106
|
def prepare_target(client, target)
|
107
107
|
# target open and reserve fields
|
108
|
-
|
108
|
+
log.debug "Going to prepare about target"
|
109
109
|
begin
|
110
110
|
unless client.targets.include?(target.escaped_name)
|
111
|
-
|
111
|
+
log.debug "opening target #{target.escaped_name}"
|
112
112
|
client.open(target.escaped_name, target.reserve_fields, target.auto_field)
|
113
|
-
|
113
|
+
log.debug "opening target #{target.escaped_name}, done."
|
114
114
|
end
|
115
115
|
|
116
116
|
reserving = target.reserve_fields
|
@@ -119,7 +119,7 @@ module Fluent::NorikraPlugin
|
|
119
119
|
if reserving[field['name']]
|
120
120
|
reserved.push(field['name'])
|
121
121
|
if reserving[field['name']] != field['type']
|
122
|
-
|
122
|
+
log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
|
123
123
|
end
|
124
124
|
end
|
125
125
|
end
|
@@ -128,7 +128,7 @@ module Fluent::NorikraPlugin
|
|
128
128
|
client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
|
129
129
|
end
|
130
130
|
rescue => e
|
131
|
-
|
131
|
+
log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
132
132
|
return false
|
133
133
|
end
|
134
134
|
|
@@ -138,7 +138,7 @@ module Fluent::NorikraPlugin
|
|
138
138
|
target.queries.each do |query|
|
139
139
|
if registered.has_key?(query.name) # query already registered
|
140
140
|
if registered[query.name] != query.expression
|
141
|
-
|
141
|
+
log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
|
142
142
|
end
|
143
143
|
next
|
144
144
|
end
|
@@ -148,7 +148,7 @@ module Fluent::NorikraPlugin
|
|
148
148
|
fetch_event_registration(query)
|
149
149
|
end
|
150
150
|
rescue => e
|
151
|
-
|
151
|
+
log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
152
152
|
end
|
153
153
|
end
|
154
154
|
|
@@ -205,7 +205,15 @@ module Fluent::NorikraPlugin
|
|
205
205
|
c = client()
|
206
206
|
|
207
207
|
events_map.each do |target, events|
|
208
|
-
|
208
|
+
begin
|
209
|
+
c.send(target, events)
|
210
|
+
rescue Norikra::RPC::ClientError => e
|
211
|
+
raise unless @drop_error_record
|
212
|
+
log.warn "Norikra server reports ClientError, and dropped", target: target, message: e.message
|
213
|
+
rescue Norikra::RPC::ServerError => e
|
214
|
+
raise unless @drop_server_error_record
|
215
|
+
log.warn "Norikra server reports ServerError, and dropped", target: target, message: e.message
|
216
|
+
end
|
209
217
|
end
|
210
218
|
end
|
211
219
|
|
@@ -21,10 +21,17 @@ module Fluent
|
|
21
21
|
config_param :target_map_tag, :bool, :default => false
|
22
22
|
config_param :target_map_key, :string, :default => nil
|
23
23
|
config_param :target_string, :string, :default => nil
|
24
|
+
config_param :drop_error_record, :default => true
|
25
|
+
config_param :drop_server_error_record, :default => false
|
24
26
|
|
25
27
|
# <default>
|
26
28
|
# <target TARGET>
|
27
29
|
|
30
|
+
# Define `log` method for v0.10.42 or earlier
|
31
|
+
unless method_defined?(:log)
|
32
|
+
define_method("log") { $log }
|
33
|
+
end
|
34
|
+
|
28
35
|
def configure(conf)
|
29
36
|
super
|
30
37
|
|
@@ -26,6 +26,8 @@ module Fluent
|
|
26
26
|
config_param :target_map_tag, :bool, :default => false
|
27
27
|
config_param :target_map_key, :string, :default => nil
|
28
28
|
config_param :target_string, :string, :default => nil
|
29
|
+
config_param :drop_error_record, :default => true
|
30
|
+
config_param :drop_server_error_record, :default => false
|
29
31
|
|
30
32
|
# <default>
|
31
33
|
# <target TARGET>
|
@@ -48,6 +50,11 @@ module Fluent
|
|
48
50
|
# tag_prefix norikra.query
|
49
51
|
# </fetch>
|
50
52
|
|
53
|
+
# Define `log` method for v0.10.42 or earlier
|
54
|
+
unless method_defined?(:log)
|
55
|
+
define_method("log") { $log }
|
56
|
+
end
|
57
|
+
|
51
58
|
def configure(conf)
|
52
59
|
super
|
53
60
|
|
@@ -129,7 +136,7 @@ module Fluent
|
|
129
136
|
end
|
130
137
|
|
131
138
|
def server_starter
|
132
|
-
|
139
|
+
log.info "starting Norikra server process #{@host}:#{@port}"
|
133
140
|
base_options = [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]
|
134
141
|
cmd,options = if @execute_jruby_path
|
135
142
|
[@execute_jruby_path, [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]]
|
@@ -144,24 +151,24 @@ module Fluent
|
|
144
151
|
exec([cmd, 'norikra(fluentd)'], *options)
|
145
152
|
end
|
146
153
|
connecting = true
|
147
|
-
|
154
|
+
log.info "trying to confirm norikra server status..."
|
148
155
|
while connecting
|
149
156
|
begin
|
150
|
-
|
157
|
+
log.debug "start to connect norikra server #{@host}:#{@port}"
|
151
158
|
client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets
|
152
159
|
# discard result: no exceptions is success
|
153
160
|
connecting = false
|
154
161
|
next
|
155
162
|
rescue HTTPClient::TimeoutError
|
156
|
-
|
163
|
+
log.debug "Norikra server test connection timeout. retrying..."
|
157
164
|
rescue Errno::ECONNREFUSED
|
158
|
-
|
165
|
+
log.debug "Norikra server test connection refused. retrying..."
|
159
166
|
rescue => e
|
160
|
-
|
167
|
+
log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
|
161
168
|
end
|
162
169
|
sleep 3
|
163
170
|
end
|
164
|
-
|
171
|
+
log.info "confirmed that norikra server #{@host}:#{@port} started."
|
165
172
|
@norikra_started = true
|
166
173
|
end
|
167
174
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-norikra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: norikra-client
|