fluent-plugin-querycombiner 0.0.0.pre

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 432eccbf19e7b303cf6457b3560a43779b7051b3
4
+ data.tar.gz: 8a3c7c461b1db05ff3968d49f46c1dcd694b7e7a
5
+ SHA512:
6
+ metadata.gz: 00f8c3f21610a21e5ad6c6887377fef64c70186cd73c1e84813240581ef9f7b2d68c140fa15543052eea7134b7086ac8f6903bb7472fc077aaa2d18fac8de5c3
7
+ data.tar.gz: a8bdcf68fe8adfb36cda540c8f691829e73476a38713f5105ddd3bf18084e5a980b162f863c696ee055d0a37cc2fc362301946e94d29f606e2717eadf660c769
data/.gitignore ADDED
@@ -0,0 +1,26 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
23
+ *~
24
+ \#*
25
+ .\#*
26
+ *.swp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-querycombiner.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,70 @@
1
+ fluent-plugin-querycombiner
2
+ ===========================
3
+ This fluentd output plugin helps you to combine multiple queries.
4
+
5
+ This plugin is based on [fluent-plugin-onlineuser](https://github.com/y-lan/fluent-plugin-onlineuser) written by [Yuyang Lan](https://github.com/y-lan).
6
+
7
+
8
+ ## Requirement
9
+ * a running Redis
10
+
11
+
12
+ ## Get started
13
+
14
+ ```
15
+ <match combiner.**>
16
+ type query_combiner
17
+ tag combined.test
18
+
19
+ flush_interval 0.5
20
+
21
+ host localhost
22
+ port 6379
23
+ db_index 0
24
+ redis_retry 3
25
+
26
+ query_identify session-id, task-id
27
+ query_ttl 3 # sec
28
+ buffer_size 10 # queries
29
+
30
+ <catch>
31
+ condition status == 'recog-init'
32
+ replace time => time_init, status => status_init
33
+ </catch>
34
+
35
+ <prolong>
36
+ condition status == 'recog-break'
37
+ </prolong>
38
+
39
+ <dump>
40
+ condition status == 'recog-finish'
41
+ replace time => time_finish, result => result_finish, status => status_finish
42
+ </dump>
43
+
44
+ <release>
45
+ condition status == 'recog-error'
46
+ </release>
47
+
48
+ </match>
49
+ ```
50
+
51
+ ## Configuration
52
+ #### host, port, db_index
53
+ The basic information for connecting to Redis. By default it's **redis://127.0.0.1:6379/0**
54
+
55
+ #### redis_retry
56
+ How many times should the plugin retry when performing a redis operation before raising a error.
57
+ By default it's 3.
58
+
59
+ ### session_timeout
60
+ The inactive expire time in seconds. By default it's 1800 (30 minutes).
61
+
62
+
63
+ ### tag
64
+ The tag prefix for emitted event messages. By default it's `query_combiner`.
65
+
66
+ ## Copyright
67
+
68
+ Copyright:: Copyright (c) 2014- Takahiro Kamatani
69
+
70
+ License:: Apache License, Version 2.0
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+ Rake::TestTask.new(:test) do |test|
5
+ test.libs << 'lib' << 'test'
6
+ test.pattern = 'test/**/test_*.rb'
7
+ test.verbose = true
8
+ end
9
+
10
+ task :default => :test
@@ -0,0 +1,23 @@
1
+ # coding: utf-8
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+
4
+ Gem::Specification.new do |spec|
5
+ spec.name = "fluent-plugin-querycombiner"
6
+ spec.version = "0.0.0.pre"
7
+ spec.authors = ["Takahiro Kamatani"]
8
+ spec.email = ["buhii314@gmail.com"]
9
+ spec.description = %q{Fluent plugin to combine multiple queries.}
10
+ spec.summary = spec.description
11
+ spec.homepage = "https://github.com/buhii/fluent-plugin-querycombiner"
12
+ spec.license = "Apache License, Version 2.0"
13
+ spec.has_rdoc = false
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_development_dependency "rake"
21
+ spec.add_runtime_dependency "fluentd", "~> 0.10.0"
22
+ spec.add_runtime_dependency "redis"
23
+ end
@@ -0,0 +1,259 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Fluent
4
+ class QueryCombinerOutput < BufferedOutput
5
+ Fluent::Plugin.register_output('query_combiner', self)
6
+
7
+ config_param :host, :string, :default => 'localhost'
8
+ config_param :port, :integer, :default => 6379
9
+ config_param :db_index, :integer, :default => 0
10
+ config_param :redis_retry, :integer, :default => 3
11
+
12
+ config_param :redis_key_prefix, :string, :default => 'query_combiner:'
13
+ config_param :query_identify, :string, :default => 'session-id'
14
+ config_param :query_ttl, :integer, :default => 1800
15
+ config_param :buffer_size, :integer, :default => 100
16
+
17
+ config_param :flush_interval, :integer, :default => 60
18
+ config_param :remove_interval, :integer, :default => 10
19
+ config_param :tag, :string, :default => "query_combiner"
20
+
21
+ def initialize
22
+ super
23
+ require 'redis'
24
+ require 'msgpack'
25
+ require 'json'
26
+ require 'rubygems'
27
+ end
28
+
29
+ def configure(conf)
30
+ super
31
+ @host = conf.has_key?('host') ? conf['host'] : 'localhost'
32
+ @port = conf.has_key?('port') ? conf['port'].to_i : 6379
33
+ @db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
34
+
35
+ @query_identify = @query_identify.split(',').map { |qid| qid.strip }
36
+
37
+ # Create functions for each conditions
38
+ @_cond_funcs = {}
39
+ @_replace_keys = {}
40
+
41
+ def get_arguments(eval_str)
42
+ eval_str.scan(/[\"\']?[a-zA-Z][\w\d\.\-\_]*[\"\']?/).uniq.select{|x|
43
+ not (x.start_with?('\'') or x.start_with?('\"')) and \
44
+ not %w{and or xor not}.include? x
45
+ }
46
+ end
47
+
48
+ def parse_replace_expr(element_name, condition_name, str)
49
+ result = {}
50
+ str.split(',').each{|cond|
51
+ before, after = cond.split('=>').map{|var| var.strip}
52
+ result[before] = after
53
+ if not (before.length > 0 and after.length > 0)
54
+ raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
55
+ end
56
+ }
57
+ if result.none?
58
+ raise Fluent::ConfigError, "SyntaxError at replace condition `#{element_name}`: #{condition_name}"
59
+ end
60
+ result
61
+ end
62
+
63
+ def create_func(var, expr)
64
+ begin
65
+ f_argv = get_arguments(expr)
66
+ f = eval('lambda {|' + f_argv.join(',') + '| ' + expr + '}')
67
+ return [f, f_argv]
68
+ rescue SyntaxError
69
+ raise Fluent::ConfigError, "SyntaxError at condition `#{var}`: #{expr}"
70
+ end
71
+ end
72
+ conf.elements.select { |element|
73
+ %w{catch prolong dump release}.include? element.name
74
+ }.each { |element|
75
+ element.each_pair { |var, expr|
76
+ element.has_key?(var) # to suppress unread configuration warning
77
+
78
+ if var == 'condition'
79
+ formula, f_argv = create_func(var, expr)
80
+ @_cond_funcs[element.name] = [f_argv, formula]
81
+
82
+ elsif var == 'replace'
83
+ if %w{catch dump}.include? element.name
84
+ @_replace_keys[element.name] = parse_replace_expr(element.name, var, expr)
85
+ else
86
+ raise Fluent::ConfigError, "`replace` configuration in #{element.name}: only allowed in `catch` and `dump`"
87
+ end
88
+ end
89
+ }
90
+ }
91
+ end
92
+
93
+ def has_all_keys?(record, argv)
94
+ argv.each {|var|
95
+ if not record.has_key?(var)
96
+ return false
97
+ end
98
+ }
99
+ true
100
+ end
101
+
102
+ def exec_func(record, f_argv, formula)
103
+ argv = []
104
+ f_argv.each {|v|
105
+ argv.push(record[v])
106
+ }
107
+ return formula.call(*argv)
108
+ end
109
+
110
+ def start
111
+ super
112
+
113
+ begin
114
+ gem "hiredis"
115
+ @redis = Redis.new(
116
+ :host => @host, :port => @port, :driver => :hiredis,
117
+ :thread_safe => true, :db => @db_index
118
+ )
119
+ rescue LoadError
120
+ @redis = Redis.new(
121
+ :host => @host, :port => @port,
122
+ :thread_safe => true, :db => @db_index
123
+ )
124
+ end
125
+
126
+ start_watch
127
+ end
128
+
129
+ def shutdown
130
+ @redis.quit
131
+ end
132
+
133
+ def tryOnRedis(method, *args)
134
+ tries = 0
135
+ begin
136
+ @redis.send(method, *args) if @redis.respond_to? method
137
+ rescue Redis::CommandError => e
138
+ tries += 1
139
+ # retry 3 times
140
+ retry if tries <= @redis_retry
141
+ $log.warn %Q[redis command retry failed : #{method}(#{args.join(', ')})]
142
+ raise e.message
143
+ end
144
+ end
145
+
146
+ def start_watch
147
+ @watcher = Thread.new(&method(:watch))
148
+ end
149
+
150
+ def format(tag, time, record)
151
+ [tag, time, record].to_msgpack
152
+ end
153
+
154
+ def do_catch(qid, record, time)
155
+ # replace record keys
156
+ @_replace_keys['catch'].each_pair { |before, after|
157
+ record[after] = record[before]
158
+ record.delete(before)
159
+ }
160
+ # save record
161
+ tryOnRedis 'set', @redis_key_prefix + qid, JSON.dump(record)
162
+ # update qid's timestamp
163
+ tryOnRedis 'zadd', @redis_key_prefix, time, qid
164
+ tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
165
+ end
166
+
167
+ def do_prolong(qid, time)
168
+ if (tryOnRedis 'exists', @redis_key_prefix + qid)
169
+ # update qid's timestamp
170
+ tryOnRedis 'zadd', @redis_key_prefix, time, qid
171
+ tryOnRedis 'expire', @redis_key_prefix + qid, @query_ttl
172
+ end
173
+ end
174
+
175
+ def do_dump(qid, record)
176
+ if (tryOnRedis 'exists', @redis_key_prefix + qid)
177
+ # replace record keys
178
+ @_replace_keys['dump'].each_pair { |before, after|
179
+ record[after] = record[before]
180
+ record.delete(before)
181
+ }
182
+
183
+ # emit
184
+ catched_record = JSON.load(tryOnRedis('get', @redis_key_prefix + qid))
185
+ combined_record = catched_record.merge(record)
186
+ Fluent::Engine.emit @tag, Fluent::Engine.now, combined_record
187
+
188
+ # remove qid
189
+ do_release(qid)
190
+ end
191
+ end
192
+
193
+ def do_release(qid)
194
+ tryOnRedis 'del', @redis_key_prefix + qid
195
+ tryOnRedis 'zrem', @redis_key_prefix, qid
196
+ end
197
+
198
+ def extract_qid(record)
199
+ qid = []
200
+ @query_identify.each { |attr|
201
+ if record.has_key?(attr)
202
+ qid.push(record[attr])
203
+ else
204
+ return nil
205
+ end
206
+ }
207
+ qid.join(':')
208
+ end
209
+
210
+ def write(chunk)
211
+
212
+ begin
213
+ chunk.msgpack_each do |(tag, time, record)|
214
+ if (qid = extract_qid record)
215
+
216
+ @_cond_funcs.each_pair { |cond, argv_and_func|
217
+ argv, func = argv_and_func
218
+ if exec_func(record, argv, func)
219
+ case cond
220
+ when "catch"
221
+ do_catch(qid, record, time)
222
+ when "prolong"
223
+ do_prolong(qid, time)
224
+ when "dump"
225
+ do_dump(qid, record)
226
+ when "release"
227
+ do_release(qid)
228
+ end
229
+ break # very important!
230
+ end
231
+ }
232
+ end
233
+ end
234
+
235
+ end
236
+ end
237
+
238
+ def watch
239
+ @last_checked = Fluent::Engine.now
240
+ tick = @remove_interval
241
+ while true
242
+ sleep 0.5
243
+ if Fluent::Engine.now - @last_checked >= tick
244
+ now = Fluent::Engine.now
245
+ to_expire = now - @query_ttl
246
+
247
+ # Delete expired qids
248
+ tryOnRedis 'zremrangebyscore', @redis_key_prefix, '-inf', to_expire
249
+
250
+ # Delete buffer_size over qids
251
+ tryOnRedis 'zremrangebyrank', @redis_key_prefix, 0, -@buffer_size
252
+
253
+ @last_checked = now
254
+ end
255
+ end
256
+ end
257
+
258
+ end
259
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+ begin
4
+ Bundler.setup(:default, :development)
5
+ rescue Bundler::BundlerError => e
6
+ $stderr.puts e.message
7
+ $stderr.puts "Run `bundle install` to install missing gems"
8
+ exit e.status_code
9
+ end
10
+ require 'test/unit'
11
+
12
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
14
+ require 'fluent/test'
15
+ unless ENV.has_key?('VERBOSE')
16
+ nulllogger = Object.new
17
+ nulllogger.instance_eval {|obj|
18
+ def method_missing(method, *args)
19
+ # pass
20
+ end
21
+ }
22
+ $log = nulllogger
23
+ end
24
+
25
+ require 'fluent/plugin/out_query_combiner'
26
+
27
+ class Test::Unit::TestCase
28
+ end
@@ -0,0 +1,25 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'helper'
3
+
4
+ class QueryCombinerOutputTest < Test::Unit::TestCase
5
+ def setup
6
+ Fluent::Test.setup
7
+ end
8
+
9
+ CONFIG = %[
10
+ ]
11
+
12
+ def create_driver(conf = CONFIG, tag='test.input')
13
+ Fluent::Test::OutputTestDriver.new(Fluent::QueryCombinerOutput, tag).configure(conf)
14
+ end
15
+
16
+ def test_configure
17
+ assert_raise(Fluent::ConfigError) {
18
+ d = create_driver('')
19
+ }
20
+ end
21
+
22
+ def test_write
23
+ d = create_driver
24
+ end
25
+ end
metadata ADDED
@@ -0,0 +1,96 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-querycombiner
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0.pre
5
+ platform: ruby
6
+ authors:
7
+ - Takahiro Kamatani
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-07-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rake
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: fluentd
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 0.10.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.10.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: redis
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description: Fluent plugin to combine multiple queries.
56
+ email:
57
+ - buhii314@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - ".gitignore"
63
+ - Gemfile
64
+ - README.md
65
+ - Rakefile
66
+ - fluent-plugin-querycombiner.gemspec
67
+ - lib/fluent/plugin/out_query_combiner.rb
68
+ - test/helper.rb
69
+ - test/plugin/test_out_query_combiner.rb
70
+ homepage: https://github.com/buhii/fluent-plugin-querycombiner
71
+ licenses:
72
+ - Apache License, Version 2.0
73
+ metadata: {}
74
+ post_install_message:
75
+ rdoc_options: []
76
+ require_paths:
77
+ - lib
78
+ required_ruby_version: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ required_rubygems_version: !ruby/object:Gem::Requirement
84
+ requirements:
85
+ - - ">"
86
+ - !ruby/object:Gem::Version
87
+ version: 1.3.1
88
+ requirements: []
89
+ rubyforge_project:
90
+ rubygems_version: 2.2.2
91
+ signing_key:
92
+ specification_version: 4
93
+ summary: Fluent plugin to combine multiple queries.
94
+ test_files:
95
+ - test/helper.rb
96
+ - test/plugin/test_out_query_combiner.rb