fluent-plugin-querycombiner 0.0.0.pre → 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE.txt +13 -0
- data/NOTICE +6 -0
- data/README.md +179 -23
- data/fluent-plugin-querycombiner.gemspec +1 -1
- data/lib/fluent/plugin/out_query_combiner.rb +9 -2
- data/test/plugin/test_out_query_combiner.rb +307 -4
- metadata +6 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 65fedb30907434824303a1a024616f1bc3431d3e
|
|
4
|
+
data.tar.gz: 12a5292be841fc2c060892f0fc588a0172ce01df
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: fee92e2a0ffbe5c606663d55c7de4773ebd5d44b976f3523cd467dde981347a87ed752f803c0e4206070ca025a6db478703a7e0b4662771289467ed831132c57
|
|
7
|
+
data.tar.gz: 546fa4062b3ab8e6b2820c29062f3ad615b3ab43e1acc5209570e614c68ae4479241a73fe3b090064e1b6b424acd5734d17b52d3b72caf200193468bc315980f
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Copyright (c) 2014- Takahiro Kamatani
|
|
2
|
+
|
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
you may not use this file except in compliance with the License.
|
|
5
|
+
You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
See the License for the specific language governing permissions and
|
|
13
|
+
limitations under the License.
|
data/NOTICE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
fluent-plugin-querycombiner
|
|
2
|
+
|
|
3
|
+
Copyright 2014- Takahiro Kamatani
|
|
4
|
+
|
|
5
|
+
The portion of this product was originally developed by Yuyang Lan, for fluent-plugin-onlineuser.
|
|
6
|
+
fluent-plugin-onlineuser is licensed under Apache License v2.0 and available at https://github.com/y-lan/fluent-plugin-onlineuser
|
data/README.md
CHANGED
|
@@ -8,60 +8,216 @@ This plugin is based on [fluent-plugin-onlineuser](https://github.com/y-lan/flue
|
|
|
8
8
|
## Requirement
|
|
9
9
|
* a running Redis
|
|
10
10
|
|
|
11
|
+
## Installation
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
```
|
|
14
|
+
$ fluent-gem install fluent-plugin-querycombiner
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
## Tutorial
|
|
19
|
+
### Simple combination
|
|
20
|
+
|
|
21
|
+
Suppose you have the sequence of event messages like:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
{
|
|
25
|
+
'event_id': '01234567',
|
|
26
|
+
'status': 'event-start',
|
|
27
|
+
'started_at': '2001-02-03T04:05:06Z',
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
and:
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
{
|
|
35
|
+
'event_id': '01234567',
|
|
36
|
+
'status': 'event-finish',
|
|
37
|
+
'finished_at': '2001-02-03T04:15:11Z',
|
|
38
|
+
}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Now you can combine these messages with this configuration:
|
|
13
42
|
|
|
14
43
|
```
|
|
15
|
-
<match
|
|
44
|
+
<match event.**>
|
|
16
45
|
type query_combiner
|
|
17
46
|
tag combined.test
|
|
18
47
|
|
|
19
|
-
|
|
20
|
-
|
|
48
|
+
# redis settings
|
|
21
49
|
host localhost
|
|
22
50
|
port 6379
|
|
23
51
|
db_index 0
|
|
24
|
-
redis_retry 3
|
|
25
52
|
|
|
26
|
-
query_identify
|
|
27
|
-
query_ttl
|
|
28
|
-
buffer_size
|
|
53
|
+
query_identify event_id # field to combine together
|
|
54
|
+
query_ttl 3600 # messages time-to-live[sec]
|
|
55
|
+
buffer_size 1000 # max queries to store in redis
|
|
29
56
|
|
|
30
57
|
<catch>
|
|
31
|
-
condition status == '
|
|
32
|
-
replace time => time_init, status => status_init
|
|
58
|
+
condition status == 'event-start'
|
|
33
59
|
</catch>
|
|
34
60
|
|
|
35
|
-
<
|
|
36
|
-
condition status == '
|
|
37
|
-
</
|
|
61
|
+
<dump>
|
|
62
|
+
condition status == 'event-finish'
|
|
63
|
+
</dump>
|
|
64
|
+
|
|
65
|
+
</match>
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Combined results will be:
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
{
|
|
72
|
+
"event_id": "01234567",
|
|
73
|
+
"status": "event-finish",
|
|
74
|
+
"started_at": "2001-02-03T04:05:06Z",
|
|
75
|
+
"finished_at": "2001-02-03T04:05:06Z"
|
|
76
|
+
}
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Replace some field names
|
|
80
|
+
|
|
81
|
+
If messages has the same fields, these are overwritten in the combination process. You can use `replace` sentence in `<catch>` and `<dump>` blocks to avoid overwrite such fields.
|
|
82
|
+
|
|
83
|
+
For example, you have some event messages like:
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
{
|
|
87
|
+
'event_id': '01234567',
|
|
88
|
+
'status': 'event-start',
|
|
89
|
+
'time': '2001-02-03T04:05:06Z',
|
|
90
|
+
}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
and:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
{
|
|
97
|
+
'event_id': '01234567',
|
|
98
|
+
'status': 'event-finish',
|
|
99
|
+
'time': '2001-02-03T04:15:11Z',
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
You can keep `time` fields which defined both `event-start` and `event-finish` by using `replace` sentence.
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
<match event.**>
|
|
107
|
+
(...type, tag and redis configuration...)
|
|
108
|
+
|
|
109
|
+
query_identify event_id # field to combine together
|
|
110
|
+
query_ttl 3600 # messages time-to-live[sec]
|
|
111
|
+
buffer_size 1000 # max queries to store in redis
|
|
112
|
+
|
|
113
|
+
<catch>
|
|
114
|
+
condition status == 'event-start'
|
|
115
|
+
replace time => time_start
|
|
116
|
+
|
|
117
|
+
</catch>
|
|
38
118
|
|
|
39
119
|
<dump>
|
|
40
|
-
condition status == '
|
|
41
|
-
replace time => time_finish
|
|
120
|
+
condition status == 'event-finish'
|
|
121
|
+
replace time => time_finish
|
|
42
122
|
</dump>
|
|
43
123
|
|
|
124
|
+
</match>
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Combined results will be:
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
{
|
|
131
|
+
"event_id": "01234567",
|
|
132
|
+
"status": "event-finish",
|
|
133
|
+
"time_start": "2001-02-03T04:05:06Z",
|
|
134
|
+
"time_finish": "2001-02-03T04:15:11Z"
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### \<release\> block
|
|
139
|
+
|
|
140
|
+
In previous examples, messages with `"status": "event-start"` will be watched by plugin immediately.
|
|
141
|
+
|
|
142
|
+
Suppose some error events occur and you don't want to watch or combine these messages.
|
|
143
|
+
|
|
144
|
+
In this case `<release>` block will be useful.
|
|
145
|
+
|
|
146
|
+
For example, your error messages are such like:
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
{
|
|
150
|
+
"event_id": "01234567",
|
|
151
|
+
"status": "event-error",
|
|
152
|
+
"time": "2001-02-03T04:05:06Z"
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Append this `<release>` block to the configuration and error events will not be watched or combined:
|
|
157
|
+
|
|
158
|
+
```
|
|
44
159
|
<release>
|
|
45
|
-
condition status == '
|
|
160
|
+
condition status == 'event-error'
|
|
46
161
|
</release>
|
|
162
|
+
```
|
|
47
163
|
|
|
48
|
-
|
|
164
|
+
You cannot use `replace` sentence in the `<release>` block.
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
### \<prolong\> block
|
|
168
|
+
|
|
169
|
+
Suppose your `query_ttl` is **600** (10 minutes) and almost events are finished within **10 minutes**. But occasionally very-long events occur which finish about **1 hours**. These very-long events send `status: 'event-continue'` messages every 5 minutes for keep-alive.
|
|
170
|
+
|
|
171
|
+
In this case you can use `<prolong>` block to reset expired time.
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
<prolong>
|
|
175
|
+
condition status == 'event-continue'
|
|
176
|
+
</prolong>
|
|
49
177
|
```
|
|
50
178
|
|
|
179
|
+
You cannot use `replace` sentence in the `<prolong>` block.
|
|
180
|
+
|
|
181
|
+
Also you cannot combine messages which defined `<prolong>` blocks.
|
|
182
|
+
|
|
183
|
+
|
|
51
184
|
## Configuration
|
|
52
|
-
|
|
185
|
+
|
|
186
|
+
### tag
|
|
187
|
+
The tag prefix for emitted event messages. By default it's `query_combiner`.
|
|
188
|
+
|
|
189
|
+
### host, port, db_index
|
|
53
190
|
The basic information for connecting to Redis. By default it's **redis://127.0.0.1:6379/0**
|
|
54
191
|
|
|
55
|
-
|
|
192
|
+
### redis_retry
|
|
56
193
|
How many times should the plugin retry when performing a redis operation before raising a error.
|
|
57
194
|
By default it's 3.
|
|
58
195
|
|
|
59
|
-
###
|
|
60
|
-
The inactive expire time in seconds. By default it's 1800 (30 minutes).
|
|
196
|
+
### querl_ttl
|
|
197
|
+
The inactive expire time in seconds. By default it's **1800** (30 minutes).
|
|
61
198
|
|
|
199
|
+
### buffer_size
|
|
200
|
+
The max queries to store in redis. By default it's **1000**.
|
|
201
|
+
|
|
202
|
+
### remove_interval
|
|
203
|
+
The interval time to delete expired or overflowed queries which configured by `query_ttl` and `buffer_size`. By default it's 10 [sec].
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
### redis_key_prefix
|
|
207
|
+
|
|
208
|
+
The key prefix for data stored in Redis. By default it's `query_combiner:`.
|
|
209
|
+
|
|
210
|
+
### query_identify
|
|
211
|
+
|
|
212
|
+
Indicates how to extract the query identity from event record.
|
|
213
|
+
It can be set as a single field name or multiple field names join by comma (`,`).
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
## TODO
|
|
217
|
+
|
|
218
|
+
- Multi-query combination
|
|
219
|
+
- Support hyphen `-` and dollar `$` contained field names
|
|
62
220
|
|
|
63
|
-
### tag
|
|
64
|
-
The tag prefix for emitted event messages. By default it's `query_combiner`.
|
|
65
221
|
|
|
66
222
|
## Copyright
|
|
67
223
|
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
|
3
3
|
|
|
4
4
|
Gem::Specification.new do |spec|
|
|
5
5
|
spec.name = "fluent-plugin-querycombiner"
|
|
6
|
-
spec.version = "0.0.
|
|
6
|
+
spec.version = "0.0.1"
|
|
7
7
|
spec.authors = ["Takahiro Kamatani"]
|
|
8
8
|
spec.email = ["buhii314@gmail.com"]
|
|
9
9
|
spec.description = %q{Fluent plugin to combine multiple queries.}
|
|
@@ -12,7 +12,7 @@ module Fluent
|
|
|
12
12
|
config_param :redis_key_prefix, :string, :default => 'query_combiner:'
|
|
13
13
|
config_param :query_identify, :string, :default => 'session-id'
|
|
14
14
|
config_param :query_ttl, :integer, :default => 1800
|
|
15
|
-
config_param :buffer_size, :integer, :default =>
|
|
15
|
+
config_param :buffer_size, :integer, :default => 1000
|
|
16
16
|
|
|
17
17
|
config_param :flush_interval, :integer, :default => 60
|
|
18
18
|
config_param :remove_interval, :integer, :default => 10
|
|
@@ -36,7 +36,10 @@ module Fluent
|
|
|
36
36
|
|
|
37
37
|
# Create functions for each conditions
|
|
38
38
|
@_cond_funcs = {}
|
|
39
|
-
@_replace_keys = {
|
|
39
|
+
@_replace_keys = {
|
|
40
|
+
'catch' => {},
|
|
41
|
+
'dump' => {},
|
|
42
|
+
}
|
|
40
43
|
|
|
41
44
|
def get_arguments(eval_str)
|
|
42
45
|
eval_str.scan(/[\"\']?[a-zA-Z][\w\d\.\-\_]*[\"\']?/).uniq.select{|x|
|
|
@@ -88,6 +91,10 @@ module Fluent
|
|
|
88
91
|
end
|
|
89
92
|
}
|
|
90
93
|
}
|
|
94
|
+
|
|
95
|
+
if not (@_cond_funcs.has_key?('catch') and @_cond_funcs.has_key?('dump'))
|
|
96
|
+
raise Fluent::ConfigError, "Must have <catch> and <dump> blocks"
|
|
97
|
+
end
|
|
91
98
|
end
|
|
92
99
|
|
|
93
100
|
def has_all_keys?(record, argv)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
require 'helper'
|
|
3
|
+
require 'redis'
|
|
3
4
|
|
|
4
5
|
class QueryCombinerOutputTest < Test::Unit::TestCase
|
|
5
6
|
def setup
|
|
@@ -7,19 +8,321 @@ class QueryCombinerOutputTest < Test::Unit::TestCase
|
|
|
7
8
|
end
|
|
8
9
|
|
|
9
10
|
CONFIG = %[
|
|
11
|
+
query_identify event_id
|
|
12
|
+
<catch>
|
|
13
|
+
condition status == 'start'
|
|
14
|
+
replace time => time_start
|
|
15
|
+
</catch>
|
|
16
|
+
|
|
17
|
+
<dump>
|
|
18
|
+
condition status == 'finish'
|
|
19
|
+
replace time => time_finish
|
|
20
|
+
</dump>
|
|
10
21
|
]
|
|
11
22
|
|
|
12
|
-
|
|
13
|
-
|
|
23
|
+
@redis
|
|
24
|
+
|
|
25
|
+
def setup
|
|
26
|
+
Fluent::Test.setup
|
|
27
|
+
@redis = Redis.new(:host => 'localhost', :port => 6379, :thread_safe => true, :db => 0)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def teardown
|
|
31
|
+
@redis.quit
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def create_driver(conf, tag='test')
|
|
35
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::QueryCombinerOutput, tag).configure(conf)
|
|
14
36
|
end
|
|
15
37
|
|
|
16
38
|
def test_configure
|
|
17
39
|
assert_raise(Fluent::ConfigError) {
|
|
18
40
|
d = create_driver('')
|
|
19
41
|
}
|
|
42
|
+
# Must have <catch> and <dump> conditions
|
|
43
|
+
assert_raise(Fluent::ConfigError) {
|
|
44
|
+
d = create_driver %[
|
|
45
|
+
query_identify event_id
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
assert_raise(Fluent::ConfigError) {
|
|
49
|
+
d = create_driver %[
|
|
50
|
+
query_identify event_id
|
|
51
|
+
<catch>
|
|
52
|
+
condition status == 'start'
|
|
53
|
+
</catch>
|
|
54
|
+
]
|
|
55
|
+
}
|
|
56
|
+
assert_raise(Fluent::ConfigError) {
|
|
57
|
+
d = create_driver %[
|
|
58
|
+
query_identify event_id
|
|
59
|
+
<dump>
|
|
60
|
+
condition status == 'finish'
|
|
61
|
+
</dump>
|
|
62
|
+
]
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# `replace` configuration only allowed in <catch> and <dump>
|
|
66
|
+
assert_raise(Fluent::ConfigError) {
|
|
67
|
+
d = create_driver %[
|
|
68
|
+
query_identify event_id
|
|
69
|
+
<catch>
|
|
70
|
+
condition status == 'start'
|
|
71
|
+
replace hoge => hoge_start
|
|
72
|
+
</catch>
|
|
73
|
+
<release>
|
|
74
|
+
condition status == 'error'
|
|
75
|
+
replace hoge => hoge_error
|
|
76
|
+
</release>
|
|
77
|
+
<dump>
|
|
78
|
+
condition status == 'finish'
|
|
79
|
+
replace hoge => hoge_finish
|
|
80
|
+
</dump>
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def test_readme_sample_basic_example
|
|
87
|
+
d = create_driver %[
|
|
88
|
+
query_identify event_id
|
|
89
|
+
query_ttl 3600 # time to live[sec]
|
|
90
|
+
buffer_size 1000 # queries
|
|
91
|
+
|
|
92
|
+
<catch>
|
|
93
|
+
condition status == 'event-start'
|
|
94
|
+
</catch>
|
|
95
|
+
|
|
96
|
+
<dump>
|
|
97
|
+
condition status == 'event-finish'
|
|
98
|
+
</dump>
|
|
99
|
+
]
|
|
100
|
+
time = Time.now.to_i
|
|
101
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-start", "started_at"=>"2001-02-03T04:05:06Z"}, time)
|
|
102
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-finish", "finished_at"=>"2001-02-03T04:05:06Z"}, time)
|
|
103
|
+
d.run
|
|
104
|
+
assert_equal d.emits.length, 1
|
|
105
|
+
assert_equal d.emits[0][2], {
|
|
106
|
+
"event_id"=>"01234567",
|
|
107
|
+
"status"=>"event-finish",
|
|
108
|
+
"started_at"=>"2001-02-03T04:05:06Z",
|
|
109
|
+
"finished_at"=>"2001-02-03T04:05:06Z"}
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def test_readme_sample_replace_sentence
|
|
113
|
+
d = create_driver %[
|
|
114
|
+
query_identify event_id
|
|
115
|
+
query_ttl 3600 # time to live[sec]
|
|
116
|
+
buffer_size 1000 # queries
|
|
117
|
+
|
|
118
|
+
<catch>
|
|
119
|
+
condition status == 'event-start'
|
|
120
|
+
replace time => time_start
|
|
121
|
+
</catch>
|
|
122
|
+
|
|
123
|
+
<dump>
|
|
124
|
+
condition status == 'event-finish'
|
|
125
|
+
replace time => time_finish
|
|
126
|
+
</dump>
|
|
127
|
+
]
|
|
128
|
+
time = Time.now.to_i
|
|
129
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-start", "time"=>"2001-02-03T04:05:06Z"}, time)
|
|
130
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-finish", "time"=>"2001-02-03T04:15:11Z"}, time)
|
|
131
|
+
d.run
|
|
132
|
+
assert_equal d.emits.length, 1
|
|
133
|
+
assert_equal d.emits[0][2], {
|
|
134
|
+
"event_id"=>"01234567",
|
|
135
|
+
"status"=>"event-finish",
|
|
136
|
+
"time_start"=>"2001-02-03T04:05:06Z",
|
|
137
|
+
"time_finish"=>"2001-02-03T04:15:11Z"}
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def test_readme_sample_release
|
|
141
|
+
d = create_driver %[
|
|
142
|
+
query_identify event_id
|
|
143
|
+
query_ttl 3600 # time to live[sec]
|
|
144
|
+
buffer_size 1000 # queries
|
|
145
|
+
|
|
146
|
+
<catch>
|
|
147
|
+
condition status == 'event-start'
|
|
148
|
+
</catch>
|
|
149
|
+
|
|
150
|
+
<dump>
|
|
151
|
+
condition status == 'event-finish'
|
|
152
|
+
</dump>
|
|
153
|
+
|
|
154
|
+
<release>
|
|
155
|
+
condition status == 'event-error'
|
|
156
|
+
</release>
|
|
157
|
+
]
|
|
158
|
+
time = Time.now.to_i
|
|
159
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-start", "time"=>"2001-02-03T04:05:06Z"}, time)
|
|
160
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-error", "time"=>"2001-02-03T04:05:06Z"}, time)
|
|
161
|
+
d.run
|
|
162
|
+
assert_equal d.emits.length, 0
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def test_readme_sample_prolong
|
|
166
|
+
d = create_driver %[
|
|
167
|
+
query_identify event_id
|
|
168
|
+
query_ttl 3600 # time to live[sec]
|
|
169
|
+
buffer_size 1000 # queries
|
|
170
|
+
|
|
171
|
+
<catch>
|
|
172
|
+
condition status == 'event-start'
|
|
173
|
+
</catch>
|
|
174
|
+
|
|
175
|
+
<dump>
|
|
176
|
+
condition status == 'event-finish'
|
|
177
|
+
</dump>
|
|
178
|
+
|
|
179
|
+
<prolong>
|
|
180
|
+
condition status == 'event-continue'
|
|
181
|
+
</prolong>
|
|
182
|
+
|
|
183
|
+
<release>
|
|
184
|
+
condition status == 'event-error'
|
|
185
|
+
</release>
|
|
186
|
+
]
|
|
187
|
+
time = Time.now.to_i
|
|
188
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-start", "time"=>"2001-02-03T04:05:06Z"}, time)
|
|
189
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-continue", "time"=>"2001-02-03T04:05:07Z"}, time)
|
|
190
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-continue", "time"=>"2001-02-03T04:05:08Z"}, time)
|
|
191
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-continue", "time"=>"2001-02-03T04:05:09Z"}, time)
|
|
192
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-continue", "time"=>"2001-02-03T04:05:10Z"}, time)
|
|
193
|
+
d.emit({"event_id"=>"01234567", "status"=>"event-finish", "time"=>"2001-02-03T04:05:11Z"}, time)
|
|
194
|
+
d.run
|
|
195
|
+
assert_equal d.emits.length, 1
|
|
196
|
+
assert_equal d.emits[0][2], {
|
|
197
|
+
"event_id"=>"01234567",
|
|
198
|
+
"status"=>"event-finish",
|
|
199
|
+
"time"=>"2001-02-03T04:05:11Z"}
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def test_simple_events
|
|
203
|
+
d = create_driver CONFIG
|
|
204
|
+
time = Time.now.to_i
|
|
205
|
+
d.emit({"event_id"=>"001", "status"=>"start", "time"=>"21:00"}, time)
|
|
206
|
+
d.emit({"event_id"=>"002", "status"=>"start", "time"=>"22:00"}, time)
|
|
207
|
+
d.emit({"event_id"=>"001", "status"=>"finish", "time"=>"23:00"}, time)
|
|
208
|
+
d.emit({"event_id"=>"002", "status"=>"finish", "time"=>"24:00"}, time)
|
|
209
|
+
d.run
|
|
210
|
+
assert_equal d.emits[0][2], {
|
|
211
|
+
"event_id"=>"001",
|
|
212
|
+
"status"=>"finish",
|
|
213
|
+
"time_start"=>"21:00",
|
|
214
|
+
"time_finish"=>"23:00"}
|
|
215
|
+
assert_equal d.emits[1][2], {
|
|
216
|
+
"event_id"=>"002",
|
|
217
|
+
"status"=>"finish",
|
|
218
|
+
"time_start"=>"22:00",
|
|
219
|
+
"time_finish"=>"24:00"}
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def test_catch_dump_release
|
|
223
|
+
d = create_driver %[
|
|
224
|
+
buffer_size 1001
|
|
225
|
+
query_identify event_id
|
|
226
|
+
|
|
227
|
+
<catch>
|
|
228
|
+
condition status == 'start'
|
|
229
|
+
replace time => time_start
|
|
230
|
+
</catch>
|
|
231
|
+
|
|
232
|
+
<dump>
|
|
233
|
+
condition status == 'finish'
|
|
234
|
+
replace time => time_finish
|
|
235
|
+
</dump>
|
|
236
|
+
|
|
237
|
+
<release>
|
|
238
|
+
condition status == 'error'
|
|
239
|
+
</release>
|
|
240
|
+
]
|
|
241
|
+
def emit(d, event_id, status, t)
|
|
242
|
+
d.emit({"event_id"=>event_id, "status"=>status, "time"=>t}, Time.now.to_i)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
(0..1000).each { |num|
|
|
246
|
+
emit(d, num, "start", "21:00")
|
|
247
|
+
}
|
|
248
|
+
finish_list = []
|
|
249
|
+
(0..1000).each { |num|
|
|
250
|
+
status = if rand >= 0.5 then
|
|
251
|
+
finish_list.push(num)
|
|
252
|
+
"finish"
|
|
253
|
+
else
|
|
254
|
+
"error"
|
|
255
|
+
end
|
|
256
|
+
emit(d, num, status, "22:00")
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
d.run
|
|
260
|
+
finish_list.each_with_index { |num, index|
|
|
261
|
+
assert_equal d.emits[index][2], {
|
|
262
|
+
"event_id" => num,
|
|
263
|
+
"status" => "finish",
|
|
264
|
+
"time_start" => "21:00",
|
|
265
|
+
"time_finish" => "22:00",
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
assert_equal d.emits.size, finish_list.size
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def test_multi_query_identifier
|
|
272
|
+
d = create_driver %[
|
|
273
|
+
buffer_size 1001
|
|
274
|
+
query_identify aid, bid, cid
|
|
275
|
+
|
|
276
|
+
<catch>
|
|
277
|
+
condition status == 'start'
|
|
278
|
+
</catch>
|
|
279
|
+
|
|
280
|
+
<dump>
|
|
281
|
+
condition status == 'finish'
|
|
282
|
+
</dump>
|
|
283
|
+
]
|
|
284
|
+
def emit(d, aid, bid, cid, status, t)
|
|
285
|
+
d.emit(
|
|
286
|
+
{"aid"=>aid, "bid"=>bid, "cid"=>cid, "status"=>status, "time"=>t},
|
|
287
|
+
Time.now.to_i
|
|
288
|
+
)
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
finish_list = []
|
|
292
|
+
(0..1000).each { |num|
|
|
293
|
+
aid = (rand * 1000).to_i
|
|
294
|
+
bid = (rand * 1000).to_i
|
|
295
|
+
cid = (rand * 1000).to_i
|
|
296
|
+
emit(d, aid, bid, cid, "start", "22:00")
|
|
297
|
+
finish_list.push([aid, bid, cid])
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
t_list = []
|
|
301
|
+
finish_list.each { |ids|
|
|
302
|
+
t = (rand * 100000).to_i
|
|
303
|
+
emit(d, ids[0], ids[1], ids[2], "finish", t)
|
|
304
|
+
t_list.push(t)
|
|
305
|
+
}
|
|
306
|
+
d.run
|
|
307
|
+
|
|
308
|
+
finish_list.each_with_index { |ids, index|
|
|
309
|
+
assert_equal d.emits[index][2], {
|
|
310
|
+
"aid" => ids[0],
|
|
311
|
+
"bid" => ids[1],
|
|
312
|
+
"cid" => ids[2],
|
|
313
|
+
"status" => "finish",
|
|
314
|
+
"time" => t_list[index]
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
assert_equal d.emits.size, finish_list.size
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def test_buffer_size
|
|
321
|
+
|
|
20
322
|
end
|
|
21
323
|
|
|
22
|
-
def
|
|
23
|
-
|
|
324
|
+
def test_query_ttl
|
|
325
|
+
|
|
24
326
|
end
|
|
327
|
+
|
|
25
328
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-querycombiner
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Takahiro Kamatani
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2014-07-
|
|
11
|
+
date: 2014-07-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -61,6 +61,8 @@ extra_rdoc_files: []
|
|
|
61
61
|
files:
|
|
62
62
|
- ".gitignore"
|
|
63
63
|
- Gemfile
|
|
64
|
+
- LICENSE.txt
|
|
65
|
+
- NOTICE
|
|
64
66
|
- README.md
|
|
65
67
|
- Rakefile
|
|
66
68
|
- fluent-plugin-querycombiner.gemspec
|
|
@@ -82,9 +84,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
82
84
|
version: '0'
|
|
83
85
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
84
86
|
requirements:
|
|
85
|
-
- - "
|
|
87
|
+
- - ">="
|
|
86
88
|
- !ruby/object:Gem::Version
|
|
87
|
-
version:
|
|
89
|
+
version: '0'
|
|
88
90
|
requirements: []
|
|
89
91
|
rubyforge_project:
|
|
90
92
|
rubygems_version: 2.2.2
|