fluent-plugin-groonga 1.0.5 → 1.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/README.md +4 -4
- data/doc/text/news.md +12 -0
- data/fluent-plugin-groonga.gemspec +2 -2
- data/lib/fluent/plugin/out_groonga.rb +258 -21
- data/sample/store-apache.conf +239 -0
- data/sample/store.conf +1 -1
- data/test/output/test_table_definition.rb +167 -0
- data/test/output/test_type_guesser.rb +21 -2
- data/test/run-test.rb +2 -7
- data/test/test_output.rb +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc471bf3943996d9224a54431bf81a5323c53dcd
|
4
|
+
data.tar.gz: 3c37826e7793bbcab4d8009c6d0555aa74577858
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1dc9a521003ddc42ccc0ff98e88468931b4d9400e5d7cb5e592a836d1be655a858a396a9275c1058a8c221041854c3d5e69e481781fdc475e0911d1595301d3
|
7
|
+
data.tar.gz: 63e6a01cafd4bcb3e104ec80bcd53a48a4d375f29f95a292b4f214e360c287a846d41f120dd5e3b9ed52ef587f60a5cd7283dad7c03aa14f07b9d80f2adf6807
|
data/Gemfile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- mode: ruby; coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2012 Kouhei Sutou <kou@clear-code.com>
|
3
|
+
# Copyright (C) 2012-2014 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
#
|
5
5
|
# This library is free software; you can redistribute it and/or
|
6
6
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -15,6 +15,6 @@
|
|
15
15
|
# License along with this library; if not, write to the Free Software
|
16
16
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
17
|
|
18
|
-
source
|
18
|
+
source "https://rubygems.org/"
|
19
19
|
|
20
20
|
gemspec
|
data/README.md
CHANGED
@@ -42,7 +42,7 @@ into Groonga:
|
|
42
42
|
|
43
43
|
<match log.**>
|
44
44
|
type groonga
|
45
|
-
|
45
|
+
store_table logs
|
46
46
|
|
47
47
|
protocol http
|
48
48
|
host 127.0.0.1
|
@@ -88,13 +88,13 @@ define schema in Groonga before running Fluentd. You just run Groonga.
|
|
88
88
|
|
89
89
|
There is one required parameter:
|
90
90
|
|
91
|
-
* `
|
91
|
+
* `store_table`: It specifies table name for storing logs.
|
92
92
|
|
93
93
|
Here is a minimum configuration:
|
94
94
|
|
95
95
|
<match log.**>
|
96
96
|
type groonga
|
97
|
-
|
97
|
+
store_table logs
|
98
98
|
</match>
|
99
99
|
|
100
100
|
The configuration stores logs into `logs` table in Groonga that runs
|
@@ -115,7 +115,7 @@ Here is a configuration that specifies optional parameters explicitly:
|
|
115
115
|
|
116
116
|
<match log.**>
|
117
117
|
type groonga
|
118
|
-
|
118
|
+
store_table logs
|
119
119
|
|
120
120
|
protocol http
|
121
121
|
host 127.0.0.1
|
data/doc/text/news.md
CHANGED
@@ -2,6 +2,18 @@
|
|
2
2
|
|
3
3
|
# News
|
4
4
|
|
5
|
+
## 1.0.6: 2014-11-05
|
6
|
+
|
7
|
+
### Improvements
|
8
|
+
|
9
|
+
* out: Renamed `table` parameter name to `store_table`.
|
10
|
+
`table` parameter is still usable for backward compatibility.
|
11
|
+
* out: Supported table definition by `<table>` configuration.
|
12
|
+
See sample/store-apache.conf for details.
|
13
|
+
* out: Supported specifying column type and creating indexes for auto
|
14
|
+
created columns by `<mapping>` configuration.
|
15
|
+
See sample/store-apache.conf for details.
|
16
|
+
|
5
17
|
## 1.0.5: 2014-10-21
|
6
18
|
|
7
19
|
### Improvements
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |spec|
|
19
19
|
spec.name = "fluent-plugin-groonga"
|
20
|
-
spec.version = "1.0.
|
20
|
+
spec.version = "1.0.6"
|
21
21
|
spec.authors = ["Kouhei Sutou"]
|
22
22
|
spec.email = ["kou@clear-code.com"]
|
23
23
|
spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
|
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.require_paths = ["lib"]
|
36
36
|
|
37
37
|
spec.add_runtime_dependency("fluentd")
|
38
|
-
spec.add_runtime_dependency("groonga-client")
|
38
|
+
spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
|
39
39
|
spec.add_runtime_dependency("groonga-command-parser")
|
40
40
|
|
41
41
|
spec.add_development_dependency("rake")
|
@@ -37,20 +37,65 @@ module Fluent
|
|
37
37
|
raise ConfigError, "must be http, gqtp or command: <#{value}>"
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
|
+
# alias is just for backward compatibility
|
42
|
+
config_param :store_table, :string, :default => nil, :alias => :table
|
43
|
+
|
44
|
+
config_section :table,
|
45
|
+
:param_name => "tables",
|
46
|
+
:required => false,
|
47
|
+
:multi => true do
|
48
|
+
config_param :name, :string
|
49
|
+
config_param :flags, :string, :default => nil
|
50
|
+
config_param :key_type, :string, :default => nil
|
51
|
+
config_param :default_tokenizer, :string, :default => nil
|
52
|
+
config_param :token_filters, :string, :default => nil
|
53
|
+
config_param :normalizer, :string, :default => nil
|
54
|
+
config_section :index,
|
55
|
+
:param_name => "indexes",
|
56
|
+
:required => false,
|
57
|
+
:multi => true do
|
58
|
+
config_param :name, :string
|
59
|
+
config_param :source_table, :string
|
60
|
+
config_param :source_columns, :string
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
config_section :mapping,
|
65
|
+
:param_name => "mappings",
|
66
|
+
:required => false,
|
67
|
+
:multi => true do
|
68
|
+
config_param :name, :string
|
69
|
+
config_param :type, :string, :default => nil
|
70
|
+
config_section :index,
|
71
|
+
:param_name => "indexes",
|
72
|
+
:required => false,
|
73
|
+
:multi => true do
|
74
|
+
config_param :table, :string
|
75
|
+
config_param :name, :string
|
76
|
+
config_param :flags, :string, :default => nil
|
77
|
+
end
|
78
|
+
end
|
41
79
|
|
42
80
|
def configure(conf)
|
43
81
|
super
|
44
82
|
@client = create_client(@protocol)
|
45
83
|
@client.configure(conf)
|
46
84
|
|
47
|
-
@
|
85
|
+
@schema = Schema.new(@client, @store_table, @mappings)
|
86
|
+
@emitter = Emitter.new(@client, @store_table, @schema)
|
87
|
+
|
88
|
+
@tables = @tables.collect do |table|
|
89
|
+
TableDefinition.new(table)
|
90
|
+
end
|
48
91
|
end
|
49
92
|
|
50
93
|
def start
|
51
94
|
super
|
52
95
|
@client.start
|
53
96
|
@emitter.start
|
97
|
+
tables_creator = TablesCreator.new(@client, @tables)
|
98
|
+
tables_creator.create
|
54
99
|
end
|
55
100
|
|
56
101
|
def shutdown
|
@@ -77,18 +122,171 @@ module Fluent
|
|
77
122
|
end
|
78
123
|
end
|
79
124
|
|
125
|
+
class TableDefinition
|
126
|
+
def initialize(raw)
|
127
|
+
@raw = raw
|
128
|
+
end
|
129
|
+
|
130
|
+
def name
|
131
|
+
@raw[:name]
|
132
|
+
end
|
133
|
+
|
134
|
+
def flags
|
135
|
+
parse_flags(@raw[:flags] || "TABLE_NO_KEY")
|
136
|
+
end
|
137
|
+
|
138
|
+
def key_type
|
139
|
+
@raw[:key_type]
|
140
|
+
end
|
141
|
+
|
142
|
+
def default_tokenizer
|
143
|
+
@raw[:default_tokenizer]
|
144
|
+
end
|
145
|
+
|
146
|
+
def token_filters
|
147
|
+
parse_items(@raw[:token_filters] || "")
|
148
|
+
end
|
149
|
+
|
150
|
+
def normalizer
|
151
|
+
@raw[:normalizer]
|
152
|
+
end
|
153
|
+
|
154
|
+
def indexes
|
155
|
+
(@raw[:indexes] || []).collect do |raw|
|
156
|
+
IndexDefinition.new(self, raw)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def use_n_gram_tokenizer?
|
161
|
+
/\AToken(?:Uni|Bi|Tri)gram/ === default_tokenizer.to_s
|
162
|
+
end
|
163
|
+
|
164
|
+
def have_difference?(table)
|
165
|
+
return true if table.name != name
|
166
|
+
|
167
|
+
table_flags = (parse_flags(table.flags) - ["PERSISTENT"])
|
168
|
+
return true if table_flags.sort != flags.sort
|
169
|
+
|
170
|
+
return true if table.domain != key_type
|
171
|
+
|
172
|
+
return true if table.default_tokenizer != default_tokenizer
|
173
|
+
|
174
|
+
# TODO
|
175
|
+
# return true if table.token_filters.sort != token_filters.sort
|
176
|
+
|
177
|
+
return true if table.normalizer != normalizer
|
178
|
+
|
179
|
+
false
|
180
|
+
end
|
181
|
+
|
182
|
+
def to_create_arguments
|
183
|
+
arguments = {
|
184
|
+
"name" => name,
|
185
|
+
"flags" => flags.join("|"),
|
186
|
+
"key_type" => key_type,
|
187
|
+
"default_tokenizer" => default_tokenizer,
|
188
|
+
# TODO
|
189
|
+
# "token_filters" => token_filters.join("|"),
|
190
|
+
"normalizer" => normalizer,
|
191
|
+
}
|
192
|
+
arguments.keys.each do |key|
|
193
|
+
value = arguments[key]
|
194
|
+
arguments.delete(key) if value.nil? or value.empty?
|
195
|
+
end
|
196
|
+
arguments
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
def parse_flags(flags)
|
201
|
+
if flags.is_a?(Array)
|
202
|
+
flags
|
203
|
+
else
|
204
|
+
flags.strip.split(/\s*\|\s*/)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def parse_items(items)
|
209
|
+
if items.is_a?(Array)
|
210
|
+
items
|
211
|
+
else
|
212
|
+
items.strip.split(/\s*,\s*/)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
class IndexDefinition
|
217
|
+
def initialize(table, raw)
|
218
|
+
@table = table
|
219
|
+
@raw = raw
|
220
|
+
end
|
221
|
+
|
222
|
+
def name
|
223
|
+
@raw[:name]
|
224
|
+
end
|
225
|
+
|
226
|
+
def source_table
|
227
|
+
@raw[:source_table]
|
228
|
+
end
|
229
|
+
|
230
|
+
def source_columns
|
231
|
+
@raw[:source_columns]
|
232
|
+
end
|
233
|
+
|
234
|
+
def flags
|
235
|
+
_flags = ["COLUMN_INDEX"]
|
236
|
+
_flags << "WITH_POSITION" if @table.use_n_gram_tokenizer?
|
237
|
+
_flags << "WITH_SECTION" if source_columns.size >= 2
|
238
|
+
_flags
|
239
|
+
end
|
240
|
+
|
241
|
+
def to_create_arguments
|
242
|
+
{
|
243
|
+
"table" => @table.name,
|
244
|
+
"name" => name,
|
245
|
+
"flags" => flags.join,
|
246
|
+
"type" => source_table,
|
247
|
+
"source" => source_columns,
|
248
|
+
}
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
class TablesCreator
|
254
|
+
def initialize(client, definitions)
|
255
|
+
@client = client
|
256
|
+
@definitions = definitions
|
257
|
+
end
|
258
|
+
|
259
|
+
def create
|
260
|
+
return if @definitions.empty?
|
261
|
+
|
262
|
+
table_list = @client.execute("table_list")
|
263
|
+
@definitions.each do |definition|
|
264
|
+
existing_table = table_list.find do |table|
|
265
|
+
table.name == definition.name
|
266
|
+
end
|
267
|
+
if existing_table
|
268
|
+
next unless definition.have_difference?(existing_table)
|
269
|
+
# TODO: Is it OK?
|
270
|
+
@client.execute("table_remove", "name" => definition.name)
|
271
|
+
end
|
272
|
+
|
273
|
+
@client.execute("table_create", definition.to_create_arguments)
|
274
|
+
definition.indexes.each do |index|
|
275
|
+
@client.execute("column_create", index.to_create_arguments)
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
80
281
|
class Schema
|
81
|
-
def initialize(client, table_name)
|
282
|
+
def initialize(client, table_name, mappings)
|
82
283
|
@client = client
|
83
284
|
@table_name = table_name
|
285
|
+
@mappings = mappings
|
84
286
|
@table = nil
|
85
287
|
@columns = nil
|
86
288
|
end
|
87
289
|
|
88
|
-
def populate
|
89
|
-
# TODO
|
90
|
-
end
|
91
|
-
|
92
290
|
def update(records)
|
93
291
|
ensure_table
|
94
292
|
ensure_columns
|
@@ -120,7 +318,6 @@ module Fluent
|
|
120
318
|
if target_table
|
121
319
|
@table = Table.new(@table_name, target_table.domain)
|
122
320
|
else
|
123
|
-
# TODO: Check response
|
124
321
|
@client.execute("table_create",
|
125
322
|
"name" => @table_name,
|
126
323
|
"flags" => "TABLE_NO_KEY")
|
@@ -142,20 +339,37 @@ module Fluent
|
|
142
339
|
end
|
143
340
|
|
144
341
|
def create_column(name, sample_values)
|
342
|
+
mapping = @mappings.find do |mapping|
|
343
|
+
mapping.name == name
|
344
|
+
end
|
345
|
+
if mapping
|
346
|
+
value_type = mapping[:type]
|
347
|
+
end
|
145
348
|
guesser = TypeGuesser.new(sample_values)
|
146
|
-
value_type
|
349
|
+
value_type ||= guesser.guess
|
147
350
|
vector_p = guesser.vector?
|
148
351
|
if vector_p
|
149
352
|
flags = "COLUMN_VECTOR"
|
150
353
|
else
|
151
354
|
flags = "COLUMN_SCALAR"
|
152
355
|
end
|
153
|
-
# TODO: Check response
|
154
356
|
@client.execute("column_create",
|
155
357
|
"table" => @table_name,
|
156
358
|
"name" => name,
|
157
359
|
"flags" => flags,
|
158
360
|
"type" => value_type)
|
361
|
+
if mapping
|
362
|
+
mapping.indexes.each do |index|
|
363
|
+
index_flags = ["COLUMN_INDEX", index[:flags]].compact
|
364
|
+
@client.execute("column_create",
|
365
|
+
"table" => index[:table],
|
366
|
+
"name" => index[:table],
|
367
|
+
"flags" => index_flags.join("|"),
|
368
|
+
"type" => @table_name,
|
369
|
+
"source" => name)
|
370
|
+
end
|
371
|
+
end
|
372
|
+
|
159
373
|
Column.new(name, value_type, vector_p)
|
160
374
|
end
|
161
375
|
|
@@ -170,8 +384,10 @@ module Fluent
|
|
170
384
|
return "Int64" if int64_values?
|
171
385
|
return "Float" if float_values?
|
172
386
|
return "WGS84GeoPoint" if geo_point_values?
|
387
|
+
return "LongText" if long_text_values?
|
388
|
+
return "Text" if text_values?
|
173
389
|
|
174
|
-
"
|
390
|
+
"ShortText"
|
175
391
|
end
|
176
392
|
|
177
393
|
def vector?
|
@@ -254,6 +470,22 @@ module Fluent
|
|
254
470
|
/\A-?\d+(?:\.\d+)[,x]-?\d+(?:\.\d+)\z/ =~ sample_value
|
255
471
|
end
|
256
472
|
end
|
473
|
+
|
474
|
+
MAX_SHORT_TEXT_SIZE = 2 ** 12
|
475
|
+
MAX_TEXT_SIZE = 2 ** 16
|
476
|
+
def text_values?
|
477
|
+
@sample_values.any? do |sample_value|
|
478
|
+
sample_value.is_a?(String) and
|
479
|
+
sample_value.bytesize > MAX_SHORT_TEXT_SIZE
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
def long_text_values?
|
484
|
+
@sample_values.any? do |sample_value|
|
485
|
+
sample_value.is_a?(String) and
|
486
|
+
sample_value.bytesize > MAX_TEXT_SIZE
|
487
|
+
end
|
488
|
+
end
|
257
489
|
end
|
258
490
|
|
259
491
|
class Table
|
@@ -273,14 +505,13 @@ module Fluent
|
|
273
505
|
end
|
274
506
|
|
275
507
|
class Emitter
|
276
|
-
def initialize(client, table)
|
508
|
+
def initialize(client, table, schema)
|
277
509
|
@client = client
|
278
510
|
@table = table
|
279
|
-
@schema =
|
511
|
+
@schema = schema
|
280
512
|
end
|
281
513
|
|
282
514
|
def start
|
283
|
-
@schema = Schema.new(@client, @table)
|
284
515
|
end
|
285
516
|
|
286
517
|
def shutdown
|
@@ -352,7 +583,13 @@ module Fluent
|
|
352
583
|
:host => @host,
|
353
584
|
:port => @port,
|
354
585
|
:backend => :synchronous)
|
355
|
-
@client.execute(command)
|
586
|
+
response = @client.execute(command)
|
587
|
+
unless response.success?
|
588
|
+
$log.error("[output][groonga][error]",
|
589
|
+
:status_code => response.status_code,
|
590
|
+
:message => response.message)
|
591
|
+
end
|
592
|
+
response
|
356
593
|
end
|
357
594
|
end
|
358
595
|
|
@@ -453,14 +690,14 @@ module Fluent
|
|
453
690
|
end
|
454
691
|
|
455
692
|
unless output_message.empty?
|
456
|
-
|
457
|
-
|
458
|
-
|
693
|
+
$log.debug("[output][groonga][output]",
|
694
|
+
:context => context,
|
695
|
+
:message => output_message)
|
459
696
|
end
|
460
697
|
unless error_message.empty?
|
461
|
-
|
462
|
-
|
463
|
-
|
698
|
+
$log.error("[output][groonga][error]",
|
699
|
+
:context => context,
|
700
|
+
:message => error_message)
|
464
701
|
end
|
465
702
|
end
|
466
703
|
end
|
@@ -0,0 +1,239 @@
|
|
1
|
+
<source>
|
2
|
+
type forward
|
3
|
+
</source>
|
4
|
+
|
5
|
+
<source>
|
6
|
+
type tail
|
7
|
+
path /var/log/apache2/access.log
|
8
|
+
pos_file /tmp/apache_access.pos
|
9
|
+
tag apache.raw.log.apache.access
|
10
|
+
format apache2
|
11
|
+
# read_from_head true
|
12
|
+
</source>
|
13
|
+
|
14
|
+
<match apache.**>
|
15
|
+
type record_reformer
|
16
|
+
enable_ruby false
|
17
|
+
|
18
|
+
tag ${tag_suffix[1]}
|
19
|
+
|
20
|
+
<record>
|
21
|
+
remote ${host}
|
22
|
+
</record>
|
23
|
+
</match>
|
24
|
+
|
25
|
+
<match raw.log.**>
|
26
|
+
type record_reformer
|
27
|
+
enable_ruby false
|
28
|
+
|
29
|
+
tag ${tag_suffix[1]}
|
30
|
+
|
31
|
+
<record>
|
32
|
+
host ${hostname}
|
33
|
+
type ${tag_suffix[2]}
|
34
|
+
timestamp ${time}
|
35
|
+
</record>
|
36
|
+
</match>
|
37
|
+
|
38
|
+
<match log.**>
|
39
|
+
type groonga
|
40
|
+
store_table Logs
|
41
|
+
|
42
|
+
protocol http
|
43
|
+
host 127.0.0.1
|
44
|
+
|
45
|
+
buffer_type file
|
46
|
+
buffer_path /tmp/buffer
|
47
|
+
flush_interval 1
|
48
|
+
|
49
|
+
<table>
|
50
|
+
name Codes
|
51
|
+
flags TABLE_PAT_KEY
|
52
|
+
key_type Int32
|
53
|
+
</table>
|
54
|
+
|
55
|
+
<table>
|
56
|
+
name Hosts
|
57
|
+
flags TABLE_PAT_KEY
|
58
|
+
key_type ShortText
|
59
|
+
normalizer NormalizerAuto
|
60
|
+
</table>
|
61
|
+
|
62
|
+
<table>
|
63
|
+
name URLs
|
64
|
+
flags TABLE_PAT_KEY
|
65
|
+
key_type ShortText
|
66
|
+
</table>
|
67
|
+
|
68
|
+
<table>
|
69
|
+
name Paths
|
70
|
+
flags TABLE_PAT_KEY
|
71
|
+
key_type ShortText
|
72
|
+
</table>
|
73
|
+
|
74
|
+
<table>
|
75
|
+
name UserAgents
|
76
|
+
flags TABLE_PAT_KEY
|
77
|
+
key_type ShortText
|
78
|
+
</table>
|
79
|
+
|
80
|
+
<table>
|
81
|
+
name Methods
|
82
|
+
flags TABLE_HASH_KEY
|
83
|
+
key_type ShortText
|
84
|
+
normalizer NormalizerAuto
|
85
|
+
</table>
|
86
|
+
|
87
|
+
<table>
|
88
|
+
name Remotes
|
89
|
+
flags TABLE_PAT_KEY
|
90
|
+
key_type ShortText
|
91
|
+
</table>
|
92
|
+
|
93
|
+
<table>
|
94
|
+
name Sizes
|
95
|
+
flags TABLE_PAT_KEY
|
96
|
+
key_type Int32
|
97
|
+
</table>
|
98
|
+
|
99
|
+
<table>
|
100
|
+
name Timestamps
|
101
|
+
flags TABLE_PAT_KEY
|
102
|
+
key_type Time
|
103
|
+
</table>
|
104
|
+
|
105
|
+
<table>
|
106
|
+
name Types
|
107
|
+
flags TABLE_PAT_KEY
|
108
|
+
key_type ShortText
|
109
|
+
</table>
|
110
|
+
|
111
|
+
<table>
|
112
|
+
name Terms
|
113
|
+
flags TABLE_PAT_KEY
|
114
|
+
key_type ShortText
|
115
|
+
default_tokenizer TokenBigram
|
116
|
+
normalizer NormalizerAuto
|
117
|
+
<index>
|
118
|
+
name host_index
|
119
|
+
source_table Hosts
|
120
|
+
source_columns _key
|
121
|
+
</index>
|
122
|
+
<index>
|
123
|
+
name url_index
|
124
|
+
source_table URLs
|
125
|
+
source_columns _key
|
126
|
+
</index>
|
127
|
+
<index>
|
128
|
+
name path_index
|
129
|
+
source_table Paths
|
130
|
+
source_columns _key
|
131
|
+
</index>
|
132
|
+
<index>
|
133
|
+
name user_agent_index
|
134
|
+
source_table UserAgents
|
135
|
+
source_columns _key
|
136
|
+
</index>
|
137
|
+
</table>
|
138
|
+
|
139
|
+
<mapping>
|
140
|
+
name agent
|
141
|
+
type UserAgents
|
142
|
+
<index>
|
143
|
+
table Terms
|
144
|
+
name logs_agent_index
|
145
|
+
flags WITH_POSITION
|
146
|
+
</index>
|
147
|
+
</mapping>
|
148
|
+
|
149
|
+
<mapping>
|
150
|
+
name code
|
151
|
+
type Codes
|
152
|
+
<index>
|
153
|
+
table Codes
|
154
|
+
name logs_index
|
155
|
+
</index>
|
156
|
+
</mapping>
|
157
|
+
|
158
|
+
<mapping>
|
159
|
+
name host
|
160
|
+
type Hosts
|
161
|
+
<index>
|
162
|
+
table Hosts
|
163
|
+
name hosts_index
|
164
|
+
</index>
|
165
|
+
</mapping>
|
166
|
+
|
167
|
+
<mapping>
|
168
|
+
name message
|
169
|
+
type Text
|
170
|
+
<index>
|
171
|
+
table Terms
|
172
|
+
name logs_message_index
|
173
|
+
flags WITH_POSITION
|
174
|
+
</index>
|
175
|
+
</mapping>
|
176
|
+
|
177
|
+
<mapping>
|
178
|
+
name method
|
179
|
+
type Methods
|
180
|
+
<index>
|
181
|
+
table Methods
|
182
|
+
name logs_index
|
183
|
+
</index>
|
184
|
+
</mapping>
|
185
|
+
|
186
|
+
<mapping>
|
187
|
+
name path
|
188
|
+
type Paths
|
189
|
+
<index>
|
190
|
+
table Paths
|
191
|
+
name logs_index
|
192
|
+
</index>
|
193
|
+
</mapping>
|
194
|
+
|
195
|
+
<mapping>
|
196
|
+
name referer
|
197
|
+
type URLs
|
198
|
+
<index>
|
199
|
+
table URLs
|
200
|
+
name logs_index
|
201
|
+
</index>
|
202
|
+
</mapping>
|
203
|
+
|
204
|
+
<mapping>
|
205
|
+
name remote
|
206
|
+
type Remotes
|
207
|
+
<index>
|
208
|
+
table Remotes
|
209
|
+
name logs_index
|
210
|
+
</index>
|
211
|
+
</mapping>
|
212
|
+
|
213
|
+
<mapping>
|
214
|
+
name size
|
215
|
+
type Int32
|
216
|
+
<index>
|
217
|
+
table Sizes
|
218
|
+
name logs_index
|
219
|
+
</index>
|
220
|
+
</mapping>
|
221
|
+
|
222
|
+
<mapping>
|
223
|
+
name timestamp
|
224
|
+
type Time
|
225
|
+
<index>
|
226
|
+
table Timestamps
|
227
|
+
name logs_index
|
228
|
+
</index>
|
229
|
+
</mapping>
|
230
|
+
|
231
|
+
<mapping>
|
232
|
+
name type
|
233
|
+
type Types
|
234
|
+
<index>
|
235
|
+
table Types
|
236
|
+
name logs_index
|
237
|
+
</index>
|
238
|
+
</mapping>
|
239
|
+
</match>
|
data/sample/store.conf
CHANGED
@@ -0,0 +1,167 @@
|
|
1
|
+
# Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "fluent/plugin/out_groonga"
|
17
|
+
|
18
|
+
class OutputTypeTableDefinitionTest < Test::Unit::TestCase
|
19
|
+
def definition(raw={})
|
20
|
+
Fluent::GroongaOutput::TableDefinition.new(raw)
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case "readers" do
|
24
|
+
sub_test_case "\#name" do
|
25
|
+
test "specified" do
|
26
|
+
assert_equal("Tags", definition(:name => "Tags").name)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
sub_test_case "\#flags" do
|
31
|
+
test "default" do
|
32
|
+
assert_equal(["TABLE_NO_KEY"],
|
33
|
+
definition.flags)
|
34
|
+
end
|
35
|
+
|
36
|
+
test "one" do
|
37
|
+
assert_equal(["TABLE_PAT_KEY"],
|
38
|
+
definition(:flags => "TABLE_PAT_KEY").flags)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
sub_test_case "\#key_type" do
|
43
|
+
test "default" do
|
44
|
+
assert_nil(definition.key_type)
|
45
|
+
end
|
46
|
+
|
47
|
+
test "specified" do
|
48
|
+
assert_equal("ShortText",
|
49
|
+
definition(:key_type => "ShortText").key_type)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
sub_test_case "\#default_tokenizer" do
|
54
|
+
def read_default_tokenizer(input)
|
55
|
+
definition(:default_tokenizer => input).default_tokenizer
|
56
|
+
end
|
57
|
+
|
58
|
+
test "default" do
|
59
|
+
assert_nil(definition.default_tokenizer)
|
60
|
+
end
|
61
|
+
|
62
|
+
test "specified" do
|
63
|
+
assert_equal("TokenBigram",
|
64
|
+
read_default_tokenizer("TokenBigram"))
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
sub_test_case "\#token_filters" do
|
69
|
+
def read_token_filters(input)
|
70
|
+
definition(:token_filters => input).token_filters
|
71
|
+
end
|
72
|
+
|
73
|
+
test "default" do
|
74
|
+
assert_equal([], definition.token_filters)
|
75
|
+
end
|
76
|
+
|
77
|
+
test "one" do
|
78
|
+
assert_equal(["TokenFilterStem"],
|
79
|
+
read_token_filters("TokenFilterStem"))
|
80
|
+
end
|
81
|
+
|
82
|
+
test "multiple" do
|
83
|
+
assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
|
84
|
+
read_token_filters("TokenFilterStem,TokenFilterStopWord"))
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
sub_test_case "\#normalizer" do
|
89
|
+
def read_normalizer(input)
|
90
|
+
definition(:normalizer => input).normalizer
|
91
|
+
end
|
92
|
+
|
93
|
+
test "default" do
|
94
|
+
assert_nil(definition.normalizer)
|
95
|
+
end
|
96
|
+
|
97
|
+
test "specified" do
|
98
|
+
assert_equal("NormalizerAuto",
|
99
|
+
read_normalizer("NormalizerAuto"))
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
sub_test_case "\#have_difference?" do
|
105
|
+
def setup
|
106
|
+
@existing_table = Groonga::Client::Response::TableList::Table.new
|
107
|
+
@existing_table.id = 260
|
108
|
+
@existing_table.name = "Paths"
|
109
|
+
@existing_table.path = "/var/lib/groonga/db/db.0000104"
|
110
|
+
@existing_table.flags = "TABLE_PAT_KEY|PERSISTENT"
|
111
|
+
@existing_table.domain = "ShortText"
|
112
|
+
@existing_table.range = nil
|
113
|
+
@existing_table.default_tokenizer = nil
|
114
|
+
@existing_table.normalizer = nil
|
115
|
+
end
|
116
|
+
|
117
|
+
def have_difference?(raw={})
|
118
|
+
default_raw = {
|
119
|
+
:name => @existing_table.name,
|
120
|
+
:flags => @existing_table.flags.gsub(/\|PERSISTENT/, ""),
|
121
|
+
:key_type => @existing_table.domain,
|
122
|
+
:default_tokenizer => @existing_table.default_tokenizer,
|
123
|
+
:normalizer => @existing_table.normalizer,
|
124
|
+
}
|
125
|
+
raw = default_raw.merge(raw)
|
126
|
+
definition(raw).have_difference?(@existing_table)
|
127
|
+
end
|
128
|
+
|
129
|
+
test "no difference" do
|
130
|
+
assert do
|
131
|
+
not have_difference?
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
sub_test_case "difference" do
|
136
|
+
test "name" do
|
137
|
+
assert do
|
138
|
+
have_difference?(:name => "Difference")
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
test "flags" do
|
143
|
+
assert do
|
144
|
+
have_difference?(:flags => "TABLE_NO_KEY")
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
test "key_type" do
|
149
|
+
assert do
|
150
|
+
have_difference?(:key_type => "UInt32")
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
test "default_tokenizer" do
|
155
|
+
assert do
|
156
|
+
have_difference?(:default_tokenizer => "TokenBigram")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
test "normalizer" do
|
161
|
+
assert do
|
162
|
+
have_difference?(:normalizer => "NormalizerAuto")
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -122,11 +122,30 @@ class OutputTypeGuesserTest < Test::Unit::TestCase
|
|
122
122
|
end
|
123
123
|
end
|
124
124
|
|
125
|
+
sub_test_case "ShortText" do
|
126
|
+
test "max" do
|
127
|
+
message = "X" * (2 ** 12)
|
128
|
+
assert_equal("ShortText", guess([message]))
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
125
132
|
sub_test_case "Text" do
|
126
|
-
test "
|
127
|
-
message = "
|
133
|
+
test "min" do
|
134
|
+
message = "X" * (2 ** 12 + 1)
|
128
135
|
assert_equal("Text", guess([message]))
|
129
136
|
end
|
137
|
+
|
138
|
+
test "max" do
|
139
|
+
message = "X" * (2 ** 16)
|
140
|
+
assert_equal("Text", guess([message]))
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
sub_test_case "LongText" do
|
145
|
+
test "min" do
|
146
|
+
message = "X" * (2 ** 16 + 1)
|
147
|
+
assert_equal("LongText", guess([message]))
|
148
|
+
end
|
130
149
|
end
|
131
150
|
end
|
132
151
|
end
|
data/test/run-test.rb
CHANGED
@@ -30,13 +30,8 @@ Test::Unit::Priority.enable
|
|
30
30
|
|
31
31
|
$LOAD_PATH.unshift(lib_dir)
|
32
32
|
|
33
|
-
require "fluent/
|
34
|
-
$log = Fluent::Log.new($stdout, Fluent::Log::LEVEL_WARN)
|
35
|
-
|
36
|
-
Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
|
37
|
-
require file.sub(/\.rb$/, '')
|
38
|
-
end
|
33
|
+
require "fluent/test"
|
39
34
|
|
40
35
|
ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
|
41
36
|
|
42
|
-
exit
|
37
|
+
exit(Test::Unit::AutoRunner.run(true, test_dir))
|
data/test/test_output.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-groonga
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.1.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.1.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: groonga-command-parser
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -157,7 +157,9 @@ files:
|
|
157
157
|
- sample/command.conf
|
158
158
|
- sample/gqtp.conf
|
159
159
|
- sample/http.conf
|
160
|
+
- sample/store-apache.conf
|
160
161
|
- sample/store.conf
|
162
|
+
- test/output/test_table_definition.rb
|
161
163
|
- test/output/test_type_guesser.rb
|
162
164
|
- test/run-test.rb
|
163
165
|
- test/test_input.rb
|
@@ -192,4 +194,5 @@ test_files:
|
|
192
194
|
- test/test_input.rb
|
193
195
|
- test/run-test.rb
|
194
196
|
- test/output/test_type_guesser.rb
|
197
|
+
- test/output/test_table_definition.rb
|
195
198
|
has_rdoc:
|