fluent-plugin-groonga 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -2
- data/README.md +4 -4
- data/doc/text/news.md +12 -0
- data/fluent-plugin-groonga.gemspec +2 -2
- data/lib/fluent/plugin/out_groonga.rb +258 -21
- data/sample/store-apache.conf +239 -0
- data/sample/store.conf +1 -1
- data/test/output/test_table_definition.rb +167 -0
- data/test/output/test_type_guesser.rb +21 -2
- data/test/run-test.rb +2 -7
- data/test/test_output.rb +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc471bf3943996d9224a54431bf81a5323c53dcd
|
4
|
+
data.tar.gz: 3c37826e7793bbcab4d8009c6d0555aa74577858
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1dc9a521003ddc42ccc0ff98e88468931b4d9400e5d7cb5e592a836d1be655a858a396a9275c1058a8c221041854c3d5e69e481781fdc475e0911d1595301d3
|
7
|
+
data.tar.gz: 63e6a01cafd4bcb3e104ec80bcd53a48a4d375f29f95a292b4f214e360c287a846d41f120dd5e3b9ed52ef587f60a5cd7283dad7c03aa14f07b9d80f2adf6807
|
data/Gemfile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- mode: ruby; coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2012 Kouhei Sutou <kou@clear-code.com>
|
3
|
+
# Copyright (C) 2012-2014 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
#
|
5
5
|
# This library is free software; you can redistribute it and/or
|
6
6
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -15,6 +15,6 @@
|
|
15
15
|
# License along with this library; if not, write to the Free Software
|
16
16
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
17
|
|
18
|
-
source
|
18
|
+
source "https://rubygems.org/"
|
19
19
|
|
20
20
|
gemspec
|
data/README.md
CHANGED
@@ -42,7 +42,7 @@ into Groonga:
|
|
42
42
|
|
43
43
|
<match log.**>
|
44
44
|
type groonga
|
45
|
-
|
45
|
+
store_table logs
|
46
46
|
|
47
47
|
protocol http
|
48
48
|
host 127.0.0.1
|
@@ -88,13 +88,13 @@ define schema in Groonga before running Fluentd. You just run Groonga.
|
|
88
88
|
|
89
89
|
There is one required parameter:
|
90
90
|
|
91
|
-
* `
|
91
|
+
* `store_table`: It specifies table name for storing logs.
|
92
92
|
|
93
93
|
Here is a minimum configuration:
|
94
94
|
|
95
95
|
<match log.**>
|
96
96
|
type groonga
|
97
|
-
|
97
|
+
store_table logs
|
98
98
|
</match>
|
99
99
|
|
100
100
|
The configuration stores logs into `logs` table in Groonga that runs
|
@@ -115,7 +115,7 @@ Here is a configuration that specifies optional parameters explicitly:
|
|
115
115
|
|
116
116
|
<match log.**>
|
117
117
|
type groonga
|
118
|
-
|
118
|
+
store_table logs
|
119
119
|
|
120
120
|
protocol http
|
121
121
|
host 127.0.0.1
|
data/doc/text/news.md
CHANGED
@@ -2,6 +2,18 @@
|
|
2
2
|
|
3
3
|
# News
|
4
4
|
|
5
|
+
## 1.0.6: 2014-11-05
|
6
|
+
|
7
|
+
### Improvements
|
8
|
+
|
9
|
+
* out: Renamed `table` parameter name to `store_table`.
|
10
|
+
`table` parameter is still usable for backward compatibility.
|
11
|
+
* out: Supported table definition by `<table>` configuration.
|
12
|
+
See sample/store-apache.conf for details.
|
13
|
+
* out: Supported specifying column type and creating indexes for auto
|
14
|
+
created columns by `<mapping>` configuration.
|
15
|
+
See sample/store-apache.conf for details.
|
16
|
+
|
5
17
|
## 1.0.5: 2014-10-21
|
6
18
|
|
7
19
|
### Improvements
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |spec|
|
19
19
|
spec.name = "fluent-plugin-groonga"
|
20
|
-
spec.version = "1.0.
|
20
|
+
spec.version = "1.0.6"
|
21
21
|
spec.authors = ["Kouhei Sutou"]
|
22
22
|
spec.email = ["kou@clear-code.com"]
|
23
23
|
spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
|
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.require_paths = ["lib"]
|
36
36
|
|
37
37
|
spec.add_runtime_dependency("fluentd")
|
38
|
-
spec.add_runtime_dependency("groonga-client")
|
38
|
+
spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
|
39
39
|
spec.add_runtime_dependency("groonga-command-parser")
|
40
40
|
|
41
41
|
spec.add_development_dependency("rake")
|
@@ -37,20 +37,65 @@ module Fluent
|
|
37
37
|
raise ConfigError, "must be http, gqtp or command: <#{value}>"
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
|
+
# alias is just for backward compatibility
|
42
|
+
config_param :store_table, :string, :default => nil, :alias => :table
|
43
|
+
|
44
|
+
config_section :table,
|
45
|
+
:param_name => "tables",
|
46
|
+
:required => false,
|
47
|
+
:multi => true do
|
48
|
+
config_param :name, :string
|
49
|
+
config_param :flags, :string, :default => nil
|
50
|
+
config_param :key_type, :string, :default => nil
|
51
|
+
config_param :default_tokenizer, :string, :default => nil
|
52
|
+
config_param :token_filters, :string, :default => nil
|
53
|
+
config_param :normalizer, :string, :default => nil
|
54
|
+
config_section :index,
|
55
|
+
:param_name => "indexes",
|
56
|
+
:required => false,
|
57
|
+
:multi => true do
|
58
|
+
config_param :name, :string
|
59
|
+
config_param :source_table, :string
|
60
|
+
config_param :source_columns, :string
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
config_section :mapping,
|
65
|
+
:param_name => "mappings",
|
66
|
+
:required => false,
|
67
|
+
:multi => true do
|
68
|
+
config_param :name, :string
|
69
|
+
config_param :type, :string, :default => nil
|
70
|
+
config_section :index,
|
71
|
+
:param_name => "indexes",
|
72
|
+
:required => false,
|
73
|
+
:multi => true do
|
74
|
+
config_param :table, :string
|
75
|
+
config_param :name, :string
|
76
|
+
config_param :flags, :string, :default => nil
|
77
|
+
end
|
78
|
+
end
|
41
79
|
|
42
80
|
def configure(conf)
|
43
81
|
super
|
44
82
|
@client = create_client(@protocol)
|
45
83
|
@client.configure(conf)
|
46
84
|
|
47
|
-
@
|
85
|
+
@schema = Schema.new(@client, @store_table, @mappings)
|
86
|
+
@emitter = Emitter.new(@client, @store_table, @schema)
|
87
|
+
|
88
|
+
@tables = @tables.collect do |table|
|
89
|
+
TableDefinition.new(table)
|
90
|
+
end
|
48
91
|
end
|
49
92
|
|
50
93
|
def start
|
51
94
|
super
|
52
95
|
@client.start
|
53
96
|
@emitter.start
|
97
|
+
tables_creator = TablesCreator.new(@client, @tables)
|
98
|
+
tables_creator.create
|
54
99
|
end
|
55
100
|
|
56
101
|
def shutdown
|
@@ -77,18 +122,171 @@ module Fluent
|
|
77
122
|
end
|
78
123
|
end
|
79
124
|
|
125
|
+
class TableDefinition
|
126
|
+
def initialize(raw)
|
127
|
+
@raw = raw
|
128
|
+
end
|
129
|
+
|
130
|
+
def name
|
131
|
+
@raw[:name]
|
132
|
+
end
|
133
|
+
|
134
|
+
def flags
|
135
|
+
parse_flags(@raw[:flags] || "TABLE_NO_KEY")
|
136
|
+
end
|
137
|
+
|
138
|
+
def key_type
|
139
|
+
@raw[:key_type]
|
140
|
+
end
|
141
|
+
|
142
|
+
def default_tokenizer
|
143
|
+
@raw[:default_tokenizer]
|
144
|
+
end
|
145
|
+
|
146
|
+
def token_filters
|
147
|
+
parse_items(@raw[:token_filters] || "")
|
148
|
+
end
|
149
|
+
|
150
|
+
def normalizer
|
151
|
+
@raw[:normalizer]
|
152
|
+
end
|
153
|
+
|
154
|
+
def indexes
|
155
|
+
(@raw[:indexes] || []).collect do |raw|
|
156
|
+
IndexDefinition.new(self, raw)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def use_n_gram_tokenizer?
|
161
|
+
/\AToken(?:Uni|Bi|Tri)gram/ === default_tokenizer.to_s
|
162
|
+
end
|
163
|
+
|
164
|
+
def have_difference?(table)
|
165
|
+
return true if table.name != name
|
166
|
+
|
167
|
+
table_flags = (parse_flags(table.flags) - ["PERSISTENT"])
|
168
|
+
return true if table_flags.sort != flags.sort
|
169
|
+
|
170
|
+
return true if table.domain != key_type
|
171
|
+
|
172
|
+
return true if table.default_tokenizer != default_tokenizer
|
173
|
+
|
174
|
+
# TODO
|
175
|
+
# return true if table.token_filters.sort != token_filters.sort
|
176
|
+
|
177
|
+
return true if table.normalizer != normalizer
|
178
|
+
|
179
|
+
false
|
180
|
+
end
|
181
|
+
|
182
|
+
def to_create_arguments
|
183
|
+
arguments = {
|
184
|
+
"name" => name,
|
185
|
+
"flags" => flags.join("|"),
|
186
|
+
"key_type" => key_type,
|
187
|
+
"default_tokenizer" => default_tokenizer,
|
188
|
+
# TODO
|
189
|
+
# "token_filters" => token_filters.join("|"),
|
190
|
+
"normalizer" => normalizer,
|
191
|
+
}
|
192
|
+
arguments.keys.each do |key|
|
193
|
+
value = arguments[key]
|
194
|
+
arguments.delete(key) if value.nil? or value.empty?
|
195
|
+
end
|
196
|
+
arguments
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
def parse_flags(flags)
|
201
|
+
if flags.is_a?(Array)
|
202
|
+
flags
|
203
|
+
else
|
204
|
+
flags.strip.split(/\s*\|\s*/)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
def parse_items(items)
|
209
|
+
if items.is_a?(Array)
|
210
|
+
items
|
211
|
+
else
|
212
|
+
items.strip.split(/\s*,\s*/)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
class IndexDefinition
|
217
|
+
def initialize(table, raw)
|
218
|
+
@table = table
|
219
|
+
@raw = raw
|
220
|
+
end
|
221
|
+
|
222
|
+
def name
|
223
|
+
@raw[:name]
|
224
|
+
end
|
225
|
+
|
226
|
+
def source_table
|
227
|
+
@raw[:source_table]
|
228
|
+
end
|
229
|
+
|
230
|
+
def source_columns
|
231
|
+
@raw[:source_columns]
|
232
|
+
end
|
233
|
+
|
234
|
+
def flags
|
235
|
+
_flags = ["COLUMN_INDEX"]
|
236
|
+
_flags << "WITH_POSITION" if @table.use_n_gram_tokenizer?
|
237
|
+
_flags << "WITH_SECTION" if source_columns.size >= 2
|
238
|
+
_flags
|
239
|
+
end
|
240
|
+
|
241
|
+
def to_create_arguments
|
242
|
+
{
|
243
|
+
"table" => @table.name,
|
244
|
+
"name" => name,
|
245
|
+
"flags" => flags.join,
|
246
|
+
"type" => source_table,
|
247
|
+
"source" => source_columns,
|
248
|
+
}
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
class TablesCreator
|
254
|
+
def initialize(client, definitions)
|
255
|
+
@client = client
|
256
|
+
@definitions = definitions
|
257
|
+
end
|
258
|
+
|
259
|
+
def create
|
260
|
+
return if @definitions.empty?
|
261
|
+
|
262
|
+
table_list = @client.execute("table_list")
|
263
|
+
@definitions.each do |definition|
|
264
|
+
existing_table = table_list.find do |table|
|
265
|
+
table.name == definition.name
|
266
|
+
end
|
267
|
+
if existing_table
|
268
|
+
next unless definition.have_difference?(existing_table)
|
269
|
+
# TODO: Is it OK?
|
270
|
+
@client.execute("table_remove", "name" => definition.name)
|
271
|
+
end
|
272
|
+
|
273
|
+
@client.execute("table_create", definition.to_create_arguments)
|
274
|
+
definition.indexes.each do |index|
|
275
|
+
@client.execute("column_create", index.to_create_arguments)
|
276
|
+
end
|
277
|
+
end
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
80
281
|
class Schema
|
81
|
-
def initialize(client, table_name)
|
282
|
+
def initialize(client, table_name, mappings)
|
82
283
|
@client = client
|
83
284
|
@table_name = table_name
|
285
|
+
@mappings = mappings
|
84
286
|
@table = nil
|
85
287
|
@columns = nil
|
86
288
|
end
|
87
289
|
|
88
|
-
def populate
|
89
|
-
# TODO
|
90
|
-
end
|
91
|
-
|
92
290
|
def update(records)
|
93
291
|
ensure_table
|
94
292
|
ensure_columns
|
@@ -120,7 +318,6 @@ module Fluent
|
|
120
318
|
if target_table
|
121
319
|
@table = Table.new(@table_name, target_table.domain)
|
122
320
|
else
|
123
|
-
# TODO: Check response
|
124
321
|
@client.execute("table_create",
|
125
322
|
"name" => @table_name,
|
126
323
|
"flags" => "TABLE_NO_KEY")
|
@@ -142,20 +339,37 @@ module Fluent
|
|
142
339
|
end
|
143
340
|
|
144
341
|
def create_column(name, sample_values)
|
342
|
+
mapping = @mappings.find do |mapping|
|
343
|
+
mapping.name == name
|
344
|
+
end
|
345
|
+
if mapping
|
346
|
+
value_type = mapping[:type]
|
347
|
+
end
|
145
348
|
guesser = TypeGuesser.new(sample_values)
|
146
|
-
value_type
|
349
|
+
value_type ||= guesser.guess
|
147
350
|
vector_p = guesser.vector?
|
148
351
|
if vector_p
|
149
352
|
flags = "COLUMN_VECTOR"
|
150
353
|
else
|
151
354
|
flags = "COLUMN_SCALAR"
|
152
355
|
end
|
153
|
-
# TODO: Check response
|
154
356
|
@client.execute("column_create",
|
155
357
|
"table" => @table_name,
|
156
358
|
"name" => name,
|
157
359
|
"flags" => flags,
|
158
360
|
"type" => value_type)
|
361
|
+
if mapping
|
362
|
+
mapping.indexes.each do |index|
|
363
|
+
index_flags = ["COLUMN_INDEX", index[:flags]].compact
|
364
|
+
@client.execute("column_create",
|
365
|
+
"table" => index[:table],
|
366
|
+
"name" => index[:table],
|
367
|
+
"flags" => index_flags.join("|"),
|
368
|
+
"type" => @table_name,
|
369
|
+
"source" => name)
|
370
|
+
end
|
371
|
+
end
|
372
|
+
|
159
373
|
Column.new(name, value_type, vector_p)
|
160
374
|
end
|
161
375
|
|
@@ -170,8 +384,10 @@ module Fluent
|
|
170
384
|
return "Int64" if int64_values?
|
171
385
|
return "Float" if float_values?
|
172
386
|
return "WGS84GeoPoint" if geo_point_values?
|
387
|
+
return "LongText" if long_text_values?
|
388
|
+
return "Text" if text_values?
|
173
389
|
|
174
|
-
"
|
390
|
+
"ShortText"
|
175
391
|
end
|
176
392
|
|
177
393
|
def vector?
|
@@ -254,6 +470,22 @@ module Fluent
|
|
254
470
|
/\A-?\d+(?:\.\d+)[,x]-?\d+(?:\.\d+)\z/ =~ sample_value
|
255
471
|
end
|
256
472
|
end
|
473
|
+
|
474
|
+
MAX_SHORT_TEXT_SIZE = 2 ** 12
|
475
|
+
MAX_TEXT_SIZE = 2 ** 16
|
476
|
+
def text_values?
|
477
|
+
@sample_values.any? do |sample_value|
|
478
|
+
sample_value.is_a?(String) and
|
479
|
+
sample_value.bytesize > MAX_SHORT_TEXT_SIZE
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
def long_text_values?
|
484
|
+
@sample_values.any? do |sample_value|
|
485
|
+
sample_value.is_a?(String) and
|
486
|
+
sample_value.bytesize > MAX_TEXT_SIZE
|
487
|
+
end
|
488
|
+
end
|
257
489
|
end
|
258
490
|
|
259
491
|
class Table
|
@@ -273,14 +505,13 @@ module Fluent
|
|
273
505
|
end
|
274
506
|
|
275
507
|
class Emitter
|
276
|
-
def initialize(client, table)
|
508
|
+
def initialize(client, table, schema)
|
277
509
|
@client = client
|
278
510
|
@table = table
|
279
|
-
@schema =
|
511
|
+
@schema = schema
|
280
512
|
end
|
281
513
|
|
282
514
|
def start
|
283
|
-
@schema = Schema.new(@client, @table)
|
284
515
|
end
|
285
516
|
|
286
517
|
def shutdown
|
@@ -352,7 +583,13 @@ module Fluent
|
|
352
583
|
:host => @host,
|
353
584
|
:port => @port,
|
354
585
|
:backend => :synchronous)
|
355
|
-
@client.execute(command)
|
586
|
+
response = @client.execute(command)
|
587
|
+
unless response.success?
|
588
|
+
$log.error("[output][groonga][error]",
|
589
|
+
:status_code => response.status_code,
|
590
|
+
:message => response.message)
|
591
|
+
end
|
592
|
+
response
|
356
593
|
end
|
357
594
|
end
|
358
595
|
|
@@ -453,14 +690,14 @@ module Fluent
|
|
453
690
|
end
|
454
691
|
|
455
692
|
unless output_message.empty?
|
456
|
-
|
457
|
-
|
458
|
-
|
693
|
+
$log.debug("[output][groonga][output]",
|
694
|
+
:context => context,
|
695
|
+
:message => output_message)
|
459
696
|
end
|
460
697
|
unless error_message.empty?
|
461
|
-
|
462
|
-
|
463
|
-
|
698
|
+
$log.error("[output][groonga][error]",
|
699
|
+
:context => context,
|
700
|
+
:message => error_message)
|
464
701
|
end
|
465
702
|
end
|
466
703
|
end
|
@@ -0,0 +1,239 @@
|
|
1
|
+
<source>
|
2
|
+
type forward
|
3
|
+
</source>
|
4
|
+
|
5
|
+
<source>
|
6
|
+
type tail
|
7
|
+
path /var/log/apache2/access.log
|
8
|
+
pos_file /tmp/apache_access.pos
|
9
|
+
tag apache.raw.log.apache.access
|
10
|
+
format apache2
|
11
|
+
# read_from_head true
|
12
|
+
</source>
|
13
|
+
|
14
|
+
<match apache.**>
|
15
|
+
type record_reformer
|
16
|
+
enable_ruby false
|
17
|
+
|
18
|
+
tag ${tag_suffix[1]}
|
19
|
+
|
20
|
+
<record>
|
21
|
+
remote ${host}
|
22
|
+
</record>
|
23
|
+
</match>
|
24
|
+
|
25
|
+
<match raw.log.**>
|
26
|
+
type record_reformer
|
27
|
+
enable_ruby false
|
28
|
+
|
29
|
+
tag ${tag_suffix[1]}
|
30
|
+
|
31
|
+
<record>
|
32
|
+
host ${hostname}
|
33
|
+
type ${tag_suffix[2]}
|
34
|
+
timestamp ${time}
|
35
|
+
</record>
|
36
|
+
</match>
|
37
|
+
|
38
|
+
<match log.**>
|
39
|
+
type groonga
|
40
|
+
store_table Logs
|
41
|
+
|
42
|
+
protocol http
|
43
|
+
host 127.0.0.1
|
44
|
+
|
45
|
+
buffer_type file
|
46
|
+
buffer_path /tmp/buffer
|
47
|
+
flush_interval 1
|
48
|
+
|
49
|
+
<table>
|
50
|
+
name Codes
|
51
|
+
flags TABLE_PAT_KEY
|
52
|
+
key_type Int32
|
53
|
+
</table>
|
54
|
+
|
55
|
+
<table>
|
56
|
+
name Hosts
|
57
|
+
flags TABLE_PAT_KEY
|
58
|
+
key_type ShortText
|
59
|
+
normalizer NormalizerAuto
|
60
|
+
</table>
|
61
|
+
|
62
|
+
<table>
|
63
|
+
name URLs
|
64
|
+
flags TABLE_PAT_KEY
|
65
|
+
key_type ShortText
|
66
|
+
</table>
|
67
|
+
|
68
|
+
<table>
|
69
|
+
name Paths
|
70
|
+
flags TABLE_PAT_KEY
|
71
|
+
key_type ShortText
|
72
|
+
</table>
|
73
|
+
|
74
|
+
<table>
|
75
|
+
name UserAgents
|
76
|
+
flags TABLE_PAT_KEY
|
77
|
+
key_type ShortText
|
78
|
+
</table>
|
79
|
+
|
80
|
+
<table>
|
81
|
+
name Methods
|
82
|
+
flags TABLE_HASH_KEY
|
83
|
+
key_type ShortText
|
84
|
+
normalizer NormalizerAuto
|
85
|
+
</table>
|
86
|
+
|
87
|
+
<table>
|
88
|
+
name Remotes
|
89
|
+
flags TABLE_PAT_KEY
|
90
|
+
key_type ShortText
|
91
|
+
</table>
|
92
|
+
|
93
|
+
<table>
|
94
|
+
name Sizes
|
95
|
+
flags TABLE_PAT_KEY
|
96
|
+
key_type Int32
|
97
|
+
</table>
|
98
|
+
|
99
|
+
<table>
|
100
|
+
name Timestamps
|
101
|
+
flags TABLE_PAT_KEY
|
102
|
+
key_type Time
|
103
|
+
</table>
|
104
|
+
|
105
|
+
<table>
|
106
|
+
name Types
|
107
|
+
flags TABLE_PAT_KEY
|
108
|
+
key_type ShortText
|
109
|
+
</table>
|
110
|
+
|
111
|
+
<table>
|
112
|
+
name Terms
|
113
|
+
flags TABLE_PAT_KEY
|
114
|
+
key_type ShortText
|
115
|
+
default_tokenizer TokenBigram
|
116
|
+
normalizer NormalizerAuto
|
117
|
+
<index>
|
118
|
+
name host_index
|
119
|
+
source_table Hosts
|
120
|
+
source_columns _key
|
121
|
+
</index>
|
122
|
+
<index>
|
123
|
+
name url_index
|
124
|
+
source_table URLs
|
125
|
+
source_columns _key
|
126
|
+
</index>
|
127
|
+
<index>
|
128
|
+
name path_index
|
129
|
+
source_table Paths
|
130
|
+
source_columns _key
|
131
|
+
</index>
|
132
|
+
<index>
|
133
|
+
name user_agent_index
|
134
|
+
source_table UserAgents
|
135
|
+
source_columns _key
|
136
|
+
</index>
|
137
|
+
</table>
|
138
|
+
|
139
|
+
<mapping>
|
140
|
+
name agent
|
141
|
+
type UserAgents
|
142
|
+
<index>
|
143
|
+
table Terms
|
144
|
+
name logs_agent_index
|
145
|
+
flags WITH_POSITION
|
146
|
+
</index>
|
147
|
+
</mapping>
|
148
|
+
|
149
|
+
<mapping>
|
150
|
+
name code
|
151
|
+
type Codes
|
152
|
+
<index>
|
153
|
+
table Codes
|
154
|
+
name logs_index
|
155
|
+
</index>
|
156
|
+
</mapping>
|
157
|
+
|
158
|
+
<mapping>
|
159
|
+
name host
|
160
|
+
type Hosts
|
161
|
+
<index>
|
162
|
+
table Hosts
|
163
|
+
name hosts_index
|
164
|
+
</index>
|
165
|
+
</mapping>
|
166
|
+
|
167
|
+
<mapping>
|
168
|
+
name message
|
169
|
+
type Text
|
170
|
+
<index>
|
171
|
+
table Terms
|
172
|
+
name logs_message_index
|
173
|
+
flags WITH_POSITION
|
174
|
+
</index>
|
175
|
+
</mapping>
|
176
|
+
|
177
|
+
<mapping>
|
178
|
+
name method
|
179
|
+
type Methods
|
180
|
+
<index>
|
181
|
+
table Methods
|
182
|
+
name logs_index
|
183
|
+
</index>
|
184
|
+
</mapping>
|
185
|
+
|
186
|
+
<mapping>
|
187
|
+
name path
|
188
|
+
type Paths
|
189
|
+
<index>
|
190
|
+
table Paths
|
191
|
+
name logs_index
|
192
|
+
</index>
|
193
|
+
</mapping>
|
194
|
+
|
195
|
+
<mapping>
|
196
|
+
name referer
|
197
|
+
type URLs
|
198
|
+
<index>
|
199
|
+
table URLs
|
200
|
+
name logs_index
|
201
|
+
</index>
|
202
|
+
</mapping>
|
203
|
+
|
204
|
+
<mapping>
|
205
|
+
name remote
|
206
|
+
type Remotes
|
207
|
+
<index>
|
208
|
+
table Remotes
|
209
|
+
name logs_index
|
210
|
+
</index>
|
211
|
+
</mapping>
|
212
|
+
|
213
|
+
<mapping>
|
214
|
+
name size
|
215
|
+
type Int32
|
216
|
+
<index>
|
217
|
+
table Sizes
|
218
|
+
name logs_index
|
219
|
+
</index>
|
220
|
+
</mapping>
|
221
|
+
|
222
|
+
<mapping>
|
223
|
+
name timestamp
|
224
|
+
type Time
|
225
|
+
<index>
|
226
|
+
table Timestamps
|
227
|
+
name logs_index
|
228
|
+
</index>
|
229
|
+
</mapping>
|
230
|
+
|
231
|
+
<mapping>
|
232
|
+
name type
|
233
|
+
type Types
|
234
|
+
<index>
|
235
|
+
table Types
|
236
|
+
name logs_index
|
237
|
+
</index>
|
238
|
+
</mapping>
|
239
|
+
</match>
|
data/sample/store.conf
CHANGED
@@ -0,0 +1,167 @@
|
|
1
|
+
# Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "fluent/plugin/out_groonga"
|
17
|
+
|
18
|
+
class OutputTypeTableDefinitionTest < Test::Unit::TestCase
|
19
|
+
def definition(raw={})
|
20
|
+
Fluent::GroongaOutput::TableDefinition.new(raw)
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case "readers" do
|
24
|
+
sub_test_case "\#name" do
|
25
|
+
test "specified" do
|
26
|
+
assert_equal("Tags", definition(:name => "Tags").name)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
sub_test_case "\#flags" do
|
31
|
+
test "default" do
|
32
|
+
assert_equal(["TABLE_NO_KEY"],
|
33
|
+
definition.flags)
|
34
|
+
end
|
35
|
+
|
36
|
+
test "one" do
|
37
|
+
assert_equal(["TABLE_PAT_KEY"],
|
38
|
+
definition(:flags => "TABLE_PAT_KEY").flags)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
sub_test_case "\#key_type" do
|
43
|
+
test "default" do
|
44
|
+
assert_nil(definition.key_type)
|
45
|
+
end
|
46
|
+
|
47
|
+
test "specified" do
|
48
|
+
assert_equal("ShortText",
|
49
|
+
definition(:key_type => "ShortText").key_type)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
sub_test_case "\#default_tokenizer" do
|
54
|
+
def read_default_tokenizer(input)
|
55
|
+
definition(:default_tokenizer => input).default_tokenizer
|
56
|
+
end
|
57
|
+
|
58
|
+
test "default" do
|
59
|
+
assert_nil(definition.default_tokenizer)
|
60
|
+
end
|
61
|
+
|
62
|
+
test "specified" do
|
63
|
+
assert_equal("TokenBigram",
|
64
|
+
read_default_tokenizer("TokenBigram"))
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
sub_test_case "\#token_filters" do
|
69
|
+
def read_token_filters(input)
|
70
|
+
definition(:token_filters => input).token_filters
|
71
|
+
end
|
72
|
+
|
73
|
+
test "default" do
|
74
|
+
assert_equal([], definition.token_filters)
|
75
|
+
end
|
76
|
+
|
77
|
+
test "one" do
|
78
|
+
assert_equal(["TokenFilterStem"],
|
79
|
+
read_token_filters("TokenFilterStem"))
|
80
|
+
end
|
81
|
+
|
82
|
+
test "multiple" do
|
83
|
+
assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
|
84
|
+
read_token_filters("TokenFilterStem,TokenFilterStopWord"))
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
sub_test_case "\#normalizer" do
|
89
|
+
def read_normalizer(input)
|
90
|
+
definition(:normalizer => input).normalizer
|
91
|
+
end
|
92
|
+
|
93
|
+
test "default" do
|
94
|
+
assert_nil(definition.normalizer)
|
95
|
+
end
|
96
|
+
|
97
|
+
test "specified" do
|
98
|
+
assert_equal("NormalizerAuto",
|
99
|
+
read_normalizer("NormalizerAuto"))
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
sub_test_case "\#have_difference?" do
|
105
|
+
def setup
|
106
|
+
@existing_table = Groonga::Client::Response::TableList::Table.new
|
107
|
+
@existing_table.id = 260
|
108
|
+
@existing_table.name = "Paths"
|
109
|
+
@existing_table.path = "/var/lib/groonga/db/db.0000104"
|
110
|
+
@existing_table.flags = "TABLE_PAT_KEY|PERSISTENT"
|
111
|
+
@existing_table.domain = "ShortText"
|
112
|
+
@existing_table.range = nil
|
113
|
+
@existing_table.default_tokenizer = nil
|
114
|
+
@existing_table.normalizer = nil
|
115
|
+
end
|
116
|
+
|
117
|
+
def have_difference?(raw={})
|
118
|
+
default_raw = {
|
119
|
+
:name => @existing_table.name,
|
120
|
+
:flags => @existing_table.flags.gsub(/\|PERSISTENT/, ""),
|
121
|
+
:key_type => @existing_table.domain,
|
122
|
+
:default_tokenizer => @existing_table.default_tokenizer,
|
123
|
+
:normalizer => @existing_table.normalizer,
|
124
|
+
}
|
125
|
+
raw = default_raw.merge(raw)
|
126
|
+
definition(raw).have_difference?(@existing_table)
|
127
|
+
end
|
128
|
+
|
129
|
+
test "no difference" do
|
130
|
+
assert do
|
131
|
+
not have_difference?
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
sub_test_case "difference" do
|
136
|
+
test "name" do
|
137
|
+
assert do
|
138
|
+
have_difference?(:name => "Difference")
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
test "flags" do
|
143
|
+
assert do
|
144
|
+
have_difference?(:flags => "TABLE_NO_KEY")
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
test "key_type" do
|
149
|
+
assert do
|
150
|
+
have_difference?(:key_type => "UInt32")
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
test "default_tokenizer" do
|
155
|
+
assert do
|
156
|
+
have_difference?(:default_tokenizer => "TokenBigram")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
test "normalizer" do
|
161
|
+
assert do
|
162
|
+
have_difference?(:normalizer => "NormalizerAuto")
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
@@ -122,11 +122,30 @@ class OutputTypeGuesserTest < Test::Unit::TestCase
|
|
122
122
|
end
|
123
123
|
end
|
124
124
|
|
125
|
+
sub_test_case "ShortText" do
|
126
|
+
test "max" do
|
127
|
+
message = "X" * (2 ** 12)
|
128
|
+
assert_equal("ShortText", guess([message]))
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
125
132
|
sub_test_case "Text" do
|
126
|
-
test "
|
127
|
-
message = "
|
133
|
+
test "min" do
|
134
|
+
message = "X" * (2 ** 12 + 1)
|
128
135
|
assert_equal("Text", guess([message]))
|
129
136
|
end
|
137
|
+
|
138
|
+
test "max" do
|
139
|
+
message = "X" * (2 ** 16)
|
140
|
+
assert_equal("Text", guess([message]))
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
sub_test_case "LongText" do
|
145
|
+
test "min" do
|
146
|
+
message = "X" * (2 ** 16 + 1)
|
147
|
+
assert_equal("LongText", guess([message]))
|
148
|
+
end
|
130
149
|
end
|
131
150
|
end
|
132
151
|
end
|
data/test/run-test.rb
CHANGED
@@ -30,13 +30,8 @@ Test::Unit::Priority.enable
|
|
30
30
|
|
31
31
|
$LOAD_PATH.unshift(lib_dir)
|
32
32
|
|
33
|
-
require "fluent/
|
34
|
-
$log = Fluent::Log.new($stdout, Fluent::Log::LEVEL_WARN)
|
35
|
-
|
36
|
-
Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
|
37
|
-
require file.sub(/\.rb$/, '')
|
38
|
-
end
|
33
|
+
require "fluent/test"
|
39
34
|
|
40
35
|
ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
|
41
36
|
|
42
|
-
exit
|
37
|
+
exit(Test::Unit::AutoRunner.run(true, test_dir))
|
data/test/test_output.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-groonga
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-11-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 0.1.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 0.1.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: groonga-command-parser
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -157,7 +157,9 @@ files:
|
|
157
157
|
- sample/command.conf
|
158
158
|
- sample/gqtp.conf
|
159
159
|
- sample/http.conf
|
160
|
+
- sample/store-apache.conf
|
160
161
|
- sample/store.conf
|
162
|
+
- test/output/test_table_definition.rb
|
161
163
|
- test/output/test_type_guesser.rb
|
162
164
|
- test/run-test.rb
|
163
165
|
- test/test_input.rb
|
@@ -192,4 +194,5 @@ test_files:
|
|
192
194
|
- test/test_input.rb
|
193
195
|
- test/run-test.rb
|
194
196
|
- test/output/test_type_guesser.rb
|
197
|
+
- test/output/test_table_definition.rb
|
195
198
|
has_rdoc:
|