fluent-plugin-groonga 1.0.5 → 1.0.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2bddd43990cc83433776467a275c7c3ac0e16407
4
- data.tar.gz: 0259005297bc33bd6476eb34d5bdae3cbbcfd978
3
+ metadata.gz: fc471bf3943996d9224a54431bf81a5323c53dcd
4
+ data.tar.gz: 3c37826e7793bbcab4d8009c6d0555aa74577858
5
5
  SHA512:
6
- metadata.gz: 1eddbfd3cdcc690f38e5544d31c9ff29a9df51b3758c3bcdc3de096636b0bb2aac8a24312d476be0b7a6780fca415d219e0547b3f57b1e46a533dbaaa79903a6
7
- data.tar.gz: 15a9fcfe04e8d18bc1c6190abbe718c8939bd6cdf6dcc027302277e504f05b71bb8db0ab39d092d00c4da3404dde014dd2ae9982f471e19699368193369ccfe0
6
+ metadata.gz: d1dc9a521003ddc42ccc0ff98e88468931b4d9400e5d7cb5e592a836d1be655a858a396a9275c1058a8c221041854c3d5e69e481781fdc475e0911d1595301d3
7
+ data.tar.gz: 63e6a01cafd4bcb3e104ec80bcd53a48a4d375f29f95a292b4f214e360c287a846d41f120dd5e3b9ed52ef587f60a5cd7283dad7c03aa14f07b9d80f2adf6807
data/Gemfile CHANGED
@@ -1,6 +1,6 @@
1
1
  # -*- mode: ruby; coding: utf-8 -*-
2
2
  #
3
- # Copyright (C) 2012 Kouhei Sutou <kou@clear-code.com>
3
+ # Copyright (C) 2012-2014 Kouhei Sutou <kou@clear-code.com>
4
4
  #
5
5
  # This library is free software; you can redistribute it and/or
6
6
  # modify it under the terms of the GNU Lesser General Public
@@ -15,6 +15,6 @@
15
15
  # License along with this library; if not, write to the Free Software
16
16
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
 
18
- source :rubygems
18
+ source "https://rubygems.org/"
19
19
 
20
20
  gemspec
data/README.md CHANGED
@@ -42,7 +42,7 @@ into Groonga:
42
42
 
43
43
  <match log.**>
44
44
  type groonga
45
- table logs
45
+ store_table logs
46
46
 
47
47
  protocol http
48
48
  host 127.0.0.1
@@ -88,13 +88,13 @@ define schema in Groonga before running Fluentd. You just run Groonga.
88
88
 
89
89
  There is one required parameter:
90
90
 
91
- * `table`: It specifies table name for storing logs.
91
+ * `store_table`: It specifies table name for storing logs.
92
92
 
93
93
  Here is a minimum configuration:
94
94
 
95
95
  <match log.**>
96
96
  type groonga
97
- table logs
97
+ store_table logs
98
98
  </match>
99
99
 
100
100
  The configuration stores logs into `logs` table in Groonga that runs
@@ -115,7 +115,7 @@ Here is a configuration that specifies optional parameters explicitly:
115
115
 
116
116
  <match log.**>
117
117
  type groonga
118
- table logs
118
+ store_table logs
119
119
 
120
120
  protocol http
121
121
  host 127.0.0.1
@@ -2,6 +2,18 @@
2
2
 
3
3
  # News
4
4
 
5
+ ## 1.0.6: 2014-11-05
6
+
7
+ ### Improvements
8
+
9
+ * out: Renamed `table` parameter name to `store_table`.
10
+ `table` parameter is still usable for backward compatibility.
11
+ * out: Supported table definition by `<table>` configuration.
12
+ See sample/store-apache.conf for details.
13
+ * out: Supported specifying column type and creating indexes for auto
14
+ created columns by `<mapping>` configuration.
15
+ See sample/store-apache.conf for details.
16
+
5
17
  ## 1.0.5: 2014-10-21
6
18
 
7
19
  ### Improvements
@@ -17,7 +17,7 @@
17
17
 
18
18
  Gem::Specification.new do |spec|
19
19
  spec.name = "fluent-plugin-groonga"
20
- spec.version = "1.0.5"
20
+ spec.version = "1.0.6"
21
21
  spec.authors = ["Kouhei Sutou"]
22
22
  spec.email = ["kou@clear-code.com"]
23
23
  spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
35
35
  spec.require_paths = ["lib"]
36
36
 
37
37
  spec.add_runtime_dependency("fluentd")
38
- spec.add_runtime_dependency("groonga-client")
38
+ spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
39
39
  spec.add_runtime_dependency("groonga-command-parser")
40
40
 
41
41
  spec.add_development_dependency("rake")
@@ -37,20 +37,65 @@ module Fluent
37
37
  raise ConfigError, "must be http, gqtp or command: <#{value}>"
38
38
  end
39
39
  end
40
- config_param :table, :string, :default => nil
40
+
41
+ # alias is just for backward compatibility
42
+ config_param :store_table, :string, :default => nil, :alias => :table
43
+
44
+ config_section :table,
45
+ :param_name => "tables",
46
+ :required => false,
47
+ :multi => true do
48
+ config_param :name, :string
49
+ config_param :flags, :string, :default => nil
50
+ config_param :key_type, :string, :default => nil
51
+ config_param :default_tokenizer, :string, :default => nil
52
+ config_param :token_filters, :string, :default => nil
53
+ config_param :normalizer, :string, :default => nil
54
+ config_section :index,
55
+ :param_name => "indexes",
56
+ :required => false,
57
+ :multi => true do
58
+ config_param :name, :string
59
+ config_param :source_table, :string
60
+ config_param :source_columns, :string
61
+ end
62
+ end
63
+
64
+ config_section :mapping,
65
+ :param_name => "mappings",
66
+ :required => false,
67
+ :multi => true do
68
+ config_param :name, :string
69
+ config_param :type, :string, :default => nil
70
+ config_section :index,
71
+ :param_name => "indexes",
72
+ :required => false,
73
+ :multi => true do
74
+ config_param :table, :string
75
+ config_param :name, :string
76
+ config_param :flags, :string, :default => nil
77
+ end
78
+ end
41
79
 
42
80
  def configure(conf)
43
81
  super
44
82
  @client = create_client(@protocol)
45
83
  @client.configure(conf)
46
84
 
47
- @emitter = Emitter.new(@client, @table)
85
+ @schema = Schema.new(@client, @store_table, @mappings)
86
+ @emitter = Emitter.new(@client, @store_table, @schema)
87
+
88
+ @tables = @tables.collect do |table|
89
+ TableDefinition.new(table)
90
+ end
48
91
  end
49
92
 
50
93
  def start
51
94
  super
52
95
  @client.start
53
96
  @emitter.start
97
+ tables_creator = TablesCreator.new(@client, @tables)
98
+ tables_creator.create
54
99
  end
55
100
 
56
101
  def shutdown
@@ -77,18 +122,171 @@ module Fluent
77
122
  end
78
123
  end
79
124
 
125
+ class TableDefinition
126
+ def initialize(raw)
127
+ @raw = raw
128
+ end
129
+
130
+ def name
131
+ @raw[:name]
132
+ end
133
+
134
+ def flags
135
+ parse_flags(@raw[:flags] || "TABLE_NO_KEY")
136
+ end
137
+
138
+ def key_type
139
+ @raw[:key_type]
140
+ end
141
+
142
+ def default_tokenizer
143
+ @raw[:default_tokenizer]
144
+ end
145
+
146
+ def token_filters
147
+ parse_items(@raw[:token_filters] || "")
148
+ end
149
+
150
+ def normalizer
151
+ @raw[:normalizer]
152
+ end
153
+
154
+ def indexes
155
+ (@raw[:indexes] || []).collect do |raw|
156
+ IndexDefinition.new(self, raw)
157
+ end
158
+ end
159
+
160
+ def use_n_gram_tokenizer?
161
+ /\AToken(?:Uni|Bi|Tri)gram/ === default_tokenizer.to_s
162
+ end
163
+
164
+ def have_difference?(table)
165
+ return true if table.name != name
166
+
167
+ table_flags = (parse_flags(table.flags) - ["PERSISTENT"])
168
+ return true if table_flags.sort != flags.sort
169
+
170
+ return true if table.domain != key_type
171
+
172
+ return true if table.default_tokenizer != default_tokenizer
173
+
174
+ # TODO
175
+ # return true if table.token_filters.sort != token_filters.sort
176
+
177
+ return true if table.normalizer != normalizer
178
+
179
+ false
180
+ end
181
+
182
+ def to_create_arguments
183
+ arguments = {
184
+ "name" => name,
185
+ "flags" => flags.join("|"),
186
+ "key_type" => key_type,
187
+ "default_tokenizer" => default_tokenizer,
188
+ # TODO
189
+ # "token_filters" => token_filters.join("|"),
190
+ "normalizer" => normalizer,
191
+ }
192
+ arguments.keys.each do |key|
193
+ value = arguments[key]
194
+ arguments.delete(key) if value.nil? or value.empty?
195
+ end
196
+ arguments
197
+ end
198
+
199
+ private
200
+ def parse_flags(flags)
201
+ if flags.is_a?(Array)
202
+ flags
203
+ else
204
+ flags.strip.split(/\s*\|\s*/)
205
+ end
206
+ end
207
+
208
+ def parse_items(items)
209
+ if items.is_a?(Array)
210
+ items
211
+ else
212
+ items.strip.split(/\s*,\s*/)
213
+ end
214
+ end
215
+
216
+ class IndexDefinition
217
+ def initialize(table, raw)
218
+ @table = table
219
+ @raw = raw
220
+ end
221
+
222
+ def name
223
+ @raw[:name]
224
+ end
225
+
226
+ def source_table
227
+ @raw[:source_table]
228
+ end
229
+
230
+ def source_columns
231
+ @raw[:source_columns]
232
+ end
233
+
234
+ def flags
235
+ _flags = ["COLUMN_INDEX"]
236
+ _flags << "WITH_POSITION" if @table.use_n_gram_tokenizer?
237
+ _flags << "WITH_SECTION" if source_columns.size >= 2
238
+ _flags
239
+ end
240
+
241
+ def to_create_arguments
242
+ {
243
+ "table" => @table.name,
244
+ "name" => name,
245
+ "flags" => flags.join,
246
+ "type" => source_table,
247
+ "source" => source_columns,
248
+ }
249
+ end
250
+ end
251
+ end
252
+
253
+ class TablesCreator
254
+ def initialize(client, definitions)
255
+ @client = client
256
+ @definitions = definitions
257
+ end
258
+
259
+ def create
260
+ return if @definitions.empty?
261
+
262
+ table_list = @client.execute("table_list")
263
+ @definitions.each do |definition|
264
+ existing_table = table_list.find do |table|
265
+ table.name == definition.name
266
+ end
267
+ if existing_table
268
+ next unless definition.have_difference?(existing_table)
269
+ # TODO: Is it OK?
270
+ @client.execute("table_remove", "name" => definition.name)
271
+ end
272
+
273
+ @client.execute("table_create", definition.to_create_arguments)
274
+ definition.indexes.each do |index|
275
+ @client.execute("column_create", index.to_create_arguments)
276
+ end
277
+ end
278
+ end
279
+ end
280
+
80
281
  class Schema
81
- def initialize(client, table_name)
282
+ def initialize(client, table_name, mappings)
82
283
  @client = client
83
284
  @table_name = table_name
285
+ @mappings = mappings
84
286
  @table = nil
85
287
  @columns = nil
86
288
  end
87
289
 
88
- def populate
89
- # TODO
90
- end
91
-
92
290
  def update(records)
93
291
  ensure_table
94
292
  ensure_columns
@@ -120,7 +318,6 @@ module Fluent
120
318
  if target_table
121
319
  @table = Table.new(@table_name, target_table.domain)
122
320
  else
123
- # TODO: Check response
124
321
  @client.execute("table_create",
125
322
  "name" => @table_name,
126
323
  "flags" => "TABLE_NO_KEY")
@@ -142,20 +339,37 @@ module Fluent
142
339
  end
143
340
 
144
341
  def create_column(name, sample_values)
342
+ mapping = @mappings.find do |mapping|
343
+ mapping.name == name
344
+ end
345
+ if mapping
346
+ value_type = mapping[:type]
347
+ end
145
348
  guesser = TypeGuesser.new(sample_values)
146
- value_type = guesser.guess
349
+ value_type ||= guesser.guess
147
350
  vector_p = guesser.vector?
148
351
  if vector_p
149
352
  flags = "COLUMN_VECTOR"
150
353
  else
151
354
  flags = "COLUMN_SCALAR"
152
355
  end
153
- # TODO: Check response
154
356
  @client.execute("column_create",
155
357
  "table" => @table_name,
156
358
  "name" => name,
157
359
  "flags" => flags,
158
360
  "type" => value_type)
361
+ if mapping
362
+ mapping.indexes.each do |index|
363
+ index_flags = ["COLUMN_INDEX", index[:flags]].compact
364
+ @client.execute("column_create",
365
+ "table" => index[:table],
366
+ "name" => index[:table],
367
+ "flags" => index_flags.join("|"),
368
+ "type" => @table_name,
369
+ "source" => name)
370
+ end
371
+ end
372
+
159
373
  Column.new(name, value_type, vector_p)
160
374
  end
161
375
 
@@ -170,8 +384,10 @@ module Fluent
170
384
  return "Int64" if int64_values?
171
385
  return "Float" if float_values?
172
386
  return "WGS84GeoPoint" if geo_point_values?
387
+ return "LongText" if long_text_values?
388
+ return "Text" if text_values?
173
389
 
174
- "Text"
390
+ "ShortText"
175
391
  end
176
392
 
177
393
  def vector?
@@ -254,6 +470,22 @@ module Fluent
254
470
  /\A-?\d+(?:\.\d+)[,x]-?\d+(?:\.\d+)\z/ =~ sample_value
255
471
  end
256
472
  end
473
+
474
+ MAX_SHORT_TEXT_SIZE = 2 ** 12
475
+ MAX_TEXT_SIZE = 2 ** 16
476
+ def text_values?
477
+ @sample_values.any? do |sample_value|
478
+ sample_value.is_a?(String) and
479
+ sample_value.bytesize > MAX_SHORT_TEXT_SIZE
480
+ end
481
+ end
482
+
483
+ def long_text_values?
484
+ @sample_values.any? do |sample_value|
485
+ sample_value.is_a?(String) and
486
+ sample_value.bytesize > MAX_TEXT_SIZE
487
+ end
488
+ end
257
489
  end
258
490
 
259
491
  class Table
@@ -273,14 +505,13 @@ module Fluent
273
505
  end
274
506
 
275
507
  class Emitter
276
- def initialize(client, table)
508
+ def initialize(client, table, schema)
277
509
  @client = client
278
510
  @table = table
279
- @schema = nil
511
+ @schema = schema
280
512
  end
281
513
 
282
514
  def start
283
- @schema = Schema.new(@client, @table)
284
515
  end
285
516
 
286
517
  def shutdown
@@ -352,7 +583,13 @@ module Fluent
352
583
  :host => @host,
353
584
  :port => @port,
354
585
  :backend => :synchronous)
355
- @client.execute(command)
586
+ response = @client.execute(command)
587
+ unless response.success?
588
+ $log.error("[output][groonga][error]",
589
+ :status_code => response.status_code,
590
+ :message => response.message)
591
+ end
592
+ response
356
593
  end
357
594
  end
358
595
 
@@ -453,14 +690,14 @@ module Fluent
453
690
  end
454
691
 
455
692
  unless output_message.empty?
456
- Engine.log.debug("[output][groonga][output]",
457
- :context => context,
458
- :message => output_message)
693
+ $log.debug("[output][groonga][output]",
694
+ :context => context,
695
+ :message => output_message)
459
696
  end
460
697
  unless error_message.empty?
461
- Engine.log.error("[output][groonga][error]",
462
- :context => context,
463
- :message => error_message)
698
+ $log.error("[output][groonga][error]",
699
+ :context => context,
700
+ :message => error_message)
464
701
  end
465
702
  end
466
703
  end
@@ -0,0 +1,239 @@
1
+ <source>
2
+ type forward
3
+ </source>
4
+
5
+ <source>
6
+ type tail
7
+ path /var/log/apache2/access.log
8
+ pos_file /tmp/apache_access.pos
9
+ tag apache.raw.log.apache.access
10
+ format apache2
11
+ # read_from_head true
12
+ </source>
13
+
14
+ <match apache.**>
15
+ type record_reformer
16
+ enable_ruby false
17
+
18
+ tag ${tag_suffix[1]}
19
+
20
+ <record>
21
+ remote ${host}
22
+ </record>
23
+ </match>
24
+
25
+ <match raw.log.**>
26
+ type record_reformer
27
+ enable_ruby false
28
+
29
+ tag ${tag_suffix[1]}
30
+
31
+ <record>
32
+ host ${hostname}
33
+ type ${tag_suffix[2]}
34
+ timestamp ${time}
35
+ </record>
36
+ </match>
37
+
38
+ <match log.**>
39
+ type groonga
40
+ store_table Logs
41
+
42
+ protocol http
43
+ host 127.0.0.1
44
+
45
+ buffer_type file
46
+ buffer_path /tmp/buffer
47
+ flush_interval 1
48
+
49
+ <table>
50
+ name Codes
51
+ flags TABLE_PAT_KEY
52
+ key_type Int32
53
+ </table>
54
+
55
+ <table>
56
+ name Hosts
57
+ flags TABLE_PAT_KEY
58
+ key_type ShortText
59
+ normalizer NormalizerAuto
60
+ </table>
61
+
62
+ <table>
63
+ name URLs
64
+ flags TABLE_PAT_KEY
65
+ key_type ShortText
66
+ </table>
67
+
68
+ <table>
69
+ name Paths
70
+ flags TABLE_PAT_KEY
71
+ key_type ShortText
72
+ </table>
73
+
74
+ <table>
75
+ name UserAgents
76
+ flags TABLE_PAT_KEY
77
+ key_type ShortText
78
+ </table>
79
+
80
+ <table>
81
+ name Methods
82
+ flags TABLE_HASH_KEY
83
+ key_type ShortText
84
+ normalizer NormalizerAuto
85
+ </table>
86
+
87
+ <table>
88
+ name Remotes
89
+ flags TABLE_PAT_KEY
90
+ key_type ShortText
91
+ </table>
92
+
93
+ <table>
94
+ name Sizes
95
+ flags TABLE_PAT_KEY
96
+ key_type Int32
97
+ </table>
98
+
99
+ <table>
100
+ name Timestamps
101
+ flags TABLE_PAT_KEY
102
+ key_type Time
103
+ </table>
104
+
105
+ <table>
106
+ name Types
107
+ flags TABLE_PAT_KEY
108
+ key_type ShortText
109
+ </table>
110
+
111
+ <table>
112
+ name Terms
113
+ flags TABLE_PAT_KEY
114
+ key_type ShortText
115
+ default_tokenizer TokenBigram
116
+ normalizer NormalizerAuto
117
+ <index>
118
+ name host_index
119
+ source_table Hosts
120
+ source_columns _key
121
+ </index>
122
+ <index>
123
+ name url_index
124
+ source_table URLs
125
+ source_columns _key
126
+ </index>
127
+ <index>
128
+ name path_index
129
+ source_table Paths
130
+ source_columns _key
131
+ </index>
132
+ <index>
133
+ name user_agent_index
134
+ source_table UserAgents
135
+ source_columns _key
136
+ </index>
137
+ </table>
138
+
139
+ <mapping>
140
+ name agent
141
+ type UserAgents
142
+ <index>
143
+ table Terms
144
+ name logs_agent_index
145
+ flags WITH_POSITION
146
+ </index>
147
+ </mapping>
148
+
149
+ <mapping>
150
+ name code
151
+ type Codes
152
+ <index>
153
+ table Codes
154
+ name logs_index
155
+ </index>
156
+ </mapping>
157
+
158
+ <mapping>
159
+ name host
160
+ type Hosts
161
+ <index>
162
+ table Hosts
163
+ name hosts_index
164
+ </index>
165
+ </mapping>
166
+
167
+ <mapping>
168
+ name message
169
+ type Text
170
+ <index>
171
+ table Terms
172
+ name logs_message_index
173
+ flags WITH_POSITION
174
+ </index>
175
+ </mapping>
176
+
177
+ <mapping>
178
+ name method
179
+ type Methods
180
+ <index>
181
+ table Methods
182
+ name logs_index
183
+ </index>
184
+ </mapping>
185
+
186
+ <mapping>
187
+ name path
188
+ type Paths
189
+ <index>
190
+ table Paths
191
+ name logs_index
192
+ </index>
193
+ </mapping>
194
+
195
+ <mapping>
196
+ name referer
197
+ type URLs
198
+ <index>
199
+ table URLs
200
+ name logs_index
201
+ </index>
202
+ </mapping>
203
+
204
+ <mapping>
205
+ name remote
206
+ type Remotes
207
+ <index>
208
+ table Remotes
209
+ name logs_index
210
+ </index>
211
+ </mapping>
212
+
213
+ <mapping>
214
+ name size
215
+ type Int32
216
+ <index>
217
+ table Sizes
218
+ name logs_index
219
+ </index>
220
+ </mapping>
221
+
222
+ <mapping>
223
+ name timestamp
224
+ type Time
225
+ <index>
226
+ table Timestamps
227
+ name logs_index
228
+ </index>
229
+ </mapping>
230
+
231
+ <mapping>
232
+ name type
233
+ type Types
234
+ <index>
235
+ table Types
236
+ name logs_index
237
+ </index>
238
+ </mapping>
239
+ </match>
@@ -4,7 +4,7 @@
4
4
 
5
5
  <match log.*>
6
6
  type groonga
7
- table Logs
7
+ store_table Logs
8
8
 
9
9
  protocol http
10
10
  host 127.0.0.1
@@ -0,0 +1,167 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "fluent/plugin/out_groonga"
17
+
18
+ class OutputTypeTableDefinitionTest < Test::Unit::TestCase
19
+ def definition(raw={})
20
+ Fluent::GroongaOutput::TableDefinition.new(raw)
21
+ end
22
+
23
+ sub_test_case "readers" do
24
+ sub_test_case "\#name" do
25
+ test "specified" do
26
+ assert_equal("Tags", definition(:name => "Tags").name)
27
+ end
28
+ end
29
+
30
+ sub_test_case "\#flags" do
31
+ test "default" do
32
+ assert_equal(["TABLE_NO_KEY"],
33
+ definition.flags)
34
+ end
35
+
36
+ test "one" do
37
+ assert_equal(["TABLE_PAT_KEY"],
38
+ definition(:flags => "TABLE_PAT_KEY").flags)
39
+ end
40
+ end
41
+
42
+ sub_test_case "\#key_type" do
43
+ test "default" do
44
+ assert_nil(definition.key_type)
45
+ end
46
+
47
+ test "specified" do
48
+ assert_equal("ShortText",
49
+ definition(:key_type => "ShortText").key_type)
50
+ end
51
+ end
52
+
53
+ sub_test_case "\#default_tokenizer" do
54
+ def read_default_tokenizer(input)
55
+ definition(:default_tokenizer => input).default_tokenizer
56
+ end
57
+
58
+ test "default" do
59
+ assert_nil(definition.default_tokenizer)
60
+ end
61
+
62
+ test "specified" do
63
+ assert_equal("TokenBigram",
64
+ read_default_tokenizer("TokenBigram"))
65
+ end
66
+ end
67
+
68
+ sub_test_case "\#token_filters" do
69
+ def read_token_filters(input)
70
+ definition(:token_filters => input).token_filters
71
+ end
72
+
73
+ test "default" do
74
+ assert_equal([], definition.token_filters)
75
+ end
76
+
77
+ test "one" do
78
+ assert_equal(["TokenFilterStem"],
79
+ read_token_filters("TokenFilterStem"))
80
+ end
81
+
82
+ test "multiple" do
83
+ assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
84
+ read_token_filters("TokenFilterStem,TokenFilterStopWord"))
85
+ end
86
+ end
87
+
88
+ sub_test_case "\#normalizer" do
89
+ def read_normalizer(input)
90
+ definition(:normalizer => input).normalizer
91
+ end
92
+
93
+ test "default" do
94
+ assert_nil(definition.normalizer)
95
+ end
96
+
97
+ test "specified" do
98
+ assert_equal("NormalizerAuto",
99
+ read_normalizer("NormalizerAuto"))
100
+ end
101
+ end
102
+ end
103
+
104
+ sub_test_case "\#have_difference?" do
105
+ def setup
106
+ @existing_table = Groonga::Client::Response::TableList::Table.new
107
+ @existing_table.id = 260
108
+ @existing_table.name = "Paths"
109
+ @existing_table.path = "/var/lib/groonga/db/db.0000104"
110
+ @existing_table.flags = "TABLE_PAT_KEY|PERSISTENT"
111
+ @existing_table.domain = "ShortText"
112
+ @existing_table.range = nil
113
+ @existing_table.default_tokenizer = nil
114
+ @existing_table.normalizer = nil
115
+ end
116
+
117
+ def have_difference?(raw={})
118
+ default_raw = {
119
+ :name => @existing_table.name,
120
+ :flags => @existing_table.flags.gsub(/\|PERSISTENT/, ""),
121
+ :key_type => @existing_table.domain,
122
+ :default_tokenizer => @existing_table.default_tokenizer,
123
+ :normalizer => @existing_table.normalizer,
124
+ }
125
+ raw = default_raw.merge(raw)
126
+ definition(raw).have_difference?(@existing_table)
127
+ end
128
+
129
+ test "no difference" do
130
+ assert do
131
+ not have_difference?
132
+ end
133
+ end
134
+
135
+ sub_test_case "difference" do
136
+ test "name" do
137
+ assert do
138
+ have_difference?(:name => "Difference")
139
+ end
140
+ end
141
+
142
+ test "flags" do
143
+ assert do
144
+ have_difference?(:flags => "TABLE_NO_KEY")
145
+ end
146
+ end
147
+
148
+ test "key_type" do
149
+ assert do
150
+ have_difference?(:key_type => "UInt32")
151
+ end
152
+ end
153
+
154
+ test "default_tokenizer" do
155
+ assert do
156
+ have_difference?(:default_tokenizer => "TokenBigram")
157
+ end
158
+ end
159
+
160
+ test "normalizer" do
161
+ assert do
162
+ have_difference?(:normalizer => "NormalizerAuto")
163
+ end
164
+ end
165
+ end
166
+ end
167
+ end
@@ -122,11 +122,30 @@ class OutputTypeGuesserTest < Test::Unit::TestCase
122
122
  end
123
123
  end
124
124
 
125
+ sub_test_case "ShortText" do
126
+ test "max" do
127
+ message = "X" * (2 ** 12)
128
+ assert_equal("ShortText", guess([message]))
129
+ end
130
+ end
131
+
125
132
  sub_test_case "Text" do
126
- test "message" do
127
- message = "failed to load data"
133
+ test "min" do
134
+ message = "X" * (2 ** 12 + 1)
128
135
  assert_equal("Text", guess([message]))
129
136
  end
137
+
138
+ test "max" do
139
+ message = "X" * (2 ** 16)
140
+ assert_equal("Text", guess([message]))
141
+ end
142
+ end
143
+
144
+ sub_test_case "LongText" do
145
+ test "min" do
146
+ message = "X" * (2 ** 16 + 1)
147
+ assert_equal("LongText", guess([message]))
148
+ end
130
149
  end
131
150
  end
132
151
  end
@@ -30,13 +30,8 @@ Test::Unit::Priority.enable
30
30
 
31
31
  $LOAD_PATH.unshift(lib_dir)
32
32
 
33
- require "fluent/log"
34
- $log = Fluent::Log.new($stdout, Fluent::Log::LEVEL_WARN)
35
-
36
- Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
37
- require file.sub(/\.rb$/, '')
38
- end
33
+ require "fluent/test"
39
34
 
40
35
  ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
41
36
 
42
- exit Test::Unit::AutoRunner.run
37
+ exit(Test::Unit::AutoRunner.run(true, test_dir))
@@ -111,7 +111,7 @@ EOC
111
111
  def configuration
112
112
  <<-CONFIGURATION
113
113
  #{super}
114
- table Logs
114
+ store_table Logs
115
115
  CONFIGURATION
116
116
  end
117
117
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-groonga
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-10-21 00:00:00.000000000 Z
11
+ date: 2014-11-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: 0.1.0
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: 0.1.0
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: groonga-command-parser
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -157,7 +157,9 @@ files:
157
157
  - sample/command.conf
158
158
  - sample/gqtp.conf
159
159
  - sample/http.conf
160
+ - sample/store-apache.conf
160
161
  - sample/store.conf
162
+ - test/output/test_table_definition.rb
161
163
  - test/output/test_type_guesser.rb
162
164
  - test/run-test.rb
163
165
  - test/test_input.rb
@@ -192,4 +194,5 @@ test_files:
192
194
  - test/test_input.rb
193
195
  - test/run-test.rb
194
196
  - test/output/test_type_guesser.rb
197
+ - test/output/test_table_definition.rb
195
198
  has_rdoc: