fluent-plugin-groonga 1.0.5 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 2bddd43990cc83433776467a275c7c3ac0e16407
4
- data.tar.gz: 0259005297bc33bd6476eb34d5bdae3cbbcfd978
3
+ metadata.gz: fc471bf3943996d9224a54431bf81a5323c53dcd
4
+ data.tar.gz: 3c37826e7793bbcab4d8009c6d0555aa74577858
5
5
  SHA512:
6
- metadata.gz: 1eddbfd3cdcc690f38e5544d31c9ff29a9df51b3758c3bcdc3de096636b0bb2aac8a24312d476be0b7a6780fca415d219e0547b3f57b1e46a533dbaaa79903a6
7
- data.tar.gz: 15a9fcfe04e8d18bc1c6190abbe718c8939bd6cdf6dcc027302277e504f05b71bb8db0ab39d092d00c4da3404dde014dd2ae9982f471e19699368193369ccfe0
6
+ metadata.gz: d1dc9a521003ddc42ccc0ff98e88468931b4d9400e5d7cb5e592a836d1be655a858a396a9275c1058a8c221041854c3d5e69e481781fdc475e0911d1595301d3
7
+ data.tar.gz: 63e6a01cafd4bcb3e104ec80bcd53a48a4d375f29f95a292b4f214e360c287a846d41f120dd5e3b9ed52ef587f60a5cd7283dad7c03aa14f07b9d80f2adf6807
data/Gemfile CHANGED
@@ -1,6 +1,6 @@
1
1
  # -*- mode: ruby; coding: utf-8 -*-
2
2
  #
3
- # Copyright (C) 2012 Kouhei Sutou <kou@clear-code.com>
3
+ # Copyright (C) 2012-2014 Kouhei Sutou <kou@clear-code.com>
4
4
  #
5
5
  # This library is free software; you can redistribute it and/or
6
6
  # modify it under the terms of the GNU Lesser General Public
@@ -15,6 +15,6 @@
15
15
  # License along with this library; if not, write to the Free Software
16
16
  # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17
17
 
18
- source :rubygems
18
+ source "https://rubygems.org/"
19
19
 
20
20
  gemspec
data/README.md CHANGED
@@ -42,7 +42,7 @@ into Groonga:
42
42
 
43
43
  <match log.**>
44
44
  type groonga
45
- table logs
45
+ store_table logs
46
46
 
47
47
  protocol http
48
48
  host 127.0.0.1
@@ -88,13 +88,13 @@ define schema in Groonga before running Fluentd. You just run Groonga.
88
88
 
89
89
  There is one required parameter:
90
90
 
91
- * `table`: It specifies table name for storing logs.
91
+ * `store_table`: It specifies table name for storing logs.
92
92
 
93
93
  Here is a minimum configuration:
94
94
 
95
95
  <match log.**>
96
96
  type groonga
97
- table logs
97
+ store_table logs
98
98
  </match>
99
99
 
100
100
  The configuration stores logs into `logs` table in Groonga that runs
@@ -115,7 +115,7 @@ Here is a configuration that specifies optional parameters explicitly:
115
115
 
116
116
  <match log.**>
117
117
  type groonga
118
- table logs
118
+ store_table logs
119
119
 
120
120
  protocol http
121
121
  host 127.0.0.1
@@ -2,6 +2,18 @@
2
2
 
3
3
  # News
4
4
 
5
+ ## 1.0.6: 2014-11-05
6
+
7
+ ### Improvements
8
+
9
+ * out: Renamed `table` parameter name to `store_table`.
10
+ `table` parameter is still usable for backward compatibility.
11
+ * out: Supported table definition by `<table>` configuration.
12
+ See sample/store-apache.conf for details.
13
+ * out: Supported specifying column type and creating indexes for auto
14
+ created columns by `<mapping>` configuration.
15
+ See sample/store-apache.conf for details.
16
+
5
17
  ## 1.0.5: 2014-10-21
6
18
 
7
19
  ### Improvements
@@ -17,7 +17,7 @@
17
17
 
18
18
  Gem::Specification.new do |spec|
19
19
  spec.name = "fluent-plugin-groonga"
20
- spec.version = "1.0.5"
20
+ spec.version = "1.0.6"
21
21
  spec.authors = ["Kouhei Sutou"]
22
22
  spec.email = ["kou@clear-code.com"]
23
23
  spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
35
35
  spec.require_paths = ["lib"]
36
36
 
37
37
  spec.add_runtime_dependency("fluentd")
38
- spec.add_runtime_dependency("groonga-client")
38
+ spec.add_runtime_dependency("groonga-client", ">= 0.1.0")
39
39
  spec.add_runtime_dependency("groonga-command-parser")
40
40
 
41
41
  spec.add_development_dependency("rake")
@@ -37,20 +37,65 @@ module Fluent
37
37
  raise ConfigError, "must be http, gqtp or command: <#{value}>"
38
38
  end
39
39
  end
40
- config_param :table, :string, :default => nil
40
+
41
+ # alias is just for backward compatibility
42
+ config_param :store_table, :string, :default => nil, :alias => :table
43
+
44
+ config_section :table,
45
+ :param_name => "tables",
46
+ :required => false,
47
+ :multi => true do
48
+ config_param :name, :string
49
+ config_param :flags, :string, :default => nil
50
+ config_param :key_type, :string, :default => nil
51
+ config_param :default_tokenizer, :string, :default => nil
52
+ config_param :token_filters, :string, :default => nil
53
+ config_param :normalizer, :string, :default => nil
54
+ config_section :index,
55
+ :param_name => "indexes",
56
+ :required => false,
57
+ :multi => true do
58
+ config_param :name, :string
59
+ config_param :source_table, :string
60
+ config_param :source_columns, :string
61
+ end
62
+ end
63
+
64
+ config_section :mapping,
65
+ :param_name => "mappings",
66
+ :required => false,
67
+ :multi => true do
68
+ config_param :name, :string
69
+ config_param :type, :string, :default => nil
70
+ config_section :index,
71
+ :param_name => "indexes",
72
+ :required => false,
73
+ :multi => true do
74
+ config_param :table, :string
75
+ config_param :name, :string
76
+ config_param :flags, :string, :default => nil
77
+ end
78
+ end
41
79
 
42
80
  def configure(conf)
43
81
  super
44
82
  @client = create_client(@protocol)
45
83
  @client.configure(conf)
46
84
 
47
- @emitter = Emitter.new(@client, @table)
85
+ @schema = Schema.new(@client, @store_table, @mappings)
86
+ @emitter = Emitter.new(@client, @store_table, @schema)
87
+
88
+ @tables = @tables.collect do |table|
89
+ TableDefinition.new(table)
90
+ end
48
91
  end
49
92
 
50
93
  def start
51
94
  super
52
95
  @client.start
53
96
  @emitter.start
97
+ tables_creator = TablesCreator.new(@client, @tables)
98
+ tables_creator.create
54
99
  end
55
100
 
56
101
  def shutdown
@@ -77,18 +122,171 @@ module Fluent
77
122
  end
78
123
  end
79
124
 
125
+ class TableDefinition
126
+ def initialize(raw)
127
+ @raw = raw
128
+ end
129
+
130
+ def name
131
+ @raw[:name]
132
+ end
133
+
134
+ def flags
135
+ parse_flags(@raw[:flags] || "TABLE_NO_KEY")
136
+ end
137
+
138
+ def key_type
139
+ @raw[:key_type]
140
+ end
141
+
142
+ def default_tokenizer
143
+ @raw[:default_tokenizer]
144
+ end
145
+
146
+ def token_filters
147
+ parse_items(@raw[:token_filters] || "")
148
+ end
149
+
150
+ def normalizer
151
+ @raw[:normalizer]
152
+ end
153
+
154
+ def indexes
155
+ (@raw[:indexes] || []).collect do |raw|
156
+ IndexDefinition.new(self, raw)
157
+ end
158
+ end
159
+
160
+ def use_n_gram_tokenizer?
161
+ /\AToken(?:Uni|Bi|Tri)gram/ === default_tokenizer.to_s
162
+ end
163
+
164
+ def have_difference?(table)
165
+ return true if table.name != name
166
+
167
+ table_flags = (parse_flags(table.flags) - ["PERSISTENT"])
168
+ return true if table_flags.sort != flags.sort
169
+
170
+ return true if table.domain != key_type
171
+
172
+ return true if table.default_tokenizer != default_tokenizer
173
+
174
+ # TODO
175
+ # return true if table.token_filters.sort != token_filters.sort
176
+
177
+ return true if table.normalizer != normalizer
178
+
179
+ false
180
+ end
181
+
182
+ def to_create_arguments
183
+ arguments = {
184
+ "name" => name,
185
+ "flags" => flags.join("|"),
186
+ "key_type" => key_type,
187
+ "default_tokenizer" => default_tokenizer,
188
+ # TODO
189
+ # "token_filters" => token_filters.join("|"),
190
+ "normalizer" => normalizer,
191
+ }
192
+ arguments.keys.each do |key|
193
+ value = arguments[key]
194
+ arguments.delete(key) if value.nil? or value.empty?
195
+ end
196
+ arguments
197
+ end
198
+
199
+ private
200
+ def parse_flags(flags)
201
+ if flags.is_a?(Array)
202
+ flags
203
+ else
204
+ flags.strip.split(/\s*\|\s*/)
205
+ end
206
+ end
207
+
208
+ def parse_items(items)
209
+ if items.is_a?(Array)
210
+ items
211
+ else
212
+ items.strip.split(/\s*,\s*/)
213
+ end
214
+ end
215
+
216
+ class IndexDefinition
217
+ def initialize(table, raw)
218
+ @table = table
219
+ @raw = raw
220
+ end
221
+
222
+ def name
223
+ @raw[:name]
224
+ end
225
+
226
+ def source_table
227
+ @raw[:source_table]
228
+ end
229
+
230
+ def source_columns
231
+ @raw[:source_columns]
232
+ end
233
+
234
+ def flags
235
+ _flags = ["COLUMN_INDEX"]
236
+ _flags << "WITH_POSITION" if @table.use_n_gram_tokenizer?
237
+ _flags << "WITH_SECTION" if source_columns.size >= 2
238
+ _flags
239
+ end
240
+
241
+ def to_create_arguments
242
+ {
243
+ "table" => @table.name,
244
+ "name" => name,
245
+ "flags" => flags.join,
246
+ "type" => source_table,
247
+ "source" => source_columns,
248
+ }
249
+ end
250
+ end
251
+ end
252
+
253
+ class TablesCreator
254
+ def initialize(client, definitions)
255
+ @client = client
256
+ @definitions = definitions
257
+ end
258
+
259
+ def create
260
+ return if @definitions.empty?
261
+
262
+ table_list = @client.execute("table_list")
263
+ @definitions.each do |definition|
264
+ existing_table = table_list.find do |table|
265
+ table.name == definition.name
266
+ end
267
+ if existing_table
268
+ next unless definition.have_difference?(existing_table)
269
+ # TODO: Is it OK?
270
+ @client.execute("table_remove", "name" => definition.name)
271
+ end
272
+
273
+ @client.execute("table_create", definition.to_create_arguments)
274
+ definition.indexes.each do |index|
275
+ @client.execute("column_create", index.to_create_arguments)
276
+ end
277
+ end
278
+ end
279
+ end
280
+
80
281
  class Schema
81
- def initialize(client, table_name)
282
+ def initialize(client, table_name, mappings)
82
283
  @client = client
83
284
  @table_name = table_name
285
+ @mappings = mappings
84
286
  @table = nil
85
287
  @columns = nil
86
288
  end
87
289
 
88
- def populate
89
- # TODO
90
- end
91
-
92
290
  def update(records)
93
291
  ensure_table
94
292
  ensure_columns
@@ -120,7 +318,6 @@ module Fluent
120
318
  if target_table
121
319
  @table = Table.new(@table_name, target_table.domain)
122
320
  else
123
- # TODO: Check response
124
321
  @client.execute("table_create",
125
322
  "name" => @table_name,
126
323
  "flags" => "TABLE_NO_KEY")
@@ -142,20 +339,37 @@ module Fluent
142
339
  end
143
340
 
144
341
  def create_column(name, sample_values)
342
+ mapping = @mappings.find do |mapping|
343
+ mapping.name == name
344
+ end
345
+ if mapping
346
+ value_type = mapping[:type]
347
+ end
145
348
  guesser = TypeGuesser.new(sample_values)
146
- value_type = guesser.guess
349
+ value_type ||= guesser.guess
147
350
  vector_p = guesser.vector?
148
351
  if vector_p
149
352
  flags = "COLUMN_VECTOR"
150
353
  else
151
354
  flags = "COLUMN_SCALAR"
152
355
  end
153
- # TODO: Check response
154
356
  @client.execute("column_create",
155
357
  "table" => @table_name,
156
358
  "name" => name,
157
359
  "flags" => flags,
158
360
  "type" => value_type)
361
+ if mapping
362
+ mapping.indexes.each do |index|
363
+ index_flags = ["COLUMN_INDEX", index[:flags]].compact
364
+ @client.execute("column_create",
365
+ "table" => index[:table],
366
+ "name" => index[:table],
367
+ "flags" => index_flags.join("|"),
368
+ "type" => @table_name,
369
+ "source" => name)
370
+ end
371
+ end
372
+
159
373
  Column.new(name, value_type, vector_p)
160
374
  end
161
375
 
@@ -170,8 +384,10 @@ module Fluent
170
384
  return "Int64" if int64_values?
171
385
  return "Float" if float_values?
172
386
  return "WGS84GeoPoint" if geo_point_values?
387
+ return "LongText" if long_text_values?
388
+ return "Text" if text_values?
173
389
 
174
- "Text"
390
+ "ShortText"
175
391
  end
176
392
 
177
393
  def vector?
@@ -254,6 +470,22 @@ module Fluent
254
470
  /\A-?\d+(?:\.\d+)[,x]-?\d+(?:\.\d+)\z/ =~ sample_value
255
471
  end
256
472
  end
473
+
474
+ MAX_SHORT_TEXT_SIZE = 2 ** 12
475
+ MAX_TEXT_SIZE = 2 ** 16
476
+ def text_values?
477
+ @sample_values.any? do |sample_value|
478
+ sample_value.is_a?(String) and
479
+ sample_value.bytesize > MAX_SHORT_TEXT_SIZE
480
+ end
481
+ end
482
+
483
+ def long_text_values?
484
+ @sample_values.any? do |sample_value|
485
+ sample_value.is_a?(String) and
486
+ sample_value.bytesize > MAX_TEXT_SIZE
487
+ end
488
+ end
257
489
  end
258
490
 
259
491
  class Table
@@ -273,14 +505,13 @@ module Fluent
273
505
  end
274
506
 
275
507
  class Emitter
276
- def initialize(client, table)
508
+ def initialize(client, table, schema)
277
509
  @client = client
278
510
  @table = table
279
- @schema = nil
511
+ @schema = schema
280
512
  end
281
513
 
282
514
  def start
283
- @schema = Schema.new(@client, @table)
284
515
  end
285
516
 
286
517
  def shutdown
@@ -352,7 +583,13 @@ module Fluent
352
583
  :host => @host,
353
584
  :port => @port,
354
585
  :backend => :synchronous)
355
- @client.execute(command)
586
+ response = @client.execute(command)
587
+ unless response.success?
588
+ $log.error("[output][groonga][error]",
589
+ :status_code => response.status_code,
590
+ :message => response.message)
591
+ end
592
+ response
356
593
  end
357
594
  end
358
595
 
@@ -453,14 +690,14 @@ module Fluent
453
690
  end
454
691
 
455
692
  unless output_message.empty?
456
- Engine.log.debug("[output][groonga][output]",
457
- :context => context,
458
- :message => output_message)
693
+ $log.debug("[output][groonga][output]",
694
+ :context => context,
695
+ :message => output_message)
459
696
  end
460
697
  unless error_message.empty?
461
- Engine.log.error("[output][groonga][error]",
462
- :context => context,
463
- :message => error_message)
698
+ $log.error("[output][groonga][error]",
699
+ :context => context,
700
+ :message => error_message)
464
701
  end
465
702
  end
466
703
  end
@@ -0,0 +1,239 @@
1
+ <source>
2
+ type forward
3
+ </source>
4
+
5
+ <source>
6
+ type tail
7
+ path /var/log/apache2/access.log
8
+ pos_file /tmp/apache_access.pos
9
+ tag apache.raw.log.apache.access
10
+ format apache2
11
+ # read_from_head true
12
+ </source>
13
+
14
+ <match apache.**>
15
+ type record_reformer
16
+ enable_ruby false
17
+
18
+ tag ${tag_suffix[1]}
19
+
20
+ <record>
21
+ remote ${host}
22
+ </record>
23
+ </match>
24
+
25
+ <match raw.log.**>
26
+ type record_reformer
27
+ enable_ruby false
28
+
29
+ tag ${tag_suffix[1]}
30
+
31
+ <record>
32
+ host ${hostname}
33
+ type ${tag_suffix[2]}
34
+ timestamp ${time}
35
+ </record>
36
+ </match>
37
+
38
+ <match log.**>
39
+ type groonga
40
+ store_table Logs
41
+
42
+ protocol http
43
+ host 127.0.0.1
44
+
45
+ buffer_type file
46
+ buffer_path /tmp/buffer
47
+ flush_interval 1
48
+
49
+ <table>
50
+ name Codes
51
+ flags TABLE_PAT_KEY
52
+ key_type Int32
53
+ </table>
54
+
55
+ <table>
56
+ name Hosts
57
+ flags TABLE_PAT_KEY
58
+ key_type ShortText
59
+ normalizer NormalizerAuto
60
+ </table>
61
+
62
+ <table>
63
+ name URLs
64
+ flags TABLE_PAT_KEY
65
+ key_type ShortText
66
+ </table>
67
+
68
+ <table>
69
+ name Paths
70
+ flags TABLE_PAT_KEY
71
+ key_type ShortText
72
+ </table>
73
+
74
+ <table>
75
+ name UserAgents
76
+ flags TABLE_PAT_KEY
77
+ key_type ShortText
78
+ </table>
79
+
80
+ <table>
81
+ name Methods
82
+ flags TABLE_HASH_KEY
83
+ key_type ShortText
84
+ normalizer NormalizerAuto
85
+ </table>
86
+
87
+ <table>
88
+ name Remotes
89
+ flags TABLE_PAT_KEY
90
+ key_type ShortText
91
+ </table>
92
+
93
+ <table>
94
+ name Sizes
95
+ flags TABLE_PAT_KEY
96
+ key_type Int32
97
+ </table>
98
+
99
+ <table>
100
+ name Timestamps
101
+ flags TABLE_PAT_KEY
102
+ key_type Time
103
+ </table>
104
+
105
+ <table>
106
+ name Types
107
+ flags TABLE_PAT_KEY
108
+ key_type ShortText
109
+ </table>
110
+
111
+ <table>
112
+ name Terms
113
+ flags TABLE_PAT_KEY
114
+ key_type ShortText
115
+ default_tokenizer TokenBigram
116
+ normalizer NormalizerAuto
117
+ <index>
118
+ name host_index
119
+ source_table Hosts
120
+ source_columns _key
121
+ </index>
122
+ <index>
123
+ name url_index
124
+ source_table URLs
125
+ source_columns _key
126
+ </index>
127
+ <index>
128
+ name path_index
129
+ source_table Paths
130
+ source_columns _key
131
+ </index>
132
+ <index>
133
+ name user_agent_index
134
+ source_table UserAgents
135
+ source_columns _key
136
+ </index>
137
+ </table>
138
+
139
+ <mapping>
140
+ name agent
141
+ type UserAgents
142
+ <index>
143
+ table Terms
144
+ name logs_agent_index
145
+ flags WITH_POSITION
146
+ </index>
147
+ </mapping>
148
+
149
+ <mapping>
150
+ name code
151
+ type Codes
152
+ <index>
153
+ table Codes
154
+ name logs_index
155
+ </index>
156
+ </mapping>
157
+
158
+ <mapping>
159
+ name host
160
+ type Hosts
161
+ <index>
162
+ table Hosts
163
+ name hosts_index
164
+ </index>
165
+ </mapping>
166
+
167
+ <mapping>
168
+ name message
169
+ type Text
170
+ <index>
171
+ table Terms
172
+ name logs_message_index
173
+ flags WITH_POSITION
174
+ </index>
175
+ </mapping>
176
+
177
+ <mapping>
178
+ name method
179
+ type Methods
180
+ <index>
181
+ table Methods
182
+ name logs_index
183
+ </index>
184
+ </mapping>
185
+
186
+ <mapping>
187
+ name path
188
+ type Paths
189
+ <index>
190
+ table Paths
191
+ name logs_index
192
+ </index>
193
+ </mapping>
194
+
195
+ <mapping>
196
+ name referer
197
+ type URLs
198
+ <index>
199
+ table URLs
200
+ name logs_index
201
+ </index>
202
+ </mapping>
203
+
204
+ <mapping>
205
+ name remote
206
+ type Remotes
207
+ <index>
208
+ table Remotes
209
+ name logs_index
210
+ </index>
211
+ </mapping>
212
+
213
+ <mapping>
214
+ name size
215
+ type Int32
216
+ <index>
217
+ table Sizes
218
+ name logs_index
219
+ </index>
220
+ </mapping>
221
+
222
+ <mapping>
223
+ name timestamp
224
+ type Time
225
+ <index>
226
+ table Timestamps
227
+ name logs_index
228
+ </index>
229
+ </mapping>
230
+
231
+ <mapping>
232
+ name type
233
+ type Types
234
+ <index>
235
+ table Types
236
+ name logs_index
237
+ </index>
238
+ </mapping>
239
+ </match>
@@ -4,7 +4,7 @@
4
4
 
5
5
  <match log.*>
6
6
  type groonga
7
- table Logs
7
+ store_table Logs
8
8
 
9
9
  protocol http
10
10
  host 127.0.0.1
@@ -0,0 +1,167 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "fluent/plugin/out_groonga"
17
+
18
+ class OutputTypeTableDefinitionTest < Test::Unit::TestCase
19
+ def definition(raw={})
20
+ Fluent::GroongaOutput::TableDefinition.new(raw)
21
+ end
22
+
23
+ sub_test_case "readers" do
24
+ sub_test_case "\#name" do
25
+ test "specified" do
26
+ assert_equal("Tags", definition(:name => "Tags").name)
27
+ end
28
+ end
29
+
30
+ sub_test_case "\#flags" do
31
+ test "default" do
32
+ assert_equal(["TABLE_NO_KEY"],
33
+ definition.flags)
34
+ end
35
+
36
+ test "one" do
37
+ assert_equal(["TABLE_PAT_KEY"],
38
+ definition(:flags => "TABLE_PAT_KEY").flags)
39
+ end
40
+ end
41
+
42
+ sub_test_case "\#key_type" do
43
+ test "default" do
44
+ assert_nil(definition.key_type)
45
+ end
46
+
47
+ test "specified" do
48
+ assert_equal("ShortText",
49
+ definition(:key_type => "ShortText").key_type)
50
+ end
51
+ end
52
+
53
+ sub_test_case "\#default_tokenizer" do
54
+ def read_default_tokenizer(input)
55
+ definition(:default_tokenizer => input).default_tokenizer
56
+ end
57
+
58
+ test "default" do
59
+ assert_nil(definition.default_tokenizer)
60
+ end
61
+
62
+ test "specified" do
63
+ assert_equal("TokenBigram",
64
+ read_default_tokenizer("TokenBigram"))
65
+ end
66
+ end
67
+
68
+ sub_test_case "\#token_filters" do
69
+ def read_token_filters(input)
70
+ definition(:token_filters => input).token_filters
71
+ end
72
+
73
+ test "default" do
74
+ assert_equal([], definition.token_filters)
75
+ end
76
+
77
+ test "one" do
78
+ assert_equal(["TokenFilterStem"],
79
+ read_token_filters("TokenFilterStem"))
80
+ end
81
+
82
+ test "multiple" do
83
+ assert_equal(["TokenFilterStem", "TokenFilterStopWord"],
84
+ read_token_filters("TokenFilterStem,TokenFilterStopWord"))
85
+ end
86
+ end
87
+
88
+ sub_test_case "\#normalizer" do
89
+ def read_normalizer(input)
90
+ definition(:normalizer => input).normalizer
91
+ end
92
+
93
+ test "default" do
94
+ assert_nil(definition.normalizer)
95
+ end
96
+
97
+ test "specified" do
98
+ assert_equal("NormalizerAuto",
99
+ read_normalizer("NormalizerAuto"))
100
+ end
101
+ end
102
+ end
103
+
104
+ sub_test_case "\#have_difference?" do
105
+ def setup
106
+ @existing_table = Groonga::Client::Response::TableList::Table.new
107
+ @existing_table.id = 260
108
+ @existing_table.name = "Paths"
109
+ @existing_table.path = "/var/lib/groonga/db/db.0000104"
110
+ @existing_table.flags = "TABLE_PAT_KEY|PERSISTENT"
111
+ @existing_table.domain = "ShortText"
112
+ @existing_table.range = nil
113
+ @existing_table.default_tokenizer = nil
114
+ @existing_table.normalizer = nil
115
+ end
116
+
117
+ def have_difference?(raw={})
118
+ default_raw = {
119
+ :name => @existing_table.name,
120
+ :flags => @existing_table.flags.gsub(/\|PERSISTENT/, ""),
121
+ :key_type => @existing_table.domain,
122
+ :default_tokenizer => @existing_table.default_tokenizer,
123
+ :normalizer => @existing_table.normalizer,
124
+ }
125
+ raw = default_raw.merge(raw)
126
+ definition(raw).have_difference?(@existing_table)
127
+ end
128
+
129
+ test "no difference" do
130
+ assert do
131
+ not have_difference?
132
+ end
133
+ end
134
+
135
+ sub_test_case "difference" do
136
+ test "name" do
137
+ assert do
138
+ have_difference?(:name => "Difference")
139
+ end
140
+ end
141
+
142
+ test "flags" do
143
+ assert do
144
+ have_difference?(:flags => "TABLE_NO_KEY")
145
+ end
146
+ end
147
+
148
+ test "key_type" do
149
+ assert do
150
+ have_difference?(:key_type => "UInt32")
151
+ end
152
+ end
153
+
154
+ test "default_tokenizer" do
155
+ assert do
156
+ have_difference?(:default_tokenizer => "TokenBigram")
157
+ end
158
+ end
159
+
160
+ test "normalizer" do
161
+ assert do
162
+ have_difference?(:normalizer => "NormalizerAuto")
163
+ end
164
+ end
165
+ end
166
+ end
167
+ end
@@ -122,11 +122,30 @@ class OutputTypeGuesserTest < Test::Unit::TestCase
122
122
  end
123
123
  end
124
124
 
125
+ sub_test_case "ShortText" do
126
+ test "max" do
127
+ message = "X" * (2 ** 12)
128
+ assert_equal("ShortText", guess([message]))
129
+ end
130
+ end
131
+
125
132
  sub_test_case "Text" do
126
- test "message" do
127
- message = "failed to load data"
133
+ test "min" do
134
+ message = "X" * (2 ** 12 + 1)
128
135
  assert_equal("Text", guess([message]))
129
136
  end
137
+
138
+ test "max" do
139
+ message = "X" * (2 ** 16)
140
+ assert_equal("Text", guess([message]))
141
+ end
142
+ end
143
+
144
+ sub_test_case "LongText" do
145
+ test "min" do
146
+ message = "X" * (2 ** 16 + 1)
147
+ assert_equal("LongText", guess([message]))
148
+ end
130
149
  end
131
150
  end
132
151
  end
@@ -30,13 +30,8 @@ Test::Unit::Priority.enable
30
30
 
31
31
  $LOAD_PATH.unshift(lib_dir)
32
32
 
33
- require "fluent/log"
34
- $log = Fluent::Log.new($stdout, Fluent::Log::LEVEL_WARN)
35
-
36
- Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
37
- require file.sub(/\.rb$/, '')
38
- end
33
+ require "fluent/test"
39
34
 
40
35
  ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
41
36
 
42
- exit Test::Unit::AutoRunner.run
37
+ exit(Test::Unit::AutoRunner.run(true, test_dir))
@@ -111,7 +111,7 @@ EOC
111
111
  def configuration
112
112
  <<-CONFIGURATION
113
113
  #{super}
114
- table Logs
114
+ store_table Logs
115
115
  CONFIGURATION
116
116
  end
117
117
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-groonga
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-10-21 00:00:00.000000000 Z
11
+ date: 2014-11-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: '0'
33
+ version: 0.1.0
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: '0'
40
+ version: 0.1.0
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: groonga-command-parser
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -157,7 +157,9 @@ files:
157
157
  - sample/command.conf
158
158
  - sample/gqtp.conf
159
159
  - sample/http.conf
160
+ - sample/store-apache.conf
160
161
  - sample/store.conf
162
+ - test/output/test_table_definition.rb
161
163
  - test/output/test_type_guesser.rb
162
164
  - test/run-test.rb
163
165
  - test/test_input.rb
@@ -192,4 +194,5 @@ test_files:
192
194
  - test/test_input.rb
193
195
  - test/run-test.rb
194
196
  - test/output/test_type_guesser.rb
197
+ - test/output/test_table_definition.rb
195
198
  has_rdoc: