fluent-plugin-droonga 1.0.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. data/.travis.yml +1 -1
  2. data/Gemfile +1 -1
  3. data/fluent-plugin-droonga.gemspec +1 -1
  4. data/lib/droonga/catalog/collection_volume.rb +97 -0
  5. data/lib/droonga/catalog/dataset.rb +28 -1
  6. data/lib/droonga/catalog/errors.rb +28 -9
  7. data/lib/droonga/catalog/schema.rb +23 -2
  8. data/lib/droonga/catalog/single_volume.rb +28 -0
  9. data/lib/droonga/catalog/slice.rb +43 -0
  10. data/lib/droonga/catalog/version1.rb +3 -3
  11. data/lib/droonga/catalog/version2.rb +17 -81
  12. data/lib/droonga/catalog/version2_validator.rb +63 -0
  13. data/lib/droonga/catalog/volume.rb +33 -0
  14. data/lib/droonga/catalog/volume_collection.rb +56 -0
  15. data/lib/droonga/catalog_observer.rb +7 -19
  16. data/lib/droonga/collectors.rb +1 -1
  17. data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
  18. data/lib/droonga/dispatcher.rb +24 -18
  19. data/lib/droonga/distributed_command_planner.rb +7 -11
  20. data/lib/droonga/distributor.rb +29 -17
  21. data/lib/droonga/event_loop.rb +2 -11
  22. data/lib/droonga/fluent_message_sender.rb +51 -5
  23. data/lib/droonga/handler_runner.rb +1 -1
  24. data/lib/droonga/job_protocol.rb +20 -0
  25. data/lib/droonga/job_pusher.rb +178 -0
  26. data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
  27. data/lib/droonga/message_matcher.rb +18 -15
  28. data/lib/droonga/planner.rb +2 -3
  29. data/lib/droonga/plugins/crud.rb +1 -1
  30. data/lib/droonga/plugins/groonga/column_create.rb +4 -1
  31. data/lib/droonga/plugins/groonga/table_create.rb +1 -1
  32. data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
  33. data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
  34. data/lib/droonga/processor.rb +3 -3
  35. data/lib/droonga/reducer.rb +15 -12
  36. data/lib/droonga/searcher.rb +49 -4
  37. data/lib/droonga/server.rb +2 -0
  38. data/lib/droonga/single_step.rb +22 -7
  39. data/lib/droonga/slice.rb +7 -7
  40. data/lib/droonga/step_runner.rb +3 -2
  41. data/lib/droonga/worker.rb +10 -8
  42. data/test/command/suite/add/dimension/column.catalog.json +27 -0
  43. data/test/command/suite/add/dimension/column.expected +57 -0
  44. data/test/command/suite/add/dimension/column.test +51 -0
  45. data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
  46. data/test/command/suite/search/adjusters/multiple.expected +23 -0
  47. data/test/command/suite/search/adjusters/multiple.test +75 -0
  48. data/test/command/suite/search/adjusters/one.catalog.json +38 -0
  49. data/test/command/suite/search/adjusters/one.expected +23 -0
  50. data/test/command/suite/search/adjusters/one.test +66 -0
  51. data/test/command/suite/search/attributes/array.test +0 -2
  52. data/test/command/suite/search/attributes/hash.test +0 -2
  53. data/test/command/suite/search/complex.test +0 -2
  54. data/test/command/suite/search/condition/nested.test +0 -2
  55. data/test/command/suite/search/condition/query.test +0 -2
  56. data/test/command/suite/search/condition/script.test +0 -2
  57. data/test/command/suite/search/group/string.test +0 -4
  58. data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
  59. data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
  60. data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
  61. data/test/command/suite/search/multiple/chained.test +0 -4
  62. data/test/command/suite/search/multiple/parallel.test +0 -4
  63. data/test/command/suite/search/range/only-output.test +0 -2
  64. data/test/command/suite/search/range/only-sort.test +0 -2
  65. data/test/command/suite/search/range/sort-and-output.test +0 -2
  66. data/test/command/suite/search/range/too-large-output-offset.test +0 -2
  67. data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
  68. data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
  69. data/test/command/suite/search/response/elapsed_time.expected +15 -0
  70. data/test/command/suite/search/response/elapsed_time.test +26 -0
  71. data/test/command/suite/search/response/records/value/time.test +0 -2
  72. data/test/command/suite/search/simple.test +0 -2
  73. data/test/command/suite/search/sort/default-offset-limit.test +0 -2
  74. data/test/command/suite/search/sort/invisible-column.test +0 -2
  75. data/test/unit/catalog/test_collection_volume.rb +103 -0
  76. data/test/unit/catalog/test_dataset.rb +69 -8
  77. data/test/unit/catalog/test_schema.rb +63 -23
  78. data/test/unit/catalog/test_single_volume.rb +31 -0
  79. data/test/unit/catalog/test_slice.rb +92 -0
  80. data/test/unit/catalog/test_version1.rb +1 -1
  81. data/test/unit/catalog/test_version2.rb +1 -32
  82. data/test/unit/catalog/test_version2_validator.rb +66 -0
  83. data/test/unit/catalog/test_volume_collection.rb +50 -0
  84. data/test/unit/plugins/groonga/test_column_create.rb +4 -1
  85. data/test/unit/plugins/groonga/test_table_create.rb +1 -1
  86. data/test/unit/test_message_matcher.rb +15 -15
  87. data/test/unit/test_watch_schema.rb +1 -1
  88. metadata +107 -94
  89. checksums.yaml +0 -7
  90. data/lib/droonga/message_pusher.rb +0 -64
@@ -198,14 +198,6 @@ class CatalogSchemaTest < Test::Unit::TestCase
198
198
  {}).type)
199
199
  end
200
200
 
201
- def test_flags
202
- assert_equal(["COLUMN_SCALAR"],
203
- create_column("column_name",
204
- {
205
- "type" => "Scalar"
206
- }).flags)
207
- end
208
-
209
201
  def test_value_type
210
202
  assert_equal("ShortText",
211
203
  create_column("column_name",
@@ -222,17 +214,6 @@ class CatalogSchemaTest < Test::Unit::TestCase
222
214
  }).value_type_groonga)
223
215
  end
224
216
 
225
- def test_flags_with_column_index_options
226
- assert_equal(["COLUMN_SCALAR", "WITH_SECTION"],
227
- create_column("column_name",
228
- {
229
- "type" => "Scalar",
230
- "indexOptions" => {
231
- "section" => true
232
- }
233
- }).flags)
234
- end
235
-
236
217
  def test_to_column_create_body
237
218
  assert_equal({
238
219
  "name" => "column_name",
@@ -241,10 +222,69 @@ class CatalogSchemaTest < Test::Unit::TestCase
241
222
  "type" => "ShortText"
242
223
  },
243
224
  create_column("column_name",
244
- {
245
- "type" => "Scalar",
246
- "valueType" => "ShortText"
247
- }).to_column_create_body)
225
+ {
226
+ "type" => "Scalar",
227
+ "valueType" => "ShortText"
228
+ }).to_column_create_body)
229
+ end
230
+
231
+ class FlagsTest < self
232
+ def flags(data)
233
+ create_column("column_name", data).flags
234
+ end
235
+
236
+ def test_type
237
+ data = {
238
+ "type" => "Scalar"
239
+ }
240
+ assert_equal(["COLUMN_SCALAR"],
241
+ flags(data))
242
+ end
243
+
244
+ def test_weight_options
245
+ data = {
246
+ "type" => "Vector",
247
+ "vectorOptions" => {
248
+ "weight" => true
249
+ }
250
+ }
251
+ assert_equal(["COLUMN_VECTOR", "WITH_WEIGHT"],
252
+ flags(data))
253
+ end
254
+
255
+ def test_index_options
256
+ data = {
257
+ "type" => "Index",
258
+ "indexOptions" => {
259
+ "section" => true
260
+ }
261
+ }
262
+ assert_equal(["COLUMN_INDEX", "WITH_SECTION"],
263
+ flags(data))
264
+ end
265
+ end
266
+ end
267
+
268
+ class ColumnVectorOptionsTest < self
269
+ def create_options(data)
270
+ Droonga::Catalog::Schema::ColumnVectorOptions.new(data)
271
+ end
272
+
273
+ def test_weight
274
+ data = {
275
+ "weight" => true
276
+ }
277
+ options = create_options(data)
278
+ assert_equal(true, options.weight)
279
+ end
280
+
281
+ def test_flags
282
+ data = {
283
+ "weight" => true
284
+ }
285
+ options = create_options(data)
286
+ assert_equal(["WITH_WEIGHT"],
287
+ options.flags)
248
288
  end
249
289
  end
250
290
 
@@ -0,0 +1,31 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/single_volume"
17
+
18
+ class CatalogSingleVolumeTest < Test::Unit::TestCase
19
+ def create_single_volume(data)
20
+ Droonga::Catalog::SingleVolume.new(data)
21
+ end
22
+
23
+ def test_address
24
+ data = {
25
+ "address" => "127.0.0.1:10047/volume.000",
26
+ }
27
+ volume = create_single_volume(data)
28
+ assert_equal("127.0.0.1:10047/volume.000",
29
+ volume.address)
30
+ end
31
+ end
@@ -0,0 +1,92 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/dataset"
17
+
18
+ class CatalogSliceTest < Test::Unit::TestCase
19
+ private
20
+ def create_slice(data)
21
+ minimum_dataset_data = {
22
+ "replicas" => {
23
+ },
24
+ }
25
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
26
+ Droonga::Catalog::Slice.new(dataset, data)
27
+ end
28
+
29
+ class WeightTest < self
30
+ def test_default
31
+ data = {
32
+ }
33
+ slice = create_slice(data)
34
+ assert_equal(1, slice.weight)
35
+ end
36
+
37
+ def test_specified
38
+ data = {
39
+ "weight" => 29,
40
+ }
41
+ slice = create_slice(data)
42
+ assert_equal(29, slice.weight)
43
+ end
44
+ end
45
+
46
+ class LabelTest < self
47
+ def test_default
48
+ data = {
49
+ }
50
+ slice = create_slice(data)
51
+ assert_nil(slice.label)
52
+ end
53
+
54
+ def test_specified
55
+ data = {
56
+ "label" => "High",
57
+ }
58
+ slice = create_slice(data)
59
+ assert_equal("High", slice.label)
60
+ end
61
+ end
62
+
63
+ class BoundaryTest < self
64
+ def test_default
65
+ data = {
66
+ }
67
+ slice = create_slice(data)
68
+ assert_nil(slice.boundary)
69
+ end
70
+
71
+ def test_specified
72
+ data = {
73
+ "boundary" => "2014-03-21",
74
+ }
75
+ slice = create_slice(data)
76
+ assert_equal("2014-03-21", slice.boundary)
77
+ end
78
+ end
79
+
80
+ class VolumeTest < self
81
+ def test_single
82
+ data = {
83
+ "volume" => {
84
+ "address" => "127.0.0.1:10047/volume.000",
85
+ },
86
+ }
87
+ slice = create_slice(data)
88
+ assert_equal("127.0.0.1:10047/volume.000",
89
+ slice.volume.address)
90
+ end
91
+ end
92
+ end
@@ -15,7 +15,7 @@
15
15
 
16
16
  require "droonga/catalog/version1"
17
17
 
18
- class CatalogTestVersion1 < Test::Unit::TestCase
18
+ class CatalogVersion1Test < Test::Unit::TestCase
19
19
  class << self
20
20
  def minimum_data
21
21
  {
@@ -15,7 +15,7 @@
15
15
 
16
16
  require "droonga/catalog/version2"
17
17
 
18
- class CatalogTestVersion2 < Test::Unit::TestCase
18
+ class CatalogVersion2Test < Test::Unit::TestCase
19
19
  class << self
20
20
  def minimum_data
21
21
  {
@@ -121,35 +121,4 @@ class CatalogTestVersion2 < Test::Unit::TestCase
121
121
  end
122
122
  end
123
123
  end
124
-
125
- class DataSetTest < self
126
- class ReplicaTest < self
127
- class TotalWeightTest < self
128
- def test_three_slices
129
- replica = {
130
- "slices" => [
131
- {
132
- "weight" => 10,
133
- },
134
- {
135
- "weight" => 20,
136
- },
137
- {
138
- "weight" => 30,
139
- },
140
- ],
141
- }
142
- assert_equal(10 + 20 + 30,
143
- total_weight(replica))
144
- end
145
-
146
- private
147
- def total_weight(replica)
148
- catalog = create_catalog(minimum_data,
149
- "base-path")
150
- catalog.send(:compute_total_weight, replica)
151
- end
152
- end
153
- end
154
- end
155
124
  end
@@ -0,0 +1,66 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/version2_validator"
17
+
18
+ class CatalogVersion2ValidatorTest < Test::Unit::TestCase
19
+ def setup
20
+ @path = "catalog.json"
21
+ end
22
+
23
+ def validate(data)
24
+ validator = Droonga::Catalog::Version2Validator.new(data, @path)
25
+ validator.validate
26
+ end
27
+
28
+ def validation_error(details)
29
+ Droonga::Catalog::ValidationError.new(@path, details)
30
+ end
31
+
32
+ def detail(value_path, message)
33
+ Droonga::Catalog::ValidationError::Detail.new(value_path, message)
34
+ end
35
+
36
+ class DatasetsTest < self
37
+ def test_missing
38
+ details = [
39
+ detail("datasets", "required parameter is missing"),
40
+ ]
41
+ assert_raise(validation_error(details)) do
42
+ validate({})
43
+ end
44
+ end
45
+
46
+ class DatasetTest < self
47
+ class ReplicasTest < self
48
+ def test_missing
49
+ details = [
50
+ detail("datasets.Droonga.replicas",
51
+ "required parameter is missing"),
52
+ ]
53
+ assert_raise(validation_error(details)) do
54
+ data = {
55
+ "datasets" => {
56
+ "Droonga" => {
57
+ }
58
+ }
59
+ }
60
+ validate(data)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,50 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/volume_collection"
17
+
18
+ class CatalogVolumeCollectionTest < Test::Unit::TestCase
19
+ private
20
+ def create_volume_collection(volumes)
21
+ Droonga::Catalog::VolumeCollection.new(volumes)
22
+ end
23
+
24
+ class SelectTest < self
25
+ def setup
26
+ volumes = [
27
+ "volume1",
28
+ "volume2",
29
+ "volume3",
30
+ ]
31
+ @collection = create_volume_collection(volumes)
32
+ end
33
+
34
+ def test_top
35
+ assert_equal(["volume1"], @collection.select(:top))
36
+ end
37
+
38
+ def test_random
39
+ random_volumes = @collection.select(:random).collect do |volume|
40
+ volume.gsub(/\Avolume[123]\z/, "any volume")
41
+ end
42
+ assert_equal(["any volume"], random_volumes)
43
+ end
44
+
45
+ def test_all
46
+ assert_equal(["volume1", "volume2", "volume3"],
47
+ @collection.select(:all))
48
+ end
49
+ end
50
+ end
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2013 Droonga Project
1
+ # Copyright (C) 2013-2014 Droonga Project
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -82,6 +82,9 @@ column_create Books main_text COLUMN_SCALAR LongText
82
82
  "COLUMN_VECTOR" => {
83
83
  :flags => "COLUMN_VECTOR",
84
84
  },
85
+ "COLUMN_VECTOR|WITH_WEIGHT" => {
86
+ :flags => "COLUMN_VECTOR|WITH_WEIGHT",
87
+ },
85
88
  })
86
89
  def test_data_store_column_type(data)
87
90
  request = {
@@ -139,7 +139,7 @@ table_create Books TABLE_HASH_KEY --key_type ShortText --default_tokenizer Token
139
139
  }
140
140
  process(:table_create, request)
141
141
  assert_equal(<<-SCHEMA, dump)
142
- table_create Books TABLE_HASH_KEY|KEY_NORMALIZE --key_type ShortText
142
+ table_create Books TABLE_HASH_KEY --key_type ShortText --normalizer NormalizerAuto
143
143
  SCHEMA
144
144
  end
145
145
  end
@@ -20,34 +20,34 @@ class MessageMatcherTest < Test::Unit::TestCase
20
20
  Droonga::MessageMatcher.new(pattern)
21
21
  end
22
22
 
23
- class ResolvePathTest < self
24
- def resolve_path(path, message)
25
- matcher(nil).send(:resolve_path, path, message)
23
+ class ExtractTargetTest < self
24
+ def extract_target(path, message)
25
+ matcher([path]).send(:extract_target, message)
26
26
  end
27
27
 
28
28
  def test_nonexistent
29
29
  assert_equal(Droonga::MessageMatcher::NONEXISTENT_PATH,
30
- resolve_path("nonexistent.path", {}))
30
+ extract_target("nonexistent.path", {}))
31
31
  end
32
32
 
33
33
  def test_top_level
34
34
  assert_equal("select",
35
- resolve_path("type",
36
- {
37
- "type" => "select"
38
- }))
35
+ extract_target("type",
36
+ {
37
+ "type" => "select"
38
+ }))
39
39
  end
40
40
 
41
41
  def test_nested
42
42
  assert_equal(10,
43
- resolve_path("body.output.limit",
44
- {
45
- "body" => {
46
- "output" => {
47
- "limit" => 10,
43
+ extract_target("body.output.limit",
44
+ {
45
+ "body" => {
46
+ "output" => {
47
+ "limit" => 10,
48
+ },
48
49
  },
49
- },
50
- }))
50
+ }))
51
51
  end
52
52
  end
53
53