fluent-plugin-droonga 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. data/.travis.yml +1 -1
  2. data/Gemfile +1 -1
  3. data/fluent-plugin-droonga.gemspec +1 -1
  4. data/lib/droonga/catalog/collection_volume.rb +97 -0
  5. data/lib/droonga/catalog/dataset.rb +28 -1
  6. data/lib/droonga/catalog/errors.rb +28 -9
  7. data/lib/droonga/catalog/schema.rb +23 -2
  8. data/lib/droonga/catalog/single_volume.rb +28 -0
  9. data/lib/droonga/catalog/slice.rb +43 -0
  10. data/lib/droonga/catalog/version1.rb +3 -3
  11. data/lib/droonga/catalog/version2.rb +17 -81
  12. data/lib/droonga/catalog/version2_validator.rb +63 -0
  13. data/lib/droonga/catalog/volume.rb +33 -0
  14. data/lib/droonga/catalog/volume_collection.rb +56 -0
  15. data/lib/droonga/catalog_observer.rb +7 -19
  16. data/lib/droonga/collectors.rb +1 -1
  17. data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
  18. data/lib/droonga/dispatcher.rb +24 -18
  19. data/lib/droonga/distributed_command_planner.rb +7 -11
  20. data/lib/droonga/distributor.rb +29 -17
  21. data/lib/droonga/event_loop.rb +2 -11
  22. data/lib/droonga/fluent_message_sender.rb +51 -5
  23. data/lib/droonga/handler_runner.rb +1 -1
  24. data/lib/droonga/job_protocol.rb +20 -0
  25. data/lib/droonga/job_pusher.rb +178 -0
  26. data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
  27. data/lib/droonga/message_matcher.rb +18 -15
  28. data/lib/droonga/planner.rb +2 -3
  29. data/lib/droonga/plugins/crud.rb +1 -1
  30. data/lib/droonga/plugins/groonga/column_create.rb +4 -1
  31. data/lib/droonga/plugins/groonga/table_create.rb +1 -1
  32. data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
  33. data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
  34. data/lib/droonga/processor.rb +3 -3
  35. data/lib/droonga/reducer.rb +15 -12
  36. data/lib/droonga/searcher.rb +49 -4
  37. data/lib/droonga/server.rb +2 -0
  38. data/lib/droonga/single_step.rb +22 -7
  39. data/lib/droonga/slice.rb +7 -7
  40. data/lib/droonga/step_runner.rb +3 -2
  41. data/lib/droonga/worker.rb +10 -8
  42. data/test/command/suite/add/dimension/column.catalog.json +27 -0
  43. data/test/command/suite/add/dimension/column.expected +57 -0
  44. data/test/command/suite/add/dimension/column.test +51 -0
  45. data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
  46. data/test/command/suite/search/adjusters/multiple.expected +23 -0
  47. data/test/command/suite/search/adjusters/multiple.test +75 -0
  48. data/test/command/suite/search/adjusters/one.catalog.json +38 -0
  49. data/test/command/suite/search/adjusters/one.expected +23 -0
  50. data/test/command/suite/search/adjusters/one.test +66 -0
  51. data/test/command/suite/search/attributes/array.test +0 -2
  52. data/test/command/suite/search/attributes/hash.test +0 -2
  53. data/test/command/suite/search/complex.test +0 -2
  54. data/test/command/suite/search/condition/nested.test +0 -2
  55. data/test/command/suite/search/condition/query.test +0 -2
  56. data/test/command/suite/search/condition/script.test +0 -2
  57. data/test/command/suite/search/group/string.test +0 -4
  58. data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
  59. data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
  60. data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
  61. data/test/command/suite/search/multiple/chained.test +0 -4
  62. data/test/command/suite/search/multiple/parallel.test +0 -4
  63. data/test/command/suite/search/range/only-output.test +0 -2
  64. data/test/command/suite/search/range/only-sort.test +0 -2
  65. data/test/command/suite/search/range/sort-and-output.test +0 -2
  66. data/test/command/suite/search/range/too-large-output-offset.test +0 -2
  67. data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
  68. data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
  69. data/test/command/suite/search/response/elapsed_time.expected +15 -0
  70. data/test/command/suite/search/response/elapsed_time.test +26 -0
  71. data/test/command/suite/search/response/records/value/time.test +0 -2
  72. data/test/command/suite/search/simple.test +0 -2
  73. data/test/command/suite/search/sort/default-offset-limit.test +0 -2
  74. data/test/command/suite/search/sort/invisible-column.test +0 -2
  75. data/test/unit/catalog/test_collection_volume.rb +103 -0
  76. data/test/unit/catalog/test_dataset.rb +69 -8
  77. data/test/unit/catalog/test_schema.rb +63 -23
  78. data/test/unit/catalog/test_single_volume.rb +31 -0
  79. data/test/unit/catalog/test_slice.rb +92 -0
  80. data/test/unit/catalog/test_version1.rb +1 -1
  81. data/test/unit/catalog/test_version2.rb +1 -32
  82. data/test/unit/catalog/test_version2_validator.rb +66 -0
  83. data/test/unit/catalog/test_volume_collection.rb +50 -0
  84. data/test/unit/plugins/groonga/test_column_create.rb +4 -1
  85. data/test/unit/plugins/groonga/test_table_create.rb +1 -1
  86. data/test/unit/test_message_matcher.rb +15 -15
  87. data/test/unit/test_watch_schema.rb +1 -1
  88. metadata +107 -94
  89. checksums.yaml +0 -7
  90. data/lib/droonga/message_pusher.rb +0 -64
@@ -198,14 +198,6 @@ class CatalogSchemaTest < Test::Unit::TestCase
198
198
  {}).type)
199
199
  end
200
200
 
201
- def test_flags
202
- assert_equal(["COLUMN_SCALAR"],
203
- create_column("column_name",
204
- {
205
- "type" => "Scalar"
206
- }).flags)
207
- end
208
-
209
201
  def test_value_type
210
202
  assert_equal("ShortText",
211
203
  create_column("column_name",
@@ -222,17 +214,6 @@ class CatalogSchemaTest < Test::Unit::TestCase
222
214
  }).value_type_groonga)
223
215
  end
224
216
 
225
- def test_flags_with_column_index_options
226
- assert_equal(["COLUMN_SCALAR", "WITH_SECTION"],
227
- create_column("column_name",
228
- {
229
- "type" => "Scalar",
230
- "indexOptions" => {
231
- "section" => true
232
- }
233
- }).flags)
234
- end
235
-
236
217
  def test_to_column_create_body
237
218
  assert_equal({
238
219
  "name" => "column_name",
@@ -241,10 +222,69 @@ class CatalogSchemaTest < Test::Unit::TestCase
241
222
  "type" => "ShortText"
242
223
  },
243
224
  create_column("column_name",
244
- {
245
- "type" => "Scalar",
246
- "valueType" => "ShortText"
247
- }).to_column_create_body)
225
+ {
226
+ "type" => "Scalar",
227
+ "valueType" => "ShortText"
228
+ }).to_column_create_body)
229
+ end
230
+
231
+ class FlagsTest < self
232
+ def flags(data)
233
+ create_column("column_name", data).flags
234
+ end
235
+
236
+ def test_type
237
+ data = {
238
+ "type" => "Scalar"
239
+ }
240
+ assert_equal(["COLUMN_SCALAR"],
241
+ flags(data))
242
+ end
243
+
244
+ def test_weight_options
245
+ data = {
246
+ "type" => "Vector",
247
+ "vectorOptions" => {
248
+ "weight" => true
249
+ }
250
+ }
251
+ assert_equal(["COLUMN_VECTOR", "WITH_WEIGHT"],
252
+ flags(data))
253
+ end
254
+
255
+ def test_index_options
256
+ data = {
257
+ "type" => "Index",
258
+ "indexOptions" => {
259
+ "section" => true
260
+ }
261
+ }
262
+ assert_equal(["COLUMN_INDEX", "WITH_SECTION"],
263
+ flags(data))
264
+ end
265
+ end
266
+ end
267
+
268
+ class ColumnVectorOptionsTest < self
269
+ def create_options(data)
270
+ Droonga::Catalog::Schema::ColumnVectorOptions.new(data)
271
+ end
272
+
273
+ def test_weight
274
+ data = {
275
+ "weight" => true
276
+ }
277
+ options = create_options(data)
278
+ assert_equal(true, options.weight)
279
+ end
280
+
281
+ def test_flags
282
+ data = {
283
+ "weight" => true
284
+ }
285
+ options = create_options(data)
286
+ assert_equal(["WITH_WEIGHT"],
287
+ options.flags)
248
288
  end
249
289
  end
250
290
 
@@ -0,0 +1,31 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/single_volume"
17
+
18
+ class CatalogSingleVolumeTest < Test::Unit::TestCase
19
+ def create_single_volume(data)
20
+ Droonga::Catalog::SingleVolume.new(data)
21
+ end
22
+
23
+ def test_address
24
+ data = {
25
+ "address" => "127.0.0.1:10047/volume.000",
26
+ }
27
+ volume = create_single_volume(data)
28
+ assert_equal("127.0.0.1:10047/volume.000",
29
+ volume.address)
30
+ end
31
+ end
@@ -0,0 +1,92 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/dataset"
17
+
18
+ class CatalogSliceTest < Test::Unit::TestCase
19
+ private
20
+ def create_slice(data)
21
+ minimum_dataset_data = {
22
+ "replicas" => {
23
+ },
24
+ }
25
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
26
+ Droonga::Catalog::Slice.new(dataset, data)
27
+ end
28
+
29
+ class WeightTest < self
30
+ def test_default
31
+ data = {
32
+ }
33
+ slice = create_slice(data)
34
+ assert_equal(1, slice.weight)
35
+ end
36
+
37
+ def test_specified
38
+ data = {
39
+ "weight" => 29,
40
+ }
41
+ slice = create_slice(data)
42
+ assert_equal(29, slice.weight)
43
+ end
44
+ end
45
+
46
+ class LabelTest < self
47
+ def test_default
48
+ data = {
49
+ }
50
+ slice = create_slice(data)
51
+ assert_nil(slice.label)
52
+ end
53
+
54
+ def test_specified
55
+ data = {
56
+ "label" => "High",
57
+ }
58
+ slice = create_slice(data)
59
+ assert_equal("High", slice.label)
60
+ end
61
+ end
62
+
63
+ class BoundaryTest < self
64
+ def test_default
65
+ data = {
66
+ }
67
+ slice = create_slice(data)
68
+ assert_nil(slice.boundary)
69
+ end
70
+
71
+ def test_specified
72
+ data = {
73
+ "boundary" => "2014-03-21",
74
+ }
75
+ slice = create_slice(data)
76
+ assert_equal("2014-03-21", slice.boundary)
77
+ end
78
+ end
79
+
80
+ class VolumeTest < self
81
+ def test_single
82
+ data = {
83
+ "volume" => {
84
+ "address" => "127.0.0.1:10047/volume.000",
85
+ },
86
+ }
87
+ slice = create_slice(data)
88
+ assert_equal("127.0.0.1:10047/volume.000",
89
+ slice.volume.address)
90
+ end
91
+ end
92
+ end
@@ -15,7 +15,7 @@
15
15
 
16
16
  require "droonga/catalog/version1"
17
17
 
18
- class CatalogTestVersion1 < Test::Unit::TestCase
18
+ class CatalogVersion1Test < Test::Unit::TestCase
19
19
  class << self
20
20
  def minimum_data
21
21
  {
@@ -15,7 +15,7 @@
15
15
 
16
16
  require "droonga/catalog/version2"
17
17
 
18
- class CatalogTestVersion2 < Test::Unit::TestCase
18
+ class CatalogVersion2Test < Test::Unit::TestCase
19
19
  class << self
20
20
  def minimum_data
21
21
  {
@@ -121,35 +121,4 @@ class CatalogTestVersion2 < Test::Unit::TestCase
121
121
  end
122
122
  end
123
123
  end
124
-
125
- class DataSetTest < self
126
- class ReplicaTest < self
127
- class TotalWeightTest < self
128
- def test_three_slices
129
- replica = {
130
- "slices" => [
131
- {
132
- "weight" => 10,
133
- },
134
- {
135
- "weight" => 20,
136
- },
137
- {
138
- "weight" => 30,
139
- },
140
- ],
141
- }
142
- assert_equal(10 + 20 + 30,
143
- total_weight(replica))
144
- end
145
-
146
- private
147
- def total_weight(replica)
148
- catalog = create_catalog(minimum_data,
149
- "base-path")
150
- catalog.send(:compute_total_weight, replica)
151
- end
152
- end
153
- end
154
- end
155
124
  end
@@ -0,0 +1,66 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/version2_validator"
17
+
18
+ class CatalogVersion2ValidatorTest < Test::Unit::TestCase
19
+ def setup
20
+ @path = "catalog.json"
21
+ end
22
+
23
+ def validate(data)
24
+ validator = Droonga::Catalog::Version2Validator.new(data, @path)
25
+ validator.validate
26
+ end
27
+
28
+ def validation_error(details)
29
+ Droonga::Catalog::ValidationError.new(@path, details)
30
+ end
31
+
32
+ def detail(value_path, message)
33
+ Droonga::Catalog::ValidationError::Detail.new(value_path, message)
34
+ end
35
+
36
+ class DatasetsTest < self
37
+ def test_missing
38
+ details = [
39
+ detail("datasets", "required parameter is missing"),
40
+ ]
41
+ assert_raise(validation_error(details)) do
42
+ validate({})
43
+ end
44
+ end
45
+
46
+ class DatasetTest < self
47
+ class ReplicasTest < self
48
+ def test_missing
49
+ details = [
50
+ detail("datasets.Droonga.replicas",
51
+ "required parameter is missing"),
52
+ ]
53
+ assert_raise(validation_error(details)) do
54
+ data = {
55
+ "datasets" => {
56
+ "Droonga" => {
57
+ }
58
+ }
59
+ }
60
+ validate(data)
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,50 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/volume_collection"
17
+
18
+ class CatalogVolumeCollectionTest < Test::Unit::TestCase
19
+ private
20
+ def create_volume_collection(volumes)
21
+ Droonga::Catalog::VolumeCollection.new(volumes)
22
+ end
23
+
24
+ class SelectTest < self
25
+ def setup
26
+ volumes = [
27
+ "volume1",
28
+ "volume2",
29
+ "volume3",
30
+ ]
31
+ @collection = create_volume_collection(volumes)
32
+ end
33
+
34
+ def test_top
35
+ assert_equal(["volume1"], @collection.select(:top))
36
+ end
37
+
38
+ def test_random
39
+ random_volumes = @collection.select(:random).collect do |volume|
40
+ volume.gsub(/\Avolume[123]\z/, "any volume")
41
+ end
42
+ assert_equal(["any volume"], random_volumes)
43
+ end
44
+
45
+ def test_all
46
+ assert_equal(["volume1", "volume2", "volume3"],
47
+ @collection.select(:all))
48
+ end
49
+ end
50
+ end
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2013 Droonga Project
1
+ # Copyright (C) 2013-2014 Droonga Project
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -82,6 +82,9 @@ column_create Books main_text COLUMN_SCALAR LongText
82
82
  "COLUMN_VECTOR" => {
83
83
  :flags => "COLUMN_VECTOR",
84
84
  },
85
+ "COLUMN_VECTOR|WITH_WEIGHT" => {
86
+ :flags => "COLUMN_VECTOR|WITH_WEIGHT",
87
+ },
85
88
  })
86
89
  def test_data_store_column_type(data)
87
90
  request = {
@@ -139,7 +139,7 @@ table_create Books TABLE_HASH_KEY --key_type ShortText --default_tokenizer Token
139
139
  }
140
140
  process(:table_create, request)
141
141
  assert_equal(<<-SCHEMA, dump)
142
- table_create Books TABLE_HASH_KEY|KEY_NORMALIZE --key_type ShortText
142
+ table_create Books TABLE_HASH_KEY --key_type ShortText --normalizer NormalizerAuto
143
143
  SCHEMA
144
144
  end
145
145
  end
@@ -20,34 +20,34 @@ class MessageMatcherTest < Test::Unit::TestCase
20
20
  Droonga::MessageMatcher.new(pattern)
21
21
  end
22
22
 
23
- class ResolvePathTest < self
24
- def resolve_path(path, message)
25
- matcher(nil).send(:resolve_path, path, message)
23
+ class ExtractTargetTest < self
24
+ def extract_target(path, message)
25
+ matcher([path]).send(:extract_target, message)
26
26
  end
27
27
 
28
28
  def test_nonexistent
29
29
  assert_equal(Droonga::MessageMatcher::NONEXISTENT_PATH,
30
- resolve_path("nonexistent.path", {}))
30
+ extract_target("nonexistent.path", {}))
31
31
  end
32
32
 
33
33
  def test_top_level
34
34
  assert_equal("select",
35
- resolve_path("type",
36
- {
37
- "type" => "select"
38
- }))
35
+ extract_target("type",
36
+ {
37
+ "type" => "select"
38
+ }))
39
39
  end
40
40
 
41
41
  def test_nested
42
42
  assert_equal(10,
43
- resolve_path("body.output.limit",
44
- {
45
- "body" => {
46
- "output" => {
47
- "limit" => 10,
43
+ extract_target("body.output.limit",
44
+ {
45
+ "body" => {
46
+ "output" => {
47
+ "limit" => 10,
48
+ },
48
49
  },
49
- },
50
- }))
50
+ }))
51
51
  end
52
52
  end
53
53