fluent-plugin-droonga 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. data/.travis.yml +1 -1
  2. data/Gemfile +1 -1
  3. data/fluent-plugin-droonga.gemspec +1 -1
  4. data/lib/droonga/catalog/collection_volume.rb +97 -0
  5. data/lib/droonga/catalog/dataset.rb +28 -1
  6. data/lib/droonga/catalog/errors.rb +28 -9
  7. data/lib/droonga/catalog/schema.rb +23 -2
  8. data/lib/droonga/catalog/single_volume.rb +28 -0
  9. data/lib/droonga/catalog/slice.rb +43 -0
  10. data/lib/droonga/catalog/version1.rb +3 -3
  11. data/lib/droonga/catalog/version2.rb +17 -81
  12. data/lib/droonga/catalog/version2_validator.rb +63 -0
  13. data/lib/droonga/catalog/volume.rb +33 -0
  14. data/lib/droonga/catalog/volume_collection.rb +56 -0
  15. data/lib/droonga/catalog_observer.rb +7 -19
  16. data/lib/droonga/collectors.rb +1 -1
  17. data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
  18. data/lib/droonga/dispatcher.rb +24 -18
  19. data/lib/droonga/distributed_command_planner.rb +7 -11
  20. data/lib/droonga/distributor.rb +29 -17
  21. data/lib/droonga/event_loop.rb +2 -11
  22. data/lib/droonga/fluent_message_sender.rb +51 -5
  23. data/lib/droonga/handler_runner.rb +1 -1
  24. data/lib/droonga/job_protocol.rb +20 -0
  25. data/lib/droonga/job_pusher.rb +178 -0
  26. data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
  27. data/lib/droonga/message_matcher.rb +18 -15
  28. data/lib/droonga/planner.rb +2 -3
  29. data/lib/droonga/plugins/crud.rb +1 -1
  30. data/lib/droonga/plugins/groonga/column_create.rb +4 -1
  31. data/lib/droonga/plugins/groonga/table_create.rb +1 -1
  32. data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
  33. data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
  34. data/lib/droonga/processor.rb +3 -3
  35. data/lib/droonga/reducer.rb +15 -12
  36. data/lib/droonga/searcher.rb +49 -4
  37. data/lib/droonga/server.rb +2 -0
  38. data/lib/droonga/single_step.rb +22 -7
  39. data/lib/droonga/slice.rb +7 -7
  40. data/lib/droonga/step_runner.rb +3 -2
  41. data/lib/droonga/worker.rb +10 -8
  42. data/test/command/suite/add/dimension/column.catalog.json +27 -0
  43. data/test/command/suite/add/dimension/column.expected +57 -0
  44. data/test/command/suite/add/dimension/column.test +51 -0
  45. data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
  46. data/test/command/suite/search/adjusters/multiple.expected +23 -0
  47. data/test/command/suite/search/adjusters/multiple.test +75 -0
  48. data/test/command/suite/search/adjusters/one.catalog.json +38 -0
  49. data/test/command/suite/search/adjusters/one.expected +23 -0
  50. data/test/command/suite/search/adjusters/one.test +66 -0
  51. data/test/command/suite/search/attributes/array.test +0 -2
  52. data/test/command/suite/search/attributes/hash.test +0 -2
  53. data/test/command/suite/search/complex.test +0 -2
  54. data/test/command/suite/search/condition/nested.test +0 -2
  55. data/test/command/suite/search/condition/query.test +0 -2
  56. data/test/command/suite/search/condition/script.test +0 -2
  57. data/test/command/suite/search/group/string.test +0 -4
  58. data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
  59. data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
  60. data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
  61. data/test/command/suite/search/multiple/chained.test +0 -4
  62. data/test/command/suite/search/multiple/parallel.test +0 -4
  63. data/test/command/suite/search/range/only-output.test +0 -2
  64. data/test/command/suite/search/range/only-sort.test +0 -2
  65. data/test/command/suite/search/range/sort-and-output.test +0 -2
  66. data/test/command/suite/search/range/too-large-output-offset.test +0 -2
  67. data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
  68. data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
  69. data/test/command/suite/search/response/elapsed_time.expected +15 -0
  70. data/test/command/suite/search/response/elapsed_time.test +26 -0
  71. data/test/command/suite/search/response/records/value/time.test +0 -2
  72. data/test/command/suite/search/simple.test +0 -2
  73. data/test/command/suite/search/sort/default-offset-limit.test +0 -2
  74. data/test/command/suite/search/sort/invisible-column.test +0 -2
  75. data/test/unit/catalog/test_collection_volume.rb +103 -0
  76. data/test/unit/catalog/test_dataset.rb +69 -8
  77. data/test/unit/catalog/test_schema.rb +63 -23
  78. data/test/unit/catalog/test_single_volume.rb +31 -0
  79. data/test/unit/catalog/test_slice.rb +92 -0
  80. data/test/unit/catalog/test_version1.rb +1 -1
  81. data/test/unit/catalog/test_version2.rb +1 -32
  82. data/test/unit/catalog/test_version2_validator.rb +66 -0
  83. data/test/unit/catalog/test_volume_collection.rb +50 -0
  84. data/test/unit/plugins/groonga/test_column_create.rb +4 -1
  85. data/test/unit/plugins/groonga/test_table_create.rb +1 -1
  86. data/test/unit/test_message_matcher.rb +15 -15
  87. data/test/unit/test_watch_schema.rb +1 -1
  88. metadata +107 -94
  89. checksums.yaml +0 -7
  90. data/lib/droonga/message_pusher.rb +0 -64
@@ -0,0 +1,81 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Threads",
9
+ "key": "ruby",
10
+ "values": {
11
+ "name": "Ruby"
12
+ }
13
+ }
14
+ }
15
+ {
16
+ "type": "add",
17
+ "dataset": "Droonga",
18
+ "body": {
19
+ "table": "Threads",
20
+ "key": "groonga",
21
+ "values": {
22
+ "name": "Groonga"
23
+ }
24
+ }
25
+ }
26
+ {
27
+ "type": "add",
28
+ "dataset": "Droonga",
29
+ "body": {
30
+ "table": "Comments",
31
+ "key": "1",
32
+ "values": {
33
+ "thread": "ruby",
34
+ "content": "Hi! I'm a new Rubyist!"
35
+ }
36
+ }
37
+ }
38
+ {
39
+ "type": "add",
40
+ "dataset": "Droonga",
41
+ "body": {
42
+ "table": "Comments",
43
+ "key": "2",
44
+ "values": {
45
+ "thread": "groonga",
46
+ "content": "Groonga is great!"
47
+ }
48
+ }
49
+ }
50
+ #@enable-logging
51
+ {
52
+ "type": "search",
53
+ "dataset": "Droonga",
54
+ "body": {
55
+ "queries": {
56
+ "comments": {
57
+ "source": "Comments",
58
+ "groupBy": {
59
+ "key": "thread",
60
+ "maxNSubRecords": 10
61
+ },
62
+ "sortBy": ["_key"],
63
+ "output": {
64
+ "elements": ["count", "records"],
65
+ "limit": -1,
66
+ "attributes": [
67
+ "_key",
68
+ "_nsubrecs",
69
+ {
70
+ "label": "items",
71
+ "source": "_subrecs",
72
+ "attributes": [
73
+ "content"
74
+ ]
75
+ }
76
+ ]
77
+ }
78
+ }
79
+ }
80
+ }
81
+ }
@@ -13,8 +13,6 @@
13
13
  "sortBy": ["_key"],
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -28,8 +26,6 @@
28
26
  "sortBy": ["-_key"],
29
27
  "output": {
30
28
  "elements": [
31
- "startTime",
32
- "elapsedTime",
33
29
  "count",
34
30
  "attributes",
35
31
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -24,8 +22,6 @@
24
22
  "sortBy": ["-_key"],
25
23
  "output": {
26
24
  "elements": [
27
- "startTime",
28
- "elapsedTime",
29
25
  "count",
30
26
  "attributes",
31
27
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -0,0 +1,13 @@
1
+ {
2
+ "datasets": {
3
+ "Droonga": {
4
+ "fact": "Memos",
5
+ "schema": {
6
+ "Memos": {
7
+ "type": "Hash",
8
+ "keyType": "ShortText"
9
+ }
10
+ }
11
+ }
12
+ }
13
+ }
@@ -0,0 +1,15 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": "request-id",
6
+ "statusCode": 200,
7
+ "type": "search.result",
8
+ "body": {
9
+ "memos": {
10
+ "count": 1,
11
+ "elapsedTime": 0.0
12
+ }
13
+ }
14
+ }
15
+ ]
@@ -0,0 +1,26 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Memos",
9
+ "key": "Droonga is fun"
10
+ }
11
+ }
12
+ #@enable-logging
13
+ {
14
+ "type": "search",
15
+ "dataset": "Droonga",
16
+ "body": {
17
+ "queries": {
18
+ "memos": {
19
+ "source": "Memos",
20
+ "output": {
21
+ "elements": ["count", "elapsedTime"]
22
+ }
23
+ }
24
+ }
25
+ }
26
+ }
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -11,8 +11,6 @@
11
11
  },
12
12
  "output": {
13
13
  "elements": [
14
- "startTime",
15
- "elapsedTime",
16
14
  "count",
17
15
  "attributes",
18
16
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -0,0 +1,103 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/dataset"
17
+
18
+ class CatalogSingleVolumeTest < Test::Unit::TestCase
19
+ def create_collection_volume(data)
20
+ minimum_dataset_data = {
21
+ "replicas" => {
22
+ },
23
+ }
24
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
25
+ Droonga::Catalog::CollectionVolume.new(dataset, data)
26
+ end
27
+
28
+ class DimensionTest < self
29
+ def test_default
30
+ data = {
31
+ "slices" => [],
32
+ }
33
+ volume = create_collection_volume(data)
34
+ assert_equal("_key", volume.dimension)
35
+ end
36
+
37
+ def test_specified
38
+ data = {
39
+ "dimension" => "group",
40
+ "slices" => [],
41
+ }
42
+ volume = create_collection_volume(data)
43
+ assert_equal("group", volume.dimension)
44
+ end
45
+ end
46
+
47
+ class SlicerTest < self
48
+ def test_default
49
+ data = {
50
+ "slices" => [],
51
+ }
52
+ volume = create_collection_volume(data)
53
+ assert_equal("hash", volume.slicer)
54
+ end
55
+
56
+ def test_specified
57
+ data = {
58
+ "slicer" => "ordinal",
59
+ }
60
+ volume = create_collection_volume(data)
61
+ assert_equal("ordinal", volume.slicer)
62
+ end
63
+ end
64
+
65
+ class SlicesTest < self
66
+ def test_empty
67
+ data = {
68
+ "slices" => [],
69
+ }
70
+ volume = create_collection_volume(data)
71
+ assert_equal([], volume.slices)
72
+ end
73
+ end
74
+
75
+ class RatioOrderSlicerTest < self
76
+ class TotalWeightTest < self
77
+ def test_three_slices
78
+ data = {
79
+ "slicer" => "hash",
80
+ "slices" => [
81
+ {
82
+ "weight" => 10,
83
+ },
84
+ {
85
+ "weight" => 20,
86
+ },
87
+ {
88
+ "weight" => 30,
89
+ },
90
+ ],
91
+ }
92
+ assert_equal(10 + 20 + 30,
93
+ total_weight(data))
94
+ end
95
+
96
+ private
97
+ def total_weight(data)
98
+ volume = create_collection_volume(data)
99
+ volume.send(:compute_total_weight)
100
+ end
101
+ end
102
+ end
103
+ end
@@ -17,27 +17,88 @@ require "droonga/catalog/dataset"
17
17
 
18
18
  class CatalogDatasetTest < Test::Unit::TestCase
19
19
  private
20
- def create_dataset(dataset_name, data)
21
- Droonga::Catalog::Dataset.new(dataset_name, data)
20
+ def create_dataset(data)
21
+ Droonga::Catalog::Dataset.new("DatasetName", data)
22
22
  end
23
23
 
24
- class DatasetTest < self
25
- def test_value
24
+ class NameTest < self
25
+ def test_reader
26
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", {})
27
+ assert_equal("DatasetName", dataset.name)
28
+ end
29
+ end
30
+
31
+ class NWorkersTest < self
32
+ def test_default
33
+ data = {
34
+ }
35
+ dataset = create_dataset(data)
36
+ assert_equal(0, dataset.n_workers)
37
+ end
38
+
39
+ def test_specified
26
40
  data = {
27
41
  "nWorkers" => 2
28
42
  }
29
- dataset = create_dataset("dataset_name", data)
30
- assert_equal(2, dataset["nWorkers"])
43
+ dataset = create_dataset(data)
44
+ assert_equal(2, dataset.n_workers)
31
45
  end
46
+ end
32
47
 
33
- def test_schema
48
+ class SchemaTest < self
49
+ def test_empty
34
50
  data = {
35
51
  "schema" => {
36
52
  }
37
53
  }
38
- dataset = create_dataset("dataset_name", data)
54
+ dataset = create_dataset(data)
39
55
  assert_equal(Droonga::Catalog::Schema.new("dataset_name", {}),
40
56
  dataset.schema)
41
57
  end
42
58
  end
59
+
60
+ class PluginsTest < self
61
+ def test_default
62
+ data = {
63
+ }
64
+ dataset = create_dataset(data)
65
+ assert_equal([], dataset.plugins)
66
+ end
67
+
68
+ def test_multiple
69
+ data = {
70
+ "plugins" => ["groonga", "crud"],
71
+ }
72
+ dataset = create_dataset(data)
73
+ assert_equal(["groonga", "crud"], dataset.plugins)
74
+ end
75
+ end
76
+
77
+ class FactTest < self
78
+ def test_default
79
+ data = {
80
+ }
81
+ dataset = create_dataset(data)
82
+ assert_nil(dataset.fact)
83
+ end
84
+
85
+ def test_specified
86
+ data = {
87
+ "fact" => "Users",
88
+ }
89
+ dataset = create_dataset(data)
90
+ assert_equal("Users", dataset.fact)
91
+ end
92
+ end
93
+
94
+ class ReplicasTest < self
95
+ def test_empty
96
+ data = {
97
+ "replicas" => [],
98
+ }
99
+ dataset = create_dataset(data)
100
+ assert_equal(Droonga::Catalog::VolumeCollection.new([]),
101
+ dataset.replicas)
102
+ end
103
+ end
43
104
  end