fluent-plugin-droonga 1.0.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. data/.travis.yml +1 -1
  2. data/Gemfile +1 -1
  3. data/fluent-plugin-droonga.gemspec +1 -1
  4. data/lib/droonga/catalog/collection_volume.rb +97 -0
  5. data/lib/droonga/catalog/dataset.rb +28 -1
  6. data/lib/droonga/catalog/errors.rb +28 -9
  7. data/lib/droonga/catalog/schema.rb +23 -2
  8. data/lib/droonga/catalog/single_volume.rb +28 -0
  9. data/lib/droonga/catalog/slice.rb +43 -0
  10. data/lib/droonga/catalog/version1.rb +3 -3
  11. data/lib/droonga/catalog/version2.rb +17 -81
  12. data/lib/droonga/catalog/version2_validator.rb +63 -0
  13. data/lib/droonga/catalog/volume.rb +33 -0
  14. data/lib/droonga/catalog/volume_collection.rb +56 -0
  15. data/lib/droonga/catalog_observer.rb +7 -19
  16. data/lib/droonga/collectors.rb +1 -1
  17. data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
  18. data/lib/droonga/dispatcher.rb +24 -18
  19. data/lib/droonga/distributed_command_planner.rb +7 -11
  20. data/lib/droonga/distributor.rb +29 -17
  21. data/lib/droonga/event_loop.rb +2 -11
  22. data/lib/droonga/fluent_message_sender.rb +51 -5
  23. data/lib/droonga/handler_runner.rb +1 -1
  24. data/lib/droonga/job_protocol.rb +20 -0
  25. data/lib/droonga/job_pusher.rb +178 -0
  26. data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
  27. data/lib/droonga/message_matcher.rb +18 -15
  28. data/lib/droonga/planner.rb +2 -3
  29. data/lib/droonga/plugins/crud.rb +1 -1
  30. data/lib/droonga/plugins/groonga/column_create.rb +4 -1
  31. data/lib/droonga/plugins/groonga/table_create.rb +1 -1
  32. data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
  33. data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
  34. data/lib/droonga/processor.rb +3 -3
  35. data/lib/droonga/reducer.rb +15 -12
  36. data/lib/droonga/searcher.rb +49 -4
  37. data/lib/droonga/server.rb +2 -0
  38. data/lib/droonga/single_step.rb +22 -7
  39. data/lib/droonga/slice.rb +7 -7
  40. data/lib/droonga/step_runner.rb +3 -2
  41. data/lib/droonga/worker.rb +10 -8
  42. data/test/command/suite/add/dimension/column.catalog.json +27 -0
  43. data/test/command/suite/add/dimension/column.expected +57 -0
  44. data/test/command/suite/add/dimension/column.test +51 -0
  45. data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
  46. data/test/command/suite/search/adjusters/multiple.expected +23 -0
  47. data/test/command/suite/search/adjusters/multiple.test +75 -0
  48. data/test/command/suite/search/adjusters/one.catalog.json +38 -0
  49. data/test/command/suite/search/adjusters/one.expected +23 -0
  50. data/test/command/suite/search/adjusters/one.test +66 -0
  51. data/test/command/suite/search/attributes/array.test +0 -2
  52. data/test/command/suite/search/attributes/hash.test +0 -2
  53. data/test/command/suite/search/complex.test +0 -2
  54. data/test/command/suite/search/condition/nested.test +0 -2
  55. data/test/command/suite/search/condition/query.test +0 -2
  56. data/test/command/suite/search/condition/script.test +0 -2
  57. data/test/command/suite/search/group/string.test +0 -4
  58. data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
  59. data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
  60. data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
  61. data/test/command/suite/search/multiple/chained.test +0 -4
  62. data/test/command/suite/search/multiple/parallel.test +0 -4
  63. data/test/command/suite/search/range/only-output.test +0 -2
  64. data/test/command/suite/search/range/only-sort.test +0 -2
  65. data/test/command/suite/search/range/sort-and-output.test +0 -2
  66. data/test/command/suite/search/range/too-large-output-offset.test +0 -2
  67. data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
  68. data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
  69. data/test/command/suite/search/response/elapsed_time.expected +15 -0
  70. data/test/command/suite/search/response/elapsed_time.test +26 -0
  71. data/test/command/suite/search/response/records/value/time.test +0 -2
  72. data/test/command/suite/search/simple.test +0 -2
  73. data/test/command/suite/search/sort/default-offset-limit.test +0 -2
  74. data/test/command/suite/search/sort/invisible-column.test +0 -2
  75. data/test/unit/catalog/test_collection_volume.rb +103 -0
  76. data/test/unit/catalog/test_dataset.rb +69 -8
  77. data/test/unit/catalog/test_schema.rb +63 -23
  78. data/test/unit/catalog/test_single_volume.rb +31 -0
  79. data/test/unit/catalog/test_slice.rb +92 -0
  80. data/test/unit/catalog/test_version1.rb +1 -1
  81. data/test/unit/catalog/test_version2.rb +1 -32
  82. data/test/unit/catalog/test_version2_validator.rb +66 -0
  83. data/test/unit/catalog/test_volume_collection.rb +50 -0
  84. data/test/unit/plugins/groonga/test_column_create.rb +4 -1
  85. data/test/unit/plugins/groonga/test_table_create.rb +1 -1
  86. data/test/unit/test_message_matcher.rb +15 -15
  87. data/test/unit/test_watch_schema.rb +1 -1
  88. metadata +107 -94
  89. checksums.yaml +0 -7
  90. data/lib/droonga/message_pusher.rb +0 -64
@@ -0,0 +1,81 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Threads",
9
+ "key": "ruby",
10
+ "values": {
11
+ "name": "Ruby"
12
+ }
13
+ }
14
+ }
15
+ {
16
+ "type": "add",
17
+ "dataset": "Droonga",
18
+ "body": {
19
+ "table": "Threads",
20
+ "key": "groonga",
21
+ "values": {
22
+ "name": "Groonga"
23
+ }
24
+ }
25
+ }
26
+ {
27
+ "type": "add",
28
+ "dataset": "Droonga",
29
+ "body": {
30
+ "table": "Comments",
31
+ "key": "1",
32
+ "values": {
33
+ "thread": "ruby",
34
+ "content": "Hi! I'm a new Rubyist!"
35
+ }
36
+ }
37
+ }
38
+ {
39
+ "type": "add",
40
+ "dataset": "Droonga",
41
+ "body": {
42
+ "table": "Comments",
43
+ "key": "2",
44
+ "values": {
45
+ "thread": "groonga",
46
+ "content": "Groonga is great!"
47
+ }
48
+ }
49
+ }
50
+ #@enable-logging
51
+ {
52
+ "type": "search",
53
+ "dataset": "Droonga",
54
+ "body": {
55
+ "queries": {
56
+ "comments": {
57
+ "source": "Comments",
58
+ "groupBy": {
59
+ "key": "thread",
60
+ "maxNSubRecords": 10
61
+ },
62
+ "sortBy": ["_key"],
63
+ "output": {
64
+ "elements": ["count", "records"],
65
+ "limit": -1,
66
+ "attributes": [
67
+ "_key",
68
+ "_nsubrecs",
69
+ {
70
+ "label": "items",
71
+ "source": "_subrecs",
72
+ "attributes": [
73
+ "content"
74
+ ]
75
+ }
76
+ ]
77
+ }
78
+ }
79
+ }
80
+ }
81
+ }
@@ -13,8 +13,6 @@
13
13
  "sortBy": ["_key"],
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -28,8 +26,6 @@
28
26
  "sortBy": ["-_key"],
29
27
  "output": {
30
28
  "elements": [
31
- "startTime",
32
- "elapsedTime",
33
29
  "count",
34
30
  "attributes",
35
31
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -24,8 +22,6 @@
24
22
  "sortBy": ["-_key"],
25
23
  "output": {
26
24
  "elements": [
27
- "startTime",
28
- "elapsedTime",
29
25
  "count",
30
26
  "attributes",
31
27
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -0,0 +1,13 @@
1
+ {
2
+ "datasets": {
3
+ "Droonga": {
4
+ "fact": "Memos",
5
+ "schema": {
6
+ "Memos": {
7
+ "type": "Hash",
8
+ "keyType": "ShortText"
9
+ }
10
+ }
11
+ }
12
+ }
13
+ }
@@ -0,0 +1,15 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": "request-id",
6
+ "statusCode": 200,
7
+ "type": "search.result",
8
+ "body": {
9
+ "memos": {
10
+ "count": 1,
11
+ "elapsedTime": 0.0
12
+ }
13
+ }
14
+ }
15
+ ]
@@ -0,0 +1,26 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Memos",
9
+ "key": "Droonga is fun"
10
+ }
11
+ }
12
+ #@enable-logging
13
+ {
14
+ "type": "search",
15
+ "dataset": "Droonga",
16
+ "body": {
17
+ "queries": {
18
+ "memos": {
19
+ "source": "Memos",
20
+ "output": {
21
+ "elements": ["count", "elapsedTime"]
22
+ }
23
+ }
24
+ }
25
+ }
26
+ }
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
@@ -11,8 +11,6 @@
11
11
  },
12
12
  "output": {
13
13
  "elements": [
14
- "startTime",
15
- "elapsedTime",
16
14
  "count",
17
15
  "attributes",
18
16
  "records"
@@ -13,8 +13,6 @@
13
13
  },
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -0,0 +1,103 @@
1
+ # Copyright (C) 2014 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "droonga/catalog/dataset"
17
+
18
+ class CatalogSingleVolumeTest < Test::Unit::TestCase
19
+ def create_collection_volume(data)
20
+ minimum_dataset_data = {
21
+ "replicas" => {
22
+ },
23
+ }
24
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
25
+ Droonga::Catalog::CollectionVolume.new(dataset, data)
26
+ end
27
+
28
+ class DimensionTest < self
29
+ def test_default
30
+ data = {
31
+ "slices" => [],
32
+ }
33
+ volume = create_collection_volume(data)
34
+ assert_equal("_key", volume.dimension)
35
+ end
36
+
37
+ def test_specified
38
+ data = {
39
+ "dimension" => "group",
40
+ "slices" => [],
41
+ }
42
+ volume = create_collection_volume(data)
43
+ assert_equal("group", volume.dimension)
44
+ end
45
+ end
46
+
47
+ class SlicerTest < self
48
+ def test_default
49
+ data = {
50
+ "slices" => [],
51
+ }
52
+ volume = create_collection_volume(data)
53
+ assert_equal("hash", volume.slicer)
54
+ end
55
+
56
+ def test_specified
57
+ data = {
58
+ "slicer" => "ordinal",
59
+ }
60
+ volume = create_collection_volume(data)
61
+ assert_equal("ordinal", volume.slicer)
62
+ end
63
+ end
64
+
65
+ class SlicesTest < self
66
+ def test_empty
67
+ data = {
68
+ "slices" => [],
69
+ }
70
+ volume = create_collection_volume(data)
71
+ assert_equal([], volume.slices)
72
+ end
73
+ end
74
+
75
+ class RatioOrderSlicerTest < self
76
+ class TotalWeightTest < self
77
+ def test_three_slices
78
+ data = {
79
+ "slicer" => "hash",
80
+ "slices" => [
81
+ {
82
+ "weight" => 10,
83
+ },
84
+ {
85
+ "weight" => 20,
86
+ },
87
+ {
88
+ "weight" => 30,
89
+ },
90
+ ],
91
+ }
92
+ assert_equal(10 + 20 + 30,
93
+ total_weight(data))
94
+ end
95
+
96
+ private
97
+ def total_weight(data)
98
+ volume = create_collection_volume(data)
99
+ volume.send(:compute_total_weight)
100
+ end
101
+ end
102
+ end
103
+ end
@@ -17,27 +17,88 @@ require "droonga/catalog/dataset"
17
17
 
18
18
  class CatalogDatasetTest < Test::Unit::TestCase
19
19
  private
20
- def create_dataset(dataset_name, data)
21
- Droonga::Catalog::Dataset.new(dataset_name, data)
20
+ def create_dataset(data)
21
+ Droonga::Catalog::Dataset.new("DatasetName", data)
22
22
  end
23
23
 
24
- class DatasetTest < self
25
- def test_value
24
+ class NameTest < self
25
+ def test_reader
26
+ dataset = Droonga::Catalog::Dataset.new("DatasetName", {})
27
+ assert_equal("DatasetName", dataset.name)
28
+ end
29
+ end
30
+
31
+ class NWorkersTest < self
32
+ def test_default
33
+ data = {
34
+ }
35
+ dataset = create_dataset(data)
36
+ assert_equal(0, dataset.n_workers)
37
+ end
38
+
39
+ def test_specified
26
40
  data = {
27
41
  "nWorkers" => 2
28
42
  }
29
- dataset = create_dataset("dataset_name", data)
30
- assert_equal(2, dataset["nWorkers"])
43
+ dataset = create_dataset(data)
44
+ assert_equal(2, dataset.n_workers)
31
45
  end
46
+ end
32
47
 
33
- def test_schema
48
+ class SchemaTest < self
49
+ def test_empty
34
50
  data = {
35
51
  "schema" => {
36
52
  }
37
53
  }
38
- dataset = create_dataset("dataset_name", data)
54
+ dataset = create_dataset(data)
39
55
  assert_equal(Droonga::Catalog::Schema.new("dataset_name", {}),
40
56
  dataset.schema)
41
57
  end
42
58
  end
59
+
60
+ class PluginsTest < self
61
+ def test_default
62
+ data = {
63
+ }
64
+ dataset = create_dataset(data)
65
+ assert_equal([], dataset.plugins)
66
+ end
67
+
68
+ def test_multiple
69
+ data = {
70
+ "plugins" => ["groonga", "crud"],
71
+ }
72
+ dataset = create_dataset(data)
73
+ assert_equal(["groonga", "crud"], dataset.plugins)
74
+ end
75
+ end
76
+
77
+ class FactTest < self
78
+ def test_default
79
+ data = {
80
+ }
81
+ dataset = create_dataset(data)
82
+ assert_nil(dataset.fact)
83
+ end
84
+
85
+ def test_specified
86
+ data = {
87
+ "fact" => "Users",
88
+ }
89
+ dataset = create_dataset(data)
90
+ assert_equal("Users", dataset.fact)
91
+ end
92
+ end
93
+
94
+ class ReplicasTest < self
95
+ def test_empty
96
+ data = {
97
+ "replicas" => [],
98
+ }
99
+ dataset = create_dataset(data)
100
+ assert_equal(Droonga::Catalog::VolumeCollection.new([]),
101
+ dataset.replicas)
102
+ end
103
+ end
43
104
  end