fluent-plugin-droonga 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +1 -1
- data/Gemfile +1 -1
- data/fluent-plugin-droonga.gemspec +1 -1
- data/lib/droonga/catalog/collection_volume.rb +97 -0
- data/lib/droonga/catalog/dataset.rb +28 -1
- data/lib/droonga/catalog/errors.rb +28 -9
- data/lib/droonga/catalog/schema.rb +23 -2
- data/lib/droonga/catalog/single_volume.rb +28 -0
- data/lib/droonga/catalog/slice.rb +43 -0
- data/lib/droonga/catalog/version1.rb +3 -3
- data/lib/droonga/catalog/version2.rb +17 -81
- data/lib/droonga/catalog/version2_validator.rb +63 -0
- data/lib/droonga/catalog/volume.rb +33 -0
- data/lib/droonga/catalog/volume_collection.rb +56 -0
- data/lib/droonga/catalog_observer.rb +7 -19
- data/lib/droonga/collectors.rb +1 -1
- data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
- data/lib/droonga/dispatcher.rb +24 -18
- data/lib/droonga/distributed_command_planner.rb +7 -11
- data/lib/droonga/distributor.rb +29 -17
- data/lib/droonga/event_loop.rb +2 -11
- data/lib/droonga/fluent_message_sender.rb +51 -5
- data/lib/droonga/handler_runner.rb +1 -1
- data/lib/droonga/job_protocol.rb +20 -0
- data/lib/droonga/job_pusher.rb +178 -0
- data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
- data/lib/droonga/message_matcher.rb +18 -15
- data/lib/droonga/planner.rb +2 -3
- data/lib/droonga/plugins/crud.rb +1 -1
- data/lib/droonga/plugins/groonga/column_create.rb +4 -1
- data/lib/droonga/plugins/groonga/table_create.rb +1 -1
- data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
- data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
- data/lib/droonga/processor.rb +3 -3
- data/lib/droonga/reducer.rb +15 -12
- data/lib/droonga/searcher.rb +49 -4
- data/lib/droonga/server.rb +2 -0
- data/lib/droonga/single_step.rb +22 -7
- data/lib/droonga/slice.rb +7 -7
- data/lib/droonga/step_runner.rb +3 -2
- data/lib/droonga/worker.rb +10 -8
- data/test/command/suite/add/dimension/column.catalog.json +27 -0
- data/test/command/suite/add/dimension/column.expected +57 -0
- data/test/command/suite/add/dimension/column.test +51 -0
- data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
- data/test/command/suite/search/adjusters/multiple.expected +23 -0
- data/test/command/suite/search/adjusters/multiple.test +75 -0
- data/test/command/suite/search/adjusters/one.catalog.json +38 -0
- data/test/command/suite/search/adjusters/one.expected +23 -0
- data/test/command/suite/search/adjusters/one.test +66 -0
- data/test/command/suite/search/attributes/array.test +0 -2
- data/test/command/suite/search/attributes/hash.test +0 -2
- data/test/command/suite/search/complex.test +0 -2
- data/test/command/suite/search/condition/nested.test +0 -2
- data/test/command/suite/search/condition/query.test +0 -2
- data/test/command/suite/search/condition/script.test +0 -2
- data/test/command/suite/search/group/string.test +0 -4
- data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
- data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
- data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
- data/test/command/suite/search/multiple/chained.test +0 -4
- data/test/command/suite/search/multiple/parallel.test +0 -4
- data/test/command/suite/search/range/only-output.test +0 -2
- data/test/command/suite/search/range/only-sort.test +0 -2
- data/test/command/suite/search/range/sort-and-output.test +0 -2
- data/test/command/suite/search/range/too-large-output-offset.test +0 -2
- data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
- data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
- data/test/command/suite/search/response/elapsed_time.expected +15 -0
- data/test/command/suite/search/response/elapsed_time.test +26 -0
- data/test/command/suite/search/response/records/value/time.test +0 -2
- data/test/command/suite/search/simple.test +0 -2
- data/test/command/suite/search/sort/default-offset-limit.test +0 -2
- data/test/command/suite/search/sort/invisible-column.test +0 -2
- data/test/unit/catalog/test_collection_volume.rb +103 -0
- data/test/unit/catalog/test_dataset.rb +69 -8
- data/test/unit/catalog/test_schema.rb +63 -23
- data/test/unit/catalog/test_single_volume.rb +31 -0
- data/test/unit/catalog/test_slice.rb +92 -0
- data/test/unit/catalog/test_version1.rb +1 -1
- data/test/unit/catalog/test_version2.rb +1 -32
- data/test/unit/catalog/test_version2_validator.rb +66 -0
- data/test/unit/catalog/test_volume_collection.rb +50 -0
- data/test/unit/plugins/groonga/test_column_create.rb +4 -1
- data/test/unit/plugins/groonga/test_table_create.rb +1 -1
- data/test/unit/test_message_matcher.rb +15 -15
- data/test/unit/test_watch_schema.rb +1 -1
- metadata +107 -94
- checksums.yaml +0 -7
- data/lib/droonga/message_pusher.rb +0 -64
@@ -0,0 +1,81 @@
|
|
1
|
+
# -*- js -*-
|
2
|
+
#@require-catalog-version 2
|
3
|
+
#@disable-logging
|
4
|
+
{
|
5
|
+
"type": "add",
|
6
|
+
"dataset": "Droonga",
|
7
|
+
"body": {
|
8
|
+
"table": "Threads",
|
9
|
+
"key": "ruby",
|
10
|
+
"values": {
|
11
|
+
"name": "Ruby"
|
12
|
+
}
|
13
|
+
}
|
14
|
+
}
|
15
|
+
{
|
16
|
+
"type": "add",
|
17
|
+
"dataset": "Droonga",
|
18
|
+
"body": {
|
19
|
+
"table": "Threads",
|
20
|
+
"key": "groonga",
|
21
|
+
"values": {
|
22
|
+
"name": "Groonga"
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
{
|
27
|
+
"type": "add",
|
28
|
+
"dataset": "Droonga",
|
29
|
+
"body": {
|
30
|
+
"table": "Comments",
|
31
|
+
"key": "1",
|
32
|
+
"values": {
|
33
|
+
"thread": "ruby",
|
34
|
+
"content": "Hi! I'm a new Rubyist!"
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
38
|
+
{
|
39
|
+
"type": "add",
|
40
|
+
"dataset": "Droonga",
|
41
|
+
"body": {
|
42
|
+
"table": "Comments",
|
43
|
+
"key": "2",
|
44
|
+
"values": {
|
45
|
+
"thread": "groonga",
|
46
|
+
"content": "Groonga is great!"
|
47
|
+
}
|
48
|
+
}
|
49
|
+
}
|
50
|
+
#@enable-logging
|
51
|
+
{
|
52
|
+
"type": "search",
|
53
|
+
"dataset": "Droonga",
|
54
|
+
"body": {
|
55
|
+
"queries": {
|
56
|
+
"comments": {
|
57
|
+
"source": "Comments",
|
58
|
+
"groupBy": {
|
59
|
+
"key": "thread",
|
60
|
+
"maxNSubRecords": 10
|
61
|
+
},
|
62
|
+
"sortBy": ["_key"],
|
63
|
+
"output": {
|
64
|
+
"elements": ["count", "records"],
|
65
|
+
"limit": -1,
|
66
|
+
"attributes": [
|
67
|
+
"_key",
|
68
|
+
"_nsubrecs",
|
69
|
+
{
|
70
|
+
"label": "items",
|
71
|
+
"source": "_subrecs",
|
72
|
+
"attributes": [
|
73
|
+
"content"
|
74
|
+
]
|
75
|
+
}
|
76
|
+
]
|
77
|
+
}
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
}
|
@@ -13,8 +13,6 @@
|
|
13
13
|
"sortBy": ["_key"],
|
14
14
|
"output": {
|
15
15
|
"elements": [
|
16
|
-
"startTime",
|
17
|
-
"elapsedTime",
|
18
16
|
"count",
|
19
17
|
"attributes",
|
20
18
|
"records"
|
@@ -28,8 +26,6 @@
|
|
28
26
|
"sortBy": ["-_key"],
|
29
27
|
"output": {
|
30
28
|
"elements": [
|
31
|
-
"startTime",
|
32
|
-
"elapsedTime",
|
33
29
|
"count",
|
34
30
|
"attributes",
|
35
31
|
"records"
|
@@ -9,8 +9,6 @@
|
|
9
9
|
"sortBy": ["_key"],
|
10
10
|
"output": {
|
11
11
|
"elements": [
|
12
|
-
"startTime",
|
13
|
-
"elapsedTime",
|
14
12
|
"count",
|
15
13
|
"attributes",
|
16
14
|
"records"
|
@@ -24,8 +22,6 @@
|
|
24
22
|
"sortBy": ["-_key"],
|
25
23
|
"output": {
|
26
24
|
"elements": [
|
27
|
-
"startTime",
|
28
|
-
"elapsedTime",
|
29
25
|
"count",
|
30
26
|
"attributes",
|
31
27
|
"records"
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- js -*-
|
2
|
+
#@require-catalog-version 2
|
3
|
+
#@disable-logging
|
4
|
+
{
|
5
|
+
"type": "add",
|
6
|
+
"dataset": "Droonga",
|
7
|
+
"body": {
|
8
|
+
"table": "Memos",
|
9
|
+
"key": "Droonga is fun"
|
10
|
+
}
|
11
|
+
}
|
12
|
+
#@enable-logging
|
13
|
+
{
|
14
|
+
"type": "search",
|
15
|
+
"dataset": "Droonga",
|
16
|
+
"body": {
|
17
|
+
"queries": {
|
18
|
+
"memos": {
|
19
|
+
"source": "Memos",
|
20
|
+
"output": {
|
21
|
+
"elements": ["count", "elapsedTime"]
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/catalog/dataset"
|
17
|
+
|
18
|
+
class CatalogSingleVolumeTest < Test::Unit::TestCase
|
19
|
+
def create_collection_volume(data)
|
20
|
+
minimum_dataset_data = {
|
21
|
+
"replicas" => {
|
22
|
+
},
|
23
|
+
}
|
24
|
+
dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
|
25
|
+
Droonga::Catalog::CollectionVolume.new(dataset, data)
|
26
|
+
end
|
27
|
+
|
28
|
+
class DimensionTest < self
|
29
|
+
def test_default
|
30
|
+
data = {
|
31
|
+
"slices" => [],
|
32
|
+
}
|
33
|
+
volume = create_collection_volume(data)
|
34
|
+
assert_equal("_key", volume.dimension)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_specified
|
38
|
+
data = {
|
39
|
+
"dimension" => "group",
|
40
|
+
"slices" => [],
|
41
|
+
}
|
42
|
+
volume = create_collection_volume(data)
|
43
|
+
assert_equal("group", volume.dimension)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class SlicerTest < self
|
48
|
+
def test_default
|
49
|
+
data = {
|
50
|
+
"slices" => [],
|
51
|
+
}
|
52
|
+
volume = create_collection_volume(data)
|
53
|
+
assert_equal("hash", volume.slicer)
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_specified
|
57
|
+
data = {
|
58
|
+
"slicer" => "ordinal",
|
59
|
+
}
|
60
|
+
volume = create_collection_volume(data)
|
61
|
+
assert_equal("ordinal", volume.slicer)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class SlicesTest < self
|
66
|
+
def test_empty
|
67
|
+
data = {
|
68
|
+
"slices" => [],
|
69
|
+
}
|
70
|
+
volume = create_collection_volume(data)
|
71
|
+
assert_equal([], volume.slices)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class RatioOrderSlicerTest < self
|
76
|
+
class TotalWeightTest < self
|
77
|
+
def test_three_slices
|
78
|
+
data = {
|
79
|
+
"slicer" => "hash",
|
80
|
+
"slices" => [
|
81
|
+
{
|
82
|
+
"weight" => 10,
|
83
|
+
},
|
84
|
+
{
|
85
|
+
"weight" => 20,
|
86
|
+
},
|
87
|
+
{
|
88
|
+
"weight" => 30,
|
89
|
+
},
|
90
|
+
],
|
91
|
+
}
|
92
|
+
assert_equal(10 + 20 + 30,
|
93
|
+
total_weight(data))
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def total_weight(data)
|
98
|
+
volume = create_collection_volume(data)
|
99
|
+
volume.send(:compute_total_weight)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -17,27 +17,88 @@ require "droonga/catalog/dataset"
|
|
17
17
|
|
18
18
|
class CatalogDatasetTest < Test::Unit::TestCase
|
19
19
|
private
|
20
|
-
def create_dataset(
|
21
|
-
Droonga::Catalog::Dataset.new(
|
20
|
+
def create_dataset(data)
|
21
|
+
Droonga::Catalog::Dataset.new("DatasetName", data)
|
22
22
|
end
|
23
23
|
|
24
|
-
class
|
25
|
-
def
|
24
|
+
class NameTest < self
|
25
|
+
def test_reader
|
26
|
+
dataset = Droonga::Catalog::Dataset.new("DatasetName", {})
|
27
|
+
assert_equal("DatasetName", dataset.name)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class NWorkersTest < self
|
32
|
+
def test_default
|
33
|
+
data = {
|
34
|
+
}
|
35
|
+
dataset = create_dataset(data)
|
36
|
+
assert_equal(0, dataset.n_workers)
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_specified
|
26
40
|
data = {
|
27
41
|
"nWorkers" => 2
|
28
42
|
}
|
29
|
-
dataset = create_dataset(
|
30
|
-
assert_equal(2, dataset
|
43
|
+
dataset = create_dataset(data)
|
44
|
+
assert_equal(2, dataset.n_workers)
|
31
45
|
end
|
46
|
+
end
|
32
47
|
|
33
|
-
|
48
|
+
class SchemaTest < self
|
49
|
+
def test_empty
|
34
50
|
data = {
|
35
51
|
"schema" => {
|
36
52
|
}
|
37
53
|
}
|
38
|
-
dataset = create_dataset(
|
54
|
+
dataset = create_dataset(data)
|
39
55
|
assert_equal(Droonga::Catalog::Schema.new("dataset_name", {}),
|
40
56
|
dataset.schema)
|
41
57
|
end
|
42
58
|
end
|
59
|
+
|
60
|
+
class PluginsTest < self
|
61
|
+
def test_default
|
62
|
+
data = {
|
63
|
+
}
|
64
|
+
dataset = create_dataset(data)
|
65
|
+
assert_equal([], dataset.plugins)
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_multiple
|
69
|
+
data = {
|
70
|
+
"plugins" => ["groonga", "crud"],
|
71
|
+
}
|
72
|
+
dataset = create_dataset(data)
|
73
|
+
assert_equal(["groonga", "crud"], dataset.plugins)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
class FactTest < self
|
78
|
+
def test_default
|
79
|
+
data = {
|
80
|
+
}
|
81
|
+
dataset = create_dataset(data)
|
82
|
+
assert_nil(dataset.fact)
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_specified
|
86
|
+
data = {
|
87
|
+
"fact" => "Users",
|
88
|
+
}
|
89
|
+
dataset = create_dataset(data)
|
90
|
+
assert_equal("Users", dataset.fact)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
class ReplicasTest < self
|
95
|
+
def test_empty
|
96
|
+
data = {
|
97
|
+
"replicas" => [],
|
98
|
+
}
|
99
|
+
dataset = create_dataset(data)
|
100
|
+
assert_equal(Droonga::Catalog::VolumeCollection.new([]),
|
101
|
+
dataset.replicas)
|
102
|
+
end
|
103
|
+
end
|
43
104
|
end
|