fluent-plugin-droonga 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -1
- data/Gemfile +1 -1
- data/fluent-plugin-droonga.gemspec +1 -1
- data/lib/droonga/catalog/collection_volume.rb +97 -0
- data/lib/droonga/catalog/dataset.rb +28 -1
- data/lib/droonga/catalog/errors.rb +28 -9
- data/lib/droonga/catalog/schema.rb +23 -2
- data/lib/droonga/catalog/single_volume.rb +28 -0
- data/lib/droonga/catalog/slice.rb +43 -0
- data/lib/droonga/catalog/version1.rb +3 -3
- data/lib/droonga/catalog/version2.rb +17 -81
- data/lib/droonga/catalog/version2_validator.rb +63 -0
- data/lib/droonga/catalog/volume.rb +33 -0
- data/lib/droonga/catalog/volume_collection.rb +56 -0
- data/lib/droonga/catalog_observer.rb +7 -19
- data/lib/droonga/collectors.rb +1 -1
- data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
- data/lib/droonga/dispatcher.rb +24 -18
- data/lib/droonga/distributed_command_planner.rb +7 -11
- data/lib/droonga/distributor.rb +29 -17
- data/lib/droonga/event_loop.rb +2 -11
- data/lib/droonga/fluent_message_sender.rb +51 -5
- data/lib/droonga/handler_runner.rb +1 -1
- data/lib/droonga/job_protocol.rb +20 -0
- data/lib/droonga/job_pusher.rb +178 -0
- data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
- data/lib/droonga/message_matcher.rb +18 -15
- data/lib/droonga/planner.rb +2 -3
- data/lib/droonga/plugins/crud.rb +1 -1
- data/lib/droonga/plugins/groonga/column_create.rb +4 -1
- data/lib/droonga/plugins/groonga/table_create.rb +1 -1
- data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
- data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
- data/lib/droonga/processor.rb +3 -3
- data/lib/droonga/reducer.rb +15 -12
- data/lib/droonga/searcher.rb +49 -4
- data/lib/droonga/server.rb +2 -0
- data/lib/droonga/single_step.rb +22 -7
- data/lib/droonga/slice.rb +7 -7
- data/lib/droonga/step_runner.rb +3 -2
- data/lib/droonga/worker.rb +10 -8
- data/test/command/suite/add/dimension/column.catalog.json +27 -0
- data/test/command/suite/add/dimension/column.expected +57 -0
- data/test/command/suite/add/dimension/column.test +51 -0
- data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
- data/test/command/suite/search/adjusters/multiple.expected +23 -0
- data/test/command/suite/search/adjusters/multiple.test +75 -0
- data/test/command/suite/search/adjusters/one.catalog.json +38 -0
- data/test/command/suite/search/adjusters/one.expected +23 -0
- data/test/command/suite/search/adjusters/one.test +66 -0
- data/test/command/suite/search/attributes/array.test +0 -2
- data/test/command/suite/search/attributes/hash.test +0 -2
- data/test/command/suite/search/complex.test +0 -2
- data/test/command/suite/search/condition/nested.test +0 -2
- data/test/command/suite/search/condition/query.test +0 -2
- data/test/command/suite/search/condition/script.test +0 -2
- data/test/command/suite/search/group/string.test +0 -4
- data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
- data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
- data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
- data/test/command/suite/search/multiple/chained.test +0 -4
- data/test/command/suite/search/multiple/parallel.test +0 -4
- data/test/command/suite/search/range/only-output.test +0 -2
- data/test/command/suite/search/range/only-sort.test +0 -2
- data/test/command/suite/search/range/sort-and-output.test +0 -2
- data/test/command/suite/search/range/too-large-output-offset.test +0 -2
- data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
- data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
- data/test/command/suite/search/response/elapsed_time.expected +15 -0
- data/test/command/suite/search/response/elapsed_time.test +26 -0
- data/test/command/suite/search/response/records/value/time.test +0 -2
- data/test/command/suite/search/simple.test +0 -2
- data/test/command/suite/search/sort/default-offset-limit.test +0 -2
- data/test/command/suite/search/sort/invisible-column.test +0 -2
- data/test/unit/catalog/test_collection_volume.rb +103 -0
- data/test/unit/catalog/test_dataset.rb +69 -8
- data/test/unit/catalog/test_schema.rb +63 -23
- data/test/unit/catalog/test_single_volume.rb +31 -0
- data/test/unit/catalog/test_slice.rb +92 -0
- data/test/unit/catalog/test_version1.rb +1 -1
- data/test/unit/catalog/test_version2.rb +1 -32
- data/test/unit/catalog/test_version2_validator.rb +66 -0
- data/test/unit/catalog/test_volume_collection.rb +50 -0
- data/test/unit/plugins/groonga/test_column_create.rb +4 -1
- data/test/unit/plugins/groonga/test_table_create.rb +1 -1
- data/test/unit/test_message_matcher.rb +15 -15
- data/test/unit/test_watch_schema.rb +1 -1
- metadata +107 -94
- checksums.yaml +0 -7
- data/lib/droonga/message_pusher.rb +0 -64
@@ -0,0 +1,81 @@
|
|
1
|
+
# -*- js -*-
|
2
|
+
#@require-catalog-version 2
|
3
|
+
#@disable-logging
|
4
|
+
{
|
5
|
+
"type": "add",
|
6
|
+
"dataset": "Droonga",
|
7
|
+
"body": {
|
8
|
+
"table": "Threads",
|
9
|
+
"key": "ruby",
|
10
|
+
"values": {
|
11
|
+
"name": "Ruby"
|
12
|
+
}
|
13
|
+
}
|
14
|
+
}
|
15
|
+
{
|
16
|
+
"type": "add",
|
17
|
+
"dataset": "Droonga",
|
18
|
+
"body": {
|
19
|
+
"table": "Threads",
|
20
|
+
"key": "groonga",
|
21
|
+
"values": {
|
22
|
+
"name": "Groonga"
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
{
|
27
|
+
"type": "add",
|
28
|
+
"dataset": "Droonga",
|
29
|
+
"body": {
|
30
|
+
"table": "Comments",
|
31
|
+
"key": "1",
|
32
|
+
"values": {
|
33
|
+
"thread": "ruby",
|
34
|
+
"content": "Hi! I'm a new Rubyist!"
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
38
|
+
{
|
39
|
+
"type": "add",
|
40
|
+
"dataset": "Droonga",
|
41
|
+
"body": {
|
42
|
+
"table": "Comments",
|
43
|
+
"key": "2",
|
44
|
+
"values": {
|
45
|
+
"thread": "groonga",
|
46
|
+
"content": "Groonga is great!"
|
47
|
+
}
|
48
|
+
}
|
49
|
+
}
|
50
|
+
#@enable-logging
|
51
|
+
{
|
52
|
+
"type": "search",
|
53
|
+
"dataset": "Droonga",
|
54
|
+
"body": {
|
55
|
+
"queries": {
|
56
|
+
"comments": {
|
57
|
+
"source": "Comments",
|
58
|
+
"groupBy": {
|
59
|
+
"key": "thread",
|
60
|
+
"maxNSubRecords": 10
|
61
|
+
},
|
62
|
+
"sortBy": ["_key"],
|
63
|
+
"output": {
|
64
|
+
"elements": ["count", "records"],
|
65
|
+
"limit": -1,
|
66
|
+
"attributes": [
|
67
|
+
"_key",
|
68
|
+
"_nsubrecs",
|
69
|
+
{
|
70
|
+
"label": "items",
|
71
|
+
"source": "_subrecs",
|
72
|
+
"attributes": [
|
73
|
+
"content"
|
74
|
+
]
|
75
|
+
}
|
76
|
+
]
|
77
|
+
}
|
78
|
+
}
|
79
|
+
}
|
80
|
+
}
|
81
|
+
}
|
@@ -13,8 +13,6 @@
|
|
13
13
|
"sortBy": ["_key"],
|
14
14
|
"output": {
|
15
15
|
"elements": [
|
16
|
-
"startTime",
|
17
|
-
"elapsedTime",
|
18
16
|
"count",
|
19
17
|
"attributes",
|
20
18
|
"records"
|
@@ -28,8 +26,6 @@
|
|
28
26
|
"sortBy": ["-_key"],
|
29
27
|
"output": {
|
30
28
|
"elements": [
|
31
|
-
"startTime",
|
32
|
-
"elapsedTime",
|
33
29
|
"count",
|
34
30
|
"attributes",
|
35
31
|
"records"
|
@@ -9,8 +9,6 @@
|
|
9
9
|
"sortBy": ["_key"],
|
10
10
|
"output": {
|
11
11
|
"elements": [
|
12
|
-
"startTime",
|
13
|
-
"elapsedTime",
|
14
12
|
"count",
|
15
13
|
"attributes",
|
16
14
|
"records"
|
@@ -24,8 +22,6 @@
|
|
24
22
|
"sortBy": ["-_key"],
|
25
23
|
"output": {
|
26
24
|
"elements": [
|
27
|
-
"startTime",
|
28
|
-
"elapsedTime",
|
29
25
|
"count",
|
30
26
|
"attributes",
|
31
27
|
"records"
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- js -*-
|
2
|
+
#@require-catalog-version 2
|
3
|
+
#@disable-logging
|
4
|
+
{
|
5
|
+
"type": "add",
|
6
|
+
"dataset": "Droonga",
|
7
|
+
"body": {
|
8
|
+
"table": "Memos",
|
9
|
+
"key": "Droonga is fun"
|
10
|
+
}
|
11
|
+
}
|
12
|
+
#@enable-logging
|
13
|
+
{
|
14
|
+
"type": "search",
|
15
|
+
"dataset": "Droonga",
|
16
|
+
"body": {
|
17
|
+
"queries": {
|
18
|
+
"memos": {
|
19
|
+
"source": "Memos",
|
20
|
+
"output": {
|
21
|
+
"elements": ["count", "elapsedTime"]
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
@@ -0,0 +1,103 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/catalog/dataset"
|
17
|
+
|
18
|
+
class CatalogSingleVolumeTest < Test::Unit::TestCase
|
19
|
+
def create_collection_volume(data)
|
20
|
+
minimum_dataset_data = {
|
21
|
+
"replicas" => {
|
22
|
+
},
|
23
|
+
}
|
24
|
+
dataset = Droonga::Catalog::Dataset.new("DatasetName", minimum_dataset_data)
|
25
|
+
Droonga::Catalog::CollectionVolume.new(dataset, data)
|
26
|
+
end
|
27
|
+
|
28
|
+
class DimensionTest < self
|
29
|
+
def test_default
|
30
|
+
data = {
|
31
|
+
"slices" => [],
|
32
|
+
}
|
33
|
+
volume = create_collection_volume(data)
|
34
|
+
assert_equal("_key", volume.dimension)
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_specified
|
38
|
+
data = {
|
39
|
+
"dimension" => "group",
|
40
|
+
"slices" => [],
|
41
|
+
}
|
42
|
+
volume = create_collection_volume(data)
|
43
|
+
assert_equal("group", volume.dimension)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class SlicerTest < self
|
48
|
+
def test_default
|
49
|
+
data = {
|
50
|
+
"slices" => [],
|
51
|
+
}
|
52
|
+
volume = create_collection_volume(data)
|
53
|
+
assert_equal("hash", volume.slicer)
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_specified
|
57
|
+
data = {
|
58
|
+
"slicer" => "ordinal",
|
59
|
+
}
|
60
|
+
volume = create_collection_volume(data)
|
61
|
+
assert_equal("ordinal", volume.slicer)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class SlicesTest < self
|
66
|
+
def test_empty
|
67
|
+
data = {
|
68
|
+
"slices" => [],
|
69
|
+
}
|
70
|
+
volume = create_collection_volume(data)
|
71
|
+
assert_equal([], volume.slices)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class RatioOrderSlicerTest < self
|
76
|
+
class TotalWeightTest < self
|
77
|
+
def test_three_slices
|
78
|
+
data = {
|
79
|
+
"slicer" => "hash",
|
80
|
+
"slices" => [
|
81
|
+
{
|
82
|
+
"weight" => 10,
|
83
|
+
},
|
84
|
+
{
|
85
|
+
"weight" => 20,
|
86
|
+
},
|
87
|
+
{
|
88
|
+
"weight" => 30,
|
89
|
+
},
|
90
|
+
],
|
91
|
+
}
|
92
|
+
assert_equal(10 + 20 + 30,
|
93
|
+
total_weight(data))
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def total_weight(data)
|
98
|
+
volume = create_collection_volume(data)
|
99
|
+
volume.send(:compute_total_weight)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
@@ -17,27 +17,88 @@ require "droonga/catalog/dataset"
|
|
17
17
|
|
18
18
|
class CatalogDatasetTest < Test::Unit::TestCase
|
19
19
|
private
|
20
|
-
def create_dataset(
|
21
|
-
Droonga::Catalog::Dataset.new(
|
20
|
+
def create_dataset(data)
|
21
|
+
Droonga::Catalog::Dataset.new("DatasetName", data)
|
22
22
|
end
|
23
23
|
|
24
|
-
class
|
25
|
-
def
|
24
|
+
class NameTest < self
|
25
|
+
def test_reader
|
26
|
+
dataset = Droonga::Catalog::Dataset.new("DatasetName", {})
|
27
|
+
assert_equal("DatasetName", dataset.name)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class NWorkersTest < self
|
32
|
+
def test_default
|
33
|
+
data = {
|
34
|
+
}
|
35
|
+
dataset = create_dataset(data)
|
36
|
+
assert_equal(0, dataset.n_workers)
|
37
|
+
end
|
38
|
+
|
39
|
+
def test_specified
|
26
40
|
data = {
|
27
41
|
"nWorkers" => 2
|
28
42
|
}
|
29
|
-
dataset = create_dataset(
|
30
|
-
assert_equal(2, dataset
|
43
|
+
dataset = create_dataset(data)
|
44
|
+
assert_equal(2, dataset.n_workers)
|
31
45
|
end
|
46
|
+
end
|
32
47
|
|
33
|
-
|
48
|
+
class SchemaTest < self
|
49
|
+
def test_empty
|
34
50
|
data = {
|
35
51
|
"schema" => {
|
36
52
|
}
|
37
53
|
}
|
38
|
-
dataset = create_dataset(
|
54
|
+
dataset = create_dataset(data)
|
39
55
|
assert_equal(Droonga::Catalog::Schema.new("dataset_name", {}),
|
40
56
|
dataset.schema)
|
41
57
|
end
|
42
58
|
end
|
59
|
+
|
60
|
+
class PluginsTest < self
|
61
|
+
def test_default
|
62
|
+
data = {
|
63
|
+
}
|
64
|
+
dataset = create_dataset(data)
|
65
|
+
assert_equal([], dataset.plugins)
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_multiple
|
69
|
+
data = {
|
70
|
+
"plugins" => ["groonga", "crud"],
|
71
|
+
}
|
72
|
+
dataset = create_dataset(data)
|
73
|
+
assert_equal(["groonga", "crud"], dataset.plugins)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
class FactTest < self
|
78
|
+
def test_default
|
79
|
+
data = {
|
80
|
+
}
|
81
|
+
dataset = create_dataset(data)
|
82
|
+
assert_nil(dataset.fact)
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_specified
|
86
|
+
data = {
|
87
|
+
"fact" => "Users",
|
88
|
+
}
|
89
|
+
dataset = create_dataset(data)
|
90
|
+
assert_equal("Users", dataset.fact)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
class ReplicasTest < self
|
95
|
+
def test_empty
|
96
|
+
data = {
|
97
|
+
"replicas" => [],
|
98
|
+
}
|
99
|
+
dataset = create_dataset(data)
|
100
|
+
assert_equal(Droonga::Catalog::VolumeCollection.new([]),
|
101
|
+
dataset.replicas)
|
102
|
+
end
|
103
|
+
end
|
43
104
|
end
|