fluent-plugin-droonga 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -4
- data/benchmark/watch/benchmark-notify.rb +2 -2
- data/benchmark/watch/benchmark-scan.rb +3 -0
- data/benchmark/watch/fluentd.conf +0 -1
- data/fluent-plugin-droonga.gemspec +2 -3
- data/lib/droonga/catalog.rb +10 -124
- data/lib/droonga/catalog/base.rb +140 -0
- data/lib/droonga/catalog/version1.rb +23 -0
- data/lib/droonga/catalog_loader.rb +33 -0
- data/lib/droonga/collector.rb +2 -71
- data/lib/droonga/collector_plugin.rb +2 -34
- data/lib/droonga/dispatcher.rb +141 -196
- data/lib/droonga/distribution_planner.rb +76 -0
- data/lib/droonga/distributor.rb +5 -7
- data/lib/droonga/distributor_plugin.rb +23 -15
- data/lib/droonga/engine.rb +2 -2
- data/lib/droonga/event_loop.rb +46 -0
- data/lib/droonga/farm.rb +9 -5
- data/lib/droonga/fluent_message_sender.rb +84 -0
- data/lib/droonga/forwarder.rb +43 -53
- data/lib/droonga/handler.rb +20 -68
- data/lib/droonga/handler_message.rb +61 -0
- data/lib/droonga/handler_messenger.rb +92 -0
- data/lib/droonga/handler_plugin.rb +10 -12
- data/lib/droonga/input_adapter.rb +52 -0
- data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
- data/lib/droonga/input_message.rb +11 -11
- data/lib/droonga/logger.rb +4 -3
- data/lib/droonga/message_pack_packer.rb +62 -0
- data/lib/droonga/message_processing_error.rb +54 -0
- data/lib/droonga/message_pusher.rb +60 -0
- data/lib/droonga/message_receiver.rb +61 -0
- data/lib/droonga/output_adapter.rb +53 -0
- data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
- data/lib/droonga/output_message.rb +37 -0
- data/lib/droonga/partition.rb +27 -5
- data/lib/droonga/pluggable.rb +9 -4
- data/lib/droonga/plugin.rb +12 -3
- data/lib/droonga/plugin/collector/basic.rb +91 -18
- data/lib/droonga/plugin/distributor/crud.rb +9 -9
- data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
- data/lib/droonga/plugin/distributor/groonga.rb +5 -5
- data/lib/droonga/plugin/distributor/search.rb +4 -246
- data/lib/droonga/plugin/distributor/watch.rb +11 -6
- data/lib/droonga/plugin/handler/add.rb +69 -7
- data/lib/droonga/plugin/handler/groonga.rb +6 -6
- data/lib/droonga/plugin/handler/search.rb +5 -3
- data/lib/droonga/plugin/handler/watch.rb +19 -13
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
- data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
- data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
- data/lib/droonga/plugin_loader.rb +2 -2
- data/lib/droonga/processor.rb +21 -23
- data/lib/droonga/replier.rb +40 -0
- data/lib/droonga/searcher.rb +298 -174
- data/lib/droonga/server.rb +0 -67
- data/lib/droonga/session.rb +85 -0
- data/lib/droonga/test.rb +21 -0
- data/lib/droonga/test/stub_distributor.rb +31 -0
- data/lib/droonga/test/stub_handler.rb +37 -0
- data/lib/droonga/test/stub_handler_message.rb +35 -0
- data/lib/droonga/test/stub_handler_messenger.rb +34 -0
- data/lib/droonga/time_formatter.rb +37 -0
- data/lib/droonga/watcher.rb +1 -0
- data/lib/droonga/worker.rb +16 -19
- data/lib/fluent/plugin/out_droonga.rb +9 -9
- data/lib/groonga_command_converter.rb +5 -5
- data/sample/cluster/catalog.json +1 -1
- data/test/command/config/default/catalog.json +19 -1
- data/test/command/fixture/event.jsons +41 -0
- data/test/command/fixture/user-table.jsons +9 -0
- data/test/command/run-test.rb +2 -2
- data/test/command/suite/add/error/invalid-integer.expected +20 -0
- data/test/command/suite/add/error/invalid-integer.test +12 -0
- data/test/command/suite/add/error/invalid-time.expected +20 -0
- data/test/command/suite/add/error/invalid-time.test +12 -0
- data/test/command/suite/add/error/missing-key.expected +13 -0
- data/test/command/suite/add/error/missing-key.test +16 -0
- data/test/command/suite/add/error/missing-table.expected +13 -0
- data/test/command/suite/add/error/missing-table.test +16 -0
- data/test/command/suite/add/error/unknown-column.expected +20 -0
- data/test/command/suite/add/error/unknown-column.test +12 -0
- data/test/command/suite/add/error/unknown-table.expected +13 -0
- data/test/command/suite/add/error/unknown-table.test +17 -0
- data/test/command/suite/add/minimum.expected +1 -3
- data/test/command/suite/add/with-values.expected +1 -3
- data/test/command/suite/add/without-key.expected +1 -3
- data/test/command/suite/message/error/missing-dataset.expected +13 -0
- data/test/command/suite/message/error/missing-dataset.test +5 -0
- data/test/command/suite/message/error/unknown-command.expected +13 -0
- data/test/command/suite/message/error/unknown-command.test +6 -0
- data/test/command/suite/message/error/unknown-dataset.expected +13 -0
- data/test/command/suite/message/error/unknown-dataset.test +6 -0
- data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
- data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
- data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
- data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
- data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
- data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
- data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
- data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
- data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
- data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
- data/test/command/suite/search/error/cyclic-source.expected +18 -0
- data/test/command/suite/search/error/cyclic-source.test +12 -0
- data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
- data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
- data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
- data/test/command/suite/search/error/missing-source-parameter.test +11 -0
- data/test/command/suite/search/error/unknown-source.expected +18 -0
- data/test/command/suite/search/error/unknown-source.test +12 -0
- data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
- data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
- data/test/command/suite/search/group/limit.expected +19 -0
- data/test/command/suite/search/group/limit.test +20 -0
- data/test/command/suite/search/group/string.expected +36 -0
- data/test/command/suite/search/group/string.test +44 -0
- data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
- data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
- data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
- data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
- data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
- data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
- data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
- data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
- data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
- data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
- data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-output-offset.test +25 -0
- data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
- data/test/command/suite/search/response/records/value/time.expected +24 -0
- data/test/command/suite/search/response/records/value/time.test +24 -0
- data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
- data/test/command/suite/search/sort/default-offset-limit.test +26 -0
- data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
- data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
- data/test/command/suite/watch/subscribe.expected +12 -0
- data/test/command/suite/watch/subscribe.test +9 -0
- data/test/command/suite/watch/unsubscribe.expected +12 -0
- data/test/command/suite/watch/unsubscribe.test +9 -0
- data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
- data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
- data/test/unit/helper.rb +2 -0
- data/test/unit/plugin/collector/test_basic.rb +289 -33
- data/test/unit/plugin/distributor/test_search.rb +176 -861
- data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
- data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
- data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
- data/test/unit/plugin/handler/test_add.rb +74 -11
- data/test/unit/plugin/handler/test_groonga.rb +15 -1
- data/test/unit/plugin/handler/test_search.rb +33 -17
- data/test/unit/plugin/handler/test_watch.rb +43 -27
- data/test/unit/run-test.rb +2 -0
- data/test/unit/test_message_pack_packer.rb +51 -0
- data/test/unit/test_time_formatter.rb +29 -0
- metadata +208 -110
- data/lib/droonga/job_queue.rb +0 -87
- data/lib/droonga/job_queue_schema.rb +0 -65
- data/test/unit/test_adapter.rb +0 -51
- data/test/unit/test_job_queue_schema.rb +0 -45
@@ -67,7 +67,7 @@ module Droonga
|
|
67
67
|
end
|
68
68
|
|
69
69
|
private
|
70
|
-
def
|
70
|
+
def create_message(type, body)
|
71
71
|
id = @options[:id]
|
72
72
|
if id.nil?
|
73
73
|
id = new_unique_id
|
@@ -99,11 +99,11 @@ module Droonga
|
|
99
99
|
end
|
100
100
|
|
101
101
|
def create_table_create_command(command)
|
102
|
-
|
102
|
+
create_message("table_create", command.arguments)
|
103
103
|
end
|
104
104
|
|
105
105
|
def create_column_create_command(command)
|
106
|
-
|
106
|
+
create_message("column_create", command.arguments)
|
107
107
|
end
|
108
108
|
|
109
109
|
def split_load_command_to_add_commands(command, &block)
|
@@ -126,12 +126,12 @@ module Droonga
|
|
126
126
|
end
|
127
127
|
body[:values] = record_values unless record_values.empty?
|
128
128
|
|
129
|
-
yield
|
129
|
+
yield create_message("add", body)
|
130
130
|
end
|
131
131
|
end
|
132
132
|
|
133
133
|
def create_select_command(command)
|
134
|
-
|
134
|
+
create_message("select", command.arguments)
|
135
135
|
end
|
136
136
|
end
|
137
137
|
end
|
data/sample/cluster/catalog.json
CHANGED
@@ -35,9 +35,27 @@
|
|
35
35
|
}
|
36
36
|
}
|
37
37
|
}
|
38
|
+
},
|
39
|
+
"Watch": {
|
40
|
+
"workers": 2,
|
41
|
+
"plugins": ["search", "groonga", "add", "watch"],
|
42
|
+
"number_of_replicas": 1,
|
43
|
+
"number_of_partitions": 1,
|
44
|
+
"partition_key": "_key",
|
45
|
+
"date_range": "infinity",
|
46
|
+
"ring": {
|
47
|
+
"localhost:23041": {
|
48
|
+
"weight": 50,
|
49
|
+
"partitions": {
|
50
|
+
"2013-09-01": [
|
51
|
+
"localhost:23003/droonga.watch"
|
52
|
+
]
|
53
|
+
}
|
54
|
+
}
|
55
|
+
}
|
38
56
|
}
|
39
57
|
},
|
40
58
|
"options": {
|
41
|
-
"plugins": ["
|
59
|
+
"plugins": ["groonga"]
|
42
60
|
}
|
43
61
|
}
|
@@ -0,0 +1,41 @@
|
|
1
|
+
#@disable-logging
|
2
|
+
{
|
3
|
+
"type": "table_create",
|
4
|
+
"dataset": "Droonga",
|
5
|
+
"body": {
|
6
|
+
"name" : "Event",
|
7
|
+
"flags" : "TABLE_HASH_KEY"
|
8
|
+
}
|
9
|
+
}
|
10
|
+
{
|
11
|
+
"type": "column_create",
|
12
|
+
"dataset": "Droonga",
|
13
|
+
"body": {
|
14
|
+
"table" : "Event",
|
15
|
+
"name" : "start",
|
16
|
+
"type" : "Time"
|
17
|
+
}
|
18
|
+
}
|
19
|
+
{
|
20
|
+
"dataset": "Droonga",
|
21
|
+
"type": "add",
|
22
|
+
"body": {
|
23
|
+
"table": "Event",
|
24
|
+
"key": "Groonga Night 3",
|
25
|
+
"values": {
|
26
|
+
"start": 1354183200
|
27
|
+
}
|
28
|
+
}
|
29
|
+
}
|
30
|
+
{
|
31
|
+
"dataset": "Droonga",
|
32
|
+
"type": "add",
|
33
|
+
"body": {
|
34
|
+
"table": "Event",
|
35
|
+
"key": "Groonga Night 4",
|
36
|
+
"values": {
|
37
|
+
"start": "2013-11-29T08:00:00Z"
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
41
|
+
#@enable-logging
|
data/test/command/run-test.rb
CHANGED
@@ -29,7 +29,7 @@ lib_dir = File.expand_path(File.join(base_dir, "..", "..", "lib"))
|
|
29
29
|
|
30
30
|
drntest_options = []
|
31
31
|
drntest_options.concat(["--fluentd-options", "-I#{lib_dir}"])
|
32
|
-
drntest_options.concat(["--base", base_dir])
|
33
|
-
drntest_options.concat(ARGV
|
32
|
+
drntest_options.concat(["--base-path", base_dir])
|
33
|
+
drntest_options.concat(ARGV)
|
34
34
|
|
35
35
|
run("bundle", "exec", "drntest", *drntest_options)
|
@@ -0,0 +1,20 @@
|
|
1
|
+
[
|
2
|
+
"droonga.message",
|
3
|
+
0,
|
4
|
+
{
|
5
|
+
"inReplyTo": null,
|
6
|
+
"statusCode": 400,
|
7
|
+
"type": "add.result",
|
8
|
+
"body": {
|
9
|
+
"name": "InvalidValue",
|
10
|
+
"message": "The column \"age\" cannot store the value \"secret\".",
|
11
|
+
"detail": {
|
12
|
+
"table": "User",
|
13
|
+
"key": "key",
|
14
|
+
"values": {
|
15
|
+
"age": "secret"
|
16
|
+
}
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
]
|
@@ -0,0 +1,20 @@
|
|
1
|
+
[
|
2
|
+
"droonga.message",
|
3
|
+
0,
|
4
|
+
{
|
5
|
+
"inReplyTo": null,
|
6
|
+
"statusCode": 400,
|
7
|
+
"type": "add.result",
|
8
|
+
"body": {
|
9
|
+
"name": "InvalidValue",
|
10
|
+
"message": "The column \"birthday\" cannot store the value \"today\".",
|
11
|
+
"detail": {
|
12
|
+
"table": "User",
|
13
|
+
"key": "key",
|
14
|
+
"values": {
|
15
|
+
"birthday": "today"
|
16
|
+
}
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
]
|
@@ -0,0 +1,13 @@
|
|
1
|
+
[
|
2
|
+
"droonga.message",
|
3
|
+
0,
|
4
|
+
{
|
5
|
+
"inReplyTo": null,
|
6
|
+
"statusCode": 400,
|
7
|
+
"type": "add.result",
|
8
|
+
"body": {
|
9
|
+
"name": "MissingPrimaryKeyParameter",
|
10
|
+
"message": "\"key\" must be specified. The table \"User\" requires a primary key for a new record."
|
11
|
+
}
|
12
|
+
}
|
13
|
+
]
|
@@ -0,0 +1,20 @@
|
|
1
|
+
[
|
2
|
+
"droonga.message",
|
3
|
+
0,
|
4
|
+
{
|
5
|
+
"inReplyTo": null,
|
6
|
+
"statusCode": 404,
|
7
|
+
"type": "add.result",
|
8
|
+
"body": {
|
9
|
+
"name": "UnknownColumn",
|
10
|
+
"message": "The column \"unknown\" does not exist in the table \"User\".",
|
11
|
+
"detail": {
|
12
|
+
"table": "User",
|
13
|
+
"key": "key",
|
14
|
+
"values": {
|
15
|
+
"unknown": "unknown"
|
16
|
+
}
|
17
|
+
}
|
18
|
+
}
|
19
|
+
}
|
20
|
+
]
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#@include fixture/user-table.jsons
|
2
|
+
{
|
3
|
+
"type": "add",
|
4
|
+
"dataset": "Droonga",
|
5
|
+
"body": {
|
6
|
+
"table": "Unknown",
|
7
|
+
"key": "key",
|
8
|
+
"values": {
|
9
|
+
"name": "Alice",
|
10
|
+
"age": 20,
|
11
|
+
"email": [
|
12
|
+
"alice@example.com",
|
13
|
+
"alice@example.jp"
|
14
|
+
]
|
15
|
+
}
|
16
|
+
}
|
17
|
+
}
|
@@ -0,0 +1,13 @@
|
|
1
|
+
[
|
2
|
+
"droonga.message",
|
3
|
+
0,
|
4
|
+
{
|
5
|
+
"inReplyTo": null,
|
6
|
+
"statusCode": 400,
|
7
|
+
"type": "unknown-command.result",
|
8
|
+
"body": {
|
9
|
+
"name": "UnknownCommand",
|
10
|
+
"message": "The command \"unknown-command\" is not available for the dataset \"Droonga\"."
|
11
|
+
}
|
12
|
+
}
|
13
|
+
]
|