fluent-plugin-droonga 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -4
  3. data/benchmark/watch/benchmark-notify.rb +2 -2
  4. data/benchmark/watch/benchmark-scan.rb +3 -0
  5. data/benchmark/watch/fluentd.conf +0 -1
  6. data/fluent-plugin-droonga.gemspec +2 -3
  7. data/lib/droonga/catalog.rb +10 -124
  8. data/lib/droonga/catalog/base.rb +140 -0
  9. data/lib/droonga/catalog/version1.rb +23 -0
  10. data/lib/droonga/catalog_loader.rb +33 -0
  11. data/lib/droonga/collector.rb +2 -71
  12. data/lib/droonga/collector_plugin.rb +2 -34
  13. data/lib/droonga/dispatcher.rb +141 -196
  14. data/lib/droonga/distribution_planner.rb +76 -0
  15. data/lib/droonga/distributor.rb +5 -7
  16. data/lib/droonga/distributor_plugin.rb +23 -15
  17. data/lib/droonga/engine.rb +2 -2
  18. data/lib/droonga/event_loop.rb +46 -0
  19. data/lib/droonga/farm.rb +9 -5
  20. data/lib/droonga/fluent_message_sender.rb +84 -0
  21. data/lib/droonga/forwarder.rb +43 -53
  22. data/lib/droonga/handler.rb +20 -68
  23. data/lib/droonga/handler_message.rb +61 -0
  24. data/lib/droonga/handler_messenger.rb +92 -0
  25. data/lib/droonga/handler_plugin.rb +10 -12
  26. data/lib/droonga/input_adapter.rb +52 -0
  27. data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
  28. data/lib/droonga/input_message.rb +11 -11
  29. data/lib/droonga/logger.rb +4 -3
  30. data/lib/droonga/message_pack_packer.rb +62 -0
  31. data/lib/droonga/message_processing_error.rb +54 -0
  32. data/lib/droonga/message_pusher.rb +60 -0
  33. data/lib/droonga/message_receiver.rb +61 -0
  34. data/lib/droonga/output_adapter.rb +53 -0
  35. data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
  36. data/lib/droonga/output_message.rb +37 -0
  37. data/lib/droonga/partition.rb +27 -5
  38. data/lib/droonga/pluggable.rb +9 -4
  39. data/lib/droonga/plugin.rb +12 -3
  40. data/lib/droonga/plugin/collector/basic.rb +91 -18
  41. data/lib/droonga/plugin/distributor/crud.rb +9 -9
  42. data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
  43. data/lib/droonga/plugin/distributor/groonga.rb +5 -5
  44. data/lib/droonga/plugin/distributor/search.rb +4 -246
  45. data/lib/droonga/plugin/distributor/watch.rb +11 -6
  46. data/lib/droonga/plugin/handler/add.rb +69 -7
  47. data/lib/droonga/plugin/handler/groonga.rb +6 -6
  48. data/lib/droonga/plugin/handler/search.rb +5 -3
  49. data/lib/droonga/plugin/handler/watch.rb +19 -13
  50. data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
  51. data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
  52. data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
  53. data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
  54. data/lib/droonga/plugin_loader.rb +2 -2
  55. data/lib/droonga/processor.rb +21 -23
  56. data/lib/droonga/replier.rb +40 -0
  57. data/lib/droonga/searcher.rb +298 -174
  58. data/lib/droonga/server.rb +0 -67
  59. data/lib/droonga/session.rb +85 -0
  60. data/lib/droonga/test.rb +21 -0
  61. data/lib/droonga/test/stub_distributor.rb +31 -0
  62. data/lib/droonga/test/stub_handler.rb +37 -0
  63. data/lib/droonga/test/stub_handler_message.rb +35 -0
  64. data/lib/droonga/test/stub_handler_messenger.rb +34 -0
  65. data/lib/droonga/time_formatter.rb +37 -0
  66. data/lib/droonga/watcher.rb +1 -0
  67. data/lib/droonga/worker.rb +16 -19
  68. data/lib/fluent/plugin/out_droonga.rb +9 -9
  69. data/lib/groonga_command_converter.rb +5 -5
  70. data/sample/cluster/catalog.json +1 -1
  71. data/test/command/config/default/catalog.json +19 -1
  72. data/test/command/fixture/event.jsons +41 -0
  73. data/test/command/fixture/user-table.jsons +9 -0
  74. data/test/command/run-test.rb +2 -2
  75. data/test/command/suite/add/error/invalid-integer.expected +20 -0
  76. data/test/command/suite/add/error/invalid-integer.test +12 -0
  77. data/test/command/suite/add/error/invalid-time.expected +20 -0
  78. data/test/command/suite/add/error/invalid-time.test +12 -0
  79. data/test/command/suite/add/error/missing-key.expected +13 -0
  80. data/test/command/suite/add/error/missing-key.test +16 -0
  81. data/test/command/suite/add/error/missing-table.expected +13 -0
  82. data/test/command/suite/add/error/missing-table.test +16 -0
  83. data/test/command/suite/add/error/unknown-column.expected +20 -0
  84. data/test/command/suite/add/error/unknown-column.test +12 -0
  85. data/test/command/suite/add/error/unknown-table.expected +13 -0
  86. data/test/command/suite/add/error/unknown-table.test +17 -0
  87. data/test/command/suite/add/minimum.expected +1 -3
  88. data/test/command/suite/add/with-values.expected +1 -3
  89. data/test/command/suite/add/without-key.expected +1 -3
  90. data/test/command/suite/message/error/missing-dataset.expected +13 -0
  91. data/test/command/suite/message/error/missing-dataset.test +5 -0
  92. data/test/command/suite/message/error/unknown-command.expected +13 -0
  93. data/test/command/suite/message/error/unknown-command.test +6 -0
  94. data/test/command/suite/message/error/unknown-dataset.expected +13 -0
  95. data/test/command/suite/message/error/unknown-dataset.test +6 -0
  96. data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
  97. data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
  98. data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
  99. data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
  100. data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
  101. data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
  102. data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
  103. data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
  104. data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
  105. data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
  106. data/test/command/suite/search/error/cyclic-source.expected +18 -0
  107. data/test/command/suite/search/error/cyclic-source.test +12 -0
  108. data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
  109. data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
  110. data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
  111. data/test/command/suite/search/error/missing-source-parameter.test +11 -0
  112. data/test/command/suite/search/error/unknown-source.expected +18 -0
  113. data/test/command/suite/search/error/unknown-source.test +12 -0
  114. data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
  115. data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
  116. data/test/command/suite/search/group/limit.expected +19 -0
  117. data/test/command/suite/search/group/limit.test +20 -0
  118. data/test/command/suite/search/group/string.expected +36 -0
  119. data/test/command/suite/search/group/string.test +44 -0
  120. data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
  121. data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
  122. data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
  123. data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
  124. data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
  125. data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
  126. data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
  127. data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
  128. data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
  129. data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
  130. data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
  131. data/test/command/suite/search/range/too-large-output-offset.test +25 -0
  132. data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
  133. data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
  134. data/test/command/suite/search/response/records/value/time.expected +24 -0
  135. data/test/command/suite/search/response/records/value/time.test +24 -0
  136. data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
  137. data/test/command/suite/search/sort/default-offset-limit.test +26 -0
  138. data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
  139. data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
  140. data/test/command/suite/watch/subscribe.expected +12 -0
  141. data/test/command/suite/watch/subscribe.test +9 -0
  142. data/test/command/suite/watch/unsubscribe.expected +12 -0
  143. data/test/command/suite/watch/unsubscribe.test +9 -0
  144. data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
  145. data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
  146. data/test/unit/helper.rb +2 -0
  147. data/test/unit/plugin/collector/test_basic.rb +289 -33
  148. data/test/unit/plugin/distributor/test_search.rb +176 -861
  149. data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
  150. data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
  151. data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
  152. data/test/unit/plugin/handler/test_add.rb +74 -11
  153. data/test/unit/plugin/handler/test_groonga.rb +15 -1
  154. data/test/unit/plugin/handler/test_search.rb +33 -17
  155. data/test/unit/plugin/handler/test_watch.rb +43 -27
  156. data/test/unit/run-test.rb +2 -0
  157. data/test/unit/test_message_pack_packer.rb +51 -0
  158. data/test/unit/test_time_formatter.rb +29 -0
  159. metadata +208 -110
  160. data/lib/droonga/job_queue.rb +0 -87
  161. data/lib/droonga/job_queue_schema.rb +0 -65
  162. data/test/unit/test_adapter.rb +0 -51
  163. data/test/unit/test_job_queue_schema.rb +0 -45
@@ -67,7 +67,7 @@ module Droonga
67
67
  end
68
68
 
69
69
  private
70
- def create_envelope(type, body)
70
+ def create_message(type, body)
71
71
  id = @options[:id]
72
72
  if id.nil?
73
73
  id = new_unique_id
@@ -99,11 +99,11 @@ module Droonga
99
99
  end
100
100
 
101
101
  def create_table_create_command(command)
102
- create_envelope("table_create", command.arguments)
102
+ create_message("table_create", command.arguments)
103
103
  end
104
104
 
105
105
  def create_column_create_command(command)
106
- create_envelope("column_create", command.arguments)
106
+ create_message("column_create", command.arguments)
107
107
  end
108
108
 
109
109
  def split_load_command_to_add_commands(command, &block)
@@ -126,12 +126,12 @@ module Droonga
126
126
  end
127
127
  body[:values] = record_values unless record_values.empty?
128
128
 
129
- yield create_envelope("add", body)
129
+ yield create_message("add", body)
130
130
  end
131
131
  end
132
132
 
133
133
  def create_select_command(command)
134
- create_envelope("select", command.arguments)
134
+ create_message("select", command.arguments)
135
135
  end
136
136
  end
137
137
  end
@@ -38,6 +38,6 @@
38
38
  }
39
39
  },
40
40
  "options": {
41
- "plugins": ["select"]
41
+ "plugins": ["groonga"]
42
42
  }
43
43
  }
@@ -35,9 +35,27 @@
35
35
  }
36
36
  }
37
37
  }
38
+ },
39
+ "Watch": {
40
+ "workers": 2,
41
+ "plugins": ["search", "groonga", "add", "watch"],
42
+ "number_of_replicas": 1,
43
+ "number_of_partitions": 1,
44
+ "partition_key": "_key",
45
+ "date_range": "infinity",
46
+ "ring": {
47
+ "localhost:23041": {
48
+ "weight": 50,
49
+ "partitions": {
50
+ "2013-09-01": [
51
+ "localhost:23003/droonga.watch"
52
+ ]
53
+ }
54
+ }
55
+ }
38
56
  }
39
57
  },
40
58
  "options": {
41
- "plugins": ["select"]
59
+ "plugins": ["groonga"]
42
60
  }
43
61
  }
@@ -0,0 +1,41 @@
1
+ #@disable-logging
2
+ {
3
+ "type": "table_create",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "name" : "Event",
7
+ "flags" : "TABLE_HASH_KEY"
8
+ }
9
+ }
10
+ {
11
+ "type": "column_create",
12
+ "dataset": "Droonga",
13
+ "body": {
14
+ "table" : "Event",
15
+ "name" : "start",
16
+ "type" : "Time"
17
+ }
18
+ }
19
+ {
20
+ "dataset": "Droonga",
21
+ "type": "add",
22
+ "body": {
23
+ "table": "Event",
24
+ "key": "Groonga Night 3",
25
+ "values": {
26
+ "start": 1354183200
27
+ }
28
+ }
29
+ }
30
+ {
31
+ "dataset": "Droonga",
32
+ "type": "add",
33
+ "body": {
34
+ "table": "Event",
35
+ "key": "Groonga Night 4",
36
+ "values": {
37
+ "start": "2013-11-29T08:00:00Z"
38
+ }
39
+ }
40
+ }
41
+ #@enable-logging
@@ -35,4 +35,13 @@
35
35
  "flags" : "COLUMN_VECTOR"
36
36
  }
37
37
  }
38
+ {
39
+ "type": "column_create",
40
+ "dataset": "Droonga",
41
+ "body": {
42
+ "table" : "User",
43
+ "name" : "birthday",
44
+ "type" : "Time"
45
+ }
46
+ }
38
47
  #@enable-logging
@@ -29,7 +29,7 @@ lib_dir = File.expand_path(File.join(base_dir, "..", "..", "lib"))
29
29
 
30
30
  drntest_options = []
31
31
  drntest_options.concat(["--fluentd-options", "-I#{lib_dir}"])
32
- drntest_options.concat(["--base", base_dir])
33
- drntest_options.concat(ARGV.dup)
32
+ drntest_options.concat(["--base-path", base_dir])
33
+ drntest_options.concat(ARGV)
34
34
 
35
35
  run("bundle", "exec", "drntest", *drntest_options)
@@ -0,0 +1,20 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "InvalidValue",
10
+ "message": "The column \"age\" cannot store the value \"secret\".",
11
+ "detail": {
12
+ "table": "User",
13
+ "key": "key",
14
+ "values": {
15
+ "age": "secret"
16
+ }
17
+ }
18
+ }
19
+ }
20
+ ]
@@ -0,0 +1,12 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "User",
7
+ "key": "key",
8
+ "values": {
9
+ "age": "secret"
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,20 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "InvalidValue",
10
+ "message": "The column \"birthday\" cannot store the value \"today\".",
11
+ "detail": {
12
+ "table": "User",
13
+ "key": "key",
14
+ "values": {
15
+ "birthday": "today"
16
+ }
17
+ }
18
+ }
19
+ }
20
+ ]
@@ -0,0 +1,12 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "User",
7
+ "key": "key",
8
+ "values": {
9
+ "birthday": "today"
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "MissingPrimaryKeyParameter",
10
+ "message": "\"key\" must be specified. The table \"User\" requires a primary key for a new record."
11
+ }
12
+ }
13
+ ]
@@ -0,0 +1,16 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "User",
7
+ "values": {
8
+ "name": "Alice",
9
+ "age": 20,
10
+ "email": [
11
+ "alice@example.com",
12
+ "alice@example.jp"
13
+ ]
14
+ }
15
+ }
16
+ }
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "MissingTableParameter",
10
+ "message": "\"table\" must be specified."
11
+ }
12
+ }
13
+ ]
@@ -0,0 +1,16 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "key": "key",
7
+ "values": {
8
+ "name": "Alice",
9
+ "age": 20,
10
+ "email": [
11
+ "alice@example.com",
12
+ "alice@example.jp"
13
+ ]
14
+ }
15
+ }
16
+ }
@@ -0,0 +1,20 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 404,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "UnknownColumn",
10
+ "message": "The column \"unknown\" does not exist in the table \"User\".",
11
+ "detail": {
12
+ "table": "User",
13
+ "key": "key",
14
+ "values": {
15
+ "unknown": "unknown"
16
+ }
17
+ }
18
+ }
19
+ }
20
+ ]
@@ -0,0 +1,12 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "User",
7
+ "key": "key",
8
+ "values": {
9
+ "unknown": "unknown"
10
+ }
11
+ }
12
+ }
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 404,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "UnknownTable",
10
+ "message": "The table \"Unknown\" does not exist in the dataset."
11
+ }
12
+ }
13
+ ]
@@ -0,0 +1,17 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "Unknown",
7
+ "key": "key",
8
+ "values": {
9
+ "name": "Alice",
10
+ "age": 20,
11
+ "email": [
12
+ "alice@example.com",
13
+ "alice@example.jp"
14
+ ]
15
+ }
16
+ }
17
+ }
@@ -5,8 +5,6 @@
5
5
  "inReplyTo": null,
6
6
  "statusCode": 200,
7
7
  "type": "add.result",
8
- "body": [
9
- true
10
- ]
8
+ "body": true
11
9
  }
12
10
  ]
@@ -5,8 +5,6 @@
5
5
  "inReplyTo": null,
6
6
  "statusCode": 200,
7
7
  "type": "add.result",
8
- "body": [
9
- true
10
- ]
8
+ "body": true
11
9
  }
12
10
  ]
@@ -5,8 +5,6 @@
5
5
  "inReplyTo": null,
6
6
  "statusCode": 200,
7
7
  "type": "add.result",
8
- "body": [
9
- true
10
- ]
8
+ "body": true
11
9
  }
12
10
  ]
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "MissingDatasetParameter",
10
+ "message": "\"dataset\" must be specified."
11
+ }
12
+ }
13
+ ]
@@ -0,0 +1,5 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "add",
4
+ "body": {}
5
+ }
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 400,
7
+ "type": "unknown-command.result",
8
+ "body": {
9
+ "name": "UnknownCommand",
10
+ "message": "The command \"unknown-command\" is not available for the dataset \"Droonga\"."
11
+ }
12
+ }
13
+ ]
@@ -0,0 +1,6 @@
1
+ #@include fixture/user-table.jsons
2
+ {
3
+ "type": "unknown-command",
4
+ "dataset": "Droonga",
5
+ "body": {}
6
+ }
@@ -0,0 +1,13 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": null,
6
+ "statusCode": 404,
7
+ "type": "add.result",
8
+ "body": {
9
+ "name": "UnknownDataset",
10
+ "message": "The dataset \"Unknown\" does not exist."
11
+ }
12
+ }
13
+ ]