fluent-plugin-droonga 1.0.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. data/.travis.yml +1 -1
  2. data/Gemfile +1 -1
  3. data/fluent-plugin-droonga.gemspec +1 -1
  4. data/lib/droonga/catalog/collection_volume.rb +97 -0
  5. data/lib/droonga/catalog/dataset.rb +28 -1
  6. data/lib/droonga/catalog/errors.rb +28 -9
  7. data/lib/droonga/catalog/schema.rb +23 -2
  8. data/lib/droonga/catalog/single_volume.rb +28 -0
  9. data/lib/droonga/catalog/slice.rb +43 -0
  10. data/lib/droonga/catalog/version1.rb +3 -3
  11. data/lib/droonga/catalog/version2.rb +17 -81
  12. data/lib/droonga/catalog/version2_validator.rb +63 -0
  13. data/lib/droonga/catalog/volume.rb +33 -0
  14. data/lib/droonga/catalog/volume_collection.rb +56 -0
  15. data/lib/droonga/catalog_observer.rb +7 -19
  16. data/lib/droonga/collectors.rb +1 -1
  17. data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
  18. data/lib/droonga/dispatcher.rb +24 -18
  19. data/lib/droonga/distributed_command_planner.rb +7 -11
  20. data/lib/droonga/distributor.rb +29 -17
  21. data/lib/droonga/event_loop.rb +2 -11
  22. data/lib/droonga/fluent_message_sender.rb +51 -5
  23. data/lib/droonga/handler_runner.rb +1 -1
  24. data/lib/droonga/job_protocol.rb +20 -0
  25. data/lib/droonga/job_pusher.rb +178 -0
  26. data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
  27. data/lib/droonga/message_matcher.rb +18 -15
  28. data/lib/droonga/planner.rb +2 -3
  29. data/lib/droonga/plugins/crud.rb +1 -1
  30. data/lib/droonga/plugins/groonga/column_create.rb +4 -1
  31. data/lib/droonga/plugins/groonga/table_create.rb +1 -1
  32. data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
  33. data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
  34. data/lib/droonga/processor.rb +3 -3
  35. data/lib/droonga/reducer.rb +15 -12
  36. data/lib/droonga/searcher.rb +49 -4
  37. data/lib/droonga/server.rb +2 -0
  38. data/lib/droonga/single_step.rb +22 -7
  39. data/lib/droonga/slice.rb +7 -7
  40. data/lib/droonga/step_runner.rb +3 -2
  41. data/lib/droonga/worker.rb +10 -8
  42. data/test/command/suite/add/dimension/column.catalog.json +27 -0
  43. data/test/command/suite/add/dimension/column.expected +57 -0
  44. data/test/command/suite/add/dimension/column.test +51 -0
  45. data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
  46. data/test/command/suite/search/adjusters/multiple.expected +23 -0
  47. data/test/command/suite/search/adjusters/multiple.test +75 -0
  48. data/test/command/suite/search/adjusters/one.catalog.json +38 -0
  49. data/test/command/suite/search/adjusters/one.expected +23 -0
  50. data/test/command/suite/search/adjusters/one.test +66 -0
  51. data/test/command/suite/search/attributes/array.test +0 -2
  52. data/test/command/suite/search/attributes/hash.test +0 -2
  53. data/test/command/suite/search/complex.test +0 -2
  54. data/test/command/suite/search/condition/nested.test +0 -2
  55. data/test/command/suite/search/condition/query.test +0 -2
  56. data/test/command/suite/search/condition/script.test +0 -2
  57. data/test/command/suite/search/group/string.test +0 -4
  58. data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
  59. data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
  60. data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
  61. data/test/command/suite/search/multiple/chained.test +0 -4
  62. data/test/command/suite/search/multiple/parallel.test +0 -4
  63. data/test/command/suite/search/range/only-output.test +0 -2
  64. data/test/command/suite/search/range/only-sort.test +0 -2
  65. data/test/command/suite/search/range/sort-and-output.test +0 -2
  66. data/test/command/suite/search/range/too-large-output-offset.test +0 -2
  67. data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
  68. data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
  69. data/test/command/suite/search/response/elapsed_time.expected +15 -0
  70. data/test/command/suite/search/response/elapsed_time.test +26 -0
  71. data/test/command/suite/search/response/records/value/time.test +0 -2
  72. data/test/command/suite/search/simple.test +0 -2
  73. data/test/command/suite/search/sort/default-offset-limit.test +0 -2
  74. data/test/command/suite/search/sort/invisible-column.test +0 -2
  75. data/test/unit/catalog/test_collection_volume.rb +103 -0
  76. data/test/unit/catalog/test_dataset.rb +69 -8
  77. data/test/unit/catalog/test_schema.rb +63 -23
  78. data/test/unit/catalog/test_single_volume.rb +31 -0
  79. data/test/unit/catalog/test_slice.rb +92 -0
  80. data/test/unit/catalog/test_version1.rb +1 -1
  81. data/test/unit/catalog/test_version2.rb +1 -32
  82. data/test/unit/catalog/test_version2_validator.rb +66 -0
  83. data/test/unit/catalog/test_volume_collection.rb +50 -0
  84. data/test/unit/plugins/groonga/test_column_create.rb +4 -1
  85. data/test/unit/plugins/groonga/test_table_create.rb +1 -1
  86. data/test/unit/test_message_matcher.rb +15 -15
  87. data/test/unit/test_watch_schema.rb +1 -1
  88. metadata +107 -94
  89. checksums.yaml +0 -7
  90. data/lib/droonga/message_pusher.rb +0 -64
data/test/command/suite/add/dimension/column.test ADDED
@@ -0,0 +1,51 @@
1
+ #@require-catalog-version 2
2
+ {
3
+ "type": "add",
4
+ "dataset": "Droonga",
5
+ "body": {
6
+ "table": "Products",
7
+ "key": "Groonga",
8
+ "values": {
9
+ "category": "groonga"
10
+ }
11
+ }
12
+ }
13
+ {
14
+ "type": "add",
15
+ "dataset": "Droonga",
16
+ "body": {
17
+ "table": "Products",
18
+ "key": "Rroonga",
19
+ "values": {
20
+ "category": "groonga"
21
+ }
22
+ }
23
+ }
24
+ {
25
+ "type": "add",
26
+ "dataset": "Droonga",
27
+ "body": {
28
+ "table": "Products",
29
+ "key": "Ruby",
30
+ "values": {
31
+ "category": "ruby"
32
+ }
33
+ }
34
+ }
35
+ {
36
+ "type": "search",
37
+ "dataset": "Droonga",
38
+ "body": {
39
+ "queries": {
40
+ "products": {
41
+ "source": "Products",
42
+ "sortBy": ["_key"],
43
+ "output": {
44
+ "elements": ["records"],
45
+ "attributes": ["_key", "category"],
46
+ "limit": -1
47
+ }
48
+ }
49
+ }
50
+ }
51
+ }
data/test/command/suite/search/adjusters/multiple.catalog.json ADDED
@@ -0,0 +1,38 @@
1
+ {
2
+ "datasets": {
3
+ "Droonga": {
4
+ "fact": "Memos",
5
+ "schema": {
6
+ "Memos": {
7
+ "type": "Hash",
8
+ "keyType": "ShortText",
9
+ "columns": {
10
+ "tags": {
11
+ "type": "Vector",
12
+ "valueType": "ShortText",
13
+ "vectorOptions": {
14
+ "weight": true
15
+ }
16
+ }
17
+ }
18
+ },
19
+ "Tags": {
20
+ "type": "Hash",
21
+ "keyType": "ShortText",
22
+ "columns": {
23
+ "memos_index": {
24
+ "type": "Index",
25
+ "valueType": "Memos",
26
+ "indexOptions": {
27
+ "sources": [
28
+ "tags"
29
+ ],
30
+ "weight": true
31
+ }
32
+ }
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+ }
data/test/command/suite/search/adjusters/multiple.expected ADDED
@@ -0,0 +1,23 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": "request-id",
6
+ "statusCode": 200,
7
+ "type": "search.result",
8
+ "body": {
9
+ "memos": {
10
+ "records": [
11
+ [
12
+ "Droonga is fun.",
13
+ 61
14
+ ],
15
+ [
16
+ "Groonga is fast!",
17
+ 21
18
+ ]
19
+ ]
20
+ }
21
+ }
22
+ }
23
+ ]
data/test/command/suite/search/adjusters/multiple.test ADDED
@@ -0,0 +1,75 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Memos",
9
+ "key": "Droonga is fun.",
10
+ "values": {
11
+ "tags": [
12
+ {
13
+ "value": "droonga",
14
+ "weight": 9
15
+ },
16
+ {
17
+ "value": "groonga",
18
+ "weight": 4
19
+ }
20
+ ]
21
+ }
22
+ }
23
+ }
24
+ {
25
+ "type": "add",
26
+ "dataset": "Droonga",
27
+ "body": {
28
+ "table": "Memos",
29
+ "key": "Groonga is fast!",
30
+ "values": {
31
+ "tags": [
32
+ {
33
+ "value": "groonga",
34
+ "weight": 9
35
+ }
36
+ ]
37
+ }
38
+ }
39
+ }
40
+ #@enable-logging
41
+ {
42
+ "type": "search",
43
+ "dataset": "Droonga",
44
+ "body": {
45
+ "queries": {
46
+ "memos": {
47
+ "source": "Memos",
48
+ "condition": "true",
49
+ "adjusters": [
50
+ {
51
+ "column": "tags",
52
+ "value": "droonga",
53
+ "factor": 5
54
+ },
55
+ {
56
+ "column": "tags",
57
+ "value": "groonga",
58
+ "factor": 2
59
+ }
60
+ ],
61
+ "sortBy": ["-_score"],
62
+ "output": {
63
+ "elements": [
64
+ "records"
65
+ ],
66
+ "limit": 10,
67
+ "attributes": [
68
+ "_key",
69
+ "_score"
70
+ ]
71
+ }
72
+ }
73
+ }
74
+ }
75
+ }
data/test/command/suite/search/adjusters/one.catalog.json ADDED
@@ -0,0 +1,38 @@
1
+ {
2
+ "datasets": {
3
+ "Droonga": {
4
+ "fact": "Memos",
5
+ "schema": {
6
+ "Memos": {
7
+ "type": "Hash",
8
+ "keyType": "ShortText",
9
+ "columns": {
10
+ "tags": {
11
+ "type": "Vector",
12
+ "valueType": "ShortText",
13
+ "vectorOptions": {
14
+ "weight": true
15
+ }
16
+ }
17
+ }
18
+ },
19
+ "Tags": {
20
+ "type": "Hash",
21
+ "keyType": "ShortText",
22
+ "columns": {
23
+ "memos_index": {
24
+ "type": "Index",
25
+ "valueType": "Memos",
26
+ "indexOptions": {
27
+ "sources": [
28
+ "tags"
29
+ ],
30
+ "weight": true
31
+ }
32
+ }
33
+ }
34
+ }
35
+ }
36
+ }
37
+ }
38
+ }
data/test/command/suite/search/adjusters/one.expected ADDED
@@ -0,0 +1,23 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": "request-id",
6
+ "statusCode": 200,
7
+ "type": "search.result",
8
+ "body": {
9
+ "memos": {
10
+ "records": [
11
+ [
12
+ "Droonga is fun.",
13
+ 51
14
+ ],
15
+ [
16
+ "Groonga is fast!",
17
+ 1
18
+ ]
19
+ ]
20
+ }
21
+ }
22
+ }
23
+ ]
data/test/command/suite/search/adjusters/one.test ADDED
@@ -0,0 +1,66 @@
1
+ # -*- js -*-
2
+ #@require-catalog-version 2
3
+ #@disable-logging
4
+ {
5
+ "type": "add",
6
+ "dataset": "Droonga",
7
+ "body": {
8
+ "table": "Memos",
9
+ "key": "Droonga is fun.",
10
+ "values": {
11
+ "tags": [
12
+ {
13
+ "value": "droonga",
14
+ "weight": 9
15
+ }
16
+ ]
17
+ }
18
+ }
19
+ }
20
+ {
21
+ "type": "add",
22
+ "dataset": "Droonga",
23
+ "body": {
24
+ "table": "Memos",
25
+ "key": "Groonga is fast!",
26
+ "values": {
27
+ "tags": [
28
+ {
29
+ "value": "groonga",
30
+ "weight": 9
31
+ }
32
+ ]
33
+ }
34
+ }
35
+ }
36
+ #@enable-logging
37
+ {
38
+ "type": "search",
39
+ "dataset": "Droonga",
40
+ "body": {
41
+ "queries": {
42
+ "memos": {
43
+ "source": "Memos",
44
+ "condition": "true",
45
+ "adjusters": [
46
+ {
47
+ "column": "tags",
48
+ "value": "droonga",
49
+ "factor": 5
50
+ }
51
+ ],
52
+ "sortBy": ["-_score"],
53
+ "output": {
54
+ "elements": [
55
+ "records"
56
+ ],
57
+ "limit": 10,
58
+ "attributes": [
59
+ "_key",
60
+ "_score"
61
+ ]
62
+ }
63
+ }
64
+ }
65
+ }
66
+ }
data/test/command/suite/search/attributes/array.test CHANGED
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
data/test/command/suite/search/attributes/hash.test CHANGED
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
data/test/command/suite/search/complex.test CHANGED
@@ -9,8 +9,6 @@
9
9
  "sortBy": ["_key"],
10
10
  "output": {
11
11
  "elements": [
12
- "startTime",
13
- "elapsedTime",
14
12
  "count",
15
13
  "attributes",
16
14
  "records"
data/test/command/suite/search/condition/nested.test CHANGED
@@ -14,8 +14,6 @@
14
14
  "sortBy": ["_key"],
15
15
  "output": {
16
16
  "elements": [
17
- "startTime",
18
- "elapsedTime",
19
17
  "count",
20
18
  "attributes",
21
19
  "records"
data/test/command/suite/search/condition/query.test CHANGED
@@ -10,8 +10,6 @@
10
10
  "sortBy": ["_key"],
11
11
  "output": {
12
12
  "elements": [
13
- "startTime",
14
- "elapsedTime",
15
13
  "count",
16
14
  "attributes",
17
15
  "records"
data/test/command/suite/search/condition/script.test CHANGED
@@ -13,8 +13,6 @@
13
13
  "sortBy": ["_key"],
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
data/test/command/suite/search/group/string.test CHANGED
@@ -13,8 +13,6 @@
13
13
  "sortBy": ["_key"],
14
14
  "output": {
15
15
  "elements": [
16
- "startTime",
17
- "elapsedTime",
18
16
  "count",
19
17
  "attributes",
20
18
  "records"
@@ -29,8 +27,6 @@
29
27
  "sortBy": ["-_key"],
30
28
  "output": {
31
29
  "elements": [
32
- "startTime",
33
- "elapsedTime",
34
30
  "count",
35
31
  "attributes",
36
32
  "records"
data/test/command/suite/search/group/subrecord/with-sort.catalog.json ADDED
@@ -0,0 +1,33 @@
1
+ {
2
+ "datasets": {
3
+ "Droonga": {
4
+ "fact": "Comments",
5
+ "schema": {
6
+ "Threads": {
7
+ "type": "Hash",
8
+ "keyType": "ShortText",
9
+ "columns": {
10
+ "name": {
11
+ "type": "Scalar",
12
+ "valueType": "ShortText"
13
+ }
14
+ }
15
+ },
16
+ "Comments": {
17
+ "type": "Hash",
18
+ "keyType": "ShortText",
19
+ "columns": {
20
+ "thread": {
21
+ "type": "Scalar",
22
+ "valueType": "Threads"
23
+ },
24
+ "content": {
25
+ "type": "Scalar",
26
+ "valueType": "Text"
27
+ }
28
+ }
29
+ }
30
+ }
31
+ }
32
+ }
33
+ }
data/test/command/suite/search/group/subrecord/with-sort.expected ADDED
@@ -0,0 +1,34 @@
1
+ [
2
+ "droonga.message",
3
+ 0,
4
+ {
5
+ "inReplyTo": "request-id",
6
+ "statusCode": 200,
7
+ "type": "search.result",
8
+ "body": {
9
+ "comments": {
10
+ "count": 2,
11
+ "records": [
12
+ [
13
+ "groonga",
14
+ 1,
15
+ [
16
+ [
17
+ "Groonga is great!"
18
+ ]
19
+ ]
20
+ ],
21
+ [
22
+ "ruby",
23
+ 1,
24
+ [
25
+ [
26
+ "Hi! I'm a new Rubyist!"
27
+ ]
28
+ ]
29
+ ]
30
+ ]
31
+ }
32
+ }
33
+ }
34
+ ]