droonga-engine 1.1.0 → 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +2 -1
  3. data/Rakefile +6 -0
  4. data/bin/droonga-engine-absorb-data +14 -14
  5. data/bin/droonga-engine-catalog-generate +24 -12
  6. data/bin/droonga-engine-catalog-modify +13 -7
  7. data/bin/droonga-engine-join +8 -8
  8. data/bin/droonga-engine-set-role +1 -1
  9. data/bin/droonga-engine-unjoin +2 -2
  10. data/lib/droonga/address.rb +3 -0
  11. data/lib/droonga/cluster.rb +16 -10
  12. data/lib/droonga/command/droonga_engine_service.rb +5 -2
  13. data/lib/droonga/command/remote_command_base.rb +3 -3
  14. data/lib/droonga/distributed_command_planner.rb +11 -1
  15. data/lib/droonga/engine.rb +12 -11
  16. data/lib/droonga/engine/version.rb +2 -2
  17. data/lib/droonga/engine_node.rb +28 -28
  18. data/lib/droonga/engine_state.rb +41 -36
  19. data/lib/droonga/forward_buffer.rb +21 -10
  20. data/lib/droonga/node_role.rb +2 -0
  21. data/lib/droonga/plugins/groonga/select.rb +3 -0
  22. data/lib/droonga/plugins/search.rb +3 -1
  23. data/lib/droonga/plugins/search/distributed_search_planner.rb +17 -5
  24. data/lib/droonga/plugins/system/statistics.rb +1 -0
  25. data/lib/droonga/searcher.rb +13 -4
  26. data/test/command/config/single_slice/catalog.json +38 -0
  27. data/test/command/config/single_slice/droonga-engine.yaml +4 -0
  28. data/test/command/run-test.rb +3 -2
  29. data/test/command/suite/catalog/fetch.expected.single_slice +50 -0
  30. data/test/command/suite/dump/column/index.expected.single_slice +86 -0
  31. data/test/command/suite/dump/column/scalar.expected.single_slice +52 -0
  32. data/test/command/suite/dump/column/vector.expected.single_slice +55 -0
  33. data/test/command/suite/dump/record/scalar.expected.single_slice +52 -0
  34. data/test/command/suite/dump/record/vector/reference.expected.single_slice +117 -0
  35. data/test/command/suite/dump/table/array.expected.single_slice +39 -0
  36. data/test/command/suite/dump/table/double_array_trie.expected.single_slice +40 -0
  37. data/test/command/suite/dump/table/hash.expected.single_slice +40 -0
  38. data/test/command/suite/dump/table/patricia_trie.expected.single_slice +40 -0
  39. data/test/command/suite/message/error/missing-dataset.test +3 -0
  40. data/test/command/suite/search/condition/query/nonexistent_column.expected.single_slice +26 -0
  41. data/test/command/suite/search/condition/query/syntax_error.expected.single_slice +26 -0
  42. data/test/command/suite/search/error/unknown-source.expected.single_slice +28 -0
  43. data/test/command/suite/search/output/attributes/invalid.expected.single_slice +24 -0
  44. data/test/command/suite/system/absorb-data/records.catalog.json.single_slice +44 -0
  45. data/test/command/suite/system/absorb-data/records.expected.single_slice +32 -0
  46. data/test/command/suite/system/statistics/object/count/per-volume/empty.test +1 -0
  47. data/test/command/suite/system/statistics/object/count/record.expected.single_slice +11 -0
  48. data/test/command/suite/system/statistics/object/count/schema.expected.single_slice +11 -0
  49. data/test/unit/catalog/test_generator.rb +3 -2
  50. data/test/unit/helper.rb +2 -1
  51. data/test/unit/helper/stub_serf.rb +28 -0
  52. data/test/unit/plugins/system/statistics/test_object_count.rb +135 -0
  53. data/test/unit/plugins/system/statistics/test_object_count_per_volume.rb +149 -0
  54. data/test/unit/plugins/test_basic.rb +0 -406
  55. data/test/unit/test_address.rb +111 -10
  56. data/test/unit/test_cluster.rb +232 -0
  57. data/test/unit/test_differ.rb +49 -0
  58. data/test/unit/test_engine_node.rb +556 -0
  59. data/test/unit/test_engine_state.rb +151 -0
  60. data/test/unit/test_forward_buffer.rb +106 -0
  61. data/test/unit/test_node_name.rb +160 -0
  62. data/test/unit/test_node_role.rb +53 -0
  63. data/test/unit/test_reducer.rb +525 -0
  64. metadata +111 -49
data/lib/droonga/plugins/system/statistics.rb CHANGED
@@ -26,6 +26,7 @@ module Droonga
26
26
  counts(message.request["output"])
27
27
  end
28
28
 
29
+ private
29
30
  def counts(output)
30
31
  counts = {}
31
32
  if output and output.is_a?(Array)
data/lib/droonga/searcher.rb CHANGED
@@ -439,12 +439,17 @@ module Droonga
439
439
  limit = -1
440
440
  when Hash
441
441
  keys = parse_order_keys(sort_by["keys"])
442
- offset = sort_by["offset"]
443
- limit = sort_by["limit"]
442
+ offset = sort_by["offset"] || 0
443
+ limit = sort_by["limit"] || -1
444
444
  else
445
445
  raise '"sortBy" parameter must be a Hash or an Array'
446
446
  end
447
- @records = @records.sort(keys, :offset => offset, :limit => limit)
447
+ if offset >= @records.size
448
+ # to avoid Groonga::TooLargeOffset error, we have to create a blank result manually.
449
+ @records = @records.sort(keys, :offset => 0, :limit => 0)
450
+ else
451
+ @records = @records.sort(keys, :offset => offset, :limit => limit)
452
+ end
448
453
  logger.trace("search_query: sort: done",
449
454
  :by => sort_by)
450
455
  end
@@ -539,9 +544,13 @@ module Droonga
539
544
 
540
545
  module RecordsFormattable
541
546
  def format(output_target_attributes, records, output_limit, output_offset)
547
+ if output_offset >= records.size
548
+ # to avoid Groonga::TooLargeOffset error, we have to create a blank result manually.
549
+ output_offset = output_limit = 0
550
+ end
542
551
  cursor_options = {
543
552
  :offset => output_offset,
544
- :limit => output_limit
553
+ :limit => output_limit,
545
554
  }
546
555
  formatted_records = nil
547
556
  records.open_cursor(cursor_options) do |cursor|
data/test/command/config/single_slice/catalog.json ADDED
@@ -0,0 +1,38 @@
1
+ {
2
+ "version": 2,
3
+ "effectiveDate": "2014-02-28T00:00:00Z",
4
+ "datasets": {
5
+ "Default": {
6
+ "nWorkers": 2,
7
+ "plugins": ["groonga", "crud", "search", "dump", "system", "catalog"],
8
+ "replicas": [
9
+ {
10
+ "dimension": "_key",
11
+ "slicer": "hash",
12
+ "slices": [
13
+ {
14
+ "label": "slice000",
15
+ "weight": 100,
16
+ "volume": {
17
+ "address": "127.0.0.1:23003/droonga.000"
18
+ }
19
+ }
20
+ ]
21
+ },
22
+ {
23
+ "dimension": "_key",
24
+ "slicer": "hash",
25
+ "slices": [
26
+ {
27
+ "label": "slice010",
28
+ "weight": 100,
29
+ "volume": {
30
+ "address": "127.0.0.1:23003/droonga.010"
31
+ }
32
+ }
33
+ ]
34
+ }
35
+ ]
36
+ }
37
+ }
38
+ }
data/test/command/config/single_slice/droonga-engine.yaml ADDED
@@ -0,0 +1,4 @@
1
+ host: 127.0.0.1
2
+ port: 10031
3
+ tag: droonga
4
+ daemon: false
data/test/command/run-test.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
  #
3
- # Copyright (C) 2013 Droonga Project
3
+ # Copyright (C) 2013-2015 Droonga Project
4
4
  #
5
5
  # This library is free software; you can redistribute it and/or
6
6
  # modify it under the terms of the GNU Lesser General Public
@@ -48,4 +48,5 @@ drntest_options.concat(["--droonga-engine-options",
48
48
  "--internal-connection-lifetime=5"].join(" ")])
49
49
  drntest_options.concat(ARGV)
50
50
 
51
- run("bundle", "exec", "drntest", *drntest_options)
51
+ run(RbConfig.ruby, "-S", "bundle", "exec",
52
+ RbConfig.ruby, "-S", "drntest", *drntest_options)
data/test/command/suite/catalog/fetch.expected.single_slice ADDED
@@ -0,0 +1,50 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "statusCode": 200,
4
+ "type": "catalog.fetch.result",
5
+ "body": {
6
+ "version": 2,
7
+ "effectiveDate": "2014-02-28T00:00:00Z",
8
+ "datasets": {
9
+ "Default": {
10
+ "nWorkers": 2,
11
+ "plugins": [
12
+ "groonga",
13
+ "crud",
14
+ "search",
15
+ "dump",
16
+ "system",
17
+ "catalog"
18
+ ],
19
+ "replicas": [
20
+ {
21
+ "dimension": "_key",
22
+ "slicer": "hash",
23
+ "slices": [
24
+ {
25
+ "label": "slice000",
26
+ "weight": 100,
27
+ "volume": {
28
+ "address": "127.0.0.1:23003/droonga.000"
29
+ }
30
+ }
31
+ ]
32
+ },
33
+ {
34
+ "dimension": "_key",
35
+ "slicer": "hash",
36
+ "slices": [
37
+ {
38
+ "label": "slice010",
39
+ "weight": 100,
40
+ "volume": {
41
+ "address": "127.0.0.1:23003/droonga.010"
42
+ }
43
+ }
44
+ ]
45
+ }
46
+ ]
47
+ }
48
+ }
49
+ }
50
+ }
data/test/command/suite/dump/column/index.expected.single_slice ADDED
@@ -0,0 +1,86 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "dataset": "Default",
4
+ "date": "0000-00-00T00:00:00.000000Z",
5
+ "type": "dump.start"
6
+ }
7
+ {
8
+ "inReplyTo": "request-id",
9
+ "statusCode": 200,
10
+ "type": "dump.result",
11
+ "body": {
12
+ "started": true
13
+ }
14
+ }
15
+ {
16
+ "inReplyTo": "request-id",
17
+ "dataset": "Default",
18
+ "date": "0000-00-00T00:00:00.000000Z",
19
+ "body": {
20
+ "nMessages": 4
21
+ },
22
+ "type": "dump.forecast"
23
+ }
24
+ {
25
+ "inReplyTo": "request-id",
26
+ "dataset": "Default",
27
+ "date": "0000-00-00T00:00:00.000000Z",
28
+ "body": {
29
+ "type": "Hash",
30
+ "name": "Memos",
31
+ "keyType": "ShortText"
32
+ },
33
+ "type": "dump.table"
34
+ }
35
+ {
36
+ "inReplyTo": "request-id",
37
+ "dataset": "Default",
38
+ "date": "0000-00-00T00:00:00.000000Z",
39
+ "body": {
40
+ "type": "PatriciaTrie",
41
+ "name": "Terms",
42
+ "keyType": "ShortText",
43
+ "tokenizer": "TokenBigram",
44
+ "normalizer": "NormalizerAuto"
45
+ },
46
+ "type": "dump.table"
47
+ }
48
+ {
49
+ "inReplyTo": "request-id",
50
+ "dataset": "Default",
51
+ "date": "0000-00-00T00:00:00.000000Z",
52
+ "body": {
53
+ "table": "Memos",
54
+ "name": "content",
55
+ "type": "Scalar",
56
+ "valueType": "ShortText"
57
+ },
58
+ "type": "dump.column"
59
+ }
60
+ {
61
+ "inReplyTo": "request-id",
62
+ "dataset": "Default",
63
+ "date": "0000-00-00T00:00:00.000000Z",
64
+ "body": {
65
+ "table": "Terms",
66
+ "name": "Memos__key_content",
67
+ "type": "Index",
68
+ "valueType": "Memos",
69
+ "indexOptions": {
70
+ "section": true,
71
+ "weight": true,
72
+ "position": true,
73
+ "sources": [
74
+ "_key",
75
+ "content"
76
+ ]
77
+ }
78
+ },
79
+ "type": "dump.column"
80
+ }
81
+ {
82
+ "inReplyTo": "request-id",
83
+ "dataset": "Default",
84
+ "date": "0000-00-00T00:00:00.000000Z",
85
+ "type": "dump.end"
86
+ }
data/test/command/suite/dump/column/scalar.expected.single_slice ADDED
@@ -0,0 +1,52 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "dataset": "Default",
4
+ "date": "0000-00-00T00:00:00.000000Z",
5
+ "type": "dump.start"
6
+ }
7
+ {
8
+ "inReplyTo": "request-id",
9
+ "statusCode": 200,
10
+ "type": "dump.result",
11
+ "body": {
12
+ "started": true
13
+ }
14
+ }
15
+ {
16
+ "inReplyTo": "request-id",
17
+ "dataset": "Default",
18
+ "date": "0000-00-00T00:00:00.000000Z",
19
+ "body": {
20
+ "nMessages": 2
21
+ },
22
+ "type": "dump.forecast"
23
+ }
24
+ {
25
+ "inReplyTo": "request-id",
26
+ "dataset": "Default",
27
+ "date": "0000-00-00T00:00:00.000000Z",
28
+ "body": {
29
+ "type": "Hash",
30
+ "name": "Memos",
31
+ "keyType": "ShortText"
32
+ },
33
+ "type": "dump.table"
34
+ }
35
+ {
36
+ "inReplyTo": "request-id",
37
+ "dataset": "Default",
38
+ "date": "0000-00-00T00:00:00.000000Z",
39
+ "body": {
40
+ "table": "Memos",
41
+ "name": "title",
42
+ "type": "Scalar",
43
+ "valueType": "ShortText"
44
+ },
45
+ "type": "dump.column"
46
+ }
47
+ {
48
+ "inReplyTo": "request-id",
49
+ "dataset": "Default",
50
+ "date": "0000-00-00T00:00:00.000000Z",
51
+ "type": "dump.end"
52
+ }
data/test/command/suite/dump/column/vector.expected.single_slice ADDED
@@ -0,0 +1,55 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "dataset": "Default",
4
+ "date": "0000-00-00T00:00:00.000000Z",
5
+ "type": "dump.start"
6
+ }
7
+ {
8
+ "inReplyTo": "request-id",
9
+ "statusCode": 200,
10
+ "type": "dump.result",
11
+ "body": {
12
+ "started": true
13
+ }
14
+ }
15
+ {
16
+ "inReplyTo": "request-id",
17
+ "dataset": "Default",
18
+ "date": "0000-00-00T00:00:00.000000Z",
19
+ "body": {
20
+ "nMessages": 2
21
+ },
22
+ "type": "dump.forecast"
23
+ }
24
+ {
25
+ "inReplyTo": "request-id",
26
+ "dataset": "Default",
27
+ "date": "0000-00-00T00:00:00.000000Z",
28
+ "body": {
29
+ "type": "Hash",
30
+ "name": "Memos",
31
+ "keyType": "ShortText"
32
+ },
33
+ "type": "dump.table"
34
+ }
35
+ {
36
+ "inReplyTo": "request-id",
37
+ "dataset": "Default",
38
+ "date": "0000-00-00T00:00:00.000000Z",
39
+ "body": {
40
+ "table": "Memos",
41
+ "name": "tags",
42
+ "type": "Vector",
43
+ "valueType": "ShortText",
44
+ "vectorOptions": {
45
+ "weight": true
46
+ }
47
+ },
48
+ "type": "dump.column"
49
+ }
50
+ {
51
+ "inReplyTo": "request-id",
52
+ "dataset": "Default",
53
+ "date": "0000-00-00T00:00:00.000000Z",
54
+ "type": "dump.end"
55
+ }
data/test/command/suite/dump/record/scalar.expected.single_slice ADDED
@@ -0,0 +1,52 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "dataset": "Default",
4
+ "date": "0000-00-00T00:00:00.000000Z",
5
+ "type": "dump.start"
6
+ }
7
+ {
8
+ "inReplyTo": "request-id",
9
+ "statusCode": 200,
10
+ "type": "dump.result",
11
+ "body": {
12
+ "started": true
13
+ }
14
+ }
15
+ {
16
+ "inReplyTo": "request-id",
17
+ "dataset": "Default",
18
+ "date": "0000-00-00T00:00:00.000000Z",
19
+ "body": {
20
+ "nMessages": 2
21
+ },
22
+ "type": "dump.forecast"
23
+ }
24
+ {
25
+ "inReplyTo": "request-id",
26
+ "dataset": "Default",
27
+ "date": "0000-00-00T00:00:00.000000Z",
28
+ "body": {
29
+ "type": "Hash",
30
+ "name": "Users",
31
+ "keyType": "ShortText"
32
+ },
33
+ "type": "dump.table"
34
+ }
35
+ {
36
+ "inReplyTo": "request-id",
37
+ "dataset": "Default",
38
+ "date": "0000-00-00T00:00:00.000000Z",
39
+ "body": {
40
+ "table": "Users",
41
+ "key": "Adam",
42
+ "values": {
43
+ }
44
+ },
45
+ "type": "dump.record"
46
+ }
47
+ {
48
+ "inReplyTo": "request-id",
49
+ "dataset": "Default",
50
+ "date": "0000-00-00T00:00:00.000000Z",
51
+ "type": "dump.end"
52
+ }
data/test/command/suite/dump/record/vector/reference.expected.single_slice ADDED
@@ -0,0 +1,117 @@
1
+ {
2
+ "inReplyTo": "request-id",
3
+ "dataset": "Default",
4
+ "date": "0000-00-00T00:00:00.000000Z",
5
+ "type": "dump.start"
6
+ }
7
+ {
8
+ "inReplyTo": "request-id",
9
+ "statusCode": 200,
10
+ "type": "dump.result",
11
+ "body": {
12
+ "started": true
13
+ }
14
+ }
15
+ {
16
+ "inReplyTo": "request-id",
17
+ "dataset": "Default",
18
+ "date": "0000-00-00T00:00:00.000000Z",
19
+ "body": {
20
+ "nMessages": 7
21
+ },
22
+ "type": "dump.forecast"
23
+ }
24
+ {
25
+ "inReplyTo": "request-id",
26
+ "dataset": "Default",
27
+ "date": "0000-00-00T00:00:00.000000Z",
28
+ "body": {
29
+ "type": "Hash",
30
+ "name": "Memos",
31
+ "keyType": "ShortText"
32
+ },
33
+ "type": "dump.table"
34
+ }
35
+ {
36
+ "inReplyTo": "request-id",
37
+ "dataset": "Default",
38
+ "date": "0000-00-00T00:00:00.000000Z",
39
+ "body": {
40
+ "type": "PatriciaTrie",
41
+ "name": "Tags",
42
+ "keyType": "ShortText"
43
+ },
44
+ "type": "dump.table"
45
+ }
46
+ {
47
+ "inReplyTo": "request-id",
48
+ "dataset": "Default",
49
+ "date": "0000-00-00T00:00:00.000000Z",
50
+ "body": {
51
+ "type": "Hash",
52
+ "name": "Users",
53
+ "keyType": "ShortText"
54
+ },
55
+ "type": "dump.table"
56
+ }
57
+ {
58
+ "inReplyTo": "request-id",
59
+ "dataset": "Default",
60
+ "date": "0000-00-00T00:00:00.000000Z",
61
+ "body": {
62
+ "table": "Memos",
63
+ "name": "tags",
64
+ "type": "Vector",
65
+ "valueType": "Tags",
66
+ "vectorOptions": {
67
+ "weight": false
68
+ }
69
+ },
70
+ "type": "dump.column"
71
+ }
72
+ {
73
+ "inReplyTo": "request-id",
74
+ "dataset": "Default",
75
+ "date": "0000-00-00T00:00:00.000000Z",
76
+ "body": {
77
+ "table": "Memos",
78
+ "key": "Groonga newbie",
79
+ "values": {
80
+ "tags": [
81
+ "Groonga",
82
+ "column store"
83
+ ]
84
+ }
85
+ },
86
+ "type": "dump.record"
87
+ }
88
+ {
89
+ "inReplyTo": "request-id",
90
+ "dataset": "Default",
91
+ "date": "0000-00-00T00:00:00.000000Z",
92
+ "body": {
93
+ "table": "Tags",
94
+ "key": "Groonga",
95
+ "values": {
96
+ }
97
+ },
98
+ "type": "dump.record"
99
+ }
100
+ {
101
+ "inReplyTo": "request-id",
102
+ "dataset": "Default",
103
+ "date": "0000-00-00T00:00:00.000000Z",
104
+ "body": {
105
+ "table": "Tags",
106
+ "key": "column store",
107
+ "values": {
108
+ }
109
+ },
110
+ "type": "dump.record"
111
+ }
112
+ {
113
+ "inReplyTo": "request-id",
114
+ "dataset": "Default",
115
+ "date": "0000-00-00T00:00:00.000000Z",
116
+ "type": "dump.end"
117
+ }