droonga-engine 1.1.0 → 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +2 -1
  3. data/Rakefile +6 -0
  4. data/bin/droonga-engine-absorb-data +14 -14
  5. data/bin/droonga-engine-catalog-generate +24 -12
  6. data/bin/droonga-engine-catalog-modify +13 -7
  7. data/bin/droonga-engine-join +8 -8
  8. data/bin/droonga-engine-set-role +1 -1
  9. data/bin/droonga-engine-unjoin +2 -2
  10. data/lib/droonga/address.rb +3 -0
  11. data/lib/droonga/cluster.rb +16 -10
  12. data/lib/droonga/command/droonga_engine_service.rb +5 -2
  13. data/lib/droonga/command/remote_command_base.rb +3 -3
  14. data/lib/droonga/distributed_command_planner.rb +11 -1
  15. data/lib/droonga/engine.rb +12 -11
  16. data/lib/droonga/engine/version.rb +2 -2
  17. data/lib/droonga/engine_node.rb +28 -28
  18. data/lib/droonga/engine_state.rb +41 -36
  19. data/lib/droonga/forward_buffer.rb +21 -10
  20. data/lib/droonga/node_role.rb +2 -0
  21. data/lib/droonga/plugins/groonga/select.rb +3 -0
  22. data/lib/droonga/plugins/search.rb +3 -1
  23. data/lib/droonga/plugins/search/distributed_search_planner.rb +17 -5
  24. data/lib/droonga/plugins/system/statistics.rb +1 -0
  25. data/lib/droonga/searcher.rb +13 -4
  26. data/test/command/config/single_slice/catalog.json +38 -0
  27. data/test/command/config/single_slice/droonga-engine.yaml +4 -0
  28. data/test/command/run-test.rb +3 -2
  29. data/test/command/suite/catalog/fetch.expected.single_slice +50 -0
  30. data/test/command/suite/dump/column/index.expected.single_slice +86 -0
  31. data/test/command/suite/dump/column/scalar.expected.single_slice +52 -0
  32. data/test/command/suite/dump/column/vector.expected.single_slice +55 -0
  33. data/test/command/suite/dump/record/scalar.expected.single_slice +52 -0
  34. data/test/command/suite/dump/record/vector/reference.expected.single_slice +117 -0
  35. data/test/command/suite/dump/table/array.expected.single_slice +39 -0
  36. data/test/command/suite/dump/table/double_array_trie.expected.single_slice +40 -0
  37. data/test/command/suite/dump/table/hash.expected.single_slice +40 -0
  38. data/test/command/suite/dump/table/patricia_trie.expected.single_slice +40 -0
  39. data/test/command/suite/message/error/missing-dataset.test +3 -0
  40. data/test/command/suite/search/condition/query/nonexistent_column.expected.single_slice +26 -0
  41. data/test/command/suite/search/condition/query/syntax_error.expected.single_slice +26 -0
  42. data/test/command/suite/search/error/unknown-source.expected.single_slice +28 -0
  43. data/test/command/suite/search/output/attributes/invalid.expected.single_slice +24 -0
  44. data/test/command/suite/system/absorb-data/records.catalog.json.single_slice +44 -0
  45. data/test/command/suite/system/absorb-data/records.expected.single_slice +32 -0
  46. data/test/command/suite/system/statistics/object/count/per-volume/empty.test +1 -0
  47. data/test/command/suite/system/statistics/object/count/record.expected.single_slice +11 -0
  48. data/test/command/suite/system/statistics/object/count/schema.expected.single_slice +11 -0
  49. data/test/unit/catalog/test_generator.rb +3 -2
  50. data/test/unit/helper.rb +2 -1
  51. data/test/unit/helper/stub_serf.rb +28 -0
  52. data/test/unit/plugins/system/statistics/test_object_count.rb +135 -0
  53. data/test/unit/plugins/system/statistics/test_object_count_per_volume.rb +149 -0
  54. data/test/unit/plugins/test_basic.rb +0 -406
  55. data/test/unit/test_address.rb +111 -10
  56. data/test/unit/test_cluster.rb +232 -0
  57. data/test/unit/test_differ.rb +49 -0
  58. data/test/unit/test_engine_node.rb +556 -0
  59. data/test/unit/test_engine_state.rb +151 -0
  60. data/test/unit/test_forward_buffer.rb +106 -0
  61. data/test/unit/test_node_name.rb +160 -0
  62. data/test/unit/test_node_role.rb +53 -0
  63. data/test/unit/test_reducer.rb +525 -0
  64. metadata +111 -49
@@ -0,0 +1,151 @@
1
+ # Copyright (C) 2015 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
+
16
+ require "droonga/engine_state"
17
+
18
+ class EngineStateTest < Test::Unit::TestCase
19
+ class EngineState < Droonga::EngineState
20
+ private
21
+ def create_forwarder
22
+ nil
23
+ end
24
+
25
+ def create_replier
26
+ nil
27
+ end
28
+ end
29
+
30
+ PUBLIC_NODE_NAME = "node29:2929/droonga"
31
+ INTERNAL_NODE_NAME = "node29:12929/droonga"
32
+
33
+ def setup
34
+ @state = EngineState.new(:name => PUBLIC_NODE_NAME,
35
+ :internal_name => INTERNAL_NODE_NAME)
36
+ end
37
+
38
+ data(:public => PUBLIC_NODE_NAME,
39
+ :internal => INTERNAL_NODE_NAME,
40
+ :public_with_local => "#{PUBLIC_NODE_NAME}.\#1",
41
+ :internal_with_local => "#{INTERNAL_NODE_NAME}.\#1")
42
+ def test_local_route(route)
43
+ assert_true(@state.local_route?(route))
44
+ end
45
+
46
+ data(:foreign => "node30:2929/droonga",
47
+ :foreign_with_logal => "node30:2929/droonga.\#1")
48
+ def test_not_local_route(route)
49
+ assert_false(@state.local_route?(route))
50
+ end
51
+
52
+ data(:public => {
53
+ :route => PUBLIC_NODE_NAME,
54
+ :expected => INTERNAL_NODE_NAME,
55
+ },
56
+ :internal => {
57
+ :route => INTERNAL_NODE_NAME,
58
+ :expected => INTERNAL_NODE_NAME,
59
+ },
60
+ :public_with_local => {
61
+ :route => "#{PUBLIC_NODE_NAME}.\#1",
62
+ :expected => "#{INTERNAL_NODE_NAME}.\#1",
63
+ },
64
+ :internal_with_local => {
65
+ :route => "#{INTERNAL_NODE_NAME}.\#1",
66
+ :expected => "#{INTERNAL_NODE_NAME}.\#1",
67
+ },
68
+ :foreign => {
69
+ :route => "node30:2929/droonga.\#1",
70
+ :expected => "node30:2929/droonga.\#1",
71
+ })
72
+ def test_internal_route(data)
73
+ assert_equal(data[:expected],
74
+ @state.internal_route(data[:route]))
75
+ end
76
+
77
+ data(:public => {
78
+ :route => PUBLIC_NODE_NAME,
79
+ :expected => PUBLIC_NODE_NAME,
80
+ },
81
+ :internal => {
82
+ :route => INTERNAL_NODE_NAME,
83
+ :expected => PUBLIC_NODE_NAME,
84
+ },
85
+ :public_with_local => {
86
+ :route => "#{PUBLIC_NODE_NAME}.\#1",
87
+ :expected => "#{PUBLIC_NODE_NAME}.\#1",
88
+ },
89
+ :internal_with_local => {
90
+ :route => "#{INTERNAL_NODE_NAME}.\#1",
91
+ :expected => "#{PUBLIC_NODE_NAME}.\#1",
92
+ },
93
+ :foreign => {
94
+ :route => "node30:2929/droonga.\#1",
95
+ :expected => "node30:2929/droonga.\#1",
96
+ })
97
+ def test_public_route(data)
98
+ assert_equal(data[:expected],
99
+ @state.public_route(data[:route]))
100
+ end
101
+
102
+ data(:public => {
103
+ :route => PUBLIC_NODE_NAME,
104
+ :expected => INTERNAL_NODE_NAME,
105
+ },
106
+ :internal => {
107
+ :route => INTERNAL_NODE_NAME,
108
+ :expected => INTERNAL_NODE_NAME,
109
+ },
110
+ :public_with_local => {
111
+ :route => "#{PUBLIC_NODE_NAME}.\#1",
112
+ :expected => INTERNAL_NODE_NAME,
113
+ },
114
+ :internal_with_local => {
115
+ :route => "#{INTERNAL_NODE_NAME}.\#1",
116
+ :expected => INTERNAL_NODE_NAME,
117
+ },
118
+ :foreign => {
119
+ :route => "node30:2929/droonga.\#1",
120
+ :expected => "node30:2929/droonga",
121
+ })
122
+ def test_internal_farm_path(data)
123
+ assert_equal(data[:expected],
124
+ @state.internal_farm_path(data[:route]))
125
+ end
126
+
127
+ data(:public => {
128
+ :route => PUBLIC_NODE_NAME,
129
+ :expected => PUBLIC_NODE_NAME,
130
+ },
131
+ :internal => {
132
+ :route => INTERNAL_NODE_NAME,
133
+ :expected => PUBLIC_NODE_NAME,
134
+ },
135
+ :public_with_local => {
136
+ :route => "#{PUBLIC_NODE_NAME}.\#1",
137
+ :expected => PUBLIC_NODE_NAME,
138
+ },
139
+ :internal_with_local => {
140
+ :route => "#{INTERNAL_NODE_NAME}.\#1",
141
+ :expected => PUBLIC_NODE_NAME,
142
+ },
143
+ :foreign => {
144
+ :route => "node30:2929/droonga.\#1",
145
+ :expected => "node30:2929/droonga",
146
+ })
147
+ def test_public_farm_path(data)
148
+ assert_equal(data[:expected],
149
+ @state.public_farm_path(data[:route]))
150
+ end
151
+ end
@@ -0,0 +1,106 @@
1
+ # Copyright (C) 2015 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
+
16
+ require "time"
17
+
18
+ require "droonga/forward_buffer"
19
+
20
+ class ForwardBufferTest < Test::Unit::TestCase
21
+ class ForwardBuffer < Droonga::ForwardBuffer
22
+ attr_writer :data_directory, :serf
23
+ end
24
+
25
+ def setup
26
+ setup_temporary_directory
27
+
28
+ @buffer = ForwardBuffer.new("node29:2929/droonga")
29
+ @buffer.data_directory = @temporary_directory
30
+ @buffer.serf = StubSerf.new
31
+
32
+ @forwarded_messages = []
33
+ @buffer.on_forward = lambda do |message, destination|
34
+ @forwarded_messages << {:message => message,
35
+ :destination => destination}
36
+ end
37
+ end
38
+
39
+ def teardown
40
+ teardown_temporary_directory
41
+ end
42
+
43
+ def test_empty
44
+ assert_true(@buffer.empty?)
45
+ end
46
+
47
+ def test_not_empty
48
+ @buffer.add({}, {})
49
+ assert_false(@buffer.empty?)
50
+ end
51
+
52
+ def test_add
53
+ @buffer.add({}, {})
54
+ @buffer.add({}, {})
55
+ @buffer.add({}, {})
56
+ assert_equal(3, @buffer.buffered_messages.size)
57
+ end
58
+
59
+ class ForwardTest < self
60
+ def setup
61
+ super
62
+ @buffer.add({"date" => "2015-04-30T00:00:00.000000Z"}, {})
63
+ @buffer.add({"date" => "2015-04-30T01:00:00.000000Z"}, {})
64
+ @buffer.add({"date" => "2015-04-30T02:00:00.000000Z"}, {})
65
+ end
66
+
67
+ def test_without_boundary
68
+ @buffer.start_forward
69
+ assert_equal({
70
+ :empty => true,
71
+ :forwarded => [
72
+ {:message => {"date" => "2015-04-30T00:00:00.000000Z",
73
+ "xSender"=>"forward-buffer"},
74
+ :destination => {}},
75
+ {:message => {"date" => "2015-04-30T01:00:00.000000Z",
76
+ "xSender"=>"forward-buffer"},
77
+ :destination => {}},
78
+ {:message => {"date" => "2015-04-30T02:00:00.000000Z",
79
+ "xSender"=>"forward-buffer"},
80
+ :destination => {}},
81
+ ],
82
+ },
83
+ {
84
+ :empty => @buffer.empty?,
85
+ :forwarded => @forwarded_messages,
86
+ })
87
+ end
88
+
89
+ def test_with_boundary
90
+ @buffer.process_messages_newer_than(Time.parse("2015-04-30T01:00:00.000000Z"))
91
+ @buffer.start_forward
92
+ assert_equal({
93
+ :empty => true,
94
+ :forwarded => [
95
+ {:message => {"date" => "2015-04-30T02:00:00.000000Z",
96
+ "xSender"=>"forward-buffer"},
97
+ :destination => {}},
98
+ ],
99
+ },
100
+ {
101
+ :empty => @buffer.empty?,
102
+ :forwarded => @forwarded_messages,
103
+ })
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,160 @@
1
+ # Copyright (C) 2014-2015 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
+
16
+ require "droonga/node_name"
17
+
18
+ class NodeNameTest < Test::Unit::TestCase
19
+ def node_name(host, port, tag)
20
+ Droonga::NodeName.new(:host => host,
21
+ :port => port,
22
+ :tag => tag)
23
+ end
24
+
25
+ class ValidationTest < self
26
+ data(:ip_address => "192.168.0.1:2929/droonga",
27
+ :host_name => "node29:2929/droonga")
28
+ def test_valid(input)
29
+ assert_true(Droonga::NodeName.valid?(input))
30
+ end
31
+
32
+ data(:no_host => ":2929/droonga",
33
+ :no_port => "192.168.0.1/droonga",
34
+ :no_tag => "192.168.0.1:2929",
35
+ :blank => "",
36
+ :nil => nil)
37
+ def test_invalid(input)
38
+ assert_false(Droonga::NodeName.valid?(input))
39
+ end
40
+ end
41
+
42
+ class DefaultParameterTest < self
43
+ data(:omitted => {
44
+ :params => {
45
+ },
46
+ :host => Droonga::NodeName::DEFAULT_HOST,
47
+ :port => Droonga::NodeName::DEFAULT_PORT,
48
+ :tag => Droonga::NodeName::DEFAULT_TAG,
49
+ },
50
+ :nil => {
51
+ :params => {
52
+ :host => nil,
53
+ :port => nil,
54
+ :tag => nil,
55
+ },
56
+ :host => Droonga::NodeName::DEFAULT_HOST,
57
+ :port => Droonga::NodeName::DEFAULT_PORT,
58
+ :tag => Droonga::NodeName::DEFAULT_TAG,
59
+ },
60
+ :filled => {
61
+ :params => {
62
+ :host => "node29",
63
+ :port => 2929,
64
+ :tag => "test",
65
+ },
66
+ :host => "node29",
67
+ :port => 2929,
68
+ :tag => "test",
69
+ })
70
+ def test_default_parameter(data)
71
+ assert_equal(node_name(data[:host], data[:port], data[:tag]),
72
+ Droonga::NodeName.new(data[:params]))
73
+ end
74
+ end
75
+
76
+ class ParseTest < self
77
+ def parse(string)
78
+ Droonga::NodeName.parse(string)
79
+ end
80
+
81
+ data(:ip_address => {
82
+ :input => "192.168.0.1:2929/droonga",
83
+ :host => "192.168.0.1",
84
+ :port => 2929,
85
+ :tag => "droonga",
86
+ },
87
+ :host_name => {
88
+ :input => "node29:2929/droonga",
89
+ :host => "node29",
90
+ :port => 2929,
91
+ :tag => "droonga",
92
+ })
93
+ def test_valid_string(data)
94
+ assert_equal(node_name(data[:host], data[:port], data[:tag]),
95
+ parse(data[:input]))
96
+ end
97
+
98
+ def test_instance
99
+ assert_equal(node_name("192.168.0.1", 2929, "droonga"),
100
+ parse(node_name("192.168.0.1", 2929, "droonga")))
101
+ end
102
+
103
+ data(:no_host => ":2929/droonga",
104
+ :no_port => "192.168.0.1/droonga",
105
+ :no_tag => "192.168.0.1:2929",
106
+ :blank => "",
107
+ :nil => nil)
108
+ def test_invalid(input)
109
+ assert_raise(ArgumentError) do
110
+ parse(input)
111
+ end
112
+ end
113
+ end
114
+
115
+ class AttributeTest < self
116
+ def test_attributes
117
+ node_name = node_name("192.168.0.1", 2929, "droonga")
118
+ assert_equal({:host => "192.168.0.1",
119
+ :port => 2929,
120
+ :tag => "droonga"},
121
+ {:host => node_name.host,
122
+ :port => node_name.port,
123
+ :tag => node_name.tag})
124
+ end
125
+ end
126
+
127
+ class ComparisonTest < self
128
+ def test_node_name_vs_string
129
+ string = "192.168.0.1:2929/droonga"
130
+ node_name = node_name("192.168.0.1", 2929, "droonga")
131
+ assert_true(node_name == string)
132
+ end
133
+
134
+ #XXX This is a confusable behavior. It seems should be true
135
+ # but actually false, so you have to be careful when you
136
+ # compare string with NodeName.
137
+ def test_string_vs_node_name
138
+ string = "192.168.0.1:2929/droonga"
139
+ node_name = node_name("192.168.0.1", 2929, "droonga")
140
+ assert_false(string == node_name)
141
+ end
142
+ end
143
+
144
+ class FormatterTest < self
145
+ def test_node
146
+ assert_equal("192.168.0.1:2929/droonga",
147
+ node_name("192.168.0.1", 2929, "droonga").node)
148
+ end
149
+
150
+ def test_string
151
+ assert_equal("192.168.0.1:2929/droonga",
152
+ node_name("192.168.0.1", 2929, "droonga").to_s)
153
+ end
154
+
155
+ def test_array
156
+ assert_equal(["192.168.0.1", 2929, "droonga"],
157
+ node_name("192.168.0.1", 2929, "droonga").to_a)
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,53 @@
1
+ # Copyright (C) 2015 Droonga Project
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
15
+
16
+ require "droonga/node_role"
17
+
18
+ class NodeRoleTest < Test::Unit::TestCase
19
+ class NormalizeTest < self
20
+ data(:service_provider => {
21
+ :input => "service-provider",
22
+ :expected => Droonga::NodeRole::SERVICE_PROVIDER,
23
+ },
24
+ :absorb_source => {
25
+ :input => "absorb-source",
26
+ :expected => Droonga::NodeRole::ABSORB_SOURCE,
27
+ },
28
+ :absorb_destination => {
29
+ :input => "absorb-destination",
30
+ :expected => Droonga::NodeRole::ABSORB_DESTINATION,
31
+ },
32
+ :not_valid_for_a_node => {
33
+ :input => "any",
34
+ :expected => Droonga::NodeRole::SERVICE_PROVIDER,
35
+ },
36
+ :mixed_case => {
37
+ :input => "Absorb-Source",
38
+ :expected => Droonga::NodeRole::ABSORB_SOURCE,
39
+ },
40
+ :nil => {
41
+ :input => nil,
42
+ :expected => Droonga::NodeRole::SERVICE_PROVIDER,
43
+ },
44
+ :blank => {
45
+ :input => "",
46
+ :expected => Droonga::NodeRole::SERVICE_PROVIDER,
47
+ })
48
+ def test_normalize(data)
49
+ assert_equal(data[:expected],
50
+ Droonga::NodeRole.normalize(data[:input]))
51
+ end
52
+ end
53
+ end