fluent-plugin-droonga 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -1
- data/Gemfile +1 -1
- data/fluent-plugin-droonga.gemspec +1 -1
- data/lib/droonga/catalog/collection_volume.rb +97 -0
- data/lib/droonga/catalog/dataset.rb +28 -1
- data/lib/droonga/catalog/errors.rb +28 -9
- data/lib/droonga/catalog/schema.rb +23 -2
- data/lib/droonga/catalog/single_volume.rb +28 -0
- data/lib/droonga/catalog/slice.rb +43 -0
- data/lib/droonga/catalog/version1.rb +3 -3
- data/lib/droonga/catalog/version2.rb +17 -81
- data/lib/droonga/catalog/version2_validator.rb +63 -0
- data/lib/droonga/catalog/volume.rb +33 -0
- data/lib/droonga/catalog/volume_collection.rb +56 -0
- data/lib/droonga/catalog_observer.rb +7 -19
- data/lib/droonga/collectors.rb +1 -1
- data/lib/droonga/collectors/{add.rb → or.rb} +1 -1
- data/lib/droonga/dispatcher.rb +24 -18
- data/lib/droonga/distributed_command_planner.rb +7 -11
- data/lib/droonga/distributor.rb +29 -17
- data/lib/droonga/event_loop.rb +2 -11
- data/lib/droonga/fluent_message_sender.rb +51 -5
- data/lib/droonga/handler_runner.rb +1 -1
- data/lib/droonga/job_protocol.rb +20 -0
- data/lib/droonga/job_pusher.rb +178 -0
- data/lib/droonga/{message_receiver.rb → job_receiver.rb} +13 -6
- data/lib/droonga/message_matcher.rb +18 -15
- data/lib/droonga/planner.rb +2 -3
- data/lib/droonga/plugins/crud.rb +1 -1
- data/lib/droonga/plugins/groonga/column_create.rb +4 -1
- data/lib/droonga/plugins/groonga/table_create.rb +1 -1
- data/lib/droonga/plugins/groonga/table_remove.rb +1 -1
- data/lib/droonga/plugins/search/distributed_search_planner.rb +9 -0
- data/lib/droonga/processor.rb +3 -3
- data/lib/droonga/reducer.rb +15 -12
- data/lib/droonga/searcher.rb +49 -4
- data/lib/droonga/server.rb +2 -0
- data/lib/droonga/single_step.rb +22 -7
- data/lib/droonga/slice.rb +7 -7
- data/lib/droonga/step_runner.rb +3 -2
- data/lib/droonga/worker.rb +10 -8
- data/test/command/suite/add/dimension/column.catalog.json +27 -0
- data/test/command/suite/add/dimension/column.expected +57 -0
- data/test/command/suite/add/dimension/column.test +51 -0
- data/test/command/suite/search/adjusters/multiple.catalog.json +38 -0
- data/test/command/suite/search/adjusters/multiple.expected +23 -0
- data/test/command/suite/search/adjusters/multiple.test +75 -0
- data/test/command/suite/search/adjusters/one.catalog.json +38 -0
- data/test/command/suite/search/adjusters/one.expected +23 -0
- data/test/command/suite/search/adjusters/one.test +66 -0
- data/test/command/suite/search/attributes/array.test +0 -2
- data/test/command/suite/search/attributes/hash.test +0 -2
- data/test/command/suite/search/complex.test +0 -2
- data/test/command/suite/search/condition/nested.test +0 -2
- data/test/command/suite/search/condition/query.test +0 -2
- data/test/command/suite/search/condition/script.test +0 -2
- data/test/command/suite/search/group/string.test +0 -4
- data/test/command/suite/search/group/subrecord/with-sort.catalog.json +33 -0
- data/test/command/suite/search/group/subrecord/with-sort.expected +34 -0
- data/test/command/suite/search/group/subrecord/with-sort.test +81 -0
- data/test/command/suite/search/multiple/chained.test +0 -4
- data/test/command/suite/search/multiple/parallel.test +0 -4
- data/test/command/suite/search/range/only-output.test +0 -2
- data/test/command/suite/search/range/only-sort.test +0 -2
- data/test/command/suite/search/range/sort-and-output.test +0 -2
- data/test/command/suite/search/range/too-large-output-offset.test +0 -2
- data/test/command/suite/search/range/too-large-sort-offset.test +0 -2
- data/test/command/suite/search/response/elapsed_time.catalog.json +13 -0
- data/test/command/suite/search/response/elapsed_time.expected +15 -0
- data/test/command/suite/search/response/elapsed_time.test +26 -0
- data/test/command/suite/search/response/records/value/time.test +0 -2
- data/test/command/suite/search/simple.test +0 -2
- data/test/command/suite/search/sort/default-offset-limit.test +0 -2
- data/test/command/suite/search/sort/invisible-column.test +0 -2
- data/test/unit/catalog/test_collection_volume.rb +103 -0
- data/test/unit/catalog/test_dataset.rb +69 -8
- data/test/unit/catalog/test_schema.rb +63 -23
- data/test/unit/catalog/test_single_volume.rb +31 -0
- data/test/unit/catalog/test_slice.rb +92 -0
- data/test/unit/catalog/test_version1.rb +1 -1
- data/test/unit/catalog/test_version2.rb +1 -32
- data/test/unit/catalog/test_version2_validator.rb +66 -0
- data/test/unit/catalog/test_volume_collection.rb +50 -0
- data/test/unit/plugins/groonga/test_column_create.rb +4 -1
- data/test/unit/plugins/groonga/test_table_create.rb +1 -1
- data/test/unit/test_message_matcher.rb +15 -15
- data/test/unit/test_watch_schema.rb +1 -1
- metadata +107 -94
- checksums.yaml +0 -7
- data/lib/droonga/message_pusher.rb +0 -64
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -41,7 +41,7 @@ droonga_client_dir = File.join(parent_dir, "droonga-client-ruby")
|
|
41
41
|
if File.exist?(droonga_client_dir)
|
42
42
|
gem "droonga-client", :path => droonga_client_dir
|
43
43
|
else
|
44
|
-
gem "droonga-client", github
|
44
|
+
gem "droonga-client", :github => "droonga/droonga-client-ruby"
|
45
45
|
end
|
46
46
|
|
47
47
|
drntest_dir = File.join(parent_dir, "drntest")
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |gem|
|
19
19
|
gem.name = "fluent-plugin-droonga"
|
20
|
-
gem.version = "1.0.
|
20
|
+
gem.version = "1.0.1"
|
21
21
|
gem.authors = ["Droonga Project"]
|
22
22
|
gem.email = ["droonga@groonga.org"]
|
23
23
|
gem.description = "Droonga (distributed Groonga) plugin for Fluent event collector"
|
@@ -0,0 +1,97 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "digest/sha1"
|
17
|
+
require "zlib"
|
18
|
+
|
19
|
+
require "droonga/catalog/slice"
|
20
|
+
|
21
|
+
module Droonga
|
22
|
+
module Catalog
|
23
|
+
class CollectionVolume
|
24
|
+
def initialize(dataset, data)
|
25
|
+
@dataset = dataset
|
26
|
+
@data = data
|
27
|
+
compute_continuum if ratio_scaled_slicer?
|
28
|
+
end
|
29
|
+
|
30
|
+
def dimension
|
31
|
+
@data["dimension"] || "_key"
|
32
|
+
end
|
33
|
+
|
34
|
+
def slicer
|
35
|
+
@data["slicer"] || "hash"
|
36
|
+
end
|
37
|
+
|
38
|
+
def slices
|
39
|
+
@slices ||= @data["slices"].collect do |raw_slice|
|
40
|
+
Slice.new(@dataset, raw_slice)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def select_slices(range=0..-1)
|
45
|
+
slices.sort_by(&:label)[range]
|
46
|
+
end
|
47
|
+
|
48
|
+
def choose_slice(record)
|
49
|
+
return slices.first unless ratio_scaled_slicer?
|
50
|
+
|
51
|
+
key = record[dimension]
|
52
|
+
hash = Zlib.crc32(key)
|
53
|
+
min = 0
|
54
|
+
max = @continuum.size - 1
|
55
|
+
while (min < max)
|
56
|
+
index = (min + max) / 2
|
57
|
+
value, key = @continuum[index]
|
58
|
+
return key if value == hash
|
59
|
+
if value > hash
|
60
|
+
max = index
|
61
|
+
else
|
62
|
+
min = index + 1
|
63
|
+
end
|
64
|
+
end
|
65
|
+
@continuum[max][1]
|
66
|
+
end
|
67
|
+
|
68
|
+
def ratio_scaled_slicer?
|
69
|
+
slicer == "hash"
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
def compute_continuum
|
74
|
+
total_weight = compute_total_weight
|
75
|
+
continuum = []
|
76
|
+
n_slices = slices.size
|
77
|
+
slices.each do |slice|
|
78
|
+
weight = slice.weight
|
79
|
+
points = n_slices * 160 * weight / total_weight
|
80
|
+
points.times do |point|
|
81
|
+
hash = Digest::SHA1.hexdigest("#{@dataset.name}:#{point}")
|
82
|
+
continuum << [hash[0..7].to_i(16), slice]
|
83
|
+
end
|
84
|
+
end
|
85
|
+
@continuum = continuum.sort do |a, b|
|
86
|
+
a[0] - b[0]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def compute_total_weight
|
91
|
+
slices.reduce(0) do |result, slice|
|
92
|
+
result + slice.weight
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -14,10 +14,14 @@
|
|
14
14
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
15
|
|
16
16
|
require "droonga/catalog/schema"
|
17
|
+
require "droonga/catalog/volume"
|
18
|
+
require "droonga/catalog/volume_collection"
|
17
19
|
|
18
20
|
module Droonga
|
19
21
|
module Catalog
|
20
22
|
class Dataset
|
23
|
+
attr_reader :name
|
24
|
+
|
21
25
|
def initialize(name, data)
|
22
26
|
@name = name
|
23
27
|
@data = data
|
@@ -35,7 +39,30 @@ module Droonga
|
|
35
39
|
end
|
36
40
|
|
37
41
|
def schema
|
38
|
-
@schema ||=
|
42
|
+
@schema ||= Schema.new(@name, @data["schema"])
|
43
|
+
end
|
44
|
+
|
45
|
+
def plugins
|
46
|
+
@data["plugins"] || []
|
47
|
+
end
|
48
|
+
|
49
|
+
def fact
|
50
|
+
@data["fact"]
|
51
|
+
end
|
52
|
+
|
53
|
+
def n_workers
|
54
|
+
@data["nWorkers"] || 0
|
55
|
+
end
|
56
|
+
|
57
|
+
def replicas
|
58
|
+
@replicas ||= VolumeCollection.new(create_volumes(@data["replicas"]))
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
def create_volumes(raw_volumes)
|
63
|
+
raw_volumes.collect do |raw_volume|
|
64
|
+
Volume.create(self, raw_volume)
|
65
|
+
end
|
39
66
|
end
|
40
67
|
end
|
41
68
|
end
|
@@ -18,6 +18,25 @@ require "droonga/error"
|
|
18
18
|
module Droonga
|
19
19
|
module Catalog
|
20
20
|
class ValidationError < Error
|
21
|
+
class Detail
|
22
|
+
attr_reader :value_path, :message
|
23
|
+
def initialize(value_path, message)
|
24
|
+
@value_path = value_path
|
25
|
+
@message = message
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_reader :path, :details
|
30
|
+
def initialize(path, details)
|
31
|
+
message = "validation error: <#{path}>"
|
32
|
+
details.each do |detail|
|
33
|
+
message << "\n * #{detail.value_path}: #{detail.message}"
|
34
|
+
end
|
35
|
+
super(message)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class LegacyValidationError < Error
|
21
40
|
def initialize(message, path)
|
22
41
|
if path
|
23
42
|
super("[Validation Error <#{path}>]#{message}")
|
@@ -27,13 +46,13 @@ module Droonga
|
|
27
46
|
end
|
28
47
|
end
|
29
48
|
|
30
|
-
class MissingRequiredParameter <
|
49
|
+
class MissingRequiredParameter < LegacyValidationError
|
31
50
|
def initialize(name, path)
|
32
51
|
super("[#{name}] A required parameter is missing.", path)
|
33
52
|
end
|
34
53
|
end
|
35
54
|
|
36
|
-
class MismatchedParameterType <
|
55
|
+
class MismatchedParameterType < LegacyValidationError
|
37
56
|
def initialize(name, expected_types, actual, path)
|
38
57
|
expected_types = [expected_types] unless expected_types.is_a?(Array)
|
39
58
|
message = nil
|
@@ -49,43 +68,43 @@ module Droonga
|
|
49
68
|
end
|
50
69
|
end
|
51
70
|
|
52
|
-
class InvalidDate <
|
71
|
+
class InvalidDate < LegacyValidationError
|
53
72
|
def initialize(name, value, path)
|
54
73
|
super("[#{name}] Invalid date string: <#{value}>", path)
|
55
74
|
end
|
56
75
|
end
|
57
76
|
|
58
|
-
class NegativeNumber <
|
77
|
+
class NegativeNumber < LegacyValidationError
|
59
78
|
def initialize(name, actual, path)
|
60
79
|
super("[#{name}] A positive number is expected, but <#{actual}>", path)
|
61
80
|
end
|
62
81
|
end
|
63
82
|
|
64
|
-
class SmallerThanOne <
|
83
|
+
class SmallerThanOne < LegacyValidationError
|
65
84
|
def initialize(name, actual, path)
|
66
85
|
super("[#{name}] A number 1 or larger is expected, but <#{actual}>", path)
|
67
86
|
end
|
68
87
|
end
|
69
88
|
|
70
|
-
class FarmNotZoned <
|
89
|
+
class FarmNotZoned < LegacyValidationError
|
71
90
|
def initialize(name, zones, path)
|
72
91
|
super("The farm does not appear in zones: <#{name}>, zones=<#{zones}>", path)
|
73
92
|
end
|
74
93
|
end
|
75
94
|
|
76
|
-
class UnknownFarmInZones <
|
95
|
+
class UnknownFarmInZones < LegacyValidationError
|
77
96
|
def initialize(name, zones, path)
|
78
97
|
super("The farm is unknown: <#{name}>, zones=<#{zones}>", path)
|
79
98
|
end
|
80
99
|
end
|
81
100
|
|
82
|
-
class UnknownFarmForPartition <
|
101
|
+
class UnknownFarmForPartition < LegacyValidationError
|
83
102
|
def initialize(name, slice, path)
|
84
103
|
super("The farm is unknown: <{#name}>, slice=<#{slice}>", path)
|
85
104
|
end
|
86
105
|
end
|
87
106
|
|
88
|
-
class UnsupportedValue <
|
107
|
+
class UnsupportedValue < LegacyValidationError
|
89
108
|
def initialize(name, value, path)
|
90
109
|
super("[#{name}] Not supported value: <#{value}>", path)
|
91
110
|
end
|
@@ -18,6 +18,22 @@ require "tsort"
|
|
18
18
|
module Droonga
|
19
19
|
module Catalog
|
20
20
|
class Schema
|
21
|
+
class ColumnVectorOptions
|
22
|
+
def initialize(data)
|
23
|
+
@data = data
|
24
|
+
end
|
25
|
+
|
26
|
+
def weight
|
27
|
+
@data["weight"]
|
28
|
+
end
|
29
|
+
|
30
|
+
def flags
|
31
|
+
flags = []
|
32
|
+
flags << "WITH_WEIGHT" if weight
|
33
|
+
flags
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
21
37
|
class ColumnIndexOptions
|
22
38
|
def initialize(data)
|
23
39
|
@data = data
|
@@ -49,11 +65,12 @@ module Droonga
|
|
49
65
|
end
|
50
66
|
|
51
67
|
class Column
|
52
|
-
attr_reader :table, :name, :data, :index_options
|
68
|
+
attr_reader :table, :name, :data, :vector_options, :index_options
|
53
69
|
def initialize(table, name, data)
|
54
70
|
@table = table
|
55
71
|
@name = name
|
56
72
|
@data = data
|
73
|
+
@vector_options = ColumnVectorOptions.new(vector_options_data)
|
57
74
|
@index_options = ColumnIndexOptions.new(index_options_data)
|
58
75
|
end
|
59
76
|
|
@@ -81,7 +98,7 @@ module Droonga
|
|
81
98
|
end
|
82
99
|
|
83
100
|
def flags
|
84
|
-
[type_flag] + index_options.flags
|
101
|
+
[type_flag] + vector_options.flags + index_options.flags
|
85
102
|
end
|
86
103
|
|
87
104
|
def value_type
|
@@ -112,6 +129,10 @@ module Droonga
|
|
112
129
|
end
|
113
130
|
|
114
131
|
private
|
132
|
+
def vector_options_data
|
133
|
+
@data["vectorOptions"] || {}
|
134
|
+
end
|
135
|
+
|
115
136
|
def index_options_data
|
116
137
|
@data["indexOptions"] || {}
|
117
138
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
module Droonga
|
17
|
+
module Catalog
|
18
|
+
class SingleVolume
|
19
|
+
def initialize(data)
|
20
|
+
@data = data
|
21
|
+
end
|
22
|
+
|
23
|
+
def address
|
24
|
+
@data["address"]
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Copyright (C) 2014 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/catalog/volume"
|
17
|
+
|
18
|
+
module Droonga
|
19
|
+
module Catalog
|
20
|
+
class Slice
|
21
|
+
def initialize(dataset, data)
|
22
|
+
@dataset = dataset
|
23
|
+
@data = data
|
24
|
+
end
|
25
|
+
|
26
|
+
def weight
|
27
|
+
@data["weight"] || 1
|
28
|
+
end
|
29
|
+
|
30
|
+
def label
|
31
|
+
@data["label"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def boundary
|
35
|
+
@data["boundary"]
|
36
|
+
end
|
37
|
+
|
38
|
+
def volume
|
39
|
+
@volume ||= Volume.create(@dataset, @data["volume"])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -75,7 +75,7 @@ module Droonga
|
|
75
75
|
select_range_and_replicas(partition, args, routes)
|
76
76
|
end
|
77
77
|
when "scatter"
|
78
|
-
name = get_partition(dataset, args["
|
78
|
+
name = get_partition(dataset, args["record"]["_key"])
|
79
79
|
partition = dataset["ring"][name]
|
80
80
|
select_range_and_replicas(partition, args, routes)
|
81
81
|
end
|
@@ -166,7 +166,7 @@ module Droonga
|
|
166
166
|
def do_validation(&block)
|
167
167
|
begin
|
168
168
|
yield
|
169
|
-
rescue
|
169
|
+
rescue LegacyValidationError => error
|
170
170
|
@errors << error
|
171
171
|
end
|
172
172
|
end
|
@@ -413,7 +413,7 @@ module Droonga
|
|
413
413
|
if directory_name.nil? or directory_name.empty?
|
414
414
|
message = "\"#{partition}\" has no database name. " +
|
415
415
|
"You mus specify a database name for \"#{name}\"."
|
416
|
-
raise
|
416
|
+
raise LegacyValidationError.new(message, @path)
|
417
417
|
end
|
418
418
|
end
|
419
419
|
end
|
@@ -15,12 +15,14 @@
|
|
15
15
|
|
16
16
|
require "droonga/catalog/base"
|
17
17
|
require "droonga/catalog/dataset"
|
18
|
+
require "droonga/catalog/version2_validator"
|
18
19
|
|
19
20
|
module Droonga
|
20
21
|
module Catalog
|
21
22
|
class Version2 < Base
|
22
23
|
def initialize(data, path)
|
23
24
|
super
|
25
|
+
validate
|
24
26
|
prepare_data
|
25
27
|
end
|
26
28
|
|
@@ -33,11 +35,11 @@ module Droonga
|
|
33
35
|
pattern = Regexp.new("^#{name}\.")
|
34
36
|
results = {}
|
35
37
|
@datasets.each do |dataset_name, dataset|
|
36
|
-
n_workers = dataset
|
37
|
-
plugins = dataset
|
38
|
-
dataset
|
39
|
-
|
40
|
-
volume_address = slice
|
38
|
+
n_workers = dataset.n_workers
|
39
|
+
plugins = dataset.plugins
|
40
|
+
dataset.replicas.each do |volume|
|
41
|
+
volume.slices.each do |slice|
|
42
|
+
volume_address = slice.volume.address
|
41
43
|
if pattern =~ volume_address
|
42
44
|
path = File.join([device, $POSTMATCH, "db"])
|
43
45
|
path = File.expand_path(path, base_path)
|
@@ -60,18 +62,18 @@ module Droonga
|
|
60
62
|
dataset = dataset(name)
|
61
63
|
case args["type"]
|
62
64
|
when "broadcast"
|
63
|
-
|
64
|
-
|
65
|
-
slices = select_slices
|
65
|
+
volumes = dataset.replicas.select(args["replica"].to_sym)
|
66
|
+
volumes.each do |volume|
|
67
|
+
slices = volume.select_slices
|
66
68
|
slices.each do |slice|
|
67
|
-
routes << slice
|
69
|
+
routes << slice.volume.address
|
68
70
|
end
|
69
71
|
end
|
70
72
|
when "scatter"
|
71
|
-
|
72
|
-
|
73
|
-
slice =
|
74
|
-
routes << slice
|
73
|
+
volumes = dataset.replicas.select(args["replica"].to_sym)
|
74
|
+
volumes.each do |volume|
|
75
|
+
slice = volume.choose_slice(args["record"])
|
76
|
+
routes << slice.volume.address
|
75
77
|
end
|
76
78
|
end
|
77
79
|
routes
|
@@ -79,82 +81,16 @@ module Droonga
|
|
79
81
|
|
80
82
|
private
|
81
83
|
def validate
|
82
|
-
|
84
|
+
validator = Version2Validator.new(@data, @path)
|
85
|
+
validator.validate
|
83
86
|
end
|
84
87
|
|
85
88
|
def prepare_data
|
86
89
|
@datasets = {}
|
87
90
|
@data["datasets"].each do |name, dataset|
|
88
|
-
replicas = dataset["replicas"]
|
89
|
-
replicas.each do |replica|
|
90
|
-
total_weight = compute_total_weight(replica)
|
91
|
-
continuum = []
|
92
|
-
slices = replica["slices"]
|
93
|
-
n_slices = slices.size
|
94
|
-
slices.each do |slice|
|
95
|
-
weight = slice["weight"] || default_weight
|
96
|
-
points = n_slices * 160 * weight / total_weight
|
97
|
-
points.times do |point|
|
98
|
-
hash = Digest::SHA1.hexdigest("#{name}:#{point}")
|
99
|
-
continuum << [hash[0..7].to_i(16), slice]
|
100
|
-
end
|
101
|
-
end
|
102
|
-
replica["continuum"] = continuum.sort do |a, b|
|
103
|
-
a[0] - b[0]
|
104
|
-
end
|
105
|
-
end
|
106
91
|
@datasets[name] = Dataset.new(name, dataset)
|
107
92
|
end
|
108
93
|
end
|
109
|
-
|
110
|
-
def default_weight
|
111
|
-
1
|
112
|
-
end
|
113
|
-
|
114
|
-
def compute_total_weight(replica)
|
115
|
-
slices = replica["slices"]
|
116
|
-
slices.reduce(0) do |result, slice|
|
117
|
-
result + (slice["weight"] || default_weight)
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
def select_replicas(replicas, how)
|
122
|
-
case how
|
123
|
-
when "top"
|
124
|
-
[replicas.first]
|
125
|
-
when "random"
|
126
|
-
[replicas.sample]
|
127
|
-
when "all"
|
128
|
-
replicas
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
def select_slices(replica, range=0..-1)
|
133
|
-
sorted_slices = replica["slices"].sort_by do |slice|
|
134
|
-
slice["label"]
|
135
|
-
end
|
136
|
-
sorted_slices[range]
|
137
|
-
end
|
138
|
-
|
139
|
-
def select_slice(replica, key)
|
140
|
-
continuum = replica["continuum"]
|
141
|
-
return replica["slices"].first unless continuum
|
142
|
-
|
143
|
-
hash = Zlib.crc32(key)
|
144
|
-
min = 0
|
145
|
-
max = continuum.size - 1
|
146
|
-
while (min < max) do
|
147
|
-
index = (min + max) / 2
|
148
|
-
value, key = continuum[index]
|
149
|
-
return key if value == hash
|
150
|
-
if value > hash
|
151
|
-
max = index
|
152
|
-
else
|
153
|
-
min = index + 1
|
154
|
-
end
|
155
|
-
end
|
156
|
-
continuum[max][1]
|
157
|
-
end
|
158
94
|
end
|
159
95
|
end
|
160
96
|
end
|