fluent-plugin-droonga 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -4
- data/benchmark/watch/benchmark-notify.rb +2 -2
- data/benchmark/watch/benchmark-scan.rb +3 -0
- data/benchmark/watch/fluentd.conf +0 -1
- data/fluent-plugin-droonga.gemspec +2 -3
- data/lib/droonga/catalog.rb +10 -124
- data/lib/droonga/catalog/base.rb +140 -0
- data/lib/droonga/catalog/version1.rb +23 -0
- data/lib/droonga/catalog_loader.rb +33 -0
- data/lib/droonga/collector.rb +2 -71
- data/lib/droonga/collector_plugin.rb +2 -34
- data/lib/droonga/dispatcher.rb +141 -196
- data/lib/droonga/distribution_planner.rb +76 -0
- data/lib/droonga/distributor.rb +5 -7
- data/lib/droonga/distributor_plugin.rb +23 -15
- data/lib/droonga/engine.rb +2 -2
- data/lib/droonga/event_loop.rb +46 -0
- data/lib/droonga/farm.rb +9 -5
- data/lib/droonga/fluent_message_sender.rb +84 -0
- data/lib/droonga/forwarder.rb +43 -53
- data/lib/droonga/handler.rb +20 -68
- data/lib/droonga/handler_message.rb +61 -0
- data/lib/droonga/handler_messenger.rb +92 -0
- data/lib/droonga/handler_plugin.rb +10 -12
- data/lib/droonga/input_adapter.rb +52 -0
- data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
- data/lib/droonga/input_message.rb +11 -11
- data/lib/droonga/logger.rb +4 -3
- data/lib/droonga/message_pack_packer.rb +62 -0
- data/lib/droonga/message_processing_error.rb +54 -0
- data/lib/droonga/message_pusher.rb +60 -0
- data/lib/droonga/message_receiver.rb +61 -0
- data/lib/droonga/output_adapter.rb +53 -0
- data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
- data/lib/droonga/output_message.rb +37 -0
- data/lib/droonga/partition.rb +27 -5
- data/lib/droonga/pluggable.rb +9 -4
- data/lib/droonga/plugin.rb +12 -3
- data/lib/droonga/plugin/collector/basic.rb +91 -18
- data/lib/droonga/plugin/distributor/crud.rb +9 -9
- data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
- data/lib/droonga/plugin/distributor/groonga.rb +5 -5
- data/lib/droonga/plugin/distributor/search.rb +4 -246
- data/lib/droonga/plugin/distributor/watch.rb +11 -6
- data/lib/droonga/plugin/handler/add.rb +69 -7
- data/lib/droonga/plugin/handler/groonga.rb +6 -6
- data/lib/droonga/plugin/handler/search.rb +5 -3
- data/lib/droonga/plugin/handler/watch.rb +19 -13
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
- data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
- data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
- data/lib/droonga/plugin_loader.rb +2 -2
- data/lib/droonga/processor.rb +21 -23
- data/lib/droonga/replier.rb +40 -0
- data/lib/droonga/searcher.rb +298 -174
- data/lib/droonga/server.rb +0 -67
- data/lib/droonga/session.rb +85 -0
- data/lib/droonga/test.rb +21 -0
- data/lib/droonga/test/stub_distributor.rb +31 -0
- data/lib/droonga/test/stub_handler.rb +37 -0
- data/lib/droonga/test/stub_handler_message.rb +35 -0
- data/lib/droonga/test/stub_handler_messenger.rb +34 -0
- data/lib/droonga/time_formatter.rb +37 -0
- data/lib/droonga/watcher.rb +1 -0
- data/lib/droonga/worker.rb +16 -19
- data/lib/fluent/plugin/out_droonga.rb +9 -9
- data/lib/groonga_command_converter.rb +5 -5
- data/sample/cluster/catalog.json +1 -1
- data/test/command/config/default/catalog.json +19 -1
- data/test/command/fixture/event.jsons +41 -0
- data/test/command/fixture/user-table.jsons +9 -0
- data/test/command/run-test.rb +2 -2
- data/test/command/suite/add/error/invalid-integer.expected +20 -0
- data/test/command/suite/add/error/invalid-integer.test +12 -0
- data/test/command/suite/add/error/invalid-time.expected +20 -0
- data/test/command/suite/add/error/invalid-time.test +12 -0
- data/test/command/suite/add/error/missing-key.expected +13 -0
- data/test/command/suite/add/error/missing-key.test +16 -0
- data/test/command/suite/add/error/missing-table.expected +13 -0
- data/test/command/suite/add/error/missing-table.test +16 -0
- data/test/command/suite/add/error/unknown-column.expected +20 -0
- data/test/command/suite/add/error/unknown-column.test +12 -0
- data/test/command/suite/add/error/unknown-table.expected +13 -0
- data/test/command/suite/add/error/unknown-table.test +17 -0
- data/test/command/suite/add/minimum.expected +1 -3
- data/test/command/suite/add/with-values.expected +1 -3
- data/test/command/suite/add/without-key.expected +1 -3
- data/test/command/suite/message/error/missing-dataset.expected +13 -0
- data/test/command/suite/message/error/missing-dataset.test +5 -0
- data/test/command/suite/message/error/unknown-command.expected +13 -0
- data/test/command/suite/message/error/unknown-command.test +6 -0
- data/test/command/suite/message/error/unknown-dataset.expected +13 -0
- data/test/command/suite/message/error/unknown-dataset.test +6 -0
- data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
- data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
- data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
- data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
- data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
- data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
- data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
- data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
- data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
- data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
- data/test/command/suite/search/error/cyclic-source.expected +18 -0
- data/test/command/suite/search/error/cyclic-source.test +12 -0
- data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
- data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
- data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
- data/test/command/suite/search/error/missing-source-parameter.test +11 -0
- data/test/command/suite/search/error/unknown-source.expected +18 -0
- data/test/command/suite/search/error/unknown-source.test +12 -0
- data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
- data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
- data/test/command/suite/search/group/limit.expected +19 -0
- data/test/command/suite/search/group/limit.test +20 -0
- data/test/command/suite/search/group/string.expected +36 -0
- data/test/command/suite/search/group/string.test +44 -0
- data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
- data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
- data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
- data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
- data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
- data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
- data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
- data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
- data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
- data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
- data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-output-offset.test +25 -0
- data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
- data/test/command/suite/search/response/records/value/time.expected +24 -0
- data/test/command/suite/search/response/records/value/time.test +24 -0
- data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
- data/test/command/suite/search/sort/default-offset-limit.test +26 -0
- data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
- data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
- data/test/command/suite/watch/subscribe.expected +12 -0
- data/test/command/suite/watch/subscribe.test +9 -0
- data/test/command/suite/watch/unsubscribe.expected +12 -0
- data/test/command/suite/watch/unsubscribe.test +9 -0
- data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
- data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
- data/test/unit/helper.rb +2 -0
- data/test/unit/plugin/collector/test_basic.rb +289 -33
- data/test/unit/plugin/distributor/test_search.rb +176 -861
- data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
- data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
- data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
- data/test/unit/plugin/handler/test_add.rb +74 -11
- data/test/unit/plugin/handler/test_groonga.rb +15 -1
- data/test/unit/plugin/handler/test_search.rb +33 -17
- data/test/unit/plugin/handler/test_watch.rb +43 -27
- data/test/unit/run-test.rb +2 -0
- data/test/unit/test_message_pack_packer.rb +51 -0
- data/test/unit/test_time_formatter.rb +29 -0
- metadata +208 -110
- data/lib/droonga/job_queue.rb +0 -87
- data/lib/droonga/job_queue_schema.rb +0 -65
- data/test/unit/test_adapter.rb +0 -51
- data/test/unit/test_job_queue_schema.rb +0 -45
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f03ac63db8e9b4b082da89c26c8bbf7caea45e76
|
4
|
+
data.tar.gz: 20b6f9a80dc7f198003fb4932b483f85c6a1d976
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 06af50c670901878e0d42f72182cd9848194eb5b14f04428fbe788098b7a7b715cb57e94241742620a0bf89fa7cbea03f7e8ea825a1e4816c3aedf59e2a19e33
|
7
|
+
data.tar.gz: 69cc3370c3401f24bf1749ef71f7fe2b45cf1de4b7c06c8d5171c3650e296d75ac8576851375ddefd3856c137fa5836a414e2dafef21c03e908b0296d1e5c464
|
data/.gitignore
CHANGED
@@ -66,14 +66,14 @@ class NotifyBenchmark
|
|
66
66
|
message = DroongaBenchmark::MessageCreator.envelope_to_subscribe(WATCHING_KEYWORD)
|
67
67
|
message["body"]["subscriber"] += " #{@n_subscribers + index}"
|
68
68
|
message["body"]["route"] = @route
|
69
|
-
@client.connection.
|
69
|
+
@client.connection.send(message, :response => :one)
|
70
70
|
end
|
71
71
|
@n_subscribers += n_subscribers
|
72
72
|
end
|
73
73
|
|
74
74
|
def do_feed(target)
|
75
75
|
message = DroongaBenchmark::MessageCreator.envelope_to_feed(target)
|
76
|
-
@client.connection.send(message)
|
76
|
+
@client.connection.send(message, :response => :none)
|
77
77
|
end
|
78
78
|
end
|
79
79
|
|
@@ -19,12 +19,15 @@ require "benchmark"
|
|
19
19
|
require "fileutils"
|
20
20
|
require "optparse"
|
21
21
|
require "csv"
|
22
|
+
require "fluent/log"
|
22
23
|
|
23
24
|
require "groonga"
|
24
25
|
|
25
26
|
require "droonga/watcher"
|
26
27
|
require File.expand_path(File.join(__FILE__, "..", "..", "utils.rb"))
|
27
28
|
|
29
|
+
$log = Fluent::Log.new
|
30
|
+
|
28
31
|
class ScanBenchmark
|
29
32
|
attr_reader :n_keywords
|
30
33
|
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |gem|
|
19
19
|
gem.name = "fluent-plugin-droonga"
|
20
|
-
gem.version = "0.
|
20
|
+
gem.version = "0.8.0"
|
21
21
|
gem.authors = ["Droonga Project"]
|
22
22
|
gem.email = ["droonga@groonga.org"]
|
23
23
|
gem.description = "Droonga(distributed Groonga) plugin for Fluent event collector"
|
@@ -28,9 +28,8 @@ Gem::Specification.new do |gem|
|
|
28
28
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
29
29
|
gem.require_paths = ["lib"]
|
30
30
|
gem.add_dependency "fluentd"
|
31
|
-
gem.add_dependency "rroonga", ">= 3.0
|
31
|
+
gem.add_dependency "rroonga", ">= 3.1.0"
|
32
32
|
gem.add_dependency "groonga-command-parser"
|
33
|
-
gem.add_dependency "fluent-logger"
|
34
33
|
gem.add_dependency "serverengine"
|
35
34
|
gem.add_development_dependency "rake"
|
36
35
|
gem.add_development_dependency "bundler"
|
data/lib/droonga/catalog.rb
CHANGED
@@ -15,139 +15,25 @@
|
|
15
15
|
# License along with this library; if not, write to the Free Software
|
16
16
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
17
|
|
18
|
-
require "
|
19
|
-
require "zlib"
|
18
|
+
require "droonga/catalog_loader"
|
20
19
|
|
21
20
|
module Droonga
|
22
21
|
class << self
|
23
22
|
def catalog
|
24
|
-
@catalog ||= Catalog.
|
23
|
+
@catalog ||= Catalog.load
|
25
24
|
end
|
26
25
|
end
|
27
26
|
|
28
|
-
|
29
|
-
|
27
|
+
module Catalog
|
28
|
+
PATH = "catalog.json"
|
30
29
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
@catalog = JSON.parse(file.read)
|
38
|
-
end
|
39
|
-
@catalog["datasets"].each do |name, dataset|
|
40
|
-
number_of_partitions = dataset["number_of_partitions"]
|
41
|
-
next if number_of_partitions < 2
|
42
|
-
total_weight = dataset["ring"].reduce do |a, b|
|
43
|
-
a[1]["weight"] + b[1]["weight"]
|
44
|
-
end
|
45
|
-
continuum = []
|
46
|
-
dataset["ring"].each do |key, value|
|
47
|
-
points = number_of_partitions * 160 * value["weight"] / total_weight
|
48
|
-
points.times do |point|
|
49
|
-
hash = Digest::SHA1.hexdigest("#{key}:#{point}")
|
50
|
-
continuum << [hash[0..7].to_i(16), key]
|
51
|
-
end
|
52
|
-
end
|
53
|
-
dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
|
54
|
-
end
|
55
|
-
@options = @catalog["options"] || {}
|
56
|
-
end
|
57
|
-
|
58
|
-
def base_path
|
59
|
-
@base_path ||= File.dirname(@path)
|
60
|
-
end
|
61
|
-
|
62
|
-
def option(name)
|
63
|
-
@options[name]
|
64
|
-
end
|
65
|
-
|
66
|
-
def get_partitions(name)
|
67
|
-
device = @catalog["farms"][name]["device"]
|
68
|
-
pattern = Regexp.new("^#{name}\.")
|
69
|
-
results = {}
|
70
|
-
@catalog["datasets"].each do |key, dataset|
|
71
|
-
workers = dataset["workers"]
|
72
|
-
plugins = dataset["plugins"]
|
73
|
-
dataset["ring"].each do |key, part|
|
74
|
-
part["partitions"].each do |range, partitions|
|
75
|
-
partitions.each do |partition|
|
76
|
-
if partition =~ pattern
|
77
|
-
path = File.join([device, $POSTMATCH, "db"])
|
78
|
-
path = File.expand_path(path, base_path)
|
79
|
-
options = {
|
80
|
-
:database => path,
|
81
|
-
:n_workers => workers,
|
82
|
-
:handlers => plugins
|
83
|
-
}
|
84
|
-
results[partition] = options
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
30
|
+
class << self
|
31
|
+
def load(path=nil)
|
32
|
+
path = ENV["DROONGA_CATALOG"] || PATH
|
33
|
+
path = File.expand_path(path)
|
34
|
+
loader = CatalogLoader.new(path)
|
35
|
+
loader.load
|
89
36
|
end
|
90
|
-
return results
|
91
|
-
end
|
92
|
-
|
93
|
-
def get_routes(name, args)
|
94
|
-
routes = []
|
95
|
-
dataset = dataset(name)
|
96
|
-
return routes unless dataset
|
97
|
-
case args["type"]
|
98
|
-
when "broadcast"
|
99
|
-
dataset["ring"].each do |key, partition|
|
100
|
-
select_range_and_replicas(partition, args, routes)
|
101
|
-
end
|
102
|
-
when "scatter"
|
103
|
-
name = get_partition(dataset, args["key"])
|
104
|
-
partition = dataset["ring"][name]
|
105
|
-
select_range_and_replicas(partition, args, routes)
|
106
|
-
end
|
107
|
-
return routes
|
108
|
-
end
|
109
|
-
|
110
|
-
def get_partition(dataset, key)
|
111
|
-
continuum = dataset["continuum"]
|
112
|
-
return dataset["ring"].keys[0] unless continuum
|
113
|
-
hash = Zlib.crc32(key)
|
114
|
-
min = 0
|
115
|
-
max = continuum.size - 1
|
116
|
-
while (min < max) do
|
117
|
-
index = (min + max) / 2
|
118
|
-
value, key = continuum[index]
|
119
|
-
return key if value == hash
|
120
|
-
if value > hash
|
121
|
-
max = index
|
122
|
-
else
|
123
|
-
min = index + 1
|
124
|
-
end
|
125
|
-
end
|
126
|
-
return continuum[max][1]
|
127
|
-
end
|
128
|
-
|
129
|
-
def dataset(name)
|
130
|
-
@catalog["datasets"][name]
|
131
|
-
end
|
132
|
-
|
133
|
-
def select_range_and_replicas(partition, args, routes)
|
134
|
-
date_range = args["date_range"] || 0..-1
|
135
|
-
partition["partitions"].sort[date_range].each do |time, replicas|
|
136
|
-
case args["replica"]
|
137
|
-
when "top"
|
138
|
-
routes << replicas[0]
|
139
|
-
when "random"
|
140
|
-
routes << replicas[rand(replicas.size)]
|
141
|
-
when "all"
|
142
|
-
routes.concat(replicas)
|
143
|
-
end
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
private
|
148
|
-
def default_path
|
149
|
-
path = ENV["DROONGA_CATALOG"] || Catalog::CATALOG_FILE_PATH
|
150
|
-
File.expand_path(path)
|
151
37
|
end
|
152
38
|
end
|
153
39
|
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
# Copyright (C) 2013 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "digest/sha1"
|
17
|
+
require "zlib"
|
18
|
+
require "droonga/message_processing_error"
|
19
|
+
|
20
|
+
module Droonga
|
21
|
+
module Catalog
|
22
|
+
class Base
|
23
|
+
class UnknownDataset < NotFound
|
24
|
+
def initialize(dataset)
|
25
|
+
super("The dataset #{dataset.inspect} does not exist.")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_reader :base_path
|
30
|
+
def initialize(data, base_path)
|
31
|
+
@data = data
|
32
|
+
@base_path = base_path
|
33
|
+
|
34
|
+
@data["datasets"].each do |name, dataset|
|
35
|
+
number_of_partitions = dataset["number_of_partitions"]
|
36
|
+
next if number_of_partitions < 2
|
37
|
+
total_weight = dataset["ring"].reduce do |a, b|
|
38
|
+
a[1]["weight"] + b[1]["weight"]
|
39
|
+
end
|
40
|
+
continuum = []
|
41
|
+
dataset["ring"].each do |key, value|
|
42
|
+
points = number_of_partitions * 160 * value["weight"] / total_weight
|
43
|
+
points.times do |point|
|
44
|
+
hash = Digest::SHA1.hexdigest("#{key}:#{point}")
|
45
|
+
continuum << [hash[0..7].to_i(16), key]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
|
49
|
+
end
|
50
|
+
@options = @data["options"] || {}
|
51
|
+
end
|
52
|
+
|
53
|
+
def option(name)
|
54
|
+
@options[name]
|
55
|
+
end
|
56
|
+
|
57
|
+
def get_partitions(name)
|
58
|
+
device = @data["farms"][name]["device"]
|
59
|
+
pattern = Regexp.new("^#{name}\.")
|
60
|
+
results = {}
|
61
|
+
@data["datasets"].each do |key, dataset|
|
62
|
+
workers = dataset["workers"]
|
63
|
+
plugins = dataset["plugins"]
|
64
|
+
dataset["ring"].each do |key, part|
|
65
|
+
part["partitions"].each do |range, partitions|
|
66
|
+
partitions.each do |partition|
|
67
|
+
if partition =~ pattern
|
68
|
+
path = File.join([device, $POSTMATCH, "db"])
|
69
|
+
path = File.expand_path(path, base_path)
|
70
|
+
options = {
|
71
|
+
:database => path,
|
72
|
+
:n_workers => workers,
|
73
|
+
:handlers => plugins
|
74
|
+
}
|
75
|
+
results[partition] = options
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
return results
|
82
|
+
end
|
83
|
+
|
84
|
+
def get_routes(name, args)
|
85
|
+
routes = []
|
86
|
+
dataset = dataset(name)
|
87
|
+
case args["type"]
|
88
|
+
when "broadcast"
|
89
|
+
dataset["ring"].each do |key, partition|
|
90
|
+
select_range_and_replicas(partition, args, routes)
|
91
|
+
end
|
92
|
+
when "scatter"
|
93
|
+
name = get_partition(dataset, args["key"])
|
94
|
+
partition = dataset["ring"][name]
|
95
|
+
select_range_and_replicas(partition, args, routes)
|
96
|
+
end
|
97
|
+
return routes
|
98
|
+
end
|
99
|
+
|
100
|
+
def get_partition(dataset, key)
|
101
|
+
continuum = dataset["continuum"]
|
102
|
+
return dataset["ring"].keys[0] unless continuum
|
103
|
+
hash = Zlib.crc32(key)
|
104
|
+
min = 0
|
105
|
+
max = continuum.size - 1
|
106
|
+
while (min < max) do
|
107
|
+
index = (min + max) / 2
|
108
|
+
value, key = continuum[index]
|
109
|
+
return key if value == hash
|
110
|
+
if value > hash
|
111
|
+
max = index
|
112
|
+
else
|
113
|
+
min = index + 1
|
114
|
+
end
|
115
|
+
end
|
116
|
+
return continuum[max][1]
|
117
|
+
end
|
118
|
+
|
119
|
+
def dataset(name)
|
120
|
+
dataset = @data["datasets"][name]
|
121
|
+
raise UnknownDataset.new(name) unless dataset
|
122
|
+
dataset
|
123
|
+
end
|
124
|
+
|
125
|
+
def select_range_and_replicas(partition, args, routes)
|
126
|
+
date_range = args["date_range"] || 0..-1
|
127
|
+
partition["partitions"].sort[date_range].each do |time, replicas|
|
128
|
+
case args["replica"]
|
129
|
+
when "top"
|
130
|
+
routes << replicas[0]
|
131
|
+
when "random"
|
132
|
+
routes << replicas[rand(replicas.size)]
|
133
|
+
when "all"
|
134
|
+
routes.concat(replicas)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# Copyright (C) 2013 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "droonga/catalog/base"
|
17
|
+
|
18
|
+
module Droonga
|
19
|
+
module Catalog
|
20
|
+
class Version1 < Base
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# Copyright (C) 2013 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "json"
|
17
|
+
|
18
|
+
require "droonga/catalog/version1"
|
19
|
+
|
20
|
+
module Droonga
|
21
|
+
class CatalogLoader
|
22
|
+
def initialize(path)
|
23
|
+
@path = path
|
24
|
+
end
|
25
|
+
|
26
|
+
def load
|
27
|
+
data = File.open(@path) do |file|
|
28
|
+
JSON.parse(file.read)
|
29
|
+
end
|
30
|
+
Catalog::Version1.new(data, File.dirname(@path))
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
data/lib/droonga/collector.rb
CHANGED
@@ -22,82 +22,13 @@ module Droonga
|
|
22
22
|
class Collector
|
23
23
|
include Pluggable
|
24
24
|
|
25
|
-
def initialize
|
26
|
-
@id = id
|
27
|
-
@dispatcher = dispatcher
|
28
|
-
@components = components
|
29
|
-
@tasks = tasks
|
30
|
-
@n_dones = 0
|
31
|
-
@inputs = inputs
|
25
|
+
def initialize
|
32
26
|
load_plugins(["basic"]) # TODO: make customizable
|
33
27
|
end
|
34
28
|
|
35
|
-
def handle(name, value)
|
36
|
-
tasks = @inputs[name]
|
37
|
-
unless tasks
|
38
|
-
#TODO: result arrived before its query
|
39
|
-
return
|
40
|
-
end
|
41
|
-
tasks.each do |task|
|
42
|
-
task["n_of_inputs"] += 1 if name
|
43
|
-
component = task["component"]
|
44
|
-
type = component["type"]
|
45
|
-
command = component["command"] || ("collector_" + type)
|
46
|
-
n_of_expects = component["n_of_expects"]
|
47
|
-
synchronous = nil
|
48
|
-
if command
|
49
|
-
synchronous = true unless n_of_expects.zero?
|
50
|
-
# TODO: check if asynchronous execution is available.
|
51
|
-
message = {
|
52
|
-
"task"=>task,
|
53
|
-
"name"=>name,
|
54
|
-
"value"=>value
|
55
|
-
}
|
56
|
-
unless synchronous
|
57
|
-
descendants = {}
|
58
|
-
component["descendants"].each do |name, indices|
|
59
|
-
descendants[name] = indices.collect do |index|
|
60
|
-
@components[index]["routes"].map do |route|
|
61
|
-
@dispatcher.farm_path(route)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
message["descendants"] = descendants
|
66
|
-
message["id"] = @id
|
67
|
-
end
|
68
|
-
if @id == task["route"]
|
69
|
-
process(command, message)
|
70
|
-
else
|
71
|
-
@dispatcher.deliver(@id, task["route"], message, command, synchronous)
|
72
|
-
end
|
73
|
-
end
|
74
|
-
return if task["n_of_inputs"] < n_of_expects
|
75
|
-
#the task is done
|
76
|
-
if synchronous
|
77
|
-
result = task["values"]
|
78
|
-
post = component["post"]
|
79
|
-
@dispatcher.post(result, post) if post
|
80
|
-
component["descendants"].each do |name, indices|
|
81
|
-
message = {
|
82
|
-
"id" => @id,
|
83
|
-
"input" => name,
|
84
|
-
"value" => result[name]
|
85
|
-
}
|
86
|
-
indices.each do |index|
|
87
|
-
@components[index]["routes"].each do |route|
|
88
|
-
@dispatcher.dispatch(message, route)
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
@n_dones += 1
|
94
|
-
@dispatcher.collectors.delete(@id) if @n_dones == @tasks.size
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
29
|
private
|
99
30
|
def instantiate_plugin(name)
|
100
|
-
CollectorPlugin.repository.instantiate(name
|
31
|
+
CollectorPlugin.repository.instantiate(name)
|
101
32
|
end
|
102
33
|
|
103
34
|
def log_tag
|