fluent-plugin-droonga 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -4
- data/benchmark/watch/benchmark-notify.rb +2 -2
- data/benchmark/watch/benchmark-scan.rb +3 -0
- data/benchmark/watch/fluentd.conf +0 -1
- data/fluent-plugin-droonga.gemspec +2 -3
- data/lib/droonga/catalog.rb +10 -124
- data/lib/droonga/catalog/base.rb +140 -0
- data/lib/droonga/catalog/version1.rb +23 -0
- data/lib/droonga/catalog_loader.rb +33 -0
- data/lib/droonga/collector.rb +2 -71
- data/lib/droonga/collector_plugin.rb +2 -34
- data/lib/droonga/dispatcher.rb +141 -196
- data/lib/droonga/distribution_planner.rb +76 -0
- data/lib/droonga/distributor.rb +5 -7
- data/lib/droonga/distributor_plugin.rb +23 -15
- data/lib/droonga/engine.rb +2 -2
- data/lib/droonga/event_loop.rb +46 -0
- data/lib/droonga/farm.rb +9 -5
- data/lib/droonga/fluent_message_sender.rb +84 -0
- data/lib/droonga/forwarder.rb +43 -53
- data/lib/droonga/handler.rb +20 -68
- data/lib/droonga/handler_message.rb +61 -0
- data/lib/droonga/handler_messenger.rb +92 -0
- data/lib/droonga/handler_plugin.rb +10 -12
- data/lib/droonga/input_adapter.rb +52 -0
- data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
- data/lib/droonga/input_message.rb +11 -11
- data/lib/droonga/logger.rb +4 -3
- data/lib/droonga/message_pack_packer.rb +62 -0
- data/lib/droonga/message_processing_error.rb +54 -0
- data/lib/droonga/message_pusher.rb +60 -0
- data/lib/droonga/message_receiver.rb +61 -0
- data/lib/droonga/output_adapter.rb +53 -0
- data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
- data/lib/droonga/output_message.rb +37 -0
- data/lib/droonga/partition.rb +27 -5
- data/lib/droonga/pluggable.rb +9 -4
- data/lib/droonga/plugin.rb +12 -3
- data/lib/droonga/plugin/collector/basic.rb +91 -18
- data/lib/droonga/plugin/distributor/crud.rb +9 -9
- data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
- data/lib/droonga/plugin/distributor/groonga.rb +5 -5
- data/lib/droonga/plugin/distributor/search.rb +4 -246
- data/lib/droonga/plugin/distributor/watch.rb +11 -6
- data/lib/droonga/plugin/handler/add.rb +69 -7
- data/lib/droonga/plugin/handler/groonga.rb +6 -6
- data/lib/droonga/plugin/handler/search.rb +5 -3
- data/lib/droonga/plugin/handler/watch.rb +19 -13
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
- data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
- data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
- data/lib/droonga/plugin_loader.rb +2 -2
- data/lib/droonga/processor.rb +21 -23
- data/lib/droonga/replier.rb +40 -0
- data/lib/droonga/searcher.rb +298 -174
- data/lib/droonga/server.rb +0 -67
- data/lib/droonga/session.rb +85 -0
- data/lib/droonga/test.rb +21 -0
- data/lib/droonga/test/stub_distributor.rb +31 -0
- data/lib/droonga/test/stub_handler.rb +37 -0
- data/lib/droonga/test/stub_handler_message.rb +35 -0
- data/lib/droonga/test/stub_handler_messenger.rb +34 -0
- data/lib/droonga/time_formatter.rb +37 -0
- data/lib/droonga/watcher.rb +1 -0
- data/lib/droonga/worker.rb +16 -19
- data/lib/fluent/plugin/out_droonga.rb +9 -9
- data/lib/groonga_command_converter.rb +5 -5
- data/sample/cluster/catalog.json +1 -1
- data/test/command/config/default/catalog.json +19 -1
- data/test/command/fixture/event.jsons +41 -0
- data/test/command/fixture/user-table.jsons +9 -0
- data/test/command/run-test.rb +2 -2
- data/test/command/suite/add/error/invalid-integer.expected +20 -0
- data/test/command/suite/add/error/invalid-integer.test +12 -0
- data/test/command/suite/add/error/invalid-time.expected +20 -0
- data/test/command/suite/add/error/invalid-time.test +12 -0
- data/test/command/suite/add/error/missing-key.expected +13 -0
- data/test/command/suite/add/error/missing-key.test +16 -0
- data/test/command/suite/add/error/missing-table.expected +13 -0
- data/test/command/suite/add/error/missing-table.test +16 -0
- data/test/command/suite/add/error/unknown-column.expected +20 -0
- data/test/command/suite/add/error/unknown-column.test +12 -0
- data/test/command/suite/add/error/unknown-table.expected +13 -0
- data/test/command/suite/add/error/unknown-table.test +17 -0
- data/test/command/suite/add/minimum.expected +1 -3
- data/test/command/suite/add/with-values.expected +1 -3
- data/test/command/suite/add/without-key.expected +1 -3
- data/test/command/suite/message/error/missing-dataset.expected +13 -0
- data/test/command/suite/message/error/missing-dataset.test +5 -0
- data/test/command/suite/message/error/unknown-command.expected +13 -0
- data/test/command/suite/message/error/unknown-command.test +6 -0
- data/test/command/suite/message/error/unknown-dataset.expected +13 -0
- data/test/command/suite/message/error/unknown-dataset.test +6 -0
- data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
- data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
- data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
- data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
- data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
- data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
- data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
- data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
- data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
- data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
- data/test/command/suite/search/error/cyclic-source.expected +18 -0
- data/test/command/suite/search/error/cyclic-source.test +12 -0
- data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
- data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
- data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
- data/test/command/suite/search/error/missing-source-parameter.test +11 -0
- data/test/command/suite/search/error/unknown-source.expected +18 -0
- data/test/command/suite/search/error/unknown-source.test +12 -0
- data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
- data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
- data/test/command/suite/search/group/limit.expected +19 -0
- data/test/command/suite/search/group/limit.test +20 -0
- data/test/command/suite/search/group/string.expected +36 -0
- data/test/command/suite/search/group/string.test +44 -0
- data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
- data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
- data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
- data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
- data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
- data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
- data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
- data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
- data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
- data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
- data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-output-offset.test +25 -0
- data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
- data/test/command/suite/search/response/records/value/time.expected +24 -0
- data/test/command/suite/search/response/records/value/time.test +24 -0
- data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
- data/test/command/suite/search/sort/default-offset-limit.test +26 -0
- data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
- data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
- data/test/command/suite/watch/subscribe.expected +12 -0
- data/test/command/suite/watch/subscribe.test +9 -0
- data/test/command/suite/watch/unsubscribe.expected +12 -0
- data/test/command/suite/watch/unsubscribe.test +9 -0
- data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
- data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
- data/test/unit/helper.rb +2 -0
- data/test/unit/plugin/collector/test_basic.rb +289 -33
- data/test/unit/plugin/distributor/test_search.rb +176 -861
- data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
- data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
- data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
- data/test/unit/plugin/handler/test_add.rb +74 -11
- data/test/unit/plugin/handler/test_groonga.rb +15 -1
- data/test/unit/plugin/handler/test_search.rb +33 -17
- data/test/unit/plugin/handler/test_watch.rb +43 -27
- data/test/unit/run-test.rb +2 -0
- data/test/unit/test_message_pack_packer.rb +51 -0
- data/test/unit/test_time_formatter.rb +29 -0
- metadata +208 -110
- data/lib/droonga/job_queue.rb +0 -87
- data/lib/droonga/job_queue_schema.rb +0 -65
- data/test/unit/test_adapter.rb +0 -51
- data/test/unit/test_job_queue_schema.rb +0 -45
data/lib/droonga/job_queue.rb
DELETED
@@ -1,87 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
4
|
-
#
|
5
|
-
# This library is free software; you can redistribute it and/or
|
6
|
-
# modify it under the terms of the GNU Lesser General Public
|
7
|
-
# License version 2.1 as published by the Free Software Foundation.
|
8
|
-
#
|
9
|
-
# This library is distributed in the hope that it will be useful,
|
10
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
-
# Lesser General Public License for more details.
|
13
|
-
#
|
14
|
-
# You should have received a copy of the GNU Lesser General Public
|
15
|
-
# License along with this library; if not, write to the Free Software
|
16
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
-
|
18
|
-
require "droonga/job_queue_schema"
|
19
|
-
require "msgpack"
|
20
|
-
|
21
|
-
module Droonga
|
22
|
-
class JobQueue
|
23
|
-
class << self
|
24
|
-
def ensure_schema(database_path, queue_name)
|
25
|
-
schema = JobQueueSchema.new(database_path, queue_name)
|
26
|
-
schema.ensure_created
|
27
|
-
end
|
28
|
-
|
29
|
-
def open(database_path, queue_name)
|
30
|
-
job_queue = new(database_path, queue_name)
|
31
|
-
job_queue.open
|
32
|
-
job_queue
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def initialize(database_path, queue_name)
|
37
|
-
@database_path = database_path
|
38
|
-
@queue_name = queue_name
|
39
|
-
end
|
40
|
-
|
41
|
-
def open
|
42
|
-
@context = Groonga::Context.new
|
43
|
-
@database = @context.open_database(@database_path)
|
44
|
-
@context.encoding = :none
|
45
|
-
|
46
|
-
@queue = @context[@queue_name]
|
47
|
-
end
|
48
|
-
|
49
|
-
def push_message(message)
|
50
|
-
$log.trace("#{log_tag}: push_message: start")
|
51
|
-
packed_message = message.to_msgpack
|
52
|
-
@queue.push do |record|
|
53
|
-
record.message = packed_message
|
54
|
-
end
|
55
|
-
$log.trace("#{log_tag}: push_message: done")
|
56
|
-
end
|
57
|
-
|
58
|
-
def pull_message
|
59
|
-
packed_message = nil
|
60
|
-
@queue.pull do |record|
|
61
|
-
if record
|
62
|
-
packed_message = record.message
|
63
|
-
record.delete
|
64
|
-
end
|
65
|
-
end
|
66
|
-
return nil unless packed_message
|
67
|
-
MessagePack.unpack(packed_message)
|
68
|
-
end
|
69
|
-
|
70
|
-
def unblock
|
71
|
-
@queue.unblock
|
72
|
-
end
|
73
|
-
|
74
|
-
def close
|
75
|
-
@queue = nil
|
76
|
-
if @database
|
77
|
-
@database.close
|
78
|
-
@context.close
|
79
|
-
@database = @context = nil
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
def log_tag
|
84
|
-
"[#{Process.ppid}][#{Process.pid}] job_queue"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
@@ -1,65 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
4
|
-
#
|
5
|
-
# This library is free software; you can redistribute it and/or
|
6
|
-
# modify it under the terms of the GNU Lesser General Public
|
7
|
-
# License version 2.1 as published by the Free Software Foundation.
|
8
|
-
#
|
9
|
-
# This library is distributed in the hope that it will be useful,
|
10
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
-
# Lesser General Public License for more details.
|
13
|
-
#
|
14
|
-
# You should have received a copy of the GNU Lesser General Public
|
15
|
-
# License along with this library; if not, write to the Free Software
|
16
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
-
|
18
|
-
require "fileutils"
|
19
|
-
|
20
|
-
require "groonga"
|
21
|
-
|
22
|
-
module Droonga
|
23
|
-
class JobQueueSchema
|
24
|
-
def initialize(database_path, queue_name)
|
25
|
-
@database_path = database_path
|
26
|
-
@queue_name = queue_name
|
27
|
-
end
|
28
|
-
|
29
|
-
def ensure_created
|
30
|
-
ensure_database
|
31
|
-
ensure_queue
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
def ensure_database
|
36
|
-
return if File.exist?(@database_path)
|
37
|
-
FileUtils.mkdir_p(File.dirname(@database_path))
|
38
|
-
create_context do |context|
|
39
|
-
context.create_database(@database_path) do
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def ensure_queue
|
45
|
-
create_context do |context|
|
46
|
-
context.open_database(@database_path) do
|
47
|
-
Groonga::Schema.define(:context => context) do |schema|
|
48
|
-
schema.create_table(@queue_name, :type => :array) do |table|
|
49
|
-
table.text("message")
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def create_context
|
57
|
-
context = Groonga::Context.new
|
58
|
-
begin
|
59
|
-
yield(context)
|
60
|
-
ensure
|
61
|
-
context.close
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
data/test/unit/test_adapter.rb
DELETED
@@ -1,51 +0,0 @@
|
|
1
|
-
# Copyright (C) 2013 Droonga Project
|
2
|
-
#
|
3
|
-
# This library is free software; you can redistribute it and/or
|
4
|
-
# modify it under the terms of the GNU Lesser General Public
|
5
|
-
# License version 2.1 as published by the Free Software Foundation.
|
6
|
-
#
|
7
|
-
# This library is distributed in the hope that it will be useful,
|
8
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
-
# Lesser General Public License for more details.
|
11
|
-
#
|
12
|
-
# You should have received a copy of the GNU Lesser General Public
|
13
|
-
# License along with this library; if not, write to the Free Software
|
14
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
-
|
16
|
-
require "droonga/adapter"
|
17
|
-
|
18
|
-
class AdapterTest < Test::Unit::TestCase
|
19
|
-
class AdaptTest < self
|
20
|
-
# class GroongaAdapter < Droonga::Adapter
|
21
|
-
# command :select
|
22
|
-
# def select(request)
|
23
|
-
# post(:search) do |response|
|
24
|
-
# # do nothing
|
25
|
-
# end
|
26
|
-
# :selected
|
27
|
-
# end
|
28
|
-
# end
|
29
|
-
|
30
|
-
def setup
|
31
|
-
omit("Pending")
|
32
|
-
@dispatcher = Object.new
|
33
|
-
@groonga_adapter = GroongaAdapter.new(@dispatcher)
|
34
|
-
end
|
35
|
-
|
36
|
-
def test_called
|
37
|
-
omit("Pending")
|
38
|
-
request = nil
|
39
|
-
stub(@dispatcher).post
|
40
|
-
assert_equal(:selected, @groonga_adapter.adapt(:select, request))
|
41
|
-
end
|
42
|
-
|
43
|
-
def test_post
|
44
|
-
omit("Pending")
|
45
|
-
request = nil
|
46
|
-
response = nil
|
47
|
-
mock(@dispatcher).post(:search).yields(response)
|
48
|
-
@groonga_adapter.adapt(:select, request)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,45 +0,0 @@
|
|
1
|
-
# Copyright (C) 2013 Droonga Project
|
2
|
-
#
|
3
|
-
# This library is free software; you can redistribute it and/or
|
4
|
-
# modify it under the terms of the GNU Lesser General Public
|
5
|
-
# License version 2.1 as published by the Free Software Foundation.
|
6
|
-
#
|
7
|
-
# This library is distributed in the hope that it will be useful,
|
8
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
-
# Lesser General Public License for more details.
|
11
|
-
#
|
12
|
-
# You should have received a copy of the GNU Lesser General Public
|
13
|
-
# License along with this library; if not, write to the Free Software
|
14
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
-
|
16
|
-
require "helper"
|
17
|
-
|
18
|
-
require "droonga/job_queue_schema"
|
19
|
-
|
20
|
-
class JobQueueSchemaTest < Test::Unit::TestCase
|
21
|
-
def setup
|
22
|
-
@database_path = @temporary_directory + "droonga/db"
|
23
|
-
@queue_name = "DroongaQueue"
|
24
|
-
end
|
25
|
-
|
26
|
-
def test_ensure_created
|
27
|
-
schema = Droonga::JobQueueSchema.new(@database_path.to_s, @queue_name)
|
28
|
-
|
29
|
-
assert_not_predicate(@database_path, :exist?)
|
30
|
-
schema.ensure_created
|
31
|
-
assert_predicate(@database_path, :exist?)
|
32
|
-
|
33
|
-
context = Groonga::Context.new
|
34
|
-
dumped_commands = nil
|
35
|
-
context.open_database(@database_path.to_s) do |database|
|
36
|
-
dumped_commands = Groonga::DatabaseDumper.dump(:context => context,
|
37
|
-
:database => database)
|
38
|
-
end
|
39
|
-
context.close
|
40
|
-
assert_equal(<<-SCHEMA, dumped_commands)
|
41
|
-
table_create #{@queue_name} TABLE_NO_KEY
|
42
|
-
column_create #{@queue_name} message COLUMN_SCALAR Text
|
43
|
-
SCHEMA
|
44
|
-
end
|
45
|
-
end
|