fluent-plugin-droonga 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -4
- data/benchmark/watch/benchmark-notify.rb +2 -2
- data/benchmark/watch/benchmark-scan.rb +3 -0
- data/benchmark/watch/fluentd.conf +0 -1
- data/fluent-plugin-droonga.gemspec +2 -3
- data/lib/droonga/catalog.rb +10 -124
- data/lib/droonga/catalog/base.rb +140 -0
- data/lib/droonga/catalog/version1.rb +23 -0
- data/lib/droonga/catalog_loader.rb +33 -0
- data/lib/droonga/collector.rb +2 -71
- data/lib/droonga/collector_plugin.rb +2 -34
- data/lib/droonga/dispatcher.rb +141 -196
- data/lib/droonga/distribution_planner.rb +76 -0
- data/lib/droonga/distributor.rb +5 -7
- data/lib/droonga/distributor_plugin.rb +23 -15
- data/lib/droonga/engine.rb +2 -2
- data/lib/droonga/event_loop.rb +46 -0
- data/lib/droonga/farm.rb +9 -5
- data/lib/droonga/fluent_message_sender.rb +84 -0
- data/lib/droonga/forwarder.rb +43 -53
- data/lib/droonga/handler.rb +20 -68
- data/lib/droonga/handler_message.rb +61 -0
- data/lib/droonga/handler_messenger.rb +92 -0
- data/lib/droonga/handler_plugin.rb +10 -12
- data/lib/droonga/input_adapter.rb +52 -0
- data/lib/droonga/{adapter.rb → input_adapter_plugin.rb} +7 -13
- data/lib/droonga/input_message.rb +11 -11
- data/lib/droonga/logger.rb +4 -3
- data/lib/droonga/message_pack_packer.rb +62 -0
- data/lib/droonga/message_processing_error.rb +54 -0
- data/lib/droonga/message_pusher.rb +60 -0
- data/lib/droonga/message_receiver.rb +61 -0
- data/lib/droonga/output_adapter.rb +53 -0
- data/lib/droonga/{adapter_plugin.rb → output_adapter_plugin.rb} +3 -21
- data/lib/droonga/output_message.rb +37 -0
- data/lib/droonga/partition.rb +27 -5
- data/lib/droonga/pluggable.rb +9 -4
- data/lib/droonga/plugin.rb +12 -3
- data/lib/droonga/plugin/collector/basic.rb +91 -18
- data/lib/droonga/plugin/distributor/crud.rb +9 -9
- data/lib/droonga/plugin/distributor/distributed_search_planner.rb +401 -0
- data/lib/droonga/plugin/distributor/groonga.rb +5 -5
- data/lib/droonga/plugin/distributor/search.rb +4 -246
- data/lib/droonga/plugin/distributor/watch.rb +11 -6
- data/lib/droonga/plugin/handler/add.rb +69 -7
- data/lib/droonga/plugin/handler/groonga.rb +6 -6
- data/lib/droonga/plugin/handler/search.rb +5 -3
- data/lib/droonga/plugin/handler/watch.rb +19 -13
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga.rb +5 -11
- data/lib/droonga/plugin/{adapter → input_adapter}/groonga/select.rb +2 -36
- data/lib/droonga/plugin/output_adapter/groonga.rb +30 -0
- data/lib/droonga/plugin/output_adapter/groonga/select.rb +54 -0
- data/lib/droonga/plugin_loader.rb +2 -2
- data/lib/droonga/processor.rb +21 -23
- data/lib/droonga/replier.rb +40 -0
- data/lib/droonga/searcher.rb +298 -174
- data/lib/droonga/server.rb +0 -67
- data/lib/droonga/session.rb +85 -0
- data/lib/droonga/test.rb +21 -0
- data/lib/droonga/test/stub_distributor.rb +31 -0
- data/lib/droonga/test/stub_handler.rb +37 -0
- data/lib/droonga/test/stub_handler_message.rb +35 -0
- data/lib/droonga/test/stub_handler_messenger.rb +34 -0
- data/lib/droonga/time_formatter.rb +37 -0
- data/lib/droonga/watcher.rb +1 -0
- data/lib/droonga/worker.rb +16 -19
- data/lib/fluent/plugin/out_droonga.rb +9 -9
- data/lib/groonga_command_converter.rb +5 -5
- data/sample/cluster/catalog.json +1 -1
- data/test/command/config/default/catalog.json +19 -1
- data/test/command/fixture/event.jsons +41 -0
- data/test/command/fixture/user-table.jsons +9 -0
- data/test/command/run-test.rb +2 -2
- data/test/command/suite/add/error/invalid-integer.expected +20 -0
- data/test/command/suite/add/error/invalid-integer.test +12 -0
- data/test/command/suite/add/error/invalid-time.expected +20 -0
- data/test/command/suite/add/error/invalid-time.test +12 -0
- data/test/command/suite/add/error/missing-key.expected +13 -0
- data/test/command/suite/add/error/missing-key.test +16 -0
- data/test/command/suite/add/error/missing-table.expected +13 -0
- data/test/command/suite/add/error/missing-table.test +16 -0
- data/test/command/suite/add/error/unknown-column.expected +20 -0
- data/test/command/suite/add/error/unknown-column.test +12 -0
- data/test/command/suite/add/error/unknown-table.expected +13 -0
- data/test/command/suite/add/error/unknown-table.test +17 -0
- data/test/command/suite/add/minimum.expected +1 -3
- data/test/command/suite/add/with-values.expected +1 -3
- data/test/command/suite/add/without-key.expected +1 -3
- data/test/command/suite/message/error/missing-dataset.expected +13 -0
- data/test/command/suite/message/error/missing-dataset.test +5 -0
- data/test/command/suite/message/error/unknown-command.expected +13 -0
- data/test/command/suite/message/error/unknown-command.test +6 -0
- data/test/command/suite/message/error/unknown-dataset.expected +13 -0
- data/test/command/suite/message/error/unknown-dataset.test +6 -0
- data/test/command/suite/search/{array-attribute-label.expected → attributes/array.expected} +0 -0
- data/test/command/suite/search/{array-attribute-label.test → attributes/array.test} +0 -0
- data/test/command/suite/search/{hash-attribute-label.expected → attributes/hash.expected} +0 -0
- data/test/command/suite/search/{hash-attribute-label.test → attributes/hash.test} +0 -0
- data/test/command/suite/search/{condition-nested.expected → condition/nested.expected} +0 -0
- data/test/command/suite/search/{condition-nested.test → condition/nested.test} +0 -0
- data/test/command/suite/search/{condition-query.expected → condition/query.expected} +0 -0
- data/test/command/suite/search/{condition-query.test → condition/query.test} +0 -0
- data/test/command/suite/search/{condition-script.expected → condition/script.expected} +0 -0
- data/test/command/suite/search/{condition-script.test → condition/script.test} +0 -0
- data/test/command/suite/search/error/cyclic-source.expected +18 -0
- data/test/command/suite/search/error/cyclic-source.test +12 -0
- data/test/command/suite/search/error/deeply-cyclic-source.expected +21 -0
- data/test/command/suite/search/error/deeply-cyclic-source.test +15 -0
- data/test/command/suite/search/error/missing-source-parameter.expected +17 -0
- data/test/command/suite/search/error/missing-source-parameter.test +11 -0
- data/test/command/suite/search/error/unknown-source.expected +18 -0
- data/test/command/suite/search/error/unknown-source.test +12 -0
- data/test/command/suite/search/{minimum.expected → group/count.expected} +2 -1
- data/test/command/suite/search/{minimum.test → group/count.test} +5 -3
- data/test/command/suite/search/group/limit.expected +19 -0
- data/test/command/suite/search/group/limit.test +20 -0
- data/test/command/suite/search/group/string.expected +36 -0
- data/test/command/suite/search/group/string.test +44 -0
- data/test/command/suite/search/{chained-queries.expected → multiple/chained.expected} +0 -0
- data/test/command/suite/search/{chained-queries.test → multiple/chained.test} +0 -0
- data/test/command/suite/search/{multiple-queries.expected → multiple/parallel.expected} +0 -0
- data/test/command/suite/search/{multiple-queries.test → multiple/parallel.test} +0 -0
- data/test/command/suite/search/{output-range.expected → range/only-output.expected} +0 -0
- data/test/command/suite/search/{output-range.test → range/only-output.test} +0 -0
- data/test/command/suite/search/{sort-range.expected → range/only-sort.expected} +0 -0
- data/test/command/suite/search/{sort-range.test → range/only-sort.test} +0 -0
- data/test/command/suite/search/{sort-and-output-range.expected → range/sort-and-output.expected} +0 -0
- data/test/command/suite/search/{sort-and-output-range.test → range/sort-and-output.test} +0 -0
- data/test/command/suite/search/range/too-large-output-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-output-offset.test +25 -0
- data/test/command/suite/search/range/too-large-sort-offset.expected +16 -0
- data/test/command/suite/search/range/too-large-sort-offset.test +28 -0
- data/test/command/suite/search/response/records/value/time.expected +24 -0
- data/test/command/suite/search/response/records/value/time.test +24 -0
- data/test/command/suite/search/sort/default-offset-limit.expected +43 -0
- data/test/command/suite/search/sort/default-offset-limit.test +26 -0
- data/test/command/suite/search/{sort-with-invisible-column.expected → sort/invisible-column.expected} +0 -0
- data/test/command/suite/search/{sort-with-invisible-column.test → sort/invisible-column.test} +0 -0
- data/test/command/suite/watch/subscribe.expected +12 -0
- data/test/command/suite/watch/subscribe.test +9 -0
- data/test/command/suite/watch/unsubscribe.expected +12 -0
- data/test/command/suite/watch/unsubscribe.test +9 -0
- data/test/unit/{test_catalog.rb → catalog/test_version1.rb} +12 -4
- data/test/unit/fixtures/{catalog.json → catalog/version1.json} +0 -0
- data/test/unit/helper.rb +2 -0
- data/test/unit/plugin/collector/test_basic.rb +289 -33
- data/test/unit/plugin/distributor/test_search.rb +176 -861
- data/test/unit/plugin/distributor/test_search_planner.rb +1102 -0
- data/test/unit/plugin/handler/groonga/test_column_create.rb +17 -13
- data/test/unit/plugin/handler/groonga/test_table_create.rb +10 -10
- data/test/unit/plugin/handler/test_add.rb +74 -11
- data/test/unit/plugin/handler/test_groonga.rb +15 -1
- data/test/unit/plugin/handler/test_search.rb +33 -17
- data/test/unit/plugin/handler/test_watch.rb +43 -27
- data/test/unit/run-test.rb +2 -0
- data/test/unit/test_message_pack_packer.rb +51 -0
- data/test/unit/test_time_formatter.rb +29 -0
- metadata +208 -110
- data/lib/droonga/job_queue.rb +0 -87
- data/lib/droonga/job_queue_schema.rb +0 -65
- data/test/unit/test_adapter.rb +0 -51
- data/test/unit/test_job_queue_schema.rb +0 -45
data/lib/droonga/job_queue.rb
DELETED
@@ -1,87 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
4
|
-
#
|
5
|
-
# This library is free software; you can redistribute it and/or
|
6
|
-
# modify it under the terms of the GNU Lesser General Public
|
7
|
-
# License version 2.1 as published by the Free Software Foundation.
|
8
|
-
#
|
9
|
-
# This library is distributed in the hope that it will be useful,
|
10
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
-
# Lesser General Public License for more details.
|
13
|
-
#
|
14
|
-
# You should have received a copy of the GNU Lesser General Public
|
15
|
-
# License along with this library; if not, write to the Free Software
|
16
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
-
|
18
|
-
require "droonga/job_queue_schema"
|
19
|
-
require "msgpack"
|
20
|
-
|
21
|
-
module Droonga
|
22
|
-
class JobQueue
|
23
|
-
class << self
|
24
|
-
def ensure_schema(database_path, queue_name)
|
25
|
-
schema = JobQueueSchema.new(database_path, queue_name)
|
26
|
-
schema.ensure_created
|
27
|
-
end
|
28
|
-
|
29
|
-
def open(database_path, queue_name)
|
30
|
-
job_queue = new(database_path, queue_name)
|
31
|
-
job_queue.open
|
32
|
-
job_queue
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def initialize(database_path, queue_name)
|
37
|
-
@database_path = database_path
|
38
|
-
@queue_name = queue_name
|
39
|
-
end
|
40
|
-
|
41
|
-
def open
|
42
|
-
@context = Groonga::Context.new
|
43
|
-
@database = @context.open_database(@database_path)
|
44
|
-
@context.encoding = :none
|
45
|
-
|
46
|
-
@queue = @context[@queue_name]
|
47
|
-
end
|
48
|
-
|
49
|
-
def push_message(message)
|
50
|
-
$log.trace("#{log_tag}: push_message: start")
|
51
|
-
packed_message = message.to_msgpack
|
52
|
-
@queue.push do |record|
|
53
|
-
record.message = packed_message
|
54
|
-
end
|
55
|
-
$log.trace("#{log_tag}: push_message: done")
|
56
|
-
end
|
57
|
-
|
58
|
-
def pull_message
|
59
|
-
packed_message = nil
|
60
|
-
@queue.pull do |record|
|
61
|
-
if record
|
62
|
-
packed_message = record.message
|
63
|
-
record.delete
|
64
|
-
end
|
65
|
-
end
|
66
|
-
return nil unless packed_message
|
67
|
-
MessagePack.unpack(packed_message)
|
68
|
-
end
|
69
|
-
|
70
|
-
def unblock
|
71
|
-
@queue.unblock
|
72
|
-
end
|
73
|
-
|
74
|
-
def close
|
75
|
-
@queue = nil
|
76
|
-
if @database
|
77
|
-
@database.close
|
78
|
-
@context.close
|
79
|
-
@database = @context = nil
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
def log_tag
|
84
|
-
"[#{Process.ppid}][#{Process.pid}] job_queue"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
@@ -1,65 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
#
|
3
|
-
# Copyright (C) 2013 Droonga Project
|
4
|
-
#
|
5
|
-
# This library is free software; you can redistribute it and/or
|
6
|
-
# modify it under the terms of the GNU Lesser General Public
|
7
|
-
# License version 2.1 as published by the Free Software Foundation.
|
8
|
-
#
|
9
|
-
# This library is distributed in the hope that it will be useful,
|
10
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
12
|
-
# Lesser General Public License for more details.
|
13
|
-
#
|
14
|
-
# You should have received a copy of the GNU Lesser General Public
|
15
|
-
# License along with this library; if not, write to the Free Software
|
16
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
17
|
-
|
18
|
-
require "fileutils"
|
19
|
-
|
20
|
-
require "groonga"
|
21
|
-
|
22
|
-
module Droonga
|
23
|
-
class JobQueueSchema
|
24
|
-
def initialize(database_path, queue_name)
|
25
|
-
@database_path = database_path
|
26
|
-
@queue_name = queue_name
|
27
|
-
end
|
28
|
-
|
29
|
-
def ensure_created
|
30
|
-
ensure_database
|
31
|
-
ensure_queue
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
def ensure_database
|
36
|
-
return if File.exist?(@database_path)
|
37
|
-
FileUtils.mkdir_p(File.dirname(@database_path))
|
38
|
-
create_context do |context|
|
39
|
-
context.create_database(@database_path) do
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def ensure_queue
|
45
|
-
create_context do |context|
|
46
|
-
context.open_database(@database_path) do
|
47
|
-
Groonga::Schema.define(:context => context) do |schema|
|
48
|
-
schema.create_table(@queue_name, :type => :array) do |table|
|
49
|
-
table.text("message")
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def create_context
|
57
|
-
context = Groonga::Context.new
|
58
|
-
begin
|
59
|
-
yield(context)
|
60
|
-
ensure
|
61
|
-
context.close
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
data/test/unit/test_adapter.rb
DELETED
@@ -1,51 +0,0 @@
|
|
1
|
-
# Copyright (C) 2013 Droonga Project
|
2
|
-
#
|
3
|
-
# This library is free software; you can redistribute it and/or
|
4
|
-
# modify it under the terms of the GNU Lesser General Public
|
5
|
-
# License version 2.1 as published by the Free Software Foundation.
|
6
|
-
#
|
7
|
-
# This library is distributed in the hope that it will be useful,
|
8
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
-
# Lesser General Public License for more details.
|
11
|
-
#
|
12
|
-
# You should have received a copy of the GNU Lesser General Public
|
13
|
-
# License along with this library; if not, write to the Free Software
|
14
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
-
|
16
|
-
require "droonga/adapter"
|
17
|
-
|
18
|
-
class AdapterTest < Test::Unit::TestCase
|
19
|
-
class AdaptTest < self
|
20
|
-
# class GroongaAdapter < Droonga::Adapter
|
21
|
-
# command :select
|
22
|
-
# def select(request)
|
23
|
-
# post(:search) do |response|
|
24
|
-
# # do nothing
|
25
|
-
# end
|
26
|
-
# :selected
|
27
|
-
# end
|
28
|
-
# end
|
29
|
-
|
30
|
-
def setup
|
31
|
-
omit("Pending")
|
32
|
-
@dispatcher = Object.new
|
33
|
-
@groonga_adapter = GroongaAdapter.new(@dispatcher)
|
34
|
-
end
|
35
|
-
|
36
|
-
def test_called
|
37
|
-
omit("Pending")
|
38
|
-
request = nil
|
39
|
-
stub(@dispatcher).post
|
40
|
-
assert_equal(:selected, @groonga_adapter.adapt(:select, request))
|
41
|
-
end
|
42
|
-
|
43
|
-
def test_post
|
44
|
-
omit("Pending")
|
45
|
-
request = nil
|
46
|
-
response = nil
|
47
|
-
mock(@dispatcher).post(:search).yields(response)
|
48
|
-
@groonga_adapter.adapt(:select, request)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
@@ -1,45 +0,0 @@
|
|
1
|
-
# Copyright (C) 2013 Droonga Project
|
2
|
-
#
|
3
|
-
# This library is free software; you can redistribute it and/or
|
4
|
-
# modify it under the terms of the GNU Lesser General Public
|
5
|
-
# License version 2.1 as published by the Free Software Foundation.
|
6
|
-
#
|
7
|
-
# This library is distributed in the hope that it will be useful,
|
8
|
-
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
-
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
-
# Lesser General Public License for more details.
|
11
|
-
#
|
12
|
-
# You should have received a copy of the GNU Lesser General Public
|
13
|
-
# License along with this library; if not, write to the Free Software
|
14
|
-
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
-
|
16
|
-
require "helper"
|
17
|
-
|
18
|
-
require "droonga/job_queue_schema"
|
19
|
-
|
20
|
-
class JobQueueSchemaTest < Test::Unit::TestCase
|
21
|
-
def setup
|
22
|
-
@database_path = @temporary_directory + "droonga/db"
|
23
|
-
@queue_name = "DroongaQueue"
|
24
|
-
end
|
25
|
-
|
26
|
-
def test_ensure_created
|
27
|
-
schema = Droonga::JobQueueSchema.new(@database_path.to_s, @queue_name)
|
28
|
-
|
29
|
-
assert_not_predicate(@database_path, :exist?)
|
30
|
-
schema.ensure_created
|
31
|
-
assert_predicate(@database_path, :exist?)
|
32
|
-
|
33
|
-
context = Groonga::Context.new
|
34
|
-
dumped_commands = nil
|
35
|
-
context.open_database(@database_path.to_s) do |database|
|
36
|
-
dumped_commands = Groonga::DatabaseDumper.dump(:context => context,
|
37
|
-
:database => database)
|
38
|
-
end
|
39
|
-
context.close
|
40
|
-
assert_equal(<<-SCHEMA, dumped_commands)
|
41
|
-
table_create #{@queue_name} TABLE_NO_KEY
|
42
|
-
column_create #{@queue_name} message COLUMN_SCALAR Text
|
43
|
-
SCHEMA
|
44
|
-
end
|
45
|
-
end
|