fluent-plugin-droonga 0.0.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99aed806731b4f63bdefc707bddd4fb9c6f1d105
-  data.tar.gz: 0a4ad0ab5c237b44a81656a489744aac4b264a64
+  metadata.gz: 16568393cd233c1f8a54d3bce227e328b6e1b293
+  data.tar.gz: 48f158466e6f15a069f3f5c8f92add90f740114c
 SHA512:
-  metadata.gz: dbeb3a6142fe82e708355ef091ecb2789aa2f18058ac5a9b66089828c91bbfc1237b8f4c5b0998e7fee55a8188f321ac394a547a1608a2e4b15bd6036d24b8e5
-  data.tar.gz: 1db4e2fe92124dbd2d259dad398fb54868214e339319286ca2ce2c3bd7128b7b76b4cde19b5c6f6f262c6a703142a7a8ebd2b2c80021b381cf024d9c61f558ca
+  metadata.gz: aa5bedb70c57916313b56a3639f8567f3a3efc3a43ba9b8bd68fae5d547e91c481ab71865fa1864d9b9cd425f54488acdc60c8cc53d652754b6552ccbe6e1f40
+  data.tar.gz: 3a3c6fa0f130808929a5225ea02b5a89bf5b5b850bc2a227cf9a35d54702bb2b63857d136672bc6849455e2de350f7de95c12bdb5dac4313a67785881ad63ef4

data/.gitignore

@@ -1 +1,7 @@
 /Gemfile.lock
+/test/command/.bundle/
+/test/command/Gemfile.lock
+/test/command/bin/
+/test/command/vendor/
+/doc/
+/.yardoc/

data/.yardopts

@@ -0,0 +1,7 @@
+--output-dir doc/reference/en
+--markup markdown
+--markup-provider kramdown
+lib/**/*.rb
+-
+LICENSE.txt
+doc/text/**/*.*

data/Gemfile

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -37,4 +37,16 @@ elsif ENV["TRAVIS"] == "true"
   end
 end
 
-gem "droonga-client", github: "droonga/droonga-client-ruby"
+droonga_client_dir = File.join(parent_dir, "droonga-client-ruby")
+if File.exist?(droonga_client_dir)
+  gem "droonga-client", :path => droonga_client_dir
+else
+  gem "droonga-client", github: "droonga/droonga-client-ruby"
+end
+
+drntest_dir = File.join(parent_dir, "drntest")
+if File.exist?(drntest_dir)
+  gem "drntest", :path => drntest_dir
+else
+  gem "drntest", :github => "droonga/drntest"
+end

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (C) 2013 droonga project
+Copyright (C) 2013 Droonga Project
 
 This library is free software; you can redistribute it and/or
 modify it under the terms of the GNU Lesser General Public

data/README.md

@@ -13,6 +13,6 @@ TODO: Write usage.
 
 ## Copyright
 
-* Copyright (c) 2013 droonga project
+* Copyright (c) 2013 Droonga Project
 * License
   * GNU Lesser General Public License version 2.1

data/Rakefile

@@ -1,6 +1,6 @@
 # -*- ruby -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,11 +15,33 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "bundler/gem_tasks"
+require "bundler/gem_helper"
+require "packnga"
 
-desc "Run test"
-task :test do
-  ruby(File.join(File.dirname(__FILE__), "test", "run-test.rb"))
+base_dir = File.join(File.dirname(__FILE__))
+
+helper = Bundler::GemHelper.new(base_dir)
+helper.install
+spec = helper.gemspec
+
+Packnga::DocumentTask.new(spec) do |task|
+  task.original_language = "en"
+  task.translate_languages = ["ja"]
+end
+
+namespace :test do
+  desc "Run unit test"
+  task :unit do
+    ruby(File.join(File.dirname(__FILE__), "test", "unit", "run-test.rb"))
+  end
+
+  desc "Run command test"
+  task :command do
+    ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb"))
+  end
 end
 
+desc "Run test"
+task :test => ["test:unit", "test:command"]
+
 task :default => :test

data/benchmark/benchmark.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/benchmark/utils.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -231,12 +231,15 @@ module DroongaBenchmark
     def start
       @messages = Queue.new
       Thread.new do
-        client = @socket.accept
-        unpacker = MessagePack::Unpacker.new(client)
-        unpacker.each do |object|
-          @messages.push(object)
+        loop do
+          Thread.new(@socket.accept) do |client|
+            unpacker = MessagePack::Unpacker.new(client)
+            unpacker.each do |object|
+              @messages.push(object)
+            end
+            client.close
+          end
         end
-        client.close
       end
     end
   end

data/benchmark/watch/benchmark-notify.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -55,7 +55,7 @@ class NotifyBenchmark
     end
 
     notifications = []
-    while notifications.size != @n_times do
+    while notifications.size != @n_times
       notifications << @receiver.new_message
     end
     notifications

data/benchmark/watch/benchmark-publish.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/benchmark/watch/benchmark-scan.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/benchmark/watch/catalog.json

@@ -9,7 +9,7 @@
   },
   "datasets": {
     "Watch": {
-      "workers": 0,
+      "workers": 2,
       "plugins": ["search", "groonga", "add", "watch"],
       "number_of_replicas": 1,
       "number_of_partitions": 1,

data/bin/grn2jsons

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public

data/fluent-plugin-droonga.gemspec

@@ -1,6 +1,6 @@
 # -*- mode: ruby; coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -17,10 +17,10 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-droonga"
-  gem.version       = "0.0.2"
+  gem.version       = "0.7.0"
   gem.authors       = ["Droonga Project"]
   gem.email         = ["droonga@groonga.org"]
-  gem.description   = "Droonga(distributed groonga) plugin for Fluent event collector"
+  gem.description   = "Droonga(distributed Groonga) plugin for Fluent event collector"
   gem.summary       = gem.description
   gem.homepage      = "https://github.com/droonga/fluent-plugin-droonga"
   gem.files         = `git ls-files`.split($/)
@@ -38,4 +38,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "test-unit"
   gem.add_development_dependency "test-unit-notify"
   gem.add_development_dependency "test-unit-rr"
+  gem.add_development_dependency "packnga"
+  gem.add_development_dependency "kramdown"
 end

data/lib/droonga/adapter.rb

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2013 droonga project
+# Copyright (C) 2013 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,142 +15,25 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-require "droonga/handler"
+require "droonga/pluggable"
+require "droonga/adapter_plugin"
 
 module Droonga
-  class Adapter < Droonga::Handler
-    def scatter_all(request, key)
-      message = [{
-        "command"=> envelope["type"],
-        "dataset"=> envelope["dataset"],
-        "body"=> request,
-        "key"=> key,
-        "type"=> "scatter",
-        "replica"=> "all",
-        "post"=> true
-      }]
-      post(message, "proxy")
-    end
-
-    def broadcast_all(request)
-      message = [{
-        "command"=> envelope["type"],
-        "dataset"=> envelope["dataset"],
-        "body"=> request,
-        "type"=> "broadcast",
-        "replica"=> "all",
-        "post"=> true
-      }]
-      post(message, "proxy")
-    end
-
-    def prefer_synchronous?(command)
-      return true
-    end
-  end
-
-  class BasicAdapter < Adapter
-    Droonga::HandlerPlugin.register("adapter", self)
-
-    command :table_create
-    def table_create(request)
-      broadcast_all(request)
-    end
-
-    command :column_create
-    def column_create(request)
-      broadcast_all(request)
-    end
-
-    command "watch.feed" => :feed
-    def feed(request)
-      broadcast_all(request)
-    end
-
-    command "watch.subscribe" => :subscribe
-    def subscribe(request)
-      broadcast_all(request)
-    end
-
-    command "watch.unsubscribe" => :unsubscribe
-    def unsubscribe(request)
-      broadcast_all(request)
-    end
-
-    command :add
-    def add(request)
-      # TODO: update events must be serialized in the primary node of replicas.
-      key = request["key"] || rand.to_s
-      scatter_all(request, key)
-    end
+  class Adapter
+    include Pluggable
 
-    command :update
-    def update(request)
-      # TODO: update events must be serialized in the primary node of replicas.
-      key = request["key"] || rand.to_s
-      scatter_all(request, key)
+    def initialize(dispatcher, options={})
+      @dispatcher = dispatcher
+      load_plugins(options[:adapters] || [])
     end
 
-    command :reset
-    def add(request)
-      # TODO: update events must be serialized in the primary node of replicas.
-      key = request["key"] || rand.to_s
-      scatter_all(request, key)
+    private
+    def instantiate_plugin(name)
+      AdapterPlugin.repository.instantiate(name, @dispatcher)
     end
 
-    command :search
-    def search(request)
-      message = []
-      input_names = []
-      output_names = []
-      name_mapper = {}
-      request["queries"].each do |input_name, query|
-        output = query["output"]
-        next unless output
-        input_names << input_name
-        output_name = input_name + "_reduced"
-        output_names << output_name
-        name_mapper[output_name] = input_name
-        # TODO: offset & limit must be arranged here.
-        elements = {}
-        output["elements"].each do |element|
-          case element
-          when "count"
-            elements[element] = ["sum"]
-          when "records"
-            # TODO: must take "sortBy" section into account.
-            elements[element] = ["sort", "<"]
-          end
-        end
-        reducer = {
-          "inputs"=> [input_name],
-          "outputs"=> [output_name],
-          "type"=> "reduce",
-          "body"=> {
-            input_name=> {
-              output_name=> elements
-            }
-          }
-        }
-        message << reducer
-      end
-      gatherer = {
-        "inputs"=> output_names,
-        "type"=> "gather",
-        "body"=> name_mapper,
-        "post"=> true
-      }
-      message << gatherer
-      searcher = {
-        "dataset"=> envelope["dataset"] || request["dataset"],
-        "outputs"=> input_names,
-        "type"=> "broadcast",
-        "command"=> "search",
-        "replica"=> "random",
-        "body"=> request
-      }
-      message.push(searcher)
-      post(message, "proxy")
+    def log_tag
+      "adapter"
     end
   end
 end

data/lib/droonga/adapter_plugin.rb

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin"
+
+module Droonga
+  class AdapterPlugin < Plugin
+    extend PluginRegisterable
+
+    def initialize(dispatcher)
+      super()
+      @dispatcher = dispatcher
+    end
+
+    def add_route(route)
+      @dispatcher.add_route(route)
+    end
+
+    def post(body, destination=nil)
+      @dispatcher.post(body, destination)
+    end
+
+    def emit(value, name=nil)
+      if name
+        @output_values[name] = value
+      else
+        @output_values = value
+      end
+    end
+
+    def process(command, message)
+      @output_values = {}
+      super(command, message)
+      post(@output_values) unless @output_values.empty?
+    end
+  end
+end