fluent-plugin-droonga 0.0.2

data/benchmark/watch/catalog.json

@@ -0,0 +1,32 @@
+{
+  "effective_date": "2013-09-01T00:00:00Z",
+  "zones": ["localhost:23003/droonga"],
+  "farms": {
+    "localhost:23003/droonga": {
+      "device": ".",
+      "capacity": 10
+    }
+  },
+  "datasets": {
+    "Watch": {
+      "workers": 0,
+      "plugins": ["search", "groonga", "add", "watch"],
+      "number_of_replicas": 1,
+      "number_of_partitions": 1,
+      "partition_key": "_key",
+      "date_range": "infinity",
+      "ring": {
+        "localhost:23041": {
+          "weight": 50,
+          "partitions": {
+            "2013-09-01": [
+              "localhost:23003/droonga.watch"
+            ]
+          }
+        }
+      }
+    }
+  },
+  "options": {
+  }
+}

data/benchmark/watch/fluentd.conf

@@ -0,0 +1,12 @@
+<source>
+  type forward
+  port 23003
+</source>
+<match droonga.message>
+  name localhost:23003/droonga
+  type droonga
+  proxy true
+</match>
+<match output.message>
+  type stdout
+</match>

data/bin/grn2jsons

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga_command_converter"
+require "json"
+require "ostruct"
+require "optparse"
+
+options = OpenStruct.new
+option_parser = OptionParser.new do |parser|
+  parser.on("-i=ID", "--id=ID",
+            "base id. (optional)") do |id|
+    options.id = id
+  end
+
+  parser.on("-d=DATE", "--date=DATE",
+            "date. (optional)") do |date|
+    options.date = date
+  end
+
+  parser.on("-r=REPLYTO", "--reply-to=REPLYTO",
+            "value of replyTo field.") do |reply_to|
+    options.reply_to = reply_to
+  end
+
+  parser.on("-s=DATASET", "--dataset=DATASET",
+            "dataset.") do |dataset|
+    options.dataset = dataset
+  end
+end
+args = option_parser.parse!(ARGV)
+
+if options.reply_to.nil?
+  raise "You must specify the value of the \"replyTo\" field by --reply-to option."
+end
+if options.dataset.nil?
+  raise "You must specify the name of the dataset by --dataset option."
+end
+
+convert_options = {
+  :id => options.id,
+  :date => options.date,
+  :reply_to => options.reply_to,
+  :dataset => options.dataset,
+}
+converter = Droonga::GroongaCommandConverter.new(convert_options)
+
+source_file = args[0]
+result_file = args[1]
+
+input = nil
+if source_file.nil?
+  input = STDIN.read
+else
+  input = File.read(source_file)
+end
+
+result_file = args[1]
+
+if result_file.nil?
+  converter.convert(input) do |command|
+    puts(JSON.generate(command))
+  end
+else
+  File.open("w", result_file) do |file|
+    converter.convert(input) do |command|
+      file.puts(JSON.generate(command))
+    end
+  end
+end

data/fluent-plugin-droonga.gemspec

@@ -0,0 +1,41 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-droonga"
+  gem.version       = "0.0.2"
+  gem.authors       = ["Droonga Project"]
+  gem.email         = ["droonga@groonga.org"]
+  gem.description   = "Droonga(distributed groonga) plugin for Fluent event collector"
+  gem.summary       = gem.description
+  gem.homepage      = "https://github.com/droonga/fluent-plugin-droonga"
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "fluentd"
+  gem.add_dependency "rroonga", ">= 3.0.3"
+  gem.add_dependency "groonga-command-parser"
+  gem.add_dependency "fluent-logger"
+  gem.add_dependency "serverengine"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "droonga-client"
+  gem.add_development_dependency "test-unit"
+  gem.add_development_dependency "test-unit-notify"
+  gem.add_development_dependency "test-unit-rr"
+end

data/lib/droonga/adapter.rb

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/handler"
+
+module Droonga
+  class Adapter < Droonga::Handler
+    def scatter_all(request, key)
+      message = [{
+        "command"=> envelope["type"],
+        "dataset"=> envelope["dataset"],
+        "body"=> request,
+        "key"=> key,
+        "type"=> "scatter",
+        "replica"=> "all",
+        "post"=> true
+      }]
+      post(message, "proxy")
+    end
+
+    def broadcast_all(request)
+      message = [{
+        "command"=> envelope["type"],
+        "dataset"=> envelope["dataset"],
+        "body"=> request,
+        "type"=> "broadcast",
+        "replica"=> "all",
+        "post"=> true
+      }]
+      post(message, "proxy")
+    end
+
+    def prefer_synchronous?(command)
+      return true
+    end
+  end
+
+  class BasicAdapter < Adapter
+    Droonga::HandlerPlugin.register("adapter", self)
+
+    command :table_create
+    def table_create(request)
+      broadcast_all(request)
+    end
+
+    command :column_create
+    def column_create(request)
+      broadcast_all(request)
+    end
+
+    command "watch.feed" => :feed
+    def feed(request)
+      broadcast_all(request)
+    end
+
+    command "watch.subscribe" => :subscribe
+    def subscribe(request)
+      broadcast_all(request)
+    end
+
+    command "watch.unsubscribe" => :unsubscribe
+    def unsubscribe(request)
+      broadcast_all(request)
+    end
+
+    command :add
+    def add(request)
+      # TODO: update events must be serialized in the primary node of replicas.
+      key = request["key"] || rand.to_s
+      scatter_all(request, key)
+    end
+
+    command :update
+    def update(request)
+      # TODO: update events must be serialized in the primary node of replicas.
+      key = request["key"] || rand.to_s
+      scatter_all(request, key)
+    end
+
+    command :reset
+    def add(request)
+      # TODO: update events must be serialized in the primary node of replicas.
+      key = request["key"] || rand.to_s
+      scatter_all(request, key)
+    end
+
+    command :search
+    def search(request)
+      message = []
+      input_names = []
+      output_names = []
+      name_mapper = {}
+      request["queries"].each do |input_name, query|
+        output = query["output"]
+        next unless output
+        input_names << input_name
+        output_name = input_name + "_reduced"
+        output_names << output_name
+        name_mapper[output_name] = input_name
+        # TODO: offset & limit must be arranged here.
+        elements = {}
+        output["elements"].each do |element|
+          case element
+          when "count"
+            elements[element] = ["sum"]
+          when "records"
+            # TODO: must take "sortBy" section into account.
+            elements[element] = ["sort", "<"]
+          end
+        end
+        reducer = {
+          "inputs"=> [input_name],
+          "outputs"=> [output_name],
+          "type"=> "reduce",
+          "body"=> {
+            input_name=> {
+              output_name=> elements
+            }
+          }
+        }
+        message << reducer
+      end
+      gatherer = {
+        "inputs"=> output_names,
+        "type"=> "gather",
+        "body"=> name_mapper,
+        "post"=> true
+      }
+      message << gatherer
+      searcher = {
+        "dataset"=> envelope["dataset"] || request["dataset"],
+        "outputs"=> input_names,
+        "type"=> "broadcast",
+        "command"=> "search",
+        "replica"=> "random",
+        "body"=> request
+      }
+      message.push(searcher)
+      post(message, "proxy")
+    end
+  end
+end

data/lib/droonga/catalog.rb

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "digest/sha1"
+require "zlib"
+
+module Droonga
+  class << self
+    def catalog
+      @catalog ||= Catalog.new
+    end
+  end
+
+  class Catalog
+    CATALOG_FILE_PATH = "catalog.json"
+
+    attr_reader :path
+
+    def initialize(path=nil)
+      @path = path || default_path
+
+      open(@path) do |file|
+        @catalog = JSON.parse(file.read)
+      end
+      @catalog["datasets"].each do |name, dataset|
+        number_of_partitions = dataset["number_of_partitions"]
+        next if number_of_partitions < 2
+        total_weight = dataset["ring"].reduce do |a, b|
+          a[1]["weight"] + b[1]["weight"]
+        end
+        continuum = []
+        dataset["ring"].each do |key, value|
+          points = number_of_partitions * 160 * value["weight"] / total_weight
+          points.times do |point|
+            hash = Digest::SHA1.hexdigest("#{key}:#{point}")
+            continuum << [hash[0..7].to_i(16), key]
+          end
+        end
+        dataset["continuum"] = continuum.sort do |a, b| a[0] - b[0]; end
+      end
+      @options = @catalog["options"] || {}
+    end
+
+    def base_path
+      @base_path ||= File.dirname(@path)
+    end
+
+    def option(name)
+      @options[name]
+    end
+
+    def get_engines(name)
+      device = @catalog["farms"][name]["device"]
+      pattern = Regexp.new("^#{name}\.")
+      results = {}
+      @catalog["datasets"].each do |key, dataset|
+        workers = dataset["workers"]
+        plugins = dataset["plugins"]
+        dataset["ring"].each do |key, part|
+          part["partitions"].each do |range, partitions|
+            partitions.each do |partition|
+              if partition =~ pattern
+                path = File.join([device, $POSTMATCH, "db"])
+                path = File.expand_path(path, base_path)
+                options = {
+                  :database => path,
+                  :n_workers => workers,
+                  :handlers => plugins
+                }
+                results[partition] = options
+              end
+            end
+          end
+        end
+      end
+      return results
+    end
+
+    def get_routes(name, args)
+      routes = []
+      dataset = dataset(name)
+      return routes unless dataset
+      case args["type"]
+      when "broadcast"
+        dataset["ring"].each do |key, partition|
+          select_range_and_replicas(partition, args, routes)
+        end
+      when "scatter"
+        name = get_partition(dataset, args["key"])
+        partition = dataset["ring"][name]
+        select_range_and_replicas(partition, args, routes)
+      end
+      return routes
+    end
+
+    def get_partition(dataset, key)
+      continuum = dataset["continuum"]
+      return dataset["ring"].keys[0] unless continuum
+      hash = Zlib.crc32(key)
+      min = 0
+      max = continuum.size - 1
+      while (min < max) do
+        index = (min + max) / 2
+        value, key = continuum[index]
+        return key if value == hash
+        if value > hash
+          max = index
+        else
+          min = index + 1
+        end
+      end
+      return continuum[max][1]
+    end
+
+    def dataset(name)
+      @catalog["datasets"][name]
+    end
+
+    def select_range_and_replicas(partition, args, routes)
+      date_range = args["date_range"] || 0..-1
+      partition["partitions"].sort[date_range].each do |time, replicas|
+        case args["replica"]
+        when "top"
+          routes << replicas[0]
+        when "random"
+          routes << replicas[rand(replicas.size)]
+        when "all"
+          routes.concat(replicas)
+        end
+      end
+    end
+
+    private
+    def default_path
+      path = ENV["DROONGA_CATALOG"] || Catalog::CATALOG_FILE_PATH
+      File.expand_path(path)
+    end
+  end
+end