fluent-plugin-droonga 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 99aed806731b4f63bdefc707bddd4fb9c6f1d105
+  data.tar.gz: 0a4ad0ab5c237b44a81656a489744aac4b264a64
+SHA512:
+  metadata.gz: dbeb3a6142fe82e708355ef091ecb2789aa2f18058ac5a9b66089828c91bbfc1237b8f4c5b0998e7fee55a8188f321ac394a547a1608a2e4b15bd6036d24b8e5
+  data.tar.gz: 1db4e2fe92124dbd2d259dad398fb54868214e339319286ca2ce2c3bd7128b7b76b4cde19b5c6f6f262c6a703142a7a8ebd2b2c80021b381cf024d9c61f558ca

data/.gitignore

@@ -0,0 +1 @@
+/Gemfile.lock

data/.travis.yml

@@ -0,0 +1,7 @@
+notifications:
+  recipients:
+    - groonga-commit@lists.sourceforge.jp
+rvm:
+  - 1.9.3
+before_install:
+  - curl https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/Gemfile

@@ -0,0 +1,40 @@
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+source "https://rubygems.org"
+
+gemspec
+
+parent_dir = File.join(File.dirname(__FILE__), "..")
+local_rroonga_path = File.join(parent_dir, "rroonga")
+local_groonga_command_path = File.join(parent_dir, "groonga-command")
+local_groonga_command_parser_path = File.join(parent_dir,
+                                              "groonga-command-parser")
+if File.exist?(local_rroonga_path)
+  gem "rroonga", :path => local_rroonga_path
+  gem "groonga-command", :path => local_groonga_command_path
+  gem "groonga-command-parser", :path => local_groonga_command_parser_path
+elsif ENV["TRAVIS"] == "true"
+  require_unreleased_gems = false
+  if require_unreleased_gems
+    gem "rroonga", :git => "git://github.com/ranguba/rroonga.git"
+    gem "groonga-command",
+        :git => "git://github.com/groonga/groonga-command.git"
+    gem "groonga-command-parser",
+        :git => "git://github.com/groonga/groonga-command-parser.git"
+  end
+end
+
+gem "droonga-client", github: "droonga/droonga-client-ruby"

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (C) 2013 droonga project
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License version 2.1 as published by the Free Software Foundation.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

data/README.md

@@ -0,0 +1,18 @@
+# Fluent::Plugin::Droonga
+
+TODO: Write a gem description
+
+## What's "droonga"?
+
+Droonga is a distributed fulltext search engine, named from "distributed-groonga".
+This package provides droonga plugin for Fluent event collector.
+
+## Usage
+
+TODO: Write usage.
+
+## Copyright
+
+* Copyright (c) 2013 droonga project
+* License
+  * GNU Lesser General Public License version 2.1

data/Rakefile

@@ -0,0 +1,25 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "bundler/gem_tasks"
+
+desc "Run test"
+task :test do
+  ruby(File.join(File.dirname(__FILE__), "test", "run-test.rb"))
+end
+
+task :default => :test

data/benchmark/benchmark.rb

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/client"
+
+module Droonga
+  class Benchmark
+    class Terms
+      class << self
+        def generate
+          new.to_enum(:each)
+        end
+      end
+
+      FIRST_INITIAL_LETTER = "㐀"
+      SUFFIX = "あいうえおかきくけこ"
+      def each
+        initial_letter = FIRST_INITIAL_LETTER
+        while true do
+          yield "#{initial_letter}#{SUFFIX}"
+          initial_letter.succ!
+        end
+      end
+    end
+
+    TERMS_STEP = 1000
+
+    N_TARGETS         = 1000
+    TARGET_PADDING    = "パディング"
+    TARGET_TITLE_SIZE = 100
+    TARGET_BODY_SIZE  = 1000
+
+    def initialize(params)
+      @params = params
+      @terms = Terms.generate
+      @client = Droonga::Client.new(tag: @params[:tag], port: @params[:port])
+    end
+
+    def run
+      prepare_watching_subscribers(TERMS_STEP)
+      populate_feeds(0.1)
+      start_at = Time.now
+      @feeds.each do |feed|
+        @client.send(feed)
+      end
+      end_at = Time.now
+    end
+
+    def prepare_watching_subscribers(step)
+      step.times do
+        term = @terms.next
+        add_subscriber(term)
+        @watching_terms << term
+      end
+    end
+
+    def add_subscriber(term)
+      subscribe_envelope = envelope_to_subscribe(term)
+      @client.connection.send_receive(subscribe_envelope)
+    end
+
+    def envelope_to_subscribe(term, route=nil)
+      message = {
+        "id" => Time.now.to_f.to_s,
+        "date" => Time.now,
+        "statusCode" => 200,
+        "type" => "watch.subscribe",
+        "body" => {
+          "condition" => term,
+          "subscriber" => term,
+        },
+      }
+      unless route.nil?
+        message["body"]["route"] = route
+      end
+      message
+    end
+
+    def populate_feeds(incidence)
+      @feeds = []
+
+      n_matched_targets = (N_TARGETS.to_f * incidence).to_i
+      n_unmatched_targets = (N_TARGETS - n_matched_targets)
+
+      n_matched_targets.times do
+        @feeds << envelope_to_feed(@watching_terms.sample(1))
+      end
+
+      n_unmatched_targets.times do
+        @feeds << envelope_to_feed(@terms.next)
+      end
+    end
+
+    def envelope_to_feed(term)
+      {
+        "id" => Time.now.to_f.to_s,
+        "date" => Time.now,
+        "statusCode" => 200,
+        "type" => "watch.feed",
+        "body" => {
+          "targets" => {
+            "title" => TARGET_PADDING * (TARGET_TITLE_SIZE / TARGET_PADDING.size),
+            "body"  => term + (TARGET_PADDING * (TARGET_BODY_SIZE / TARGET_PADDING.size)),
+          },
+        },
+      }
+    end
+  end
+end

data/benchmark/utils.rb

@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 droonga project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "json"
+require "thread"
+require "droonga/watch_schema"
+
+module DroongaBenchmark
+  WATCH_DATASET = "Watch"
+
+  class WatchDatabase
+    attr_reader :context
+
+    def initialize
+      @database_dir = "/tmp/watch-benchmark"
+      @database_path = "#{@database_dir}/db"
+      FileUtils.rm_rf(@database_dir)
+      FileUtils.mkdir_p(@database_dir)
+
+      Groonga::Database.create(:path => @database_path)
+
+      @context = Groonga::Context.new
+      @context.open_database(@database_path)
+      schema = Droonga::WatchSchema.new(@context)
+      schema.ensure_created
+    end
+
+=begin
+    def subscribe_to(keywords)
+      @context.send("load --table Query")
+      @context.send("[")
+      keywords.each do |keyword|
+        @context.send("{'_key':'#{keyword}'," +
+                        "'keywords':['#{keyword}']},")
+      end
+      @context.send("]")
+
+      @context.send("load --table Subscriber")
+      @context.send("[")
+      keywords.each do |keyword|
+        @context.send("{'_key':'subscriber for #{keyword}'," +
+                        "'subscriptions':['#{keyword}']," +
+                        "'route':'0.0.0.0:0/benchamrk'},")
+      end
+      @context.send("]")
+    end
+=end
+
+    def subscribe_to(keywords)
+      queries = []
+      subscribers = []
+      keywords.each do |keyword|
+        queries << {:_key => keyword,
+                    :keywords => [keyword]}
+        subscribers << {:_key => "subscriber for #{keyword}",
+                        :subscriptions => [keyword],
+                        :route => "0.0.0.0:0/benchamrk"}
+      end
+
+      command_load_queries = [
+        "load --table Query",
+        JSON.generate(queries)
+      ]
+      command_load_subscribers = [
+        "load --table Subscriber",
+        JSON.generate(subscribers)
+      ]
+
+      @context.restore(command_load_queries.join("\n"))
+      @context.restore(command_load_subscribers.join("\n"))
+    end
+
+    def subscribe(keyword)
+      queries = @context["Query"]
+      query = queries.add(keyword, :keywords => [keyword])
+
+      subscribers = @context["Subscriber"]
+      subscribers.add("subscriber for #{keyword}",
+                      :subscriptions => [query],
+                      :route => "0.0.0.0:0/benchamrk")
+    end
+  end
+
+  class KeywordsGenerator
+    class << self
+      def generate(n_keywords)
+        new.generate(n_keywords)
+      end
+    end
+
+    def initialize
+      @generator = to_enum(:each)
+    end
+
+    def generate(n_keywords)
+      keywords = []
+      n_keywords.times do
+        keywords << @generator.next
+      end
+      keywords
+    end
+
+    def next
+      @generator.next
+    end
+
+    FIRST_INITIAL_LETTER = "㐀"
+    SUFFIX = "あいうえおかきくけこ"
+    def each
+      initial_letter = FIRST_INITIAL_LETTER
+      while true do
+        yield "#{initial_letter}#{SUFFIX}"
+        initial_letter.succ!
+      end
+    end
+  end
+
+  class TargetsGenerator
+    class << self
+      def generate(n_keywords, params)
+        new(params).generate(n_keywords)
+      end
+    end
+
+    PADDING = "パディング"
+    SIZE    = 1000
+
+    def initialize(params)
+      @keywords  = params[:keywords]
+      @incidence = params[:incidence]
+      @matched_keywords = params[:matched_keywords] || 1
+    end
+
+    def generate(n_targets)
+      targets = []
+
+      n_matched_targets = (n_targets.to_f * @incidence).to_i
+      n_unmatched_targets = (n_targets - n_matched_targets)
+
+      n_matched_targets.times do
+        targets << generate_target(@keywords.sample(@matched_keywords))
+      end
+
+      n_unmatched_targets.times do
+        targets << generate_target
+      end
+
+      targets
+    end
+
+    def generate_target(keywords=[])
+     (PADDING * (SIZE / PADDING.size)) + keywords.join("/")
+    end
+  end
+
+  class MessageCreator
+    class << self
+      def envelope_to_subscribe(keyword, route=nil)
+        message = {
+          "id" => Time.now.to_f.to_s,
+          "dataset" => WATCH_DATASET,
+          "date" => Time.now,
+          "statusCode" => 200,
+          "type" => "watch.subscribe",
+          "body" => {
+            "condition" => keyword,
+            "subscriber" => "subscriber for #{keyword}",
+          },
+        }
+        message
+      end
+
+      def envelope_to_feed(keyword)
+        {
+          "id" => Time.now.to_f.to_s,
+          "dataset" => WATCH_DATASET,
+          "date" => Time.now,
+          "statusCode" => 200,
+          "type" => "watch.feed",
+          "body" => {
+            "targets" => {
+              "keyword"  => keyword,
+            },
+          },
+        }
+      end
+    end
+  end
+
+  class MessageReceiver
+    def initialize(options={})
+      default_options = {
+        :host => "0.0.0.0",
+        :port => 0,
+      }
+      options = default_options.merge(options)
+      @socket = TCPServer.new(options[:host], options[:port])
+      start
+    end
+
+    def close
+      @socket.close
+    end
+
+    def host
+      @socket.addr[3]
+    end
+
+    def port
+      @socket.addr[1]
+    end
+
+    def new_message
+      @messages.pop
+    end
+
+    def start
+      @messages = Queue.new
+      Thread.new do
+        client = @socket.accept
+        unpacker = MessagePack::Unpacker.new(client)
+        unpacker.each do |object|
+          @messages.push(object)
+        end
+        client.close
+      end
+    end
+  end
+end