droonga-engine 1.0.1

data/.dir-locals.el

@@ -0,0 +1,3 @@
+((js-mode . ((indent-tabs-mode . nil)
+             (js-indent-level . 2)
+             (show-trailing-whitespace . t))))

data/.gitignore

@@ -0,0 +1,6 @@
+/Gemfile.lock
+/test/command/tmp
+/doc/
+/.yardoc/
+/pkg/
+/sample/cluster/0*

data/.travis.yml

@@ -0,0 +1,15 @@
+notifications:
+  recipients:
+    - groonga-commit@lists.sourceforge.jp
+env:
+  - DEFAULT_TEST_TASK=test:unit
+  - DEFAULT_TEST_TASK=test:command:default
+  - DEFAULT_TEST_TASK=test:command:version1
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+#  - ruby-head
+before_install:
+#  - GROONGA_MASTER=yes curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh
+  - curl --silent --location https://raw.github.com/groonga/groonga/master/data/travis/setup.sh | sh

data/.yardopts

@@ -0,0 +1,7 @@
+--output-dir doc/reference/en
+--markup markdown
+--markup-provider kramdown
+lib/**/*.rb
+-
+LICENSE.txt
+doc/text/**/*.*

data/Gemfile

@@ -0,0 +1,66 @@
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+source "https://rubygems.org"
+
+gemspec
+
+parent_dir = File.join(File.dirname(__FILE__), "..")
+local_rroonga_path = File.join(parent_dir, "rroonga")
+local_groonga_command_path = File.join(parent_dir, "groonga-command")
+local_groonga_command_parser_path = File.join(parent_dir,
+                                              "groonga-command-parser")
+if File.exist?(local_rroonga_path)
+  gem "rroonga", :path => local_rroonga_path
+  gem "groonga-command", :path => local_groonga_command_path
+  gem "groonga-command-parser", :path => local_groonga_command_parser_path
+elsif ENV["TRAVIS"] == "true"
+  require_unreleased_gems = true
+  if require_unreleased_gems
+    gem "rroonga", :git => "git://github.com/ranguba/rroonga.git"
+    gem "groonga-command",
+        :git => "git://github.com/groonga/groonga-command.git"
+    gem "groonga-command-parser",
+        :git => "git://github.com/groonga/groonga-command-parser.git"
+  end
+end
+
+droonga_client_dir = File.join(parent_dir, "droonga-client-ruby")
+if File.exist?(droonga_client_dir)
+  gem "droonga-client", :path => droonga_client_dir
+else
+  gem "droonga-client", :github => "droonga/droonga-client-ruby"
+end
+
+drntest_dir = File.join(parent_dir, "drntest")
+if File.exist?(drntest_dir)
+  gem "drntest", :path => drntest_dir
+else
+  gem "drntest", :github => "droonga/drntest"
+end
+
+drnbench_dir = File.join(parent_dir, "drnbench")
+if File.exist?(drnbench_dir)
+  gem "drnbench", :path => drnbench_dir
+else
+  gem "drnbench", :github => "droonga/drnbench"
+end
+
+grn2drn_dir = File.join(parent_dir, "grn2drn")
+if File.exist?(grn2drn_dir)
+  gem "grn2drn", :path => grn2drn_dir
+else
+  gem "grn2drn", :github => "droonga/grn2drn"
+end

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (C) 2013 Droonga Project
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Lesser General Public
+License version 2.1 as published by the Free Software Foundation.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public
+License along with this library; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

data/README.md

@@ -0,0 +1,17 @@
+# Droonga engine
+
+## What's "Droonga"?
+
+Droonga is a scalable data processing engine, named from "Distributed
+Groonga". Droonga can execute distributed fulltext search.
+
+## Usage
+
+See [droonga.org](http://droonga.org/). You can find overview, install
+manual, reference manual and tutorial at the site.
+
+## Copyright
+
+* Copyright (c) 2013-2014 Droonga Project
+* License
+  * GNU Lesser General Public License version 2.1

data/Rakefile

@@ -0,0 +1,64 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "bundler/gem_helper"
+require "packnga"
+
+base_dir = File.join(File.dirname(__FILE__))
+
+helper = Bundler::GemHelper.new(base_dir)
+helper.install
+spec = helper.gemspec
+
+Packnga::DocumentTask.new(spec) do |task|
+  task.original_language = "en"
+  task.translate_languages = ["ja"]
+end
+
+def run_command_test(*options)
+  ruby(File.join(File.dirname(__FILE__), "test", "command", "run-test.rb",),
+       *options)
+end
+
+namespace :test do
+  desc "Run unit test"
+  task :unit do
+    ruby(File.join(File.dirname(__FILE__), "test", "unit", "run-test.rb"))
+  end
+
+  namespace :command do
+    desc "Run command test: default"
+    task :default do
+      run_command_test
+    end
+
+    desc "Run command test: version1"
+    task :version1 do
+      run_command_test("--config", "version1")
+    end
+  end
+end
+
+desc "Run test"
+task :test => [
+  "test:unit",
+  "test:command:default",
+  "test:command:version1",
+]
+
+test_task = ENV["DEFAULT_TEST_TASK"] || "test"
+task :default => test_task

data/benchmark/benchmark.rb

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/client"
+
+module Droonga
+  class Benchmark
+    class Terms
+      class << self
+        def generate
+          new.to_enum(:each)
+        end
+      end
+
+      FIRST_INITIAL_LETTER = "㐀"
+      SUFFIX = "あいうえおかきくけこ"
+      def each
+        initial_letter = FIRST_INITIAL_LETTER
+        while true do
+          yield "#{initial_letter}#{SUFFIX}"
+          initial_letter.succ!
+        end
+      end
+    end
+
+    TERMS_STEP = 1000
+
+    N_TARGETS         = 1000
+    TARGET_PADDING    = "パディング"
+    TARGET_TITLE_SIZE = 100
+    TARGET_BODY_SIZE  = 1000
+
+    def initialize(params)
+      @params = params
+      @terms = Terms.generate
+      @client = Droonga::Client.new(tag: @params[:tag], port: @params[:port])
+    end
+
+    def run
+      prepare_watching_subscribers(TERMS_STEP)
+      populate_feeds(0.1)
+      start_at = Time.now
+      @feeds.each do |feed|
+        @client.send(feed)
+      end
+      end_at = Time.now
+    end
+
+    def prepare_watching_subscribers(step)
+      step.times do
+        term = @terms.next
+        add_subscriber(term)
+        @watching_terms << term
+      end
+    end
+
+    def add_subscriber(term)
+      subscribe_envelope = envelope_to_subscribe(term)
+      @client.connection.send_receive(subscribe_envelope)
+    end
+
+    def envelope_to_subscribe(term, route=nil)
+      message = {
+        "id" => Time.now.to_f.to_s,
+        "date" => Time.now,
+        "statusCode" => 200,
+        "type" => "watch.subscribe",
+        "body" => {
+          "condition" => term,
+          "subscriber" => term,
+        },
+      }
+      unless route.nil?
+        message["body"]["route"] = route
+      end
+      message
+    end
+
+    def populate_feeds(incidence)
+      @feeds = []
+
+      n_matched_targets = (N_TARGETS.to_f * incidence).to_i
+      n_unmatched_targets = (N_TARGETS - n_matched_targets)
+
+      n_matched_targets.times do
+        @feeds << envelope_to_feed(@watching_terms.sample(1))
+      end
+
+      n_unmatched_targets.times do
+        @feeds << envelope_to_feed(@terms.next)
+      end
+    end
+
+    def envelope_to_feed(term)
+      {
+        "id" => Time.now.to_f.to_s,
+        "date" => Time.now,
+        "statusCode" => 200,
+        "type" => "watch.feed",
+        "body" => {
+          "targets" => {
+            "title" => TARGET_PADDING * (TARGET_TITLE_SIZE / TARGET_PADDING.size),
+            "body"  => term + (TARGET_PADDING * (TARGET_BODY_SIZE / TARGET_PADDING.size)),
+          },
+        },
+      }
+    end
+  end
+end

data/benchmark/utils.rb

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "json"
+require "thread"
+require "droonga/watch_schema"
+
+module DroongaBenchmark
+  WATCH_DATASET = "Watch"
+
+  class WatchDatabase
+    attr_reader :context
+
+    def initialize
+      @database_dir = "/tmp/watch-benchmark"
+      @database_path = "#{@database_dir}/db"
+      FileUtils.rm_rf(@database_dir)
+      FileUtils.mkdir_p(@database_dir)
+
+      Groonga::Database.create(:path => @database_path)
+
+      @context = Groonga::Context.new
+      @context.open_database(@database_path)
+      schema = Droonga::WatchSchema.new(@context)
+      schema.ensure_created
+    end
+
+=begin
+    def subscribe_to(keywords)
+      @context.send("load --table Query")
+      @context.send("[")
+      keywords.each do |keyword|
+        @context.send("{'_key':'#{keyword}'," +
+                        "'keywords':['#{keyword}']},")
+      end
+      @context.send("]")
+
+      @context.send("load --table Subscriber")
+      @context.send("[")
+      keywords.each do |keyword|
+        @context.send("{'_key':'subscriber for #{keyword}'," +
+                        "'subscriptions':['#{keyword}']," +
+                        "'route':'0.0.0.0:0/benchamrk'},")
+      end
+      @context.send("]")
+    end
+=end
+
+    def subscribe_to(keywords)
+      queries = []
+      subscribers = []
+      keywords.each do |keyword|
+        queries << {:_key => keyword,
+                    :keywords => [keyword]}
+        subscribers << {:_key => "subscriber for #{keyword}",
+                        :subscriptions => [keyword],
+                        :route => "0.0.0.0:0/benchamrk"}
+      end
+
+      command_load_queries = [
+        "load --table Query",
+        JSON.generate(queries)
+      ]
+      command_load_subscribers = [
+        "load --table Subscriber",
+        JSON.generate(subscribers)
+      ]
+
+      @context.restore(command_load_queries.join("\n"))
+      @context.restore(command_load_subscribers.join("\n"))
+    end
+
+    def subscribe(keyword)
+      queries = @context["Query"]
+      query = queries.add(keyword, :keywords => [keyword])
+
+      subscribers = @context["Subscriber"]
+      subscribers.add("subscriber for #{keyword}",
+                      :subscriptions => [query],
+                      :route => "0.0.0.0:0/benchamrk")
+    end
+  end
+
+  class KeywordsGenerator
+    class << self
+      def generate(n_keywords)
+        new.generate(n_keywords)
+      end
+    end
+
+    def initialize
+      @generator = to_enum(:each)
+    end
+
+    def generate(n_keywords)
+      keywords = []
+      n_keywords.times do
+        keywords << @generator.next
+      end
+      keywords
+    end
+
+    def next
+      @generator.next
+    end
+
+    FIRST_INITIAL_LETTER = "㐀"
+    SUFFIX = "あいうえおかきくけこ"
+    def each
+      initial_letter = FIRST_INITIAL_LETTER
+      while true do
+        yield "#{initial_letter}#{SUFFIX}"
+        initial_letter.succ!
+      end
+    end
+  end
+
+  class TargetsGenerator
+    class << self
+      def generate(n_keywords, params)
+        new(params).generate(n_keywords)
+      end
+    end
+
+    PADDING = "パディング"
+    SIZE    = 1000
+
+    def initialize(params)
+      @keywords  = params[:keywords]
+      @incidence = params[:incidence]
+      @matched_keywords = params[:matched_keywords] || 1
+    end
+
+    def generate(n_targets)
+      targets = []
+
+      n_matched_targets = (n_targets.to_f * @incidence).to_i
+      n_unmatched_targets = (n_targets - n_matched_targets)
+
+      n_matched_targets.times do
+        targets << generate_target(@keywords.sample(@matched_keywords))
+      end
+
+      n_unmatched_targets.times do
+        targets << generate_target
+      end
+
+      targets
+    end
+
+    def generate_target(keywords=[])
+     (PADDING * (SIZE / PADDING.size)) + keywords.join("/")
+    end
+  end
+
+  class MessageCreator
+    class << self
+      def envelope_to_subscribe(keyword, route=nil)
+        message = {
+          "id" => Time.now.to_f.to_s,
+          "dataset" => WATCH_DATASET,
+          "date" => Time.now,
+          "statusCode" => 200,
+          "type" => "watch.subscribe",
+          "body" => {
+            "condition" => keyword,
+            "subscriber" => "subscriber for #{keyword}",
+          },
+        }
+        message
+      end
+
+      def envelope_to_feed(keyword)
+        {
+          "id" => Time.now.to_f.to_s,
+          "dataset" => WATCH_DATASET,
+          "date" => Time.now,
+          "statusCode" => 200,
+          "type" => "watch.feed",
+          "body" => {
+            "targets" => {
+              "keyword"  => keyword,
+            },
+          },
+        }
+      end
+    end
+  end
+
+  class MessageReceiver
+    def initialize(options={})
+      default_options = {
+        :host => "0.0.0.0",
+        :port => 0,
+      }
+      options = default_options.merge(options)
+      @socket = TCPServer.new(options[:host], options[:port])
+      start
+    end
+
+    def close
+      @socket.close
+    end
+
+    def host
+      @socket.addr[3]
+    end
+
+    def port
+      @socket.addr[1]
+    end
+
+    def new_message
+      @messages.pop
+    end
+
+    def start
+      @messages = Queue.new
+      Thread.new do
+        loop do
+          Thread.new(@socket.accept) do |client|
+            unpacker = MessagePack::Unpacker.new(client)
+            unpacker.each do |object|
+              @messages.push(object)
+            end
+            client.close
+          end
+        end
+      end
+    end
+  end
+end