droonga-engine 1.0.9 → 1.1.0

data/lib/droonga/test/stub_handler_messenger.rb

@@ -17,12 +17,13 @@ module Droonga
   module Test
     class StubHandlerMessenger
       attr_reader :values, :messages
-      attr_accessor :engine_state
+      attr_accessor :engine_state, :cluster
 
       def initialize
         @values = []
         @messages = []
         @engine_state = nil
+        @cluster = nil
       end
 
       def emit(value)

data/lib/droonga/timestamp.rb

@@ -0,0 +1,69 @@
+# Copyright (C) 2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "time"
+
+require "droonga/path"
+require "droonga/safe_file_writer"
+require "droonga/file_observer"
+
+module Droonga
+  module Timestamp
+    MICRO_SECONDS_DECIMAL_PLACE = 6
+
+    class << self
+      def stringify(timestamp)
+        timestamp.utc.iso8601(MICRO_SECONDS_DECIMAL_PLACE)
+      end
+
+      def last_message_timestamp=(timestamp)
+        if timestamp.is_a?(String)
+          timestamp = timestamp.strip
+          if timestamp.empty?
+            timestamp = nil
+          else
+            timestamp = Time.parse(timestamp)
+          end
+        end
+        if timestamp
+          timestamp = stringify(timestamp)
+        else
+          timestamp = ""
+        end
+        SafeFileWriter.write(Path.last_message_timestamp) do |output, file|
+          output.puts(timestamp)
+        end
+      end
+
+      def last_message_timestamp
+        file = Path.last_message_timestamp
+        return nil unless file.exist?
+        timestamp = file.read.strip
+        return nil if timestamp.nil? or timestamp.empty?
+        Time.parse(timestamp)
+      end
+
+      def run_last_message_timestamp_observer(loop, &block)
+        path = Path.last_message_timestamp
+        observer = FileObserver.new(loop, path)
+        observer.on_change = lambda do
+          yield(last_message_timestamp)
+        end
+        observer.start
+        observer
+      end
+    end
+  end
+end

data/lib/droonga/worker_process_agent.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -28,14 +28,16 @@ module Droonga
       @loop = loop
       create_input(input)
       create_output(output)
-      @on_ready = nil
-      @on_finish = nil
     end
 
     def start
       logger.trace("start: start")
       @loop.attach(@input)
+      # logger.trace("start: new input watcher attached",
+      #              :watcher => @input)
       @loop.attach(@output)
+      # logger.trace("start: new output watcher attached",
+      #              :watcher => @output)
       logger.trace("start: done")
     end
 
@@ -46,11 +48,15 @@ module Droonga
         output.write(Messages::FINISH)
         output.on_write_complete do
           output.close
+          logger.trace("stop: output detached",
+                       :watcher => output)
         end
       end
       if @input
         @input, input = nil, @input
         input.close
+        logger.trace("stop: input detached",
+                     :watcher => input)
       end
       logger.trace("stop: done")
     end
@@ -67,45 +73,49 @@ module Droonga
       @on_stop_immediately = callback
     end
 
+    def on_refresh_self_reference=(callback)
+      @on_refresh_self_reference = callback
+    end
+
+    def on_refresh_node_reference=(callback)
+      @on_refresh_node_reference = callback
+    end
+
     private
     def create_input(raw_input)
       @input = Coolio::IO.new(raw_input)
-      on_read = lambda do |data|
+      @input.on_read do |data|
         line_buffer = LineBuffer.new
         line_buffer.feed(data) do |line|
+          logger.trace("line buffer feeded", :line => line);
           case line
           when Messages::STOP_GRACEFUL
             on_stop_gracefully
           when Messages::STOP_IMMEDIATELY
             on_stop_immediately
+          when Messages::REFRESH_SELF_REFERENCE
+            on_refresh_self_reference
+          when Messages::REFRESH_NODE_REFERENCE
+            on_refresh_node_reference
           end
         end
       end
-      @input.on_read do |data|
-        on_read.call(data)
-      end
-      on_close = lambda do
+      @input.on_close do
         if @input
           @input = nil
           on_stop_immediately
         end
       end
-      @input.on_close do
-        on_close.call
-      end
     end
 
     def create_output(raw_output)
       @output = Coolio::IO.new(raw_output)
-      on_close = lambda do
+      @output.on_close do
         if @output
           @output = nil
           on_stop_immediately
         end
       end
-      @output.on_close do
-        on_close.call
-      end
     end
 
     def on_stop_gracefully
@@ -116,6 +126,14 @@ module Droonga
       @on_stop_immediately.call if @on_stop_immediately
     end
 
+    def on_refresh_self_reference
+      @on_refresh_self_reference.call if @on_refresh_self_reference
+    end
+
+    def on_refresh_node_reference
+      @on_refresh_node_reference.call if @on_refresh_node_reference
+    end
+
     def log_tag
       "worker_process_agent"
     end

data/sample/cluster-state.json

@@ -0,0 +1,8 @@
+{
+  "127.0.0.1:23003/droonga": {
+    "type": "engine",
+    "role": "service-provider",
+    "live": true,
+    "have_unprocessed_messages": false
+  }
+}

data/sample/cluster/Rakefile

@@ -17,6 +17,7 @@
 
 require "pathname"
 require "json"
+require "yaml"
 
 base_dir_path = Pathname.new(__FILE__).dirname
 top_dir_path = base_dir_path.parent.parent
@@ -48,20 +49,22 @@ class DroongaEngine
     rm_rf(working_dir_path.to_s)
     mkdir_p(working_dir_path.to_s)
     generate_catalog(node_ids)
+    generate_config
   end
 
   def start
     @pid = spawn("droonga-engine",
-                 "--base-dir", working_dir_path.to_s,
-                 "--host", host,
-                 "--port", self.class.port.to_s,
-                 "--pid-file", pid_file_path.to_s)
+                 "--base-dir", working_dir_path.to_s)
   end
 
   def stop
     Process.kill(:TERM, @pid)
     Process.waitpid(@pid)
     @pid = nil
+    if pid_path.exist?
+      pid = Integer(pid_path.read)
+      Process.kill(:TERM, pid)
+    end
   end
 
   def host
@@ -77,8 +80,16 @@ class DroongaEngine
     working_dir_path + "catalog.json"
   end
 
-  def pid_file_path
-    working_dir_path + "droonga-engine.pid"
+  def config_path
+    working_dir_path + "droonga-engine.yaml"
+  end
+
+  def pid_path_base_name
+    "droonga-engine.pid"
+  end
+
+  def pid_path
+    working_dir_path + pid_path_base_name
   end
 
   def generate_catalog(node_ids)
@@ -91,6 +102,19 @@ class DroongaEngine
        "--hosts", hosts.join(","),
        "--port", self.class.port.to_s)
   end
+
+  def generate_config
+    config = {
+      "host"      => host,
+      "port"      => self.class.port,
+      # "log_file"  => "droonga-engine.log",
+      "log_level" => "info",
+      "pid_file"  => pid_path_base_name,
+    }
+    config_path.open("w") do |file|
+      file.puts(config.to_yaml)
+    end
+  end
 end
 
 namespace :droonga do

data/test/command/fixture/integer-key-table.jsons

@@ -0,0 +1,11 @@
+#@disable-logging
+{
+  "type": "table_create",
+  "dataset": "Default",
+  "body": {
+    "name"     : "Integer",
+    "flags"    : "TABLE_PAT_KEY",
+    "key_type" : "UInt32"
+  }
+}
+#@enable-logging

data/test/command/fixture/string-key-table.jsons

@@ -0,0 +1,11 @@
+#@disable-logging
+{
+  "type": "table_create",
+  "dataset": "Default",
+  "body": {
+    "name"     : "String",
+    "flags"    : "TABLE_PAT_KEY",
+    "key_type" : "ShortText"
+  }
+}
+#@enable-logging

data/test/command/run-test.rb

@@ -39,9 +39,13 @@ end
 
 base_dir = File.dirname(__FILE__)
 lib_dir = File.expand_path(File.join(base_dir, "..", "..", "lib"))
+log_path = File.expand_path(File.join(base_dir, "tmp", "droonga-engine.log"))
 
 drntest_options = []
 drntest_options.concat(["--base-path", base_dir])
+drntest_options.concat(["--droonga-engine-options",
+                        ["--log-file=#{log_path}",
+                         "--internal-connection-lifetime=5"].join(" ")])
 drntest_options.concat(ARGV)
 
 run("bundle", "exec", "drntest", *drntest_options)

data/test/command/suite/add/error/invalid-integer.expected

@@ -4,7 +4,7 @@
   "type": "add.result",
   "body": {
     "name": "InvalidValue",
-    "message": "The column \"age\" cannot store the value \"secret\".",
+    "message": "The column \"age\" cannot store the value \"secret\": invalid argument: <User.age>: failed to cast to <Int32>: <\"secret\">: #<Groonga::FixSizeColumn ...>",
     "detail": {
       "table": "User",
       "key": "key",
@@ -18,7 +18,7 @@
       "statusCode": 400,
       "body": {
         "name": "InvalidValue",
-        "message": "The column \"age\" cannot store the value \"secret\".",
+        "message": "The column \"age\" cannot store the value \"secret\": invalid argument: <User.age>: failed to cast to <Int32>: <\"secret\">: #<Groonga::FixSizeColumn ...>",
         "detail": {
           "table": "User",
           "key": "key",
@@ -32,7 +32,7 @@
       "statusCode": 400,
       "body": {
         "name": "InvalidValue",
-        "message": "The column \"age\" cannot store the value \"secret\".",
+        "message": "The column \"age\" cannot store the value \"secret\": invalid argument: <User.age>: failed to cast to <Int32>: <\"secret\">: #<Groonga::FixSizeColumn ...>",
         "detail": {
           "table": "User",
           "key": "key",

data/test/command/suite/add/error/invalid-time.expected

@@ -4,7 +4,7 @@
   "type": "add.result",
   "body": {
     "name": "InvalidValue",
-    "message": "The column \"birthday\" cannot store the value \"today\".",
+    "message": "The column \"birthday\" cannot store the value \"today\": no time information in \"today\"",
     "detail": {
       "table": "User",
       "key": "key",
@@ -18,7 +18,7 @@
       "statusCode": 400,
       "body": {
         "name": "InvalidValue",
-        "message": "The column \"birthday\" cannot store the value \"today\".",
+        "message": "The column \"birthday\" cannot store the value \"today\": no time information in \"today\"",
         "detail": {
           "table": "User",
           "key": "key",
@@ -32,7 +32,7 @@
       "statusCode": 400,
       "body": {
         "name": "InvalidValue",
-        "message": "The column \"birthday\" cannot store the value \"today\".",
+        "message": "The column \"birthday\" cannot store the value \"today\": no time information in \"today\"",
         "detail": {
           "table": "User",
           "key": "key",

data/test/command/suite/add/key-string.expected

@@ -0,0 +1,6 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "add.result",
+  "body": true
+}

data/test/command/suite/add/key-string.test

@@ -0,0 +1,9 @@
+#@include fixture/string-key-table.jsons
+{
+  "type": "add",
+  "dataset": "Default",
+  "body": {
+    "table": "String",
+    "key": "key"
+  }
+}

data/test/command/suite/add/mismatched-key-type/acceptable/integer-for-string.expected

@@ -0,0 +1,6 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "add.result",
+  "body": true
+}

data/test/command/suite/add/mismatched-key-type/acceptable/integer-for-string.test

@@ -0,0 +1,9 @@
+#@include fixture/string-key-table.jsons
+{
+  "type": "add",
+  "dataset": "Default",
+  "body": {
+    "table": "String",
+    "key": 1
+  }
+}

data/test/command/suite/add/mismatched-key-type/acceptable/string-for-integer.expected

@@ -0,0 +1,6 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "add.result",
+  "body": true
+}

data/test/command/suite/add/mismatched-key-type/acceptable/string-for-integer.test

@@ -0,0 +1,9 @@
+#@include fixture/integer-key-table.jsons
+{
+  "type": "add",
+  "dataset": "Default",
+  "body": {
+    "table": "Integer",
+    "key": "1"
+  }
+}

data/test/command/suite/add/without-values.expected

@@ -0,0 +1,6 @@
+{
+  "inReplyTo": "request-id",
+  "statusCode": 200,
+  "type": "add.result",
+  "body": true
+}

data/test/command/suite/add/without-values.test

@@ -0,0 +1,11 @@
+#@include fixture/user-table.jsons
+{
+  "type": "add",
+  "dataset": "Default",
+  "body": {
+    "table": "User",
+    "key": "key",
+    "values": {
+    }
+  }
+}