drndump 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f360ce69bbbae294a5d10b32486910970402232
-  data.tar.gz: b2c1f9925a35b424ce739c5d7086e7c30610a697
+  metadata.gz: ffe9d62833a98f43fa9911c82c56429d74af6205
+  data.tar.gz: 62d440b218bc4d963d3c91a7a426f6e6bd3940fa
 SHA512:
-  metadata.gz: 96dfb02da9769cf3be082156bc27759891b8d2e68159ff041604159cd86be170fd6b59819ae494c2f0066b843a17fe4fe98ce015e61c1cc4da36943e38df2954
-  data.tar.gz: 34fef0d416a78cdc52861b209bef9ecb0aac8e329e856dc770e9e91a6bd5d5418573d82bcfecb998293dc0b5d918367d62763e8e6d23f0744ff5f0fd8af01b97
+  metadata.gz: 2c32f075bcf25f1104d7331aed5f9cc55545da025ced1b0ea6ce11c102f0b6a10de2afb9fc5c160c9cbcf4c5a533595611df42d9c61666c39fc1c4ee1731d98e
+  data.tar.gz: 1ea16dde38b2f88bea29c058fd2d8f54cf0e90abe255597c622f3b5455de02ade8f734b50166053569bd6b9584e7586e39a37e0d40f87dc786379a525d9c2ec9

data/doc/text/news.md

@@ -1,5 +1,12 @@
 # News
 
+## 1.0.1: 2014-04-29
+
+ * Becomes modulable. You can use internal libraries for other products.
+ * Supports new `@subscribe-until` directive, for subscription type commands like `dump`.
+   You can unsubscribe the next request following to the directive automatically with given timeout, like:
+   `#@subscribe-until 10s`
+
 ## 1.0.0: 2014-05-29
 
 The first release!!!

data/lib/drndump/command.rb

@@ -19,9 +19,8 @@ require "json"
 
 require "cool.io"
 
-require "droonga/client"
-
 require "drndump/version"
+require "drndump/dump_client"
 
 module Drndump
   class Command
@@ -106,138 +105,28 @@ module Drndump
       parser
     end
 
-    def client_options
+    def dumper_params
       {
         :host          => @host,
         :port          => @port,
         :tag           => @tag,
-        :protocol      => :droonga,
+        :dataset       => @dataset,
         :receiver_host => @receiver_host,
         :receiver_port => @receiver_port,
-        :backend       => :coolio,
-        :loop          => @loop,
       }
     end
 
     def dump
-      client = Droonga::Client.new(client_options)
-
-      error_message = nil
-      n_dumpers = 0
-
-      dump_message = {
-        "type"    => "dump",
-        "dataset" => @dataset,
+      @dumper = DumpClient.new(dumper_params)
+      dump_options = {
+        :backend => :coolio,
+        :loop    => @loop,
       }
-      client.subscribe(dump_message) do |message|
-        case message
-        when Droonga::Client::Error
-          client.close
-          error_message = message.to_s
-        else
-          case message["type"]
-          when "dump.result", "dump.error"
-            if message["statusCode"] != 200
-              client.close
-              error = message["body"]
-              error_message = "#{error['name']}: #{error['message']}"
-            end
-          when "dump.table"
-            table_create_message = convert_to_table_create_message(message)
-            puts(JSON.pretty_generate(table_create_message))
-          when "dump.column"
-            column_create_message = convert_to_column_create_message(message)
-            puts(JSON.pretty_generate(column_create_message))
-          when "dump.record"
-            add_message = message.dup
-            add_message.delete("inReplyTo")
-            add_message["type"] = "add"
-            puts(JSON.pretty_generate(add_message))
-          when "dump.start"
-            n_dumpers += 1
-          when "dump.end"
-            n_dumpers -= 1
-            client.close if n_dumpers <= 0
-          end
-        end
+      @dumper.run(dump_options) do |message|
+        puts(JSON.pretty_generate(message))
       end
       @loop.run
-
-      error_message
-    end
-
-    def convert_to_table_create_message(message)
-      body = message["body"]
-      flags = []
-      case body["type"]
-      when "Array"
-        flags << "TABLE_NO_KEY"
-      when "Hash"
-        flags << "TABLE_HASH_KEY"
-      when "PatriciaTrie"
-        flags << "TABLE_PAT_KEY"
-      when "DoubleArrayTrie"
-        flags << "TABLE_DAT_KEY"
-      end
-      table_create_message = {
-        "type"    => "table_create",
-        "dataset" => message["dataset"],
-        "body" => {
-          "name"              => body["name"],
-          "flags"             => flags.join("|"),
-          "key_type"          => body["keyType"],
-        }
-      }
-
-      if body["tokenizer"]
-        table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
-      end
-      if body["normalizer"]
-        table_create_message["body"]["normalizer"] = body["normalizer"]
-      end
-
-      table_create_message
-    end
-
-    def convert_to_column_create_message(message)
-      body = message["body"]
-      column_create_message = {
-        "type"    => "column_create",
-        "dataset" => message["dataset"],
-        "body" => {
-          "table"  => body["table"],
-          "name"   => body["name"],
-          "type"   => body["valueType"],
-        }
-      }
-
-      flags = []
-      case body["type"]
-      when "Scalar"
-        flags << "COLUMN_SCALAR"
-      when "Vector"
-        flags << "COLUMN_VECTOR"
-        vector_options = body["vectorOptions"] || {}
-        flags << "WITH_WEIGHT" if vector_options["weight"]
-      when "Index"
-        flags << "COLUMN_INDEX"
-        index_options = body["indexOptions"] || {}
-        flags << "WITH_SECTION"  if index_options["section"]
-        flags << "WITH_WEIGHT"   if index_options["weight"]
-        flags << "WITH_POSITION" if index_options["position"]
-      end
-
-      column_create_message["body"]["flags"] = flags.join("|")
-
-      if body["type"] == "Index"
-        index_options = body["indexOptions"] || {}
-        sources = index_options["sources"] || []
-        unless sources.empty?
-          column_create_message["body"]["source"] = sources.join(",")
-        end
-      end
-
-      column_create_message
+      @dumper.error_message
     end
   end
 end

data/lib/drndump/dump_client.rb

@@ -0,0 +1,298 @@
+# Copyright (C) 2014-2015 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "socket"
+
+require "droonga/client"
+
+module Drndump
+  class DumpClient
+    DEFAULT_MESSAGES_PER_SECOND = 10000 # same to droogna-engine's one
+    MIN_REPORTED_THROUGHPUT = 0.01
+
+    attr_reader :n_forecasted_messages, :n_received_messages
+    attr_reader :error_message
+    attr_writer :on_finish, :on_progress, :on_error
+
+    class NilMessage < StandardError
+    end
+
+    class InvalidMessage < StandardError
+    end
+
+    class ClientError < StandardError
+      def initialize(error)
+        super(error.inspect)
+      end
+    end
+
+    def initialize(params)
+      @host     = params[:host]
+      @port     = params[:port]
+      @tag      = params[:tag]
+      @dataset  = params[:dataset]
+
+      @receiver_host = params[:receiver_host]
+      @receiver_port = params[:receiver_port]
+
+      @n_forecasted_messages = 0
+      @n_received_messages = 0
+      @n_messages_per_second = DEFAULT_MESSAGES_PER_SECOND
+
+      @error_message = nil
+
+      @on_finish = nil
+      @on_progress = nil
+      @on_error = nil
+    end
+
+    def run(options={}, &block)
+      extra_client_options = {
+        :backend => options[:backend],
+        :loop    => options[:loop],
+      }
+      @client = Droonga::Client.new(client_options.merge(extra_client_options))
+      @client.on_error = lambda do |error|
+        on_error(ClientError.new(error))
+      end
+
+      @n_dumpers = 0
+
+      @n_messages_per_second = options[:messages_per_second] || DEFAULT_MESSAGES_PER_SECOND
+      @n_messages_per_second = [@n_messages_per_second, 1].max
+
+      @measure_start_time = Time.now
+      @previous_measure_time = @measure_start_time
+      @previous_n_received_messages = 0.0
+
+      dump_message = {
+        "type"    => "dump",
+        "dataset" => @dataset,
+        "body"    => {
+          "messagesPerSecond" => @n_messages_per_second,
+        },
+      }
+      @client.subscribe(dump_message) do |message|
+        begin
+          on_progress(message)
+          case message
+          when Droonga::Client::Error
+            client.close
+            on_error(message)
+            @error_message = message.to_s
+          when Hash
+            handle_dump_message(message) do |restore_message|
+              yield(restore_message)
+            end
+          when NilClass
+            raise NilMessage.new("nil message in dump")
+          else
+            raise InvalidMessage.new("invalid message in dump",
+                                     :message => message.inspect)
+          end
+        rescue Exception => exception
+          @client.close
+          on_error(exception)
+          @error_message = exception.to_s
+        end
+      end
+
+      @error_message
+    end
+
+    def recent_throughput
+      now = Time.now
+      n_messages = @n_received_messages - @previous_n_received_messages
+
+      if now - @previous_measure_time < 1
+        now = @previous_measure_time
+        n_messages = @previous_n_received_messages
+      else
+        @previous_measure_time = now
+        @previous_n_received_messages = n_messages.to_f
+      end
+
+      if now == @measure_start_time
+        actual_throughput = 0
+      else
+        elapsed_seconds = now - @measure_start_time
+        actual_throughput = n_messages / elapsed_seconds
+      end
+
+      [actual_throughput, MIN_REPORTED_THROUGHPUT].max
+    end
+
+    def n_remaining_messages
+      [@n_forecasted_messages - @n_received_messages, 0].max
+    end
+
+    def remaining_seconds
+      throughput = [recent_throughput, @n_messages_per_second].min
+      n_remaining_messages.to_f / throughput
+    end
+
+    ONE_MINUTE_IN_SECONDS = 60
+    ONE_HOUR_IN_SECONDS = ONE_MINUTE_IN_SECONDS * 60
+
+    def formatted_remaining_time
+      seconds  = remaining_seconds
+      hours    = (seconds / ONE_HOUR_IN_SECONDS).floor
+      seconds -= hours * ONE_HOUR_IN_SECONDS
+      minutes  = (seconds / ONE_MINUTE_IN_SECONDS).floor
+      seconds -= minutes * ONE_MINUTE_IN_SECONDS
+      sprintf("%02i:%02i:%02i", hours, minutes, seconds)
+    end
+
+    def progress_percentage
+      return 0 if @n_forecasted_messages.zero?
+      progress = @n_received_messages.to_f / @n_forecasted_messages
+      [(progress * 100).to_i, 100].min
+    end
+
+    private
+    def client_options
+      {
+        :host          => @host,
+        :port          => @port,
+        :tag           => @tag,
+        :protocol      => :droonga,
+        :receiver_host => @receiver_host,
+        :receiver_port => @receiver_port,
+      }
+    end
+
+    def handle_dump_message(message, &block)
+      case message["type"]
+      when "dump.result", "dump.error"
+        if message["statusCode"] != 200
+          @client.close
+          error = message["body"]
+          on_error(error)
+          @error_message = "#{error['name']}: #{error['message']}"
+        end
+      when "dump.table"
+        @n_received_messages += 1
+        table_create_message = convert_to_table_create_message(message)
+        yield(table_create_message)
+      when "dump.column"
+        @n_received_messages += 1
+        column_create_message = convert_to_column_create_message(message)
+        yield(column_create_message)
+      when "dump.record"
+        @n_received_messages += 1
+        add_message = message.dup
+        add_message.delete("inReplyTo")
+        add_message["type"] = "add"
+        yield(add_message)
+      when "dump.start"
+        @n_dumpers += 1
+      when "dump.end"
+        @n_dumpers -= 1
+        if @n_dumpers <= 0
+          @client.close
+          on_finish
+        end
+      when "dump.forecast"
+        @n_forecasted_messages += message["body"]["nMessages"]
+      end
+    end
+
+    def convert_to_table_create_message(message)
+      body = message["body"]
+      flags = []
+      case body["type"]
+      when "Array"
+        flags << "TABLE_NO_KEY"
+      when "Hash"
+        flags << "TABLE_HASH_KEY"
+      when "PatriciaTrie"
+        flags << "TABLE_PAT_KEY"
+      when "DoubleArrayTrie"
+        flags << "TABLE_DAT_KEY"
+      end
+      table_create_message = {
+        "type"    => "table_create",
+        "dataset" => message["dataset"],
+        "body" => {
+          "name"              => body["name"],
+          "flags"             => flags.join("|"),
+          "key_type"          => body["keyType"],
+        }
+      }
+
+      if body["tokenizer"]
+        table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
+      end
+      if body["normalizer"]
+        table_create_message["body"]["normalizer"] = body["normalizer"]
+      end
+
+      table_create_message
+    end
+
+    def convert_to_column_create_message(message)
+      body = message["body"]
+      column_create_message = {
+        "type"    => "column_create",
+        "dataset" => message["dataset"],
+        "body" => {
+          "table"  => body["table"],
+          "name"   => body["name"],
+          "type"   => body["valueType"],
+        }
+      }
+
+      flags = []
+      case body["type"]
+      when "Scalar"
+        flags << "COLUMN_SCALAR"
+      when "Vector"
+        flags << "COLUMN_VECTOR"
+        vector_options = body["vectorOptions"] || {}
+        flags << "WITH_WEIGHT" if vector_options["weight"]
+      when "Index"
+        flags << "COLUMN_INDEX"
+        index_options = body["indexOptions"] || {}
+        flags << "WITH_SECTION"  if index_options["section"]
+        flags << "WITH_WEIGHT"   if index_options["weight"]
+        flags << "WITH_POSITION" if index_options["position"]
+      end
+
+      column_create_message["body"]["flags"] = flags.join("|")
+
+      if body["type"] == "Index"
+        index_options = body["indexOptions"] || {}
+        sources = index_options["sources"] || []
+        unless sources.empty?
+          column_create_message["body"]["source"] = sources.join(",")
+        end
+      end
+
+      column_create_message
+    end
+
+    def on_finish
+      @on_finish.call if @on_finish
+    end
+
+    def on_progress(message)
+      @on_progress.call(message) if @on_progress
+    end
+
+    def on_error(error)
+      @on_error.call(error) if @on_error
+    end
+  end
+end

data/lib/drndump/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Droonga Project
+# Copyright (C) 2014-2015 Droonga Project
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module Drndump
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: drndump
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Droonga Project
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-29 00:00:00.000000000 Z
+date: 2015-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -116,14 +116,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- Gemfile
 - README.md
 - Rakefile
-- Gemfile
+- bin/drndump
+- doc/text/news.md
 - drndump.gemspec
-- lib/drndump/version.rb
 - lib/drndump/command.rb
-- doc/text/news.md
-- bin/drndump
+- lib/drndump/dump_client.rb
+- lib/drndump/version.rb
 homepage: https://github.com/droonga/drndump
 licenses:
 - GPLv3 or later
@@ -144,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Drndump is a tool to dump Droonga database. You can use it for backup.