drndump 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/doc/text/news.md +7 -0
- data/lib/drndump/command.rb +10 -121
- data/lib/drndump/dump_client.rb +298 -0
- data/lib/drndump/version.rb +2 -2
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffe9d62833a98f43fa9911c82c56429d74af6205
|
4
|
+
data.tar.gz: 62d440b218bc4d963d3c91a7a426f6e6bd3940fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2c32f075bcf25f1104d7331aed5f9cc55545da025ced1b0ea6ce11c102f0b6a10de2afb9fc5c160c9cbcf4c5a533595611df42d9c61666c39fc1c4ee1731d98e
|
7
|
+
data.tar.gz: 1ea16dde38b2f88bea29c058fd2d8f54cf0e90abe255597c622f3b5455de02ade8f734b50166053569bd6b9584e7586e39a37e0d40f87dc786379a525d9c2ec9
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 1.0.1: 2014-04-29
|
4
|
+
|
5
|
+
* Becomes modulable. You can use internal libraries for other products.
|
6
|
+
* Supports new `@subscribe-until` directive, for subscription type commands like `dump`.
|
7
|
+
You can unsubscribe the next request following to the directive automatically with given timeout, like:
|
8
|
+
`#@subscribe-until 10s`
|
9
|
+
|
3
10
|
## 1.0.0: 2014-05-29
|
4
11
|
|
5
12
|
The first release!!!
|
data/lib/drndump/command.rb
CHANGED
@@ -19,9 +19,8 @@ require "json"
|
|
19
19
|
|
20
20
|
require "cool.io"
|
21
21
|
|
22
|
-
require "droonga/client"
|
23
|
-
|
24
22
|
require "drndump/version"
|
23
|
+
require "drndump/dump_client"
|
25
24
|
|
26
25
|
module Drndump
|
27
26
|
class Command
|
@@ -106,138 +105,28 @@ module Drndump
|
|
106
105
|
parser
|
107
106
|
end
|
108
107
|
|
109
|
-
def
|
108
|
+
def dumper_params
|
110
109
|
{
|
111
110
|
:host => @host,
|
112
111
|
:port => @port,
|
113
112
|
:tag => @tag,
|
114
|
-
:
|
113
|
+
:dataset => @dataset,
|
115
114
|
:receiver_host => @receiver_host,
|
116
115
|
:receiver_port => @receiver_port,
|
117
|
-
:backend => :coolio,
|
118
|
-
:loop => @loop,
|
119
116
|
}
|
120
117
|
end
|
121
118
|
|
122
119
|
def dump
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
dump_message = {
|
129
|
-
"type" => "dump",
|
130
|
-
"dataset" => @dataset,
|
120
|
+
@dumper = DumpClient.new(dumper_params)
|
121
|
+
dump_options = {
|
122
|
+
:backend => :coolio,
|
123
|
+
:loop => @loop,
|
131
124
|
}
|
132
|
-
|
133
|
-
|
134
|
-
when Droonga::Client::Error
|
135
|
-
client.close
|
136
|
-
error_message = message.to_s
|
137
|
-
else
|
138
|
-
case message["type"]
|
139
|
-
when "dump.result", "dump.error"
|
140
|
-
if message["statusCode"] != 200
|
141
|
-
client.close
|
142
|
-
error = message["body"]
|
143
|
-
error_message = "#{error['name']}: #{error['message']}"
|
144
|
-
end
|
145
|
-
when "dump.table"
|
146
|
-
table_create_message = convert_to_table_create_message(message)
|
147
|
-
puts(JSON.pretty_generate(table_create_message))
|
148
|
-
when "dump.column"
|
149
|
-
column_create_message = convert_to_column_create_message(message)
|
150
|
-
puts(JSON.pretty_generate(column_create_message))
|
151
|
-
when "dump.record"
|
152
|
-
add_message = message.dup
|
153
|
-
add_message.delete("inReplyTo")
|
154
|
-
add_message["type"] = "add"
|
155
|
-
puts(JSON.pretty_generate(add_message))
|
156
|
-
when "dump.start"
|
157
|
-
n_dumpers += 1
|
158
|
-
when "dump.end"
|
159
|
-
n_dumpers -= 1
|
160
|
-
client.close if n_dumpers <= 0
|
161
|
-
end
|
162
|
-
end
|
125
|
+
@dumper.run(dump_options) do |message|
|
126
|
+
puts(JSON.pretty_generate(message))
|
163
127
|
end
|
164
128
|
@loop.run
|
165
|
-
|
166
|
-
error_message
|
167
|
-
end
|
168
|
-
|
169
|
-
def convert_to_table_create_message(message)
|
170
|
-
body = message["body"]
|
171
|
-
flags = []
|
172
|
-
case body["type"]
|
173
|
-
when "Array"
|
174
|
-
flags << "TABLE_NO_KEY"
|
175
|
-
when "Hash"
|
176
|
-
flags << "TABLE_HASH_KEY"
|
177
|
-
when "PatriciaTrie"
|
178
|
-
flags << "TABLE_PAT_KEY"
|
179
|
-
when "DoubleArrayTrie"
|
180
|
-
flags << "TABLE_DAT_KEY"
|
181
|
-
end
|
182
|
-
table_create_message = {
|
183
|
-
"type" => "table_create",
|
184
|
-
"dataset" => message["dataset"],
|
185
|
-
"body" => {
|
186
|
-
"name" => body["name"],
|
187
|
-
"flags" => flags.join("|"),
|
188
|
-
"key_type" => body["keyType"],
|
189
|
-
}
|
190
|
-
}
|
191
|
-
|
192
|
-
if body["tokenizer"]
|
193
|
-
table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
|
194
|
-
end
|
195
|
-
if body["normalizer"]
|
196
|
-
table_create_message["body"]["normalizer"] = body["normalizer"]
|
197
|
-
end
|
198
|
-
|
199
|
-
table_create_message
|
200
|
-
end
|
201
|
-
|
202
|
-
def convert_to_column_create_message(message)
|
203
|
-
body = message["body"]
|
204
|
-
column_create_message = {
|
205
|
-
"type" => "column_create",
|
206
|
-
"dataset" => message["dataset"],
|
207
|
-
"body" => {
|
208
|
-
"table" => body["table"],
|
209
|
-
"name" => body["name"],
|
210
|
-
"type" => body["valueType"],
|
211
|
-
}
|
212
|
-
}
|
213
|
-
|
214
|
-
flags = []
|
215
|
-
case body["type"]
|
216
|
-
when "Scalar"
|
217
|
-
flags << "COLUMN_SCALAR"
|
218
|
-
when "Vector"
|
219
|
-
flags << "COLUMN_VECTOR"
|
220
|
-
vector_options = body["vectorOptions"] || {}
|
221
|
-
flags << "WITH_WEIGHT" if vector_options["weight"]
|
222
|
-
when "Index"
|
223
|
-
flags << "COLUMN_INDEX"
|
224
|
-
index_options = body["indexOptions"] || {}
|
225
|
-
flags << "WITH_SECTION" if index_options["section"]
|
226
|
-
flags << "WITH_WEIGHT" if index_options["weight"]
|
227
|
-
flags << "WITH_POSITION" if index_options["position"]
|
228
|
-
end
|
229
|
-
|
230
|
-
column_create_message["body"]["flags"] = flags.join("|")
|
231
|
-
|
232
|
-
if body["type"] == "Index"
|
233
|
-
index_options = body["indexOptions"] || {}
|
234
|
-
sources = index_options["sources"] || []
|
235
|
-
unless sources.empty?
|
236
|
-
column_create_message["body"]["source"] = sources.join(",")
|
237
|
-
end
|
238
|
-
end
|
239
|
-
|
240
|
-
column_create_message
|
129
|
+
@dumper.error_message
|
241
130
|
end
|
242
131
|
end
|
243
132
|
end
|
@@ -0,0 +1,298 @@
|
|
1
|
+
# Copyright (C) 2014-2015 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "socket"
|
17
|
+
|
18
|
+
require "droonga/client"
|
19
|
+
|
20
|
+
module Drndump
|
21
|
+
class DumpClient
|
22
|
+
DEFAULT_MESSAGES_PER_SECOND = 10000 # same to droogna-engine's one
|
23
|
+
MIN_REPORTED_THROUGHPUT = 0.01
|
24
|
+
|
25
|
+
attr_reader :n_forecasted_messages, :n_received_messages
|
26
|
+
attr_reader :error_message
|
27
|
+
attr_writer :on_finish, :on_progress, :on_error
|
28
|
+
|
29
|
+
class NilMessage < StandardError
|
30
|
+
end
|
31
|
+
|
32
|
+
class InvalidMessage < StandardError
|
33
|
+
end
|
34
|
+
|
35
|
+
class ClientError < StandardError
|
36
|
+
def initialize(error)
|
37
|
+
super(error.inspect)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(params)
|
42
|
+
@host = params[:host]
|
43
|
+
@port = params[:port]
|
44
|
+
@tag = params[:tag]
|
45
|
+
@dataset = params[:dataset]
|
46
|
+
|
47
|
+
@receiver_host = params[:receiver_host]
|
48
|
+
@receiver_port = params[:receiver_port]
|
49
|
+
|
50
|
+
@n_forecasted_messages = 0
|
51
|
+
@n_received_messages = 0
|
52
|
+
@n_messages_per_second = DEFAULT_MESSAGES_PER_SECOND
|
53
|
+
|
54
|
+
@error_message = nil
|
55
|
+
|
56
|
+
@on_finish = nil
|
57
|
+
@on_progress = nil
|
58
|
+
@on_error = nil
|
59
|
+
end
|
60
|
+
|
61
|
+
def run(options={}, &block)
|
62
|
+
extra_client_options = {
|
63
|
+
:backend => options[:backend],
|
64
|
+
:loop => options[:loop],
|
65
|
+
}
|
66
|
+
@client = Droonga::Client.new(client_options.merge(extra_client_options))
|
67
|
+
@client.on_error = lambda do |error|
|
68
|
+
on_error(ClientError.new(error))
|
69
|
+
end
|
70
|
+
|
71
|
+
@n_dumpers = 0
|
72
|
+
|
73
|
+
@n_messages_per_second = options[:messages_per_second] || DEFAULT_MESSAGES_PER_SECOND
|
74
|
+
@n_messages_per_second = [@n_messages_per_second, 1].max
|
75
|
+
|
76
|
+
@measure_start_time = Time.now
|
77
|
+
@previous_measure_time = @measure_start_time
|
78
|
+
@previous_n_received_messages = 0.0
|
79
|
+
|
80
|
+
dump_message = {
|
81
|
+
"type" => "dump",
|
82
|
+
"dataset" => @dataset,
|
83
|
+
"body" => {
|
84
|
+
"messagesPerSecond" => @n_messages_per_second,
|
85
|
+
},
|
86
|
+
}
|
87
|
+
@client.subscribe(dump_message) do |message|
|
88
|
+
begin
|
89
|
+
on_progress(message)
|
90
|
+
case message
|
91
|
+
when Droonga::Client::Error
|
92
|
+
client.close
|
93
|
+
on_error(message)
|
94
|
+
@error_message = message.to_s
|
95
|
+
when Hash
|
96
|
+
handle_dump_message(message) do |restore_message|
|
97
|
+
yield(restore_message)
|
98
|
+
end
|
99
|
+
when NilClass
|
100
|
+
raise NilMessage.new("nil message in dump")
|
101
|
+
else
|
102
|
+
raise InvalidMessage.new("invalid message in dump",
|
103
|
+
:message => message.inspect)
|
104
|
+
end
|
105
|
+
rescue Exception => exception
|
106
|
+
@client.close
|
107
|
+
on_error(exception)
|
108
|
+
@error_message = exception.to_s
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
@error_message
|
113
|
+
end
|
114
|
+
|
115
|
+
def recent_throughput
|
116
|
+
now = Time.now
|
117
|
+
n_messages = @n_received_messages - @previous_n_received_messages
|
118
|
+
|
119
|
+
if now - @previous_measure_time < 1
|
120
|
+
now = @previous_measure_time
|
121
|
+
n_messages = @previous_n_received_messages
|
122
|
+
else
|
123
|
+
@previous_measure_time = now
|
124
|
+
@previous_n_received_messages = n_messages.to_f
|
125
|
+
end
|
126
|
+
|
127
|
+
if now == @measure_start_time
|
128
|
+
actual_throughput = 0
|
129
|
+
else
|
130
|
+
elapsed_seconds = now - @measure_start_time
|
131
|
+
actual_throughput = n_messages / elapsed_seconds
|
132
|
+
end
|
133
|
+
|
134
|
+
[actual_throughput, MIN_REPORTED_THROUGHPUT].max
|
135
|
+
end
|
136
|
+
|
137
|
+
def n_remaining_messages
|
138
|
+
[@n_forecasted_messages - @n_received_messages, 0].max
|
139
|
+
end
|
140
|
+
|
141
|
+
def remaining_seconds
|
142
|
+
throughput = [recent_throughput, @n_messages_per_second].min
|
143
|
+
n_remaining_messages.to_f / throughput
|
144
|
+
end
|
145
|
+
|
146
|
+
ONE_MINUTE_IN_SECONDS = 60
|
147
|
+
ONE_HOUR_IN_SECONDS = ONE_MINUTE_IN_SECONDS * 60
|
148
|
+
|
149
|
+
def formatted_remaining_time
|
150
|
+
seconds = remaining_seconds
|
151
|
+
hours = (seconds / ONE_HOUR_IN_SECONDS).floor
|
152
|
+
seconds -= hours * ONE_HOUR_IN_SECONDS
|
153
|
+
minutes = (seconds / ONE_MINUTE_IN_SECONDS).floor
|
154
|
+
seconds -= minutes * ONE_MINUTE_IN_SECONDS
|
155
|
+
sprintf("%02i:%02i:%02i", hours, minutes, seconds)
|
156
|
+
end
|
157
|
+
|
158
|
+
def progress_percentage
|
159
|
+
return 0 if @n_forecasted_messages.zero?
|
160
|
+
progress = @n_received_messages.to_f / @n_forecasted_messages
|
161
|
+
[(progress * 100).to_i, 100].min
|
162
|
+
end
|
163
|
+
|
164
|
+
private
|
165
|
+
def client_options
|
166
|
+
{
|
167
|
+
:host => @host,
|
168
|
+
:port => @port,
|
169
|
+
:tag => @tag,
|
170
|
+
:protocol => :droonga,
|
171
|
+
:receiver_host => @receiver_host,
|
172
|
+
:receiver_port => @receiver_port,
|
173
|
+
}
|
174
|
+
end
|
175
|
+
|
176
|
+
def handle_dump_message(message, &block)
|
177
|
+
case message["type"]
|
178
|
+
when "dump.result", "dump.error"
|
179
|
+
if message["statusCode"] != 200
|
180
|
+
@client.close
|
181
|
+
error = message["body"]
|
182
|
+
on_error(error)
|
183
|
+
@error_message = "#{error['name']}: #{error['message']}"
|
184
|
+
end
|
185
|
+
when "dump.table"
|
186
|
+
@n_received_messages += 1
|
187
|
+
table_create_message = convert_to_table_create_message(message)
|
188
|
+
yield(table_create_message)
|
189
|
+
when "dump.column"
|
190
|
+
@n_received_messages += 1
|
191
|
+
column_create_message = convert_to_column_create_message(message)
|
192
|
+
yield(column_create_message)
|
193
|
+
when "dump.record"
|
194
|
+
@n_received_messages += 1
|
195
|
+
add_message = message.dup
|
196
|
+
add_message.delete("inReplyTo")
|
197
|
+
add_message["type"] = "add"
|
198
|
+
yield(add_message)
|
199
|
+
when "dump.start"
|
200
|
+
@n_dumpers += 1
|
201
|
+
when "dump.end"
|
202
|
+
@n_dumpers -= 1
|
203
|
+
if @n_dumpers <= 0
|
204
|
+
@client.close
|
205
|
+
on_finish
|
206
|
+
end
|
207
|
+
when "dump.forecast"
|
208
|
+
@n_forecasted_messages += message["body"]["nMessages"]
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def convert_to_table_create_message(message)
|
213
|
+
body = message["body"]
|
214
|
+
flags = []
|
215
|
+
case body["type"]
|
216
|
+
when "Array"
|
217
|
+
flags << "TABLE_NO_KEY"
|
218
|
+
when "Hash"
|
219
|
+
flags << "TABLE_HASH_KEY"
|
220
|
+
when "PatriciaTrie"
|
221
|
+
flags << "TABLE_PAT_KEY"
|
222
|
+
when "DoubleArrayTrie"
|
223
|
+
flags << "TABLE_DAT_KEY"
|
224
|
+
end
|
225
|
+
table_create_message = {
|
226
|
+
"type" => "table_create",
|
227
|
+
"dataset" => message["dataset"],
|
228
|
+
"body" => {
|
229
|
+
"name" => body["name"],
|
230
|
+
"flags" => flags.join("|"),
|
231
|
+
"key_type" => body["keyType"],
|
232
|
+
}
|
233
|
+
}
|
234
|
+
|
235
|
+
if body["tokenizer"]
|
236
|
+
table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
|
237
|
+
end
|
238
|
+
if body["normalizer"]
|
239
|
+
table_create_message["body"]["normalizer"] = body["normalizer"]
|
240
|
+
end
|
241
|
+
|
242
|
+
table_create_message
|
243
|
+
end
|
244
|
+
|
245
|
+
def convert_to_column_create_message(message)
|
246
|
+
body = message["body"]
|
247
|
+
column_create_message = {
|
248
|
+
"type" => "column_create",
|
249
|
+
"dataset" => message["dataset"],
|
250
|
+
"body" => {
|
251
|
+
"table" => body["table"],
|
252
|
+
"name" => body["name"],
|
253
|
+
"type" => body["valueType"],
|
254
|
+
}
|
255
|
+
}
|
256
|
+
|
257
|
+
flags = []
|
258
|
+
case body["type"]
|
259
|
+
when "Scalar"
|
260
|
+
flags << "COLUMN_SCALAR"
|
261
|
+
when "Vector"
|
262
|
+
flags << "COLUMN_VECTOR"
|
263
|
+
vector_options = body["vectorOptions"] || {}
|
264
|
+
flags << "WITH_WEIGHT" if vector_options["weight"]
|
265
|
+
when "Index"
|
266
|
+
flags << "COLUMN_INDEX"
|
267
|
+
index_options = body["indexOptions"] || {}
|
268
|
+
flags << "WITH_SECTION" if index_options["section"]
|
269
|
+
flags << "WITH_WEIGHT" if index_options["weight"]
|
270
|
+
flags << "WITH_POSITION" if index_options["position"]
|
271
|
+
end
|
272
|
+
|
273
|
+
column_create_message["body"]["flags"] = flags.join("|")
|
274
|
+
|
275
|
+
if body["type"] == "Index"
|
276
|
+
index_options = body["indexOptions"] || {}
|
277
|
+
sources = index_options["sources"] || []
|
278
|
+
unless sources.empty?
|
279
|
+
column_create_message["body"]["source"] = sources.join(",")
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
column_create_message
|
284
|
+
end
|
285
|
+
|
286
|
+
def on_finish
|
287
|
+
@on_finish.call if @on_finish
|
288
|
+
end
|
289
|
+
|
290
|
+
def on_progress(message)
|
291
|
+
@on_progress.call(message) if @on_progress
|
292
|
+
end
|
293
|
+
|
294
|
+
def on_error(error)
|
295
|
+
@on_error.call(error) if @on_error
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
data/lib/drndump/version.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2014
|
1
|
+
# Copyright (C) 2014-2015 Droonga Project
|
2
2
|
#
|
3
3
|
# This program is free software: you can redistribute it and/or modify
|
4
4
|
# it under the terms of the GNU General Public License as published by
|
@@ -14,5 +14,5 @@
|
|
14
14
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
15
15
|
|
16
16
|
module Drndump
|
17
|
-
VERSION = "1.0.
|
17
|
+
VERSION = "1.0.1"
|
18
18
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: drndump
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Droonga Project
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-04-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|
@@ -116,14 +116,15 @@ executables:
|
|
116
116
|
extensions: []
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
|
+
- Gemfile
|
119
120
|
- README.md
|
120
121
|
- Rakefile
|
121
|
-
-
|
122
|
+
- bin/drndump
|
123
|
+
- doc/text/news.md
|
122
124
|
- drndump.gemspec
|
123
|
-
- lib/drndump/version.rb
|
124
125
|
- lib/drndump/command.rb
|
125
|
-
-
|
126
|
-
-
|
126
|
+
- lib/drndump/dump_client.rb
|
127
|
+
- lib/drndump/version.rb
|
127
128
|
homepage: https://github.com/droonga/drndump
|
128
129
|
licenses:
|
129
130
|
- GPLv3 or later
|
@@ -144,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
144
145
|
version: '0'
|
145
146
|
requirements: []
|
146
147
|
rubyforge_project:
|
147
|
-
rubygems_version: 2.
|
148
|
+
rubygems_version: 2.4.1
|
148
149
|
signing_key:
|
149
150
|
specification_version: 4
|
150
151
|
summary: Drndump is a tool to dump Droonga database. You can use it for backup.
|