drndump 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/doc/text/news.md +7 -0
- data/lib/drndump/command.rb +10 -121
- data/lib/drndump/dump_client.rb +298 -0
- data/lib/drndump/version.rb +2 -2
- metadata +8 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffe9d62833a98f43fa9911c82c56429d74af6205
|
4
|
+
data.tar.gz: 62d440b218bc4d963d3c91a7a426f6e6bd3940fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2c32f075bcf25f1104d7331aed5f9cc55545da025ced1b0ea6ce11c102f0b6a10de2afb9fc5c160c9cbcf4c5a533595611df42d9c61666c39fc1c4ee1731d98e
|
7
|
+
data.tar.gz: 1ea16dde38b2f88bea29c058fd2d8f54cf0e90abe255597c622f3b5455de02ade8f734b50166053569bd6b9584e7586e39a37e0d40f87dc786379a525d9c2ec9
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 1.0.1: 2014-04-29
|
4
|
+
|
5
|
+
* Becomes modulable. You can use internal libraries for other products.
|
6
|
+
* Supports new `@subscribe-until` directive, for subscription type commands like `dump`.
|
7
|
+
You can unsubscribe the next request following to the directive automatically with given timeout, like:
|
8
|
+
`#@subscribe-until 10s`
|
9
|
+
|
3
10
|
## 1.0.0: 2014-05-29
|
4
11
|
|
5
12
|
The first release!!!
|
data/lib/drndump/command.rb
CHANGED
@@ -19,9 +19,8 @@ require "json"
|
|
19
19
|
|
20
20
|
require "cool.io"
|
21
21
|
|
22
|
-
require "droonga/client"
|
23
|
-
|
24
22
|
require "drndump/version"
|
23
|
+
require "drndump/dump_client"
|
25
24
|
|
26
25
|
module Drndump
|
27
26
|
class Command
|
@@ -106,138 +105,28 @@ module Drndump
|
|
106
105
|
parser
|
107
106
|
end
|
108
107
|
|
109
|
-
def
|
108
|
+
def dumper_params
|
110
109
|
{
|
111
110
|
:host => @host,
|
112
111
|
:port => @port,
|
113
112
|
:tag => @tag,
|
114
|
-
:
|
113
|
+
:dataset => @dataset,
|
115
114
|
:receiver_host => @receiver_host,
|
116
115
|
:receiver_port => @receiver_port,
|
117
|
-
:backend => :coolio,
|
118
|
-
:loop => @loop,
|
119
116
|
}
|
120
117
|
end
|
121
118
|
|
122
119
|
def dump
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
dump_message = {
|
129
|
-
"type" => "dump",
|
130
|
-
"dataset" => @dataset,
|
120
|
+
@dumper = DumpClient.new(dumper_params)
|
121
|
+
dump_options = {
|
122
|
+
:backend => :coolio,
|
123
|
+
:loop => @loop,
|
131
124
|
}
|
132
|
-
|
133
|
-
|
134
|
-
when Droonga::Client::Error
|
135
|
-
client.close
|
136
|
-
error_message = message.to_s
|
137
|
-
else
|
138
|
-
case message["type"]
|
139
|
-
when "dump.result", "dump.error"
|
140
|
-
if message["statusCode"] != 200
|
141
|
-
client.close
|
142
|
-
error = message["body"]
|
143
|
-
error_message = "#{error['name']}: #{error['message']}"
|
144
|
-
end
|
145
|
-
when "dump.table"
|
146
|
-
table_create_message = convert_to_table_create_message(message)
|
147
|
-
puts(JSON.pretty_generate(table_create_message))
|
148
|
-
when "dump.column"
|
149
|
-
column_create_message = convert_to_column_create_message(message)
|
150
|
-
puts(JSON.pretty_generate(column_create_message))
|
151
|
-
when "dump.record"
|
152
|
-
add_message = message.dup
|
153
|
-
add_message.delete("inReplyTo")
|
154
|
-
add_message["type"] = "add"
|
155
|
-
puts(JSON.pretty_generate(add_message))
|
156
|
-
when "dump.start"
|
157
|
-
n_dumpers += 1
|
158
|
-
when "dump.end"
|
159
|
-
n_dumpers -= 1
|
160
|
-
client.close if n_dumpers <= 0
|
161
|
-
end
|
162
|
-
end
|
125
|
+
@dumper.run(dump_options) do |message|
|
126
|
+
puts(JSON.pretty_generate(message))
|
163
127
|
end
|
164
128
|
@loop.run
|
165
|
-
|
166
|
-
error_message
|
167
|
-
end
|
168
|
-
|
169
|
-
def convert_to_table_create_message(message)
|
170
|
-
body = message["body"]
|
171
|
-
flags = []
|
172
|
-
case body["type"]
|
173
|
-
when "Array"
|
174
|
-
flags << "TABLE_NO_KEY"
|
175
|
-
when "Hash"
|
176
|
-
flags << "TABLE_HASH_KEY"
|
177
|
-
when "PatriciaTrie"
|
178
|
-
flags << "TABLE_PAT_KEY"
|
179
|
-
when "DoubleArrayTrie"
|
180
|
-
flags << "TABLE_DAT_KEY"
|
181
|
-
end
|
182
|
-
table_create_message = {
|
183
|
-
"type" => "table_create",
|
184
|
-
"dataset" => message["dataset"],
|
185
|
-
"body" => {
|
186
|
-
"name" => body["name"],
|
187
|
-
"flags" => flags.join("|"),
|
188
|
-
"key_type" => body["keyType"],
|
189
|
-
}
|
190
|
-
}
|
191
|
-
|
192
|
-
if body["tokenizer"]
|
193
|
-
table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
|
194
|
-
end
|
195
|
-
if body["normalizer"]
|
196
|
-
table_create_message["body"]["normalizer"] = body["normalizer"]
|
197
|
-
end
|
198
|
-
|
199
|
-
table_create_message
|
200
|
-
end
|
201
|
-
|
202
|
-
def convert_to_column_create_message(message)
|
203
|
-
body = message["body"]
|
204
|
-
column_create_message = {
|
205
|
-
"type" => "column_create",
|
206
|
-
"dataset" => message["dataset"],
|
207
|
-
"body" => {
|
208
|
-
"table" => body["table"],
|
209
|
-
"name" => body["name"],
|
210
|
-
"type" => body["valueType"],
|
211
|
-
}
|
212
|
-
}
|
213
|
-
|
214
|
-
flags = []
|
215
|
-
case body["type"]
|
216
|
-
when "Scalar"
|
217
|
-
flags << "COLUMN_SCALAR"
|
218
|
-
when "Vector"
|
219
|
-
flags << "COLUMN_VECTOR"
|
220
|
-
vector_options = body["vectorOptions"] || {}
|
221
|
-
flags << "WITH_WEIGHT" if vector_options["weight"]
|
222
|
-
when "Index"
|
223
|
-
flags << "COLUMN_INDEX"
|
224
|
-
index_options = body["indexOptions"] || {}
|
225
|
-
flags << "WITH_SECTION" if index_options["section"]
|
226
|
-
flags << "WITH_WEIGHT" if index_options["weight"]
|
227
|
-
flags << "WITH_POSITION" if index_options["position"]
|
228
|
-
end
|
229
|
-
|
230
|
-
column_create_message["body"]["flags"] = flags.join("|")
|
231
|
-
|
232
|
-
if body["type"] == "Index"
|
233
|
-
index_options = body["indexOptions"] || {}
|
234
|
-
sources = index_options["sources"] || []
|
235
|
-
unless sources.empty?
|
236
|
-
column_create_message["body"]["source"] = sources.join(",")
|
237
|
-
end
|
238
|
-
end
|
239
|
-
|
240
|
-
column_create_message
|
129
|
+
@dumper.error_message
|
241
130
|
end
|
242
131
|
end
|
243
132
|
end
|
@@ -0,0 +1,298 @@
|
|
1
|
+
# Copyright (C) 2014-2015 Droonga Project
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License version 2.1 as published by the Free Software Foundation.
|
6
|
+
#
|
7
|
+
# This library is distributed in the hope that it will be useful,
|
8
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
9
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
10
|
+
# Lesser General Public License for more details.
|
11
|
+
#
|
12
|
+
# You should have received a copy of the GNU Lesser General Public
|
13
|
+
# License along with this library; if not, write to the Free Software
|
14
|
+
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
15
|
+
|
16
|
+
require "socket"
|
17
|
+
|
18
|
+
require "droonga/client"
|
19
|
+
|
20
|
+
module Drndump
|
21
|
+
class DumpClient
|
22
|
+
DEFAULT_MESSAGES_PER_SECOND = 10000 # same to droogna-engine's one
|
23
|
+
MIN_REPORTED_THROUGHPUT = 0.01
|
24
|
+
|
25
|
+
attr_reader :n_forecasted_messages, :n_received_messages
|
26
|
+
attr_reader :error_message
|
27
|
+
attr_writer :on_finish, :on_progress, :on_error
|
28
|
+
|
29
|
+
class NilMessage < StandardError
|
30
|
+
end
|
31
|
+
|
32
|
+
class InvalidMessage < StandardError
|
33
|
+
end
|
34
|
+
|
35
|
+
class ClientError < StandardError
|
36
|
+
def initialize(error)
|
37
|
+
super(error.inspect)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(params)
|
42
|
+
@host = params[:host]
|
43
|
+
@port = params[:port]
|
44
|
+
@tag = params[:tag]
|
45
|
+
@dataset = params[:dataset]
|
46
|
+
|
47
|
+
@receiver_host = params[:receiver_host]
|
48
|
+
@receiver_port = params[:receiver_port]
|
49
|
+
|
50
|
+
@n_forecasted_messages = 0
|
51
|
+
@n_received_messages = 0
|
52
|
+
@n_messages_per_second = DEFAULT_MESSAGES_PER_SECOND
|
53
|
+
|
54
|
+
@error_message = nil
|
55
|
+
|
56
|
+
@on_finish = nil
|
57
|
+
@on_progress = nil
|
58
|
+
@on_error = nil
|
59
|
+
end
|
60
|
+
|
61
|
+
def run(options={}, &block)
|
62
|
+
extra_client_options = {
|
63
|
+
:backend => options[:backend],
|
64
|
+
:loop => options[:loop],
|
65
|
+
}
|
66
|
+
@client = Droonga::Client.new(client_options.merge(extra_client_options))
|
67
|
+
@client.on_error = lambda do |error|
|
68
|
+
on_error(ClientError.new(error))
|
69
|
+
end
|
70
|
+
|
71
|
+
@n_dumpers = 0
|
72
|
+
|
73
|
+
@n_messages_per_second = options[:messages_per_second] || DEFAULT_MESSAGES_PER_SECOND
|
74
|
+
@n_messages_per_second = [@n_messages_per_second, 1].max
|
75
|
+
|
76
|
+
@measure_start_time = Time.now
|
77
|
+
@previous_measure_time = @measure_start_time
|
78
|
+
@previous_n_received_messages = 0.0
|
79
|
+
|
80
|
+
dump_message = {
|
81
|
+
"type" => "dump",
|
82
|
+
"dataset" => @dataset,
|
83
|
+
"body" => {
|
84
|
+
"messagesPerSecond" => @n_messages_per_second,
|
85
|
+
},
|
86
|
+
}
|
87
|
+
@client.subscribe(dump_message) do |message|
|
88
|
+
begin
|
89
|
+
on_progress(message)
|
90
|
+
case message
|
91
|
+
when Droonga::Client::Error
|
92
|
+
client.close
|
93
|
+
on_error(message)
|
94
|
+
@error_message = message.to_s
|
95
|
+
when Hash
|
96
|
+
handle_dump_message(message) do |restore_message|
|
97
|
+
yield(restore_message)
|
98
|
+
end
|
99
|
+
when NilClass
|
100
|
+
raise NilMessage.new("nil message in dump")
|
101
|
+
else
|
102
|
+
raise InvalidMessage.new("invalid message in dump",
|
103
|
+
:message => message.inspect)
|
104
|
+
end
|
105
|
+
rescue Exception => exception
|
106
|
+
@client.close
|
107
|
+
on_error(exception)
|
108
|
+
@error_message = exception.to_s
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
@error_message
|
113
|
+
end
|
114
|
+
|
115
|
+
def recent_throughput
|
116
|
+
now = Time.now
|
117
|
+
n_messages = @n_received_messages - @previous_n_received_messages
|
118
|
+
|
119
|
+
if now - @previous_measure_time < 1
|
120
|
+
now = @previous_measure_time
|
121
|
+
n_messages = @previous_n_received_messages
|
122
|
+
else
|
123
|
+
@previous_measure_time = now
|
124
|
+
@previous_n_received_messages = n_messages.to_f
|
125
|
+
end
|
126
|
+
|
127
|
+
if now == @measure_start_time
|
128
|
+
actual_throughput = 0
|
129
|
+
else
|
130
|
+
elapsed_seconds = now - @measure_start_time
|
131
|
+
actual_throughput = n_messages / elapsed_seconds
|
132
|
+
end
|
133
|
+
|
134
|
+
[actual_throughput, MIN_REPORTED_THROUGHPUT].max
|
135
|
+
end
|
136
|
+
|
137
|
+
def n_remaining_messages
|
138
|
+
[@n_forecasted_messages - @n_received_messages, 0].max
|
139
|
+
end
|
140
|
+
|
141
|
+
def remaining_seconds
|
142
|
+
throughput = [recent_throughput, @n_messages_per_second].min
|
143
|
+
n_remaining_messages.to_f / throughput
|
144
|
+
end
|
145
|
+
|
146
|
+
ONE_MINUTE_IN_SECONDS = 60
|
147
|
+
ONE_HOUR_IN_SECONDS = ONE_MINUTE_IN_SECONDS * 60
|
148
|
+
|
149
|
+
def formatted_remaining_time
|
150
|
+
seconds = remaining_seconds
|
151
|
+
hours = (seconds / ONE_HOUR_IN_SECONDS).floor
|
152
|
+
seconds -= hours * ONE_HOUR_IN_SECONDS
|
153
|
+
minutes = (seconds / ONE_MINUTE_IN_SECONDS).floor
|
154
|
+
seconds -= minutes * ONE_MINUTE_IN_SECONDS
|
155
|
+
sprintf("%02i:%02i:%02i", hours, minutes, seconds)
|
156
|
+
end
|
157
|
+
|
158
|
+
def progress_percentage
|
159
|
+
return 0 if @n_forecasted_messages.zero?
|
160
|
+
progress = @n_received_messages.to_f / @n_forecasted_messages
|
161
|
+
[(progress * 100).to_i, 100].min
|
162
|
+
end
|
163
|
+
|
164
|
+
private
|
165
|
+
def client_options
|
166
|
+
{
|
167
|
+
:host => @host,
|
168
|
+
:port => @port,
|
169
|
+
:tag => @tag,
|
170
|
+
:protocol => :droonga,
|
171
|
+
:receiver_host => @receiver_host,
|
172
|
+
:receiver_port => @receiver_port,
|
173
|
+
}
|
174
|
+
end
|
175
|
+
|
176
|
+
def handle_dump_message(message, &block)
|
177
|
+
case message["type"]
|
178
|
+
when "dump.result", "dump.error"
|
179
|
+
if message["statusCode"] != 200
|
180
|
+
@client.close
|
181
|
+
error = message["body"]
|
182
|
+
on_error(error)
|
183
|
+
@error_message = "#{error['name']}: #{error['message']}"
|
184
|
+
end
|
185
|
+
when "dump.table"
|
186
|
+
@n_received_messages += 1
|
187
|
+
table_create_message = convert_to_table_create_message(message)
|
188
|
+
yield(table_create_message)
|
189
|
+
when "dump.column"
|
190
|
+
@n_received_messages += 1
|
191
|
+
column_create_message = convert_to_column_create_message(message)
|
192
|
+
yield(column_create_message)
|
193
|
+
when "dump.record"
|
194
|
+
@n_received_messages += 1
|
195
|
+
add_message = message.dup
|
196
|
+
add_message.delete("inReplyTo")
|
197
|
+
add_message["type"] = "add"
|
198
|
+
yield(add_message)
|
199
|
+
when "dump.start"
|
200
|
+
@n_dumpers += 1
|
201
|
+
when "dump.end"
|
202
|
+
@n_dumpers -= 1
|
203
|
+
if @n_dumpers <= 0
|
204
|
+
@client.close
|
205
|
+
on_finish
|
206
|
+
end
|
207
|
+
when "dump.forecast"
|
208
|
+
@n_forecasted_messages += message["body"]["nMessages"]
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def convert_to_table_create_message(message)
|
213
|
+
body = message["body"]
|
214
|
+
flags = []
|
215
|
+
case body["type"]
|
216
|
+
when "Array"
|
217
|
+
flags << "TABLE_NO_KEY"
|
218
|
+
when "Hash"
|
219
|
+
flags << "TABLE_HASH_KEY"
|
220
|
+
when "PatriciaTrie"
|
221
|
+
flags << "TABLE_PAT_KEY"
|
222
|
+
when "DoubleArrayTrie"
|
223
|
+
flags << "TABLE_DAT_KEY"
|
224
|
+
end
|
225
|
+
table_create_message = {
|
226
|
+
"type" => "table_create",
|
227
|
+
"dataset" => message["dataset"],
|
228
|
+
"body" => {
|
229
|
+
"name" => body["name"],
|
230
|
+
"flags" => flags.join("|"),
|
231
|
+
"key_type" => body["keyType"],
|
232
|
+
}
|
233
|
+
}
|
234
|
+
|
235
|
+
if body["tokenizer"]
|
236
|
+
table_create_message["body"]["default_tokenizer"] = body["tokenizer"]
|
237
|
+
end
|
238
|
+
if body["normalizer"]
|
239
|
+
table_create_message["body"]["normalizer"] = body["normalizer"]
|
240
|
+
end
|
241
|
+
|
242
|
+
table_create_message
|
243
|
+
end
|
244
|
+
|
245
|
+
def convert_to_column_create_message(message)
|
246
|
+
body = message["body"]
|
247
|
+
column_create_message = {
|
248
|
+
"type" => "column_create",
|
249
|
+
"dataset" => message["dataset"],
|
250
|
+
"body" => {
|
251
|
+
"table" => body["table"],
|
252
|
+
"name" => body["name"],
|
253
|
+
"type" => body["valueType"],
|
254
|
+
}
|
255
|
+
}
|
256
|
+
|
257
|
+
flags = []
|
258
|
+
case body["type"]
|
259
|
+
when "Scalar"
|
260
|
+
flags << "COLUMN_SCALAR"
|
261
|
+
when "Vector"
|
262
|
+
flags << "COLUMN_VECTOR"
|
263
|
+
vector_options = body["vectorOptions"] || {}
|
264
|
+
flags << "WITH_WEIGHT" if vector_options["weight"]
|
265
|
+
when "Index"
|
266
|
+
flags << "COLUMN_INDEX"
|
267
|
+
index_options = body["indexOptions"] || {}
|
268
|
+
flags << "WITH_SECTION" if index_options["section"]
|
269
|
+
flags << "WITH_WEIGHT" if index_options["weight"]
|
270
|
+
flags << "WITH_POSITION" if index_options["position"]
|
271
|
+
end
|
272
|
+
|
273
|
+
column_create_message["body"]["flags"] = flags.join("|")
|
274
|
+
|
275
|
+
if body["type"] == "Index"
|
276
|
+
index_options = body["indexOptions"] || {}
|
277
|
+
sources = index_options["sources"] || []
|
278
|
+
unless sources.empty?
|
279
|
+
column_create_message["body"]["source"] = sources.join(",")
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
283
|
+
column_create_message
|
284
|
+
end
|
285
|
+
|
286
|
+
def on_finish
|
287
|
+
@on_finish.call if @on_finish
|
288
|
+
end
|
289
|
+
|
290
|
+
def on_progress(message)
|
291
|
+
@on_progress.call(message) if @on_progress
|
292
|
+
end
|
293
|
+
|
294
|
+
def on_error(error)
|
295
|
+
@on_error.call(error) if @on_error
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
data/lib/drndump/version.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2014
|
1
|
+
# Copyright (C) 2014-2015 Droonga Project
|
2
2
|
#
|
3
3
|
# This program is free software: you can redistribute it and/or modify
|
4
4
|
# it under the terms of the GNU General Public License as published by
|
@@ -14,5 +14,5 @@
|
|
14
14
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
15
15
|
|
16
16
|
module Drndump
|
17
|
-
VERSION = "1.0.
|
17
|
+
VERSION = "1.0.1"
|
18
18
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: drndump
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Droonga Project
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-04-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|
@@ -116,14 +116,15 @@ executables:
|
|
116
116
|
extensions: []
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
|
+
- Gemfile
|
119
120
|
- README.md
|
120
121
|
- Rakefile
|
121
|
-
-
|
122
|
+
- bin/drndump
|
123
|
+
- doc/text/news.md
|
122
124
|
- drndump.gemspec
|
123
|
-
- lib/drndump/version.rb
|
124
125
|
- lib/drndump/command.rb
|
125
|
-
-
|
126
|
-
-
|
126
|
+
- lib/drndump/dump_client.rb
|
127
|
+
- lib/drndump/version.rb
|
127
128
|
homepage: https://github.com/droonga/drndump
|
128
129
|
licenses:
|
129
130
|
- GPLv3 or later
|
@@ -144,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
144
145
|
version: '0'
|
145
146
|
requirements: []
|
146
147
|
rubyforge_project:
|
147
|
-
rubygems_version: 2.
|
148
|
+
rubygems_version: 2.4.1
|
148
149
|
signing_key:
|
149
150
|
specification_version: 4
|
150
151
|
summary: Drndump is a tool to dump Droonga database. You can use it for backup.
|