flydata 0.3.8 → 0.3.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +4 -4
- data/lib/flydata/command/sync.rb +55 -45
- data/lib/flydata/errors.rb +3 -0
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +2 -2
- data/lib/flydata/parser/mysql/dump_parser.rb +38 -35
- data/lib/flydata/util/mysql_util.rb +3 -0
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +7 -25
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: abffe0873e3fe1143a5053d9f02b5c61eb1e90a3
|
4
|
+
data.tar.gz: c7f54222297282bcc2c13ffa1ca18ddc45bb10ef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b81782a6f4ac5bf2c28485b56ea1912e6f8490d375cb6ffd5f70e070776691fe4d20f74cfb444672192a0440a651b73835765054fc9da617e36182e4c021655
|
7
|
+
data.tar.gz: 7ef6babb2f76e4c1e75ff700a377c1b8a9504f022fe1bbe0fe09dc30f54e629277dafe2d55cfd905967edeac55d70d0019181fc1a90000c77172f9f7325961b8
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.9
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.3.
|
5
|
+
# stub: flydata 0.3.9 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "flydata"
|
9
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.9"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2015-03-
|
14
|
+
s.date = "2015-03-21"
|
15
15
|
s.description = "FlyData Agent"
|
16
16
|
s.email = "sysadmin@flydata.com"
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -167,7 +167,7 @@ Gem::Specification.new do |s|
|
|
167
167
|
]
|
168
168
|
s.homepage = "http://flydata.com/"
|
169
169
|
s.licenses = ["All right reserved."]
|
170
|
-
s.rubygems_version = "2.
|
170
|
+
s.rubygems_version = "2.2.2"
|
171
171
|
s.summary = "FlyData Agent"
|
172
172
|
|
173
173
|
if s.respond_to? :specification_version then
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -289,9 +289,9 @@ EOS
|
|
289
289
|
unless Flydata::Preference::DataEntryPreference.conf_exists?(de)
|
290
290
|
Flydata::Command::Conf.new.copy_templates
|
291
291
|
end
|
292
|
-
generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, db_bytesize|
|
292
|
+
generate_mysqldump(de, sync_fm, !opts.dump_stream?) do |mysqldump_io, binlog_pos, db_bytesize|
|
293
293
|
sync_fm.save_sync_info(@full_initial_sync, de['mysql_data_entry_preference']['tables'])
|
294
|
-
parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize)
|
294
|
+
parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize)
|
295
295
|
end
|
296
296
|
wait_for_mysqldump_processed(dp, de, sync_fm)
|
297
297
|
end
|
@@ -362,13 +362,13 @@ EOM
|
|
362
362
|
log_info_stdout("Exporting data from database.")
|
363
363
|
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
364
364
|
if file_dump
|
365
|
-
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
365
|
+
binlog_pos = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
366
366
|
new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp)
|
367
367
|
log_info_stdout(" -> Done")
|
368
|
-
call_block_or_return_io(fp, &block)
|
368
|
+
call_block_or_return_io(fp, binlog_pos, &block)
|
369
369
|
else
|
370
370
|
Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
|
371
|
-
new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io| block.call(io, db_bytesize)}
|
371
|
+
new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump {|io, binlog_pos| block.call(io, binlog_pos, db_bytesize)}
|
372
372
|
end
|
373
373
|
else
|
374
374
|
exit 1
|
@@ -381,11 +381,11 @@ EOM
|
|
381
381
|
stat.block_size * stat.blocks_available
|
382
382
|
end
|
383
383
|
|
384
|
-
def call_block_or_return_io(fp, &block)
|
384
|
+
def call_block_or_return_io(fp, binlog_pos = nil, &block)
|
385
385
|
if block
|
386
386
|
f_io = open_file_io(fp)
|
387
387
|
begin
|
388
|
-
block.call(f_io)
|
388
|
+
block.call(f_io, binlog_pos)
|
389
389
|
return nil
|
390
390
|
ensure
|
391
391
|
f_io.close rescue nil
|
@@ -418,7 +418,7 @@ EOM
|
|
418
418
|
# <- checkpoint
|
419
419
|
#...
|
420
420
|
#CREATE TABLE ...
|
421
|
-
def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, db_bytesize = nil)
|
421
|
+
def parse_mysqldump_and_send(mysqldump_io, dp, de, sync_fm, binlog_pos, db_bytesize = nil)
|
422
422
|
# Prepare forwarder
|
423
423
|
de_tag_name = de["tag_name#{env_suffix}"]
|
424
424
|
server_port = dp['server_port']
|
@@ -435,8 +435,10 @@ EOM
|
|
435
435
|
dump_pos_info = sync_fm.load_dump_pos
|
436
436
|
option = dump_pos_info || {}
|
437
437
|
if option[:table_name] && option[:last_pos].to_i != -1
|
438
|
+
binlog_pos = option[:binlog_pos]
|
438
439
|
log_info_stdout("Resuming... Last processed table: #{option[:table_name]}")
|
439
440
|
else
|
441
|
+
option[:binlog_pos] = binlog_pos
|
440
442
|
#If its a new sync, ensure server side resources are clean
|
441
443
|
cleanup_sync_server(de, target_tables_for_api) unless opts.skip_cleanup?
|
442
444
|
end
|
@@ -448,45 +450,53 @@ EOM
|
|
448
450
|
tmp_num_inserted_record = 0
|
449
451
|
dump_fp = sync_fm.dump_file_path
|
450
452
|
dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
|
451
|
-
binlog_pos = Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
|
452
|
-
mysqldump_io,
|
453
|
-
# create table
|
454
|
-
Proc.new { |mysql_table|
|
455
|
-
tmp_num_inserted_record = 0
|
456
|
-
# dump mysql_table for resume
|
457
|
-
#TODO: make it option
|
458
|
-
sync_fm.save_mysql_table_marshal_dump(mysql_table)
|
459
|
-
},
|
460
|
-
# insert record
|
461
|
-
Proc.new { |mysql_table, values_set|
|
462
|
-
mysql_table_name = mysql_table.table_name
|
463
|
-
records = values_set.collect do |values|
|
464
|
-
json = generate_json(mysql_table, values)
|
465
|
-
{table_name: mysql_table_name, log: json}
|
466
|
-
end
|
467
|
-
ret = forwarder.emit(records)
|
468
|
-
tmp_num_inserted_record += 1
|
469
|
-
ret
|
470
|
-
},
|
471
|
-
# checkpoint
|
472
|
-
Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
473
|
-
# flush if buffer records exist
|
474
|
-
if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
|
475
|
-
forwarder.flush # send buffer data to the server before checkpoint
|
476
|
-
end
|
477
453
|
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
454
|
+
begin
|
455
|
+
Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
|
456
|
+
mysqldump_io,
|
457
|
+
# create table
|
458
|
+
Proc.new { |mysql_table|
|
459
|
+
tmp_num_inserted_record = 0
|
460
|
+
# dump mysql_table for resume
|
461
|
+
#TODO: make it option
|
462
|
+
sync_fm.save_mysql_table_marshal_dump(mysql_table)
|
463
|
+
},
|
464
|
+
# insert record
|
465
|
+
Proc.new { |mysql_table, values_set|
|
466
|
+
mysql_table_name = mysql_table.table_name
|
467
|
+
records = values_set.collect do |values|
|
468
|
+
json = generate_json(mysql_table, values)
|
469
|
+
{table_name: mysql_table_name, log: json}
|
470
|
+
end
|
471
|
+
ret = forwarder.emit(records)
|
472
|
+
tmp_num_inserted_record += 1
|
473
|
+
ret
|
474
|
+
},
|
475
|
+
# checkpoint
|
476
|
+
Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
477
|
+
# flush if buffer records exist
|
478
|
+
if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
|
479
|
+
forwarder.flush # send buffer data to the server before checkpoint
|
480
|
+
end
|
481
|
+
|
482
|
+
# show the current progress
|
483
|
+
if last_pos.to_i == -1 # stream dump
|
484
|
+
log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
|
485
|
+
else
|
486
|
+
log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
|
487
|
+
end
|
484
488
|
|
485
|
-
|
486
|
-
|
487
|
-
|
488
|
-
|
489
|
-
|
489
|
+
# save check point
|
490
|
+
table_name = mysql_table.nil? ? '' : mysql_table.table_name
|
491
|
+
sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
|
492
|
+
}
|
493
|
+
)
|
494
|
+
rescue DumpParseError =>e
|
495
|
+
ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n #{e.message}")
|
496
|
+
ee.description = " Please contact support@flydata.com to report the issue."
|
497
|
+
ee.set_backtrace e.backtrace
|
498
|
+
raise ee
|
499
|
+
end
|
490
500
|
forwarder.close
|
491
501
|
log_info_stdout(" -> Done")
|
492
502
|
sync_fm.save_dump_pos(STATUS_WAITING, '', dump_file_size, binlog_pos)
|
data/lib/flydata/errors.rb
CHANGED
@@ -4,8 +4,8 @@ require 'fluent/plugin/in_mysql_binlog'
|
|
4
4
|
require 'binlog'
|
5
5
|
require 'kodama'
|
6
6
|
|
7
|
-
# Load client library(flydata/
|
8
|
-
lib = File.expand_path(File.dirname(__FILE__), '
|
7
|
+
# Load client library(flydata-agent/lib)
|
8
|
+
lib = File.expand_path(File.join(File.dirname(__FILE__), '../../'))
|
9
9
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
10
10
|
require 'flydata'
|
11
11
|
require 'flydata/sync_file_manager'
|
@@ -41,14 +41,11 @@ module Flydata
|
|
41
41
|
|
42
42
|
class MysqlDumpGenerator
|
43
43
|
def initialize(conf)
|
44
|
-
@
|
44
|
+
@conf = conf
|
45
45
|
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = conf[sym.to_s]; h}
|
46
46
|
end
|
47
|
-
def dump(file_path)
|
48
|
-
raise "subclass must implement the method"
|
49
|
-
end
|
50
47
|
|
51
|
-
def
|
48
|
+
def dump(file_path)
|
52
49
|
raise "subclass must implement the method"
|
53
50
|
end
|
54
51
|
end
|
@@ -68,6 +65,8 @@ EOS
|
|
68
65
|
raise ArgumentError.new("file_path or block must be given.")
|
69
66
|
end
|
70
67
|
|
68
|
+
dump_cmd = generate_dump_cmd(@conf, file_path)
|
69
|
+
|
71
70
|
# RDS doesn't allow obtaining binlog position using mysqldump. Get it separately and insert it into the dump file.
|
72
71
|
table_locker = create_table_locker
|
73
72
|
table_locker.resume # Lock tables
|
@@ -78,14 +77,16 @@ EOS
|
|
78
77
|
wr_io.sync = true
|
79
78
|
wr_io.set_encoding("utf-8")
|
80
79
|
rd_io.extend(DumpStreamIO)
|
80
|
+
binlog_pos = nil
|
81
81
|
|
82
82
|
# start mysqldump
|
83
|
-
Open3.popen3
|
83
|
+
Open3.popen3 dump_cmd do |cmd_in, cmd_out, cmd_err, wait_thr|
|
84
84
|
cmd_in.close_write
|
85
85
|
cmd_out.set_encoding("utf-8") # mysqldump output must be in UTF-8
|
86
86
|
|
87
87
|
first_line = cmd_out.gets # wait until first line comes
|
88
|
-
|
88
|
+
binfile, pos = table_locker.resume
|
89
|
+
binlog_pos = {binfile: binfile, pos: pos}
|
89
90
|
|
90
91
|
threads = []
|
91
92
|
|
@@ -93,7 +94,7 @@ EOS
|
|
93
94
|
threads << Thread.new do
|
94
95
|
begin
|
95
96
|
wr_io.print(first_line) # write a first line
|
96
|
-
filter_dump_stream(cmd_out, wr_io
|
97
|
+
filter_dump_stream(cmd_out, wr_io)
|
97
98
|
ensure
|
98
99
|
wr_io.close rescue nil
|
99
100
|
end
|
@@ -109,10 +110,7 @@ EOS
|
|
109
110
|
|
110
111
|
if block
|
111
112
|
# call callback function with io if block is given
|
112
|
-
block.call(rd_io)
|
113
|
-
elsif file_path
|
114
|
-
# store data to file
|
115
|
-
open_file_stream(file_path) {|f| rd_io.each_line{|l| f.print(l)}}
|
113
|
+
block.call(rd_io, binlog_pos)
|
116
114
|
end
|
117
115
|
|
118
116
|
threads.each(&:join)
|
@@ -123,7 +121,7 @@ EOS
|
|
123
121
|
end
|
124
122
|
raise errors unless errors.empty?
|
125
123
|
end
|
126
|
-
|
124
|
+
binlog_pos
|
127
125
|
rescue
|
128
126
|
# Cleanup
|
129
127
|
FileUtils.rm(file_path) if file_path && File.exists?(file_path)
|
@@ -136,8 +134,8 @@ EOS
|
|
136
134
|
end
|
137
135
|
end
|
138
136
|
|
139
|
-
def generate_dump_cmd(conf)
|
140
|
-
Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf)
|
137
|
+
def generate_dump_cmd(conf, file_path = nil)
|
138
|
+
Util::MysqlUtil.generate_mysqldump_without_master_data_cmd(conf.merge(result_file: file_path))
|
141
139
|
end
|
142
140
|
|
143
141
|
private
|
@@ -201,25 +199,13 @@ EOS
|
|
201
199
|
result.first['Value']
|
202
200
|
end
|
203
201
|
|
204
|
-
def filter_dump_stream(cmd_out, w_io
|
205
|
-
find_insert_pos = :not_started
|
202
|
+
def filter_dump_stream(cmd_out, w_io)
|
206
203
|
cmd_out.each_line do |line|
|
207
|
-
if find_insert_pos == :not_started && /^-- Server version/ === line
|
208
|
-
find_insert_pos = :finding
|
209
|
-
elsif find_insert_pos == :finding && /^--/ === line
|
210
|
-
# wait before writing the first database queries
|
211
|
-
# insert binlog pos
|
212
|
-
change_master = CHANGE_MASTER_TEMPLATE % [binlog_file, binlog_pos]
|
213
|
-
w_io.print change_master
|
214
|
-
|
215
|
-
find_insert_pos = :found
|
216
|
-
end
|
217
204
|
w_io.print line
|
218
205
|
w_io.puts unless line.end_with?("\n")
|
219
206
|
w_io.flush
|
220
207
|
end
|
221
208
|
end
|
222
|
-
|
223
209
|
end
|
224
210
|
|
225
211
|
# Custom mysql client that sets config params (eg:-read_timeout) uniformly for all
|
@@ -258,9 +244,10 @@ EOS
|
|
258
244
|
end
|
259
245
|
|
260
246
|
attr_accessor :binlog_pos
|
247
|
+
BINLOG_INV_ERROR_CHUNK_SIZE = 250
|
261
248
|
|
262
249
|
def initialize(option = {})
|
263
|
-
@binlog_pos = option[:binlog_pos]
|
250
|
+
@binlog_pos = option[:binlog_pos] or raise ArgumentError.new("binlog position is required")
|
264
251
|
@option = option
|
265
252
|
end
|
266
253
|
|
@@ -291,10 +278,8 @@ EOS
|
|
291
278
|
state_start = Proc.new do
|
292
279
|
line = readline_proc.call
|
293
280
|
|
294
|
-
# --
|
295
|
-
|
296
|
-
if m
|
297
|
-
@binlog_pos = {binfile: m[:binfile], pos: m[:pos].to_i}
|
281
|
+
# -- Server version 5.6.21-log
|
282
|
+
if line.start_with?('-- Server version')
|
298
283
|
current_state = State::CREATE_TABLE
|
299
284
|
check_point_block.call(nil, dump_io.pos, bytesize, @binlog_pos, current_state)
|
300
285
|
end
|
@@ -478,8 +463,26 @@ EOS
|
|
478
463
|
#
|
479
464
|
# We are using the C native method that is like 'split', 'start_with?', 'regexp'
|
480
465
|
# instead of 'String#each_char' and string comparision for the performance.
|
481
|
-
# 'String#each_char' is twice as slow as the current
|
482
|
-
|
466
|
+
# 'String#each_char' is twice as slow as the current strategy.
|
467
|
+
begin
|
468
|
+
items = target_line.split(',')
|
469
|
+
rescue ArgumentError => e
|
470
|
+
# If there are any non-UTF8 characters, split will fail
|
471
|
+
# This will go through the chunk of data and cut the data up into chunks to make it
|
472
|
+
# displayable to user.
|
473
|
+
target_chunk=""
|
474
|
+
chunk_size = BINLOG_INV_ERROR_CHUNK_SIZE
|
475
|
+
chunk_repeat_count = target_line.length/chunk_size
|
476
|
+
for i in 0..chunk_repeat_count
|
477
|
+
line_chunk=target_line[(0+(i*chunk_size)..chunk_size+((i*chunk_size)-1))]
|
478
|
+
unless line_chunk.valid_encoding?
|
479
|
+
target_chunk=line_chunk
|
480
|
+
break
|
481
|
+
end
|
482
|
+
end
|
483
|
+
hex_string = target_chunk.bytes.map { |b| sprintf(", 0x%02X",b) }.join
|
484
|
+
raise DumpParseError, "raw_text: #{target_chunk}\n hex_dump: #{hex_string}"
|
485
|
+
end
|
483
486
|
index = 0
|
484
487
|
cur_state = :next_values
|
485
488
|
|
@@ -54,6 +54,9 @@ module Flydata
|
|
54
54
|
opt = option.dup
|
55
55
|
opt[:command] = 'mysqldump'
|
56
56
|
opt[:custom_option] = '--skip-lock-tables --single-transaction --hex-blob'
|
57
|
+
if opt[:result_file]
|
58
|
+
opt[:custom_option] << " --result-file=#{opt[:result_file]}"
|
59
|
+
end
|
57
60
|
generate_mysql_cmd(opt)
|
58
61
|
end
|
59
62
|
|
@@ -1,8 +1,9 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
require 'spec_helper'
|
3
|
-
require 'flydata/parser/mysql/dump_parser'
|
4
3
|
require 'open3'
|
5
4
|
require 'mysql2'
|
5
|
+
require 'flydata/command/sync'
|
6
|
+
require 'flydata/parser/mysql/dump_parser'
|
6
7
|
|
7
8
|
module Flydata
|
8
9
|
module Parser
|
@@ -33,6 +34,7 @@ module Flydata
|
|
33
34
|
let(:wait_thr) { double(:wait_thr) }
|
34
35
|
|
35
36
|
let(:default_dump_generator) { MysqlDumpGeneratorNoMasterData.new(default_conf) }
|
37
|
+
let(:binlog_pos) { { binfile: 'mysql-bin.000451', pos: 89872 } }
|
36
38
|
let(:mysql_client) do
|
37
39
|
m = double(:mysql_client)
|
38
40
|
allow(m).to receive(:query).with(/TABLES/)
|
@@ -44,12 +46,6 @@ module Flydata
|
|
44
46
|
m
|
45
47
|
end
|
46
48
|
|
47
|
-
describe '#initialize' do
|
48
|
-
subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
|
49
|
-
it { is_expected.to eq('mysqldump -h localhost -P 3306 -uadmin -p"pass" --default-character-set=utf8 --protocol=tcp --skip-lock-tables ' +
|
50
|
-
'--single-transaction --hex-blob dev users groups') }
|
51
|
-
end
|
52
|
-
|
53
49
|
describe '#dump' do
|
54
50
|
before do
|
55
51
|
expect(Mysql2::Client).to receive(:new).and_return(mysql_client)
|
@@ -59,8 +55,7 @@ module Flydata
|
|
59
55
|
it do
|
60
56
|
expect(wait_thr).to receive(:value).and_return(0)
|
61
57
|
expect(stderr).to receive(:each_line).and_yield("")
|
62
|
-
expect(default_dump_generator.dump(file_path)).to
|
63
|
-
expect(File.exists?(file_path)).to be_truthy
|
58
|
+
expect(default_dump_generator.dump(file_path)).to eq(binlog_pos)
|
64
59
|
end
|
65
60
|
end
|
66
61
|
context "when mysqldump exits with status 1" do
|
@@ -115,7 +110,7 @@ module Flydata
|
|
115
110
|
describe MysqlDumpParser do
|
116
111
|
let(:file_path) { File.join('/tmp', "flydata_sync_spec_mysqldump_parse_#{Time.now.to_i}") }
|
117
112
|
let(:dump_io) { File.open(file_path, 'r', encoding: "utf-8") }
|
118
|
-
let(:default_parser) { MysqlDumpParser.new }
|
113
|
+
let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
|
119
114
|
|
120
115
|
def generate_dump_file(content)
|
121
116
|
File.open(file_path, 'w') {|f| f.write(content)}
|
@@ -155,9 +150,9 @@ EOT
|
|
155
150
|
content.index(string) + string.bytesize + 1
|
156
151
|
end
|
157
152
|
|
158
|
-
let(:default_parser) { MysqlDumpParser.new }
|
153
|
+
let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
|
159
154
|
let(:default_binlog_pos) { {binfile: 'mysql-bin.000267', pos: 120 } }
|
160
|
-
let(:dump_pos_after_binlog_pos) { index_after(DUMP_HEADER, '
|
155
|
+
let(:dump_pos_after_binlog_pos) { index_after(DUMP_HEADER, '5.6.13-log') }
|
161
156
|
|
162
157
|
let(:create_table_block) { double('create_table_block') }
|
163
158
|
let(:insert_record_block) { double('insert_record_block') }
|
@@ -167,19 +162,6 @@ EOT
|
|
167
162
|
generate_dump_file('')
|
168
163
|
end
|
169
164
|
|
170
|
-
context 'when dump does not contain binlog pos' do
|
171
|
-
before { generate_dump_file('dummy content') }
|
172
|
-
it do
|
173
|
-
expect(create_table_block).to receive(:call).never
|
174
|
-
expect(insert_record_block).to receive(:call).never
|
175
|
-
expect(check_point_block).to receive(:call).never
|
176
|
-
binlog_pos = default_parser.parse(
|
177
|
-
dump_io, create_table_block, insert_record_block, check_point_block
|
178
|
-
)
|
179
|
-
expect(binlog_pos).to be_nil
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
165
|
context 'when dump contains only binlog pos' do
|
184
166
|
before { generate_dump_file(<<EOT
|
185
167
|
#{DUMP_HEADER}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-03-
|
15
|
+
date: 2015-03-21 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -607,7 +607,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
607
607
|
version: '0'
|
608
608
|
requirements: []
|
609
609
|
rubyforge_project:
|
610
|
-
rubygems_version: 2.
|
610
|
+
rubygems_version: 2.2.2
|
611
611
|
signing_key:
|
612
612
|
specification_version: 4
|
613
613
|
summary: FlyData Agent
|