flydata 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/flydata.gemspec +2 -2
- data/lib/flydata/command/sync.rb +10 -2
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +33 -10
- data/spec/flydata/command/sync_spec.rb +6 -6
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.6
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.6"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa"]
|
12
|
-
s.date = "2014-05-
|
12
|
+
s.date = "2014-05-22"
|
13
13
|
s.description = "FlyData Command Line Interface"
|
14
14
|
s.email = "sysadmin@flydata.co"
|
15
15
|
s.executables = ["fdmysqldump", "flydata"]
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -88,7 +88,7 @@ module Flydata
|
|
88
88
|
flydata.data_entry.buffer_stat(de['id'], env_mode)
|
89
89
|
end
|
90
90
|
|
91
|
-
DDL_DUMP_CMD_TEMPLATE = "mysqldump -d -h %s -P %s -u %s %s %s %s"
|
91
|
+
DDL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -d -h %s -P %s -u %s %s %s %s"
|
92
92
|
def do_generate_table_ddl(de)
|
93
93
|
if `which mysqldump`.empty?
|
94
94
|
raise "mysqldump is not installed. mysqldump is required to run the command"
|
@@ -668,7 +668,7 @@ EOT
|
|
668
668
|
|
669
669
|
class MysqlDumpGenerator
|
670
670
|
# host, port, username, password, database, tables
|
671
|
-
MYSQL_DUMP_CMD_TEMPLATE = "mysqldump -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 %s %s"
|
671
|
+
MYSQL_DUMP_CMD_TEMPLATE = "mysqldump --protocol=tcp -h %s -P %s -u%s %s --skip-lock-tables --single-transaction --flush-logs --hex-blob --master-data=2 %s %s"
|
672
672
|
def initialize(conf)
|
673
673
|
password = conf['password'].to_s.empty? ? "" : "-p#{conf['password']}"
|
674
674
|
tables = if conf['tables']
|
@@ -868,6 +868,8 @@ EOT
|
|
868
868
|
@binlog_pos
|
869
869
|
end
|
870
870
|
|
871
|
+
# Parse the insert line containing multiple values. (max line size is 1kb)
|
872
|
+
# ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
|
871
873
|
class InsertParser
|
872
874
|
#INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
|
873
875
|
module State
|
@@ -913,6 +915,12 @@ EOT
|
|
913
915
|
target_line = target_line.strip
|
914
916
|
start_index = target_line.index('(')
|
915
917
|
target_line = target_line[start_index..-2]
|
918
|
+
|
919
|
+
# Split insert line text with ',' and take care of ',' inside of the values later.
|
920
|
+
#
|
921
|
+
# We are using the C native method that is like 'split', 'start_with?', 'regexp'
|
922
|
+
# instead of 'String#each_char' and string comparision for the performance.
|
923
|
+
# 'String#each_char' is twice as slow as the current storategy.
|
916
924
|
items = target_line.split(',')
|
917
925
|
index = 0
|
918
926
|
cur_state = State::NEXT_VALUES
|
@@ -39,19 +39,42 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
39
39
|
super
|
40
40
|
positions_path = @sync_fm.table_positions_dir_path
|
41
41
|
Dir.mkdir positions_path unless File.exists? positions_path
|
42
|
+
rescue Binlog::Error
|
43
|
+
if (/basic_string::_M_replace_aux/ === $!.to_s)
|
44
|
+
# TODO Fix the root cause in mysql-replication-listener
|
45
|
+
$log.error <<EOS
|
46
|
+
a mysql-replication-listener error. This could have been caused by one of the following reasons.
|
47
|
+
- Failed on connect: Your host is blocked because of many connection errors; unblock with 'mysqladmin flush-hosts'
|
48
|
+
EOS
|
49
|
+
else
|
50
|
+
$log.error "unexpected mysql-replication-listener error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
|
51
|
+
end
|
52
|
+
raise
|
53
|
+
rescue Exception
|
54
|
+
$log.error "unexpected fatal error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
|
55
|
+
raise
|
56
|
+
end
|
57
|
+
|
58
|
+
def run
|
59
|
+
super
|
60
|
+
rescue
|
61
|
+
$log.error "unexpected error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
|
62
|
+
raise
|
63
|
+
rescue SignalException
|
64
|
+
$log.debug "signal exception. exception: #{$!.class.to_s}, error: #{$!.to_s}"
|
65
|
+
raise
|
66
|
+
rescue Exception
|
67
|
+
$log.error "unexpected fatal error. exception: #{$!.class.to_s}, error: #{$!.to_s}\n#{$!.backtrace.join("\n")}"
|
68
|
+
raise
|
42
69
|
end
|
43
70
|
|
44
71
|
def event_listener(event)
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
$log.error e.backtrace.join("\n")
|
52
|
-
# Not reraising a StandardError because the underlying code can't handle an error well.
|
53
|
-
raise unless e.kind_of?(StandardError)
|
54
|
-
end
|
72
|
+
@record_handler.dispatch(event)
|
73
|
+
rescue Exception
|
74
|
+
position = File.open(@position_file) {|f| f.read }
|
75
|
+
$log.error "error occured while processing #{event.event_type} event at #{position}\n#{e.message}\n#{$!.backtrace.join("\n")}"
|
76
|
+
# Not reraising a StandardError because the underlying code can't handle an error well.
|
77
|
+
raise unless e.kind_of?(StandardError)
|
55
78
|
end
|
56
79
|
|
57
80
|
end
|
@@ -50,7 +50,7 @@ module Flydata
|
|
50
50
|
context 'with full options' do
|
51
51
|
it 'issues mysqldump command with expected parameters' do
|
52
52
|
expect(IO).to receive(:popen).with(
|
53
|
-
'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
53
|
+
'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
54
54
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
55
55
|
end
|
56
56
|
end
|
@@ -70,7 +70,7 @@ module Flydata
|
|
70
70
|
end
|
71
71
|
it "uses the default port" do
|
72
72
|
expect(IO).to receive(:popen).with(
|
73
|
-
'mysqldump -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
73
|
+
'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
74
74
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
75
75
|
end
|
76
76
|
end
|
@@ -80,7 +80,7 @@ module Flydata
|
|
80
80
|
end
|
81
81
|
it "uses the specified port" do
|
82
82
|
expect(IO).to receive(:popen).with(
|
83
|
-
'mysqldump -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
83
|
+
'mysqldump --protocol=tcp -d -h localhost -P 1234 -u masashi -pwelcome sync_test table1 table2', 'r').and_call_original
|
84
84
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
85
85
|
end
|
86
86
|
end
|
@@ -100,7 +100,7 @@ module Flydata
|
|
100
100
|
end
|
101
101
|
it "call mysqldump without -p option" do
|
102
102
|
expect(IO).to receive(:popen).with(
|
103
|
-
'mysqldump -d -h localhost -P 3306 -u masashi sync_test table1 table2', 'r').and_call_original
|
103
|
+
'mysqldump --protocol=tcp -d -h localhost -P 3306 -u masashi sync_test table1 table2', 'r').and_call_original
|
104
104
|
subject.send(:do_generate_table_ddl, default_data_entry)
|
105
105
|
end
|
106
106
|
end
|
@@ -398,7 +398,7 @@ module Flydata
|
|
398
398
|
describe '#initialize' do
|
399
399
|
context 'with password' do
|
400
400
|
subject { default_dump_generator.instance_variable_get(:@dump_cmd) }
|
401
|
-
it { should eq('mysqldump -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
|
401
|
+
it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin -ppass --skip-lock-tables ' +
|
402
402
|
'--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
|
403
403
|
end
|
404
404
|
context 'without password' do
|
@@ -406,7 +406,7 @@ module Flydata
|
|
406
406
|
MysqlDumpGenerator.new(default_conf.merge({'password' => ''}))
|
407
407
|
end
|
408
408
|
subject { dump_generator.instance_variable_get(:@dump_cmd) }
|
409
|
-
it { should eq('mysqldump -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
409
|
+
it { should eq('mysqldump --protocol=tcp -h localhost -P 3306 -uadmin --skip-lock-tables ' +
|
410
410
|
'--single-transaction --flush-logs --hex-blob --master-data=2 dev users groups') }
|
411
411
|
end
|
412
412
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-05-
|
12
|
+
date: 2014-05-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rest-client
|
@@ -391,7 +391,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
391
391
|
version: '0'
|
392
392
|
segments:
|
393
393
|
- 0
|
394
|
-
hash:
|
394
|
+
hash: -2211529075960565235
|
395
395
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
396
396
|
none: false
|
397
397
|
requirements:
|