flydata 0.5.12 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/Gemfile.lock +3 -0
- data/Rakefile +12 -0
- data/VERSION +1 -1
- data/ext/flydata/flydata.h +13 -0
- data/ext/flydata/json/extconf.rb +3 -0
- data/ext/flydata/json/json_ext.cpp +137 -0
- data/ext/flydata/parser/mysql/.gitignore +1 -0
- data/ext/flydata/parser/mysql/dump_parser_ext.cpp +106 -0
- data/ext/flydata/parser/mysql/extconf.rb +3 -0
- data/ext/flydata/parser/mysql/parser.txt +121 -0
- data/ext/flydata/parser/mysql/sql_parser.cpp +414 -0
- data/ext/flydata/parser/mysql/sql_parser.h +15 -0
- data/flydata-core/lib/flydata-core/mysql/command_generator.rb +20 -9
- data/flydata-core/spec/mysql/command_generator_spec.rb +17 -17
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/sync.rb +78 -90
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +4 -1
- data/lib/flydata/json.rb +1 -0
- data/lib/flydata/json/.gitignore +1 -0
- data/lib/flydata/parser/mysql/.gitignore +1 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +21 -5
- data/spec/flydata/command/sync_spec.rb +20 -13
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +104 -0
- data/spec/flydata/sync_file_manager_spec.rb +3 -3
- data/test-suite.sh +1 -0
- metadata +38 -4
@@ -26,12 +26,12 @@ module FlydataCore
|
|
26
26
|
|
27
27
|
context 'when option is empty' do
|
28
28
|
let(:option) { {} }
|
29
|
-
it { is_expected.to eq("mysql --default-character-set=utf8 --protocol=tcp") }
|
29
|
+
it { is_expected.to eq("mysql --default-character-set=utf8 --protocol=tcp --skip-auto-rehash") }
|
30
30
|
end
|
31
31
|
|
32
32
|
context 'when all option is specified' do
|
33
33
|
it { is_expected.to eq(
|
34
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
34
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb'
|
35
35
|
) }
|
36
36
|
end
|
37
37
|
|
@@ -41,7 +41,7 @@ module FlydataCore
|
|
41
41
|
option['database'] = 'another_db'
|
42
42
|
}
|
43
43
|
it { is_expected.to eq(
|
44
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
44
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp another_db'
|
45
45
|
) }
|
46
46
|
end
|
47
47
|
|
@@ -54,54 +54,54 @@ module FlydataCore
|
|
54
54
|
|
55
55
|
context 'when password is not empty' do
|
56
56
|
before { option[:password] = "abc`def" }
|
57
|
-
it { is_expected.to eq(cmd_pswd_check % '
|
57
|
+
it { is_expected.to eq(cmd_pswd_check % 'abc\\`def' ) }
|
58
58
|
end
|
59
59
|
|
60
60
|
context 'when password includes double quate' do
|
61
61
|
before { option[:password] = "abc\"def" }
|
62
|
-
it { is_expected.to eq(cmd_pswd_check % '
|
62
|
+
it { is_expected.to eq(cmd_pswd_check % 'abc\\"def') }
|
63
63
|
end
|
64
64
|
|
65
65
|
context 'when password includes dollars sign' do
|
66
66
|
before { option[:password] = "abc$def" }
|
67
|
-
it { is_expected.to eq(cmd_pswd_check % '
|
67
|
+
it { is_expected.to eq(cmd_pswd_check % 'abc\\$def' ) }
|
68
68
|
end
|
69
69
|
|
70
70
|
context 'when password includes multiple special characters' do
|
71
71
|
before { option[:password] = "ab\"\"c$$def" }
|
72
|
-
it { is_expected.to eq(cmd_pswd_check % '
|
72
|
+
it { is_expected.to eq(cmd_pswd_check % 'ab\\"\\"c\\$\\$def' ) }
|
73
73
|
end
|
74
74
|
|
75
75
|
context 'when password does not includes special characters' do
|
76
76
|
before { option[:password] = "abcdef" }
|
77
|
-
it { is_expected.to eq(cmd_pswd_check % '
|
77
|
+
it { is_expected.to eq(cmd_pswd_check % 'abcdef' ) }
|
78
78
|
end
|
79
79
|
|
80
80
|
context 'when tables are specified' do
|
81
81
|
before { option[:tables] = %w(table_a table_b table_c) }
|
82
82
|
it { is_expected.to eq(
|
83
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
83
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp testdb table_a table_b table_c'
|
84
84
|
) }
|
85
85
|
end
|
86
86
|
|
87
87
|
context 'when ssl_ca is specified' do
|
88
88
|
before { option[:ssl_ca] = 'mysql_sync.ssl_ca.pem' }
|
89
89
|
it { is_expected.to eq(
|
90
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
90
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --ssl-ca mysql_sync.ssl_ca.pem testdb'
|
91
91
|
) }
|
92
92
|
end
|
93
93
|
|
94
94
|
context 'when ssl_cipher is specified' do
|
95
95
|
before { option[:ssl_cipher] = 'AES256-GCM-SHA384:AES256-SHA' }
|
96
96
|
it { is_expected.to eq(
|
97
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
97
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --ssl-cipher=AES256-GCM-SHA384:AES256-SHA testdb'
|
98
98
|
) }
|
99
99
|
end
|
100
100
|
|
101
101
|
context 'when no_default_option is turned on' do
|
102
102
|
before { option[:no_default_option] = true }
|
103
103
|
it { is_expected.to eq(
|
104
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
104
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword testdb'
|
105
105
|
) }
|
106
106
|
end
|
107
107
|
|
@@ -111,7 +111,7 @@ module FlydataCore
|
|
111
111
|
option[:custom_option] = '-e "SHOW GRANTS;"'
|
112
112
|
}
|
113
113
|
it { is_expected.to eq(
|
114
|
-
'mysql -h test-host -P 3306 -utestuser -
|
114
|
+
'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash -e "SHOW GRANTS;" testdb'
|
115
115
|
) }
|
116
116
|
end
|
117
117
|
end
|
@@ -120,7 +120,7 @@ module FlydataCore
|
|
120
120
|
subject { described_class.generate_mysql_ddl_dump_cmd(option) }
|
121
121
|
|
122
122
|
it { is_expected.to eq(
|
123
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
123
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp -d testdb'
|
124
124
|
) }
|
125
125
|
end
|
126
126
|
|
@@ -128,7 +128,7 @@ module FlydataCore
|
|
128
128
|
subject { described_class.generate_mysqldump_with_master_data_cmd(option) }
|
129
129
|
before { option[:tables] = %w(table_a table_b) }
|
130
130
|
it { is_expected.to eq(
|
131
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
131
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob --flush-logs --master-data=2 testdb table_a table_b'
|
132
132
|
) }
|
133
133
|
end
|
134
134
|
|
@@ -136,14 +136,14 @@ module FlydataCore
|
|
136
136
|
subject { described_class.generate_mysqldump_without_master_data_cmd(option) }
|
137
137
|
before { option[:tables] = %w(table_a table_b) }
|
138
138
|
it { is_expected.to eq(
|
139
|
-
'mysqldump -h test-host -P 3306 -utestuser -
|
139
|
+
'mysqldump -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-lock-tables --single-transaction --hex-blob testdb table_a table_b'
|
140
140
|
) }
|
141
141
|
end
|
142
142
|
|
143
143
|
describe '.generate_mysql_show_grants_cmd' do
|
144
144
|
subject { described_class.generate_mysql_show_grants_cmd(option) }
|
145
145
|
it { is_expected.to eq(
|
146
|
-
'mysql -h test-host -P 3306 -utestuser -
|
146
|
+
'mysql -h test-host -P 3306 -utestuser -ptestpassword --default-character-set=utf8 --protocol=tcp --skip-auto-rehash -e "SHOW GRANTS;" testdb'
|
147
147
|
) }
|
148
148
|
end
|
149
149
|
|
data/flydata.gemspec
CHANGED
Binary file
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -9,6 +9,7 @@ require 'flydata/command/sender'
|
|
9
9
|
require 'flydata/compatibility_check'
|
10
10
|
require 'flydata/errors'
|
11
11
|
require 'flydata/helpers'
|
12
|
+
require 'flydata/json'
|
12
13
|
require 'flydata/output/forwarder'
|
13
14
|
require 'flydata/parser/mysql/dump_parser'
|
14
15
|
require 'flydata/preference/data_entry_preference'
|
@@ -63,45 +64,7 @@ module Flydata
|
|
63
64
|
# Command: flydata sync
|
64
65
|
# - Entry method
|
65
66
|
def run(*tables)
|
66
|
-
|
67
|
-
sender = Flydata::Command::Sender.new
|
68
|
-
if (sender.process_exist?)
|
69
|
-
if tables.empty?
|
70
|
-
# full sync
|
71
|
-
log_warn_stderr("FlyData Agent is already running. If you'd like to restart FlyData Sync from scratch, run 'flydata sync:reset' first.")
|
72
|
-
else
|
73
|
-
# per-table sync
|
74
|
-
log_warn_stderr("Flydata Agent is already running. If you'd like to Sync the table(s), run 'flydata sync:flush' first.")
|
75
|
-
end
|
76
|
-
exit 1
|
77
|
-
end
|
78
|
-
|
79
|
-
fluentd_started = false
|
80
|
-
|
81
|
-
quiet_option = false
|
82
|
-
|
83
|
-
start_fluentd = Proc.new do |binlog_pos|
|
84
|
-
# Start continuous sync by starting fluentd process
|
85
|
-
unless opts.no_flydata_start?
|
86
|
-
log_info_stdout("Starting FlyData Agent...") unless quiet_option
|
87
|
-
Flydata::Command::Sender.new.start(quiet: true)
|
88
|
-
log_info_stdout(" -> Done") unless quiet_option
|
89
|
-
end
|
90
|
-
fluentd_started = true
|
91
|
-
end
|
92
|
-
|
93
|
-
quiet_option = true
|
94
|
-
# Start initial sync with check
|
95
|
-
handle_mysql_sync(tables, binlog_ready_callback: start_fluentd)
|
96
|
-
quiet_option = false
|
97
|
-
|
98
|
-
start_fluentd.call unless fluentd_started
|
99
|
-
|
100
|
-
# Show message
|
101
|
-
dashboard_url = "#{flydata.flydata_api_host}/dashboard"
|
102
|
-
redshift_console_url = "#{flydata.flydata_api_host}/redshift_clusters/query/new"
|
103
|
-
last_message = ALL_DONE_MESSAGE_TEMPLATE % [redshift_console_url, dashboard_url]
|
104
|
-
log_info_stdout(last_message)
|
67
|
+
raise "Command 'flydata sync' has been deprecated. Use 'flydata start' instead."
|
105
68
|
end
|
106
69
|
run_exclusive :run
|
107
70
|
|
@@ -459,8 +422,9 @@ EOS
|
|
459
422
|
sent_binlog_pos = nil
|
460
423
|
@full_tables.each do |table|
|
461
424
|
binlog_str = table_status_hash[table]["src_pos"]
|
462
|
-
|
463
|
-
|
425
|
+
if binlog_str.nil? || binlog_str == "-"
|
426
|
+
# the table status has no src_pos (which is rare.) Skip the table
|
427
|
+
next
|
464
428
|
end
|
465
429
|
binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
|
466
430
|
if tables.empty?
|
@@ -479,44 +443,49 @@ EOS
|
|
479
443
|
end
|
480
444
|
end
|
481
445
|
end
|
482
|
-
if
|
483
|
-
|
484
|
-
|
485
|
-
|
446
|
+
# if sent_binlog_pos is nil, it means sync has started for none of tables. No need
|
447
|
+
# to repair positions nor clean buffer data.
|
448
|
+
if sent_binlog_pos
|
449
|
+
if oldest_binlog && sent_binlog_pos < oldest_binlog
|
450
|
+
e = AgentError.new("Repair failed due to expired binlog")
|
451
|
+
e.description = <<EOS
|
452
|
+
Repair failed because the starting binlog position `#{sent_binlog_pos} no longer exists. Run full initial sync instead.
|
486
453
|
EOS
|
487
|
-
|
488
|
-
|
489
|
-
|
490
|
-
|
491
|
-
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
454
|
+
raise e
|
455
|
+
end
|
456
|
+
master_binlog_pos = FlydataCore::Mysql::BinlogPos.new(sent_binlog_pos.filename, 4)
|
457
|
+
|
458
|
+
# Delete agent buffer
|
459
|
+
log_info_stdout "Deleting data in the agent buffer..."
|
460
|
+
files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
|
461
|
+
unless files.empty?
|
462
|
+
$log.debug "Deleted buffer files\n " + files.join("\n ")
|
463
|
+
end
|
464
|
+
|
465
|
+
# Delete query queue items for the tables
|
466
|
+
log_info_stdout "Deleting data stuck in the server buffer..."
|
467
|
+
cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
|
468
|
+
|
469
|
+
# Save the positions (binlog and seq)
|
470
|
+
log_info_stdout "Fixing table positions..."
|
471
|
+
@full_tables.each do |table|
|
472
|
+
binlog_str = table_status_hash[table]["src_pos"]
|
473
|
+
if binlog_str.nil? || binlog_str == "-"
|
474
|
+
# no init sync has happened to the table. No need to set positions.
|
475
|
+
next
|
476
|
+
end
|
477
|
+
binlog_pos = FlydataCore::Mysql::BinlogPos.new(binlog_str)
|
478
|
+
pos = table_status_hash[table]["seq"]
|
479
|
+
old_binlog_pos, old_pos = save_table_positions(table, binlog_pos, pos)
|
480
|
+
if pos.to_i != old_pos.to_i && !tables.include?(table)
|
481
|
+
$log.debug "Fixed broken table position. table:#{table} pos:#{old_pos} -> #{pos}"
|
482
|
+
end
|
483
|
+
end
|
484
|
+
|
485
|
+
log_info_stdout "Fixing the master position files..."
|
486
|
+
save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
|
515
487
|
end
|
516
488
|
|
517
|
-
log_info_stdout "Fixing the master position files..."
|
518
|
-
save_master_binlog_positions(master_binlog_pos, sent_binlog_pos)
|
519
|
-
|
520
489
|
# Remove the lock file if exists.
|
521
490
|
File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
|
522
491
|
|
@@ -814,6 +783,7 @@ EOM
|
|
814
783
|
dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
|
815
784
|
|
816
785
|
begin
|
786
|
+
threads = []
|
817
787
|
Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
|
818
788
|
mysqldump_io,
|
819
789
|
# create table
|
@@ -828,7 +798,7 @@ EOM
|
|
828
798
|
mysql_table_name = mysql_table.table_name
|
829
799
|
records = values_set.collect do |values|
|
830
800
|
values = convert_to_flydata_values(mysql_table, values)
|
831
|
-
json =
|
801
|
+
json = JSON.generate_kv_pairs(mysql_table.column_names, values)
|
832
802
|
{table_name: mysql_table_name, log: json}
|
833
803
|
end
|
834
804
|
ret = forwarder.emit(records)
|
@@ -859,10 +829,11 @@ EOM
|
|
859
829
|
if mysql_table &&
|
860
830
|
state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
|
861
831
|
# all records for `mysql_table` have been sent
|
862
|
-
send_record_counts(de, sync_fm, mysql_table.table_name)
|
832
|
+
send_record_counts(de, sync_fm, mysql_table.table_name, threads)
|
863
833
|
end
|
864
834
|
}
|
865
835
|
)
|
836
|
+
threads.each{|t| t.join}
|
866
837
|
rescue DumpParseError =>e
|
867
838
|
ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n #{e.message}")
|
868
839
|
ee.description = " Please contact support@flydata.com to report the issue."
|
@@ -972,16 +943,24 @@ EOM
|
|
972
943
|
end
|
973
944
|
|
974
945
|
NUM_TABLES_IN_CHUNK = 30
|
975
|
-
def send_record_counts(de, sync_fm, table = nil)
|
946
|
+
def send_record_counts(de, sync_fm, table = nil, thread_array = nil)
|
976
947
|
stats = sync_fm.load_stats
|
977
948
|
stats = { table => stats[table] } if table # single table stats
|
978
949
|
stats.each_slice(NUM_TABLES_IN_CHUNK) do |slice|
|
979
950
|
h = Hash[slice]
|
980
|
-
send_record_counts_chunk(de, h)
|
951
|
+
send_record_counts_chunk(de, h, thread_array)
|
952
|
+
end
|
953
|
+
end
|
954
|
+
|
955
|
+
def send_record_counts_chunk(de, stats, thread_array)
|
956
|
+
if thread_array
|
957
|
+
thread_array << Thread.new { do_send_record_counts_chunk(de, stats) }
|
958
|
+
else
|
959
|
+
do_send_record_counts_chunk(de, stats)
|
981
960
|
end
|
982
961
|
end
|
983
962
|
|
984
|
-
def
|
963
|
+
def do_send_record_counts_chunk(de, stats)
|
985
964
|
retry_count = 0
|
986
965
|
retry_interval = 3
|
987
966
|
begin
|
@@ -1006,14 +985,24 @@ EOM
|
|
1006
985
|
end
|
1007
986
|
|
1008
987
|
def convert_to_flydata_values(mysql_table, values)
|
1009
|
-
types = mysql_table.
|
1010
|
-
types.size
|
988
|
+
types = mysql_table.format_types
|
989
|
+
sz = types.size
|
990
|
+
result = []
|
991
|
+
i = 0
|
992
|
+
while i < sz
|
993
|
+
result << FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_value(values[i], types[i])
|
994
|
+
i += 1
|
995
|
+
end
|
996
|
+
result
|
1011
997
|
end
|
1012
998
|
|
1013
|
-
def generate_json(
|
999
|
+
def generate_json(column_names, values)
|
1014
1000
|
h = {}
|
1015
|
-
|
1016
|
-
|
1001
|
+
sz = column_names.size
|
1002
|
+
i = 0
|
1003
|
+
while i < sz
|
1004
|
+
h[column_names[i]] = values[i]
|
1005
|
+
i += 1
|
1017
1006
|
end
|
1018
1007
|
h.to_json
|
1019
1008
|
end
|
@@ -1274,7 +1263,7 @@ Thank you for using FlyData!
|
|
1274
1263
|
|
1275
1264
|
server_sequence = table_status["seq"]
|
1276
1265
|
agent_sequence = sync_fm.get_table_position(table)
|
1277
|
-
agent_sequence = agent_sequence.to_i
|
1266
|
+
agent_sequence = agent_sequence.to_i # nil is considred 0
|
1278
1267
|
if server_sequence != agent_sequence
|
1279
1268
|
pos_mismatch_tables << {table: table, agent_seq: agent_sequence, server_seq: server_sequence}
|
1280
1269
|
end
|
@@ -1347,10 +1336,9 @@ Thank you for using FlyData!
|
|
1347
1336
|
src_pos = table_status_hash[table]["src_pos"]
|
1348
1337
|
if src_pos.nil? || src_pos == "-"
|
1349
1338
|
init_binlog_pos = sync_fm.get_table_binlog_pos_init(table)
|
1350
|
-
|
1351
|
-
|
1339
|
+
if init_binlog_pos
|
1340
|
+
table_status_hash[table]["src_pos"] = init_binlog_pos
|
1352
1341
|
end
|
1353
|
-
table_status_hash[table]["src_pos"] = init_binlog_pos
|
1354
1342
|
end
|
1355
1343
|
end
|
1356
1344
|
end
|
@@ -99,7 +99,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
99
99
|
$log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
|
100
100
|
$log.info "mysql client version: #{`mysql -V`}"
|
101
101
|
server_msg = `echo 'select version();' | #{FlydataCore::Mysql::CommandGenerator.generate_mysql_cmd(@db_opts)} 2>&1`
|
102
|
-
if ($?.exitstatus == 0)
|
102
|
+
if ($? && $?.exitstatus == 0)
|
103
103
|
$log.info "mysql server version: #{server_msg.strip}"
|
104
104
|
else
|
105
105
|
err_msg = "Failed to access mysql server... #{server_msg.strip}"
|
@@ -234,6 +234,9 @@ EOS
|
|
234
234
|
current_ssl_cipher = @secondary_ssl_cipher
|
235
235
|
$log.warn("Retry with secondary ssl cipher list due to '#{e}' - secondary_ssl_cipher: '#{@secondary_ssl_cipher}'")
|
236
236
|
retry
|
237
|
+
elsif /binlog file.*does not exist/.match(e.to_s)
|
238
|
+
$log.error("#{e.to_s}. Sync must be reset. Terminating the agent.")
|
239
|
+
Process.kill(:TERM, Process.ppid)
|
237
240
|
else
|
238
241
|
raise e
|
239
242
|
end
|
data/lib/flydata/json.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'flydata/json/json_ext'
|
@@ -0,0 +1 @@
|
|
1
|
+
json_ext.so
|
@@ -0,0 +1 @@
|
|
1
|
+
dump_parser_ext.so
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'fiber'
|
2
2
|
require 'io/wait'
|
3
|
+
require 'mysql2'
|
3
4
|
require 'flydata-core/mysql/config'
|
4
5
|
require 'flydata-core/mysql/command_generator'
|
5
6
|
|
@@ -31,13 +32,19 @@ module Flydata
|
|
31
32
|
def initialize(table_name, columns = {}, primary_keys = [])
|
32
33
|
@table_name = table_name
|
33
34
|
@columns = columns
|
35
|
+
@column_names = columns.collect{|k,v| v[:column_name]}
|
36
|
+
@format_types = columns.collect{|k,v| v[:format_type]}
|
34
37
|
@primary_keys = primary_keys
|
35
38
|
end
|
36
39
|
|
37
|
-
attr_accessor :table_name, :columns, :
|
40
|
+
attr_accessor :table_name, :columns, :column_names, :format_types,
|
41
|
+
:primary_keys
|
38
42
|
|
39
43
|
def add_column(column)
|
40
|
-
|
44
|
+
cn = column[:column_name]
|
45
|
+
@columns[cn] = column
|
46
|
+
@column_names << cn
|
47
|
+
@format_types << column[:format_type]
|
41
48
|
end
|
42
49
|
end
|
43
50
|
|
@@ -303,7 +310,7 @@ EOS
|
|
303
310
|
line = buffered_line
|
304
311
|
buffered_line = nil
|
305
312
|
else
|
306
|
-
rawline = dump_io.readline
|
313
|
+
rawline = dump_io.readline
|
307
314
|
bytesize += rawline.bytesize
|
308
315
|
line = rawline.strip
|
309
316
|
end
|
@@ -417,7 +424,14 @@ EOS
|
|
417
424
|
state_parsing_insert_record = Proc.new do
|
418
425
|
line = readline_proc.call
|
419
426
|
|
420
|
-
values_set =
|
427
|
+
values_set = nil
|
428
|
+
begin
|
429
|
+
values_set = InsertParser.new.parse(line)
|
430
|
+
rescue => e
|
431
|
+
newe = e.class.new(e.message + "line:#{line}")
|
432
|
+
newe.set_backtrace(e.backtrace)
|
433
|
+
raise newe
|
434
|
+
end
|
421
435
|
current_state = State::INSERT_RECORD
|
422
436
|
|
423
437
|
if insert_record_block.call(current_table, values_set)
|
@@ -478,7 +492,7 @@ EOS
|
|
478
492
|
def parse(line)
|
479
493
|
start_ruby_prof
|
480
494
|
bench_start_time = Time.now
|
481
|
-
|
495
|
+
_parse2(line)
|
482
496
|
ensure
|
483
497
|
stop_ruby_prof
|
484
498
|
if ENV['FLYDATA_BENCHMARK']
|
@@ -652,3 +666,5 @@ EOT
|
|
652
666
|
end
|
653
667
|
end
|
654
668
|
end
|
669
|
+
|
670
|
+
require 'flydata/parser/mysql/dump_parser_ext'
|