flydata 0.1.10 → 0.1.11
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -0
- data/Gemfile.lock +4 -0
- data/VERSION +1 -1
- data/flydata.gemspec +11 -9
- data/lib/flydata.rb +1 -1
- data/lib/flydata/cli.rb +1 -1
- data/lib/flydata/command/sync.rb +151 -15
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +13 -24
- data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +9 -25
- data/lib/flydata/fluent-plugins/mysql/binlog_position_file.rb +17 -0
- data/lib/flydata/fluent-plugins/preference.rb +1 -1
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +302 -0
- data/lib/flydata/parser_provider.rb +27 -0
- data/lib/flydata/preference/data_entry_preference.rb +1 -1
- data/lib/flydata/sync_file_manager.rb +22 -20
- data/lib/flydata/table_def.rb +2 -2
- data/lib/flydata/table_def/mysql_table_def.rb +9 -8
- data/spec/flydata/command/sync_spec.rb +1 -1
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +61 -73
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +282 -0
- data/spec/flydata/table_def/mysql_table_def_spec.rb +14 -0
- data/spec/flydata/table_def/mysqldump_test_unique_key.dump +45 -0
- data/spec/spec_helper.rb +10 -0
- metadata +25 -10
- data/lib/flydata/fluent-plugins/mysql/parser.rb +0 -2
- data/lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb +0 -65
- data/lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb +0 -49
- data/lib/flydata/fluent-plugins/mysql/parser/query_parser.rb +0 -9
- data/spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb +0 -52
- data/spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb +0 -68
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -78,6 +78,7 @@ GEM
|
|
78
78
|
multi_json (~> 1.0)
|
79
79
|
multi_xml (~> 0.5)
|
80
80
|
rack (~> 1.2)
|
81
|
+
polyglot (0.3.5)
|
81
82
|
protected_attributes (1.0.3)
|
82
83
|
activemodel (>= 4.0.0, < 5.0)
|
83
84
|
rack (1.5.2)
|
@@ -106,6 +107,8 @@ GEM
|
|
106
107
|
thread_safe (0.1.3)
|
107
108
|
atomic
|
108
109
|
timecop (0.7.1)
|
110
|
+
treetop (1.5.3)
|
111
|
+
polyglot (~> 0.3)
|
109
112
|
tzinfo (0.3.38)
|
110
113
|
yajl-ruby (1.2.0)
|
111
114
|
|
@@ -132,3 +135,4 @@ DEPENDENCIES
|
|
132
135
|
slop
|
133
136
|
sqlite3
|
134
137
|
timecop
|
138
|
+
treetop
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.11
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.1.
|
8
|
+
s.version = "0.1.11"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa"]
|
12
|
-
s.date = "2014-07-
|
12
|
+
s.date = "2014-07-29"
|
13
13
|
s.description = "FlyData Command Line Interface"
|
14
14
|
s.email = "sysadmin@flydata.co"
|
15
15
|
s.executables = ["fdmysqldump", "flydata"]
|
@@ -52,16 +52,13 @@ Gem::Specification.new do |s|
|
|
52
52
|
"lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb",
|
53
53
|
"lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb",
|
54
54
|
"lib/flydata/fluent-plugins/mysql/binlog_position.rb",
|
55
|
+
"lib/flydata/fluent-plugins/mysql/binlog_position_file.rb",
|
55
56
|
"lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb",
|
56
57
|
"lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb",
|
57
58
|
"lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb",
|
58
59
|
"lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb",
|
59
60
|
"lib/flydata/fluent-plugins/mysql/context.rb",
|
60
61
|
"lib/flydata/fluent-plugins/mysql/dml_record_handler.rb",
|
61
|
-
"lib/flydata/fluent-plugins/mysql/parser.rb",
|
62
|
-
"lib/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser.rb",
|
63
|
-
"lib/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser.rb",
|
64
|
-
"lib/flydata/fluent-plugins/mysql/parser/query_parser.rb",
|
65
62
|
"lib/flydata/fluent-plugins/out_forward_ssl.rb",
|
66
63
|
"lib/flydata/fluent-plugins/preference.rb",
|
67
64
|
"lib/flydata/flydata_crontab.sh",
|
@@ -70,6 +67,8 @@ Gem::Specification.new do |s|
|
|
70
67
|
"lib/flydata/heroku/configuration_methods.rb",
|
71
68
|
"lib/flydata/heroku/instance_methods.rb",
|
72
69
|
"lib/flydata/log_monitor.rb",
|
70
|
+
"lib/flydata/parser/mysql/mysql_alter_table.treetop",
|
71
|
+
"lib/flydata/parser_provider.rb",
|
73
72
|
"lib/flydata/preference/data_entry_preference.rb",
|
74
73
|
"lib/flydata/proxy.rb",
|
75
74
|
"lib/flydata/sync_file_manager.rb",
|
@@ -85,9 +84,8 @@ Gem::Specification.new do |s|
|
|
85
84
|
"spec/flydata/command/sync_spec.rb",
|
86
85
|
"spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb",
|
87
86
|
"spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb",
|
88
|
-
"spec/flydata/fluent-plugins/mysql/parser/alter_table_add_column_parser_spec.rb",
|
89
|
-
"spec/flydata/fluent-plugins/mysql/parser/alter_table_drop_column_parser_spec.rb",
|
90
87
|
"spec/flydata/heroku_spec.rb",
|
88
|
+
"spec/flydata/parser/mysql/alter_table_parser_spec.rb",
|
91
89
|
"spec/flydata/table_def/mysql_table_def_spec.rb",
|
92
90
|
"spec/flydata/table_def/mysqldump_test_foreign_key.dump",
|
93
91
|
"spec/flydata/table_def/mysqldump_test_table_all.dump",
|
@@ -95,6 +93,7 @@ Gem::Specification.new do |s|
|
|
95
93
|
"spec/flydata/table_def/mysqldump_test_table_enum.dump",
|
96
94
|
"spec/flydata/table_def/mysqldump_test_table_multi_pk.dump",
|
97
95
|
"spec/flydata/table_def/mysqldump_test_table_no_pk.dump",
|
96
|
+
"spec/flydata/table_def/mysqldump_test_unique_key.dump",
|
98
97
|
"spec/flydata/table_def/mysqldump_test_unsigned.dump",
|
99
98
|
"spec/flydata/table_def/redshift_table_def_spec.rb",
|
100
99
|
"spec/flydata/util/encryptor_spec.rb",
|
@@ -105,7 +104,7 @@ Gem::Specification.new do |s|
|
|
105
104
|
s.homepage = "http://flydata.co/"
|
106
105
|
s.licenses = ["All right reserved."]
|
107
106
|
s.require_paths = ["lib"]
|
108
|
-
s.rubygems_version = "1.8.
|
107
|
+
s.rubygems_version = "1.8.28"
|
109
108
|
s.summary = "FlyData CLI"
|
110
109
|
|
111
110
|
if s.respond_to? :specification_version then
|
@@ -122,6 +121,7 @@ Gem::Specification.new do |s|
|
|
122
121
|
s.add_runtime_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
|
123
122
|
s.add_runtime_dependency(%q<mysql2>, ["~> 0.3.11"])
|
124
123
|
s.add_runtime_dependency(%q<slop>, [">= 0"])
|
124
|
+
s.add_runtime_dependency(%q<treetop>, [">= 0"])
|
125
125
|
s.add_development_dependency(%q<bundler>, [">= 0"])
|
126
126
|
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
127
127
|
s.add_development_dependency(%q<rspec>, ["~> 3.0"])
|
@@ -142,6 +142,7 @@ Gem::Specification.new do |s|
|
|
142
142
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
|
143
143
|
s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
|
144
144
|
s.add_dependency(%q<slop>, [">= 0"])
|
145
|
+
s.add_dependency(%q<treetop>, [">= 0"])
|
145
146
|
s.add_dependency(%q<bundler>, [">= 0"])
|
146
147
|
s.add_dependency(%q<jeweler>, [">= 0"])
|
147
148
|
s.add_dependency(%q<rspec>, ["~> 3.0"])
|
@@ -163,6 +164,7 @@ Gem::Specification.new do |s|
|
|
163
164
|
s.add_dependency(%q<fluent-plugin-mysql-binlog>, ["~> 0.0.2"])
|
164
165
|
s.add_dependency(%q<mysql2>, ["~> 0.3.11"])
|
165
166
|
s.add_dependency(%q<slop>, [">= 0"])
|
167
|
+
s.add_dependency(%q<treetop>, [">= 0"])
|
166
168
|
s.add_dependency(%q<bundler>, [">= 0"])
|
167
169
|
s.add_dependency(%q<jeweler>, [">= 0"])
|
168
170
|
s.add_dependency(%q<rspec>, ["~> 3.0"])
|
data/lib/flydata.rb
CHANGED
data/lib/flydata/cli.rb
CHANGED
@@ -9,7 +9,7 @@ module Flydata
|
|
9
9
|
|
10
10
|
def run
|
11
11
|
unless check_environment
|
12
|
-
puts "Sorry, you have to run the installation command to use flydata. Please go to https://
|
12
|
+
puts "Sorry, you have to run the installation command to use flydata. Please go to https://console.flydata.com/ and sign up."
|
13
13
|
return
|
14
14
|
end
|
15
15
|
begin
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -52,28 +52,39 @@ module Flydata
|
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
def reset
|
56
|
-
|
55
|
+
def reset(*tables)
|
56
|
+
msg = tables.empty? ? '' : " for these tables : #{tables.join(" ")}"
|
57
|
+
return unless ask_yes_no("This resets the current sync#{msg}. Are you sure?")
|
57
58
|
sender = Flydata::Command::Sender.new
|
58
59
|
sender.flush_client_buffer # TODO We should rather delete buffer files
|
59
60
|
sender.stop
|
60
61
|
|
61
62
|
de = retrieve_data_entries.first
|
62
|
-
|
63
|
+
wait_for_server_buffer
|
64
|
+
cleanup_sync_server(de, tables) unless opts.client?
|
63
65
|
sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
|
64
|
-
[
|
66
|
+
delete_files = [
|
65
67
|
sync_fm.dump_file_path,
|
66
68
|
sync_fm.dump_pos_path,
|
67
|
-
sync_fm.binlog_path,
|
68
69
|
sync_fm.mysql_table_marshal_dump_path,
|
69
|
-
sync_fm.table_position_file_paths,
|
70
70
|
sync_fm.sync_info_file,
|
71
|
-
sync_fm.
|
72
|
-
|
71
|
+
sync_fm.table_position_file_paths(*tables),
|
72
|
+
sync_fm.table_rev_file_paths(*tables)
|
73
|
+
]
|
74
|
+
delete_files << sync_fm.binlog_path if tables.empty?
|
75
|
+
delete_files.flatten.each do |path|
|
73
76
|
FileUtils.rm(path) if File.exists?(path)
|
74
77
|
end
|
75
78
|
end
|
76
79
|
|
80
|
+
def wait_for_server_buffer
|
81
|
+
puts "Waiting for the server buffer to get empty"
|
82
|
+
while (status = check) && (status['state'] == 'processing')
|
83
|
+
print_progress(status)
|
84
|
+
sleep 10
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
77
88
|
def wait_for_server_data_processing
|
78
89
|
state = :PROCESS
|
79
90
|
puts "Uploading data to Redshift..."
|
@@ -121,6 +132,8 @@ module Flydata
|
|
121
132
|
|
122
133
|
def generate_table_ddl(*tables)
|
123
134
|
de = retrieve_data_entries.first
|
135
|
+
Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
|
136
|
+
|
124
137
|
raise "There are no data entry." unless de
|
125
138
|
case de['type']
|
126
139
|
when 'RedshiftMysqlDataEntry'
|
@@ -133,10 +146,10 @@ module Flydata
|
|
133
146
|
private
|
134
147
|
|
135
148
|
def cleanup_sync_server(de, tables = [])
|
136
|
-
puts "
|
149
|
+
puts "Cleaning the server"
|
137
150
|
flydata.data_entry.cleanup_sync(de['id'], tables)
|
138
151
|
end
|
139
|
-
|
152
|
+
|
140
153
|
def do_check(de)
|
141
154
|
flydata.data_entry.buffer_stat(de['id'], env_mode)
|
142
155
|
end
|
@@ -157,7 +170,7 @@ module Flydata
|
|
157
170
|
if mp['host'] then params << mp['host'] else raise "MySQL `host` is not defined in the data entry" end
|
158
171
|
params << (mp['port'] or '3306')
|
159
172
|
if mp['username'] then params << mp['username'] else raise "MySQL `username` is not defined is not defined in the data entry" end
|
160
|
-
params << (mp['password'] ? "-p#{mp['password']}"
|
173
|
+
params << (mp['password'].to_s.empty? ? "" : "-p#{mp['password']}")
|
161
174
|
if mp['database'] then params << mp['database'] else raise "`database` is not defined in the data entry" end
|
162
175
|
if mp['tables'] then params << mp['tables'].gsub(/,/, ' ') else raise "`tables` is not defined in the data entry" end
|
163
176
|
|
@@ -240,6 +253,7 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
240
253
|
print confirmation_text
|
241
254
|
|
242
255
|
if ask_yes_no('Start Sync?')
|
256
|
+
Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference'], fp).check
|
243
257
|
puts "Exporting data from database..."
|
244
258
|
Flydata::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference']).dump(fp)
|
245
259
|
else
|
@@ -293,11 +307,11 @@ Dump file saves contents of your tables temporarily. Make sure you have enough
|
|
293
307
|
if option[:table_name]
|
294
308
|
puts "Resuming... Last processed table: #{option[:table_name]}"
|
295
309
|
else
|
296
|
-
#If its a new sync, ensure server side resources are clean
|
297
|
-
cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(','))
|
310
|
+
#If its a new sync, ensure server side resources are clean
|
311
|
+
cleanup_sync_server(de, de['mysql_data_entry_preference']['tables'].split(','))
|
298
312
|
end
|
299
313
|
puts "Sending data to FlyData Server..."
|
300
|
-
|
314
|
+
|
301
315
|
bench_start_time = Time.now
|
302
316
|
|
303
317
|
# Start parsing dump file
|
@@ -434,7 +448,7 @@ Thank you for using FlyData!
|
|
434
448
|
sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
|
435
449
|
mp = de['mysql_data_entry_preference']
|
436
450
|
unless (rs = sync_fm.load_sync_info).nil?
|
437
|
-
mp['initial_sync'] = rs[:initial_sync]
|
451
|
+
mp['initial_sync'] = rs[:initial_sync]
|
438
452
|
mp['tables'] = rs[:tables]
|
439
453
|
end
|
440
454
|
de
|
@@ -1265,5 +1279,127 @@ EOS
|
|
1265
1279
|
end
|
1266
1280
|
end
|
1267
1281
|
end
|
1282
|
+
class CompatibilityCheck
|
1283
|
+
|
1284
|
+
class CompatibilityError < StandardError
|
1285
|
+
end
|
1286
|
+
|
1287
|
+
SELECT_QUERY_TMPLT = "SELECT %s"
|
1288
|
+
|
1289
|
+
def initialize(de_hash, dump_dir=nil)
|
1290
|
+
@db_opts = [:host, :port, :username, :password, :database].inject({}) {|h, sym| h[sym] = de_hash[sym.to_s]; h}
|
1291
|
+
@dump_dir = dump_dir
|
1292
|
+
@errors=[]
|
1293
|
+
end
|
1294
|
+
|
1295
|
+
def check
|
1296
|
+
self.methods.grep(/^check_/).each do |m|
|
1297
|
+
begin
|
1298
|
+
send(m)
|
1299
|
+
rescue CompatibilityError => e
|
1300
|
+
@errors << e
|
1301
|
+
end
|
1302
|
+
end
|
1303
|
+
print_errors
|
1304
|
+
end
|
1305
|
+
|
1306
|
+
def print_errors
|
1307
|
+
return if @errors.empty?
|
1308
|
+
puts "There may be some compatibility issues with your MySQL credentials: "
|
1309
|
+
@errors.each do |error|
|
1310
|
+
puts " * #{error.message}"
|
1311
|
+
end
|
1312
|
+
raise "Please correct these errors if you wish to run FlyData Sync"
|
1313
|
+
end
|
1314
|
+
|
1315
|
+
def check_mysql_user_compat
|
1316
|
+
client = Mysql2::Client.new(@db_opts)
|
1317
|
+
grants_sql = "SHOW GRANTS"
|
1318
|
+
correct_db = ["ON (\\*|#{@db_opts[:database]})","TO '#{@db_opts[:username]}"]
|
1319
|
+
necessary_permission_fields= ["SELECT","RELOAD","LOCK TABLES","REPLICATION SLAVE","REPLICATION CLIENT"]
|
1320
|
+
all_privileges_field= ["ALL PRIVILEGES"]
|
1321
|
+
result = client.query(grants_sql)
|
1322
|
+
# Do not catch MySQL connection problem because check should stop if no MySQL connection can be made.
|
1323
|
+
client.close
|
1324
|
+
missing_priv = necessary_permission_fields
|
1325
|
+
result.each do |res|
|
1326
|
+
# SHOW GRANTS should only return one column
|
1327
|
+
res_value = res.values.first
|
1328
|
+
if correct_db.all? {|perm| res_value.match(perm)}
|
1329
|
+
necessary_permission_fields.each do |priv|
|
1330
|
+
missing_priv.delete_at(missing_priv.index(priv)) unless res_value.match(priv)
|
1331
|
+
end
|
1332
|
+
return true if missing_priv.empty? or all_privileges_field.all? {|d| res_value.match(d)}
|
1333
|
+
end
|
1334
|
+
end
|
1335
|
+
raise CompatibilityError, "The user '#{@db_opts[:username]}' does not have the correct permissions to run FlyData Sync\n * These privileges are missing: #{missing_priv.join(", ")}"
|
1336
|
+
end
|
1337
|
+
|
1338
|
+
def check_mysql_protocol_tcp_compat
|
1339
|
+
query = "mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} -e \"SHOW GRANTS;\" --protocol=tcp"
|
1340
|
+
query << " -p#{@db_opts[:password]}" unless @db_opts[:password].to_s.empty?
|
1341
|
+
|
1342
|
+
Open3.popen3(query) do |stdin, stdout, stderr|
|
1343
|
+
stdin.close
|
1344
|
+
while !stderr.eof?
|
1345
|
+
line = stderr.gets
|
1346
|
+
unless /Warning: Using a password on the command line interface can be insecure./ === line
|
1347
|
+
raise CompatibilityError, "Cannot connect to MySQL database. Please make sure you can connect with this command:\n $ mysql -u #{@db_opts[:username]} -h #{@db_opts[:host]} -P #{@db_opts[:port]} #{@db_opts[:database]} --protocol=tcp -p"
|
1348
|
+
end
|
1349
|
+
end
|
1350
|
+
end
|
1351
|
+
end
|
1352
|
+
|
1353
|
+
def check_mysql_row_mode_compat
|
1354
|
+
sys_var_to_check = {'@@binlog_format'=>'ROW', '@@binlog_checksum'=>'NONE', '@@log_bin_use_v1_row_events'=>1}
|
1355
|
+
errors={}
|
1356
|
+
|
1357
|
+
client = Mysql2::Client.new(@db_opts)
|
1358
|
+
|
1359
|
+
begin
|
1360
|
+
sys_var_to_check.each_key do |sys_var|
|
1361
|
+
sel_query = SELECT_QUERY_TMPLT % sys_var
|
1362
|
+
begin
|
1363
|
+
result = client.query(sel_query)
|
1364
|
+
unless result.first[sys_var] == sys_var_to_check[sys_var]
|
1365
|
+
errors[sys_var]=result.first[sys_var]
|
1366
|
+
end
|
1367
|
+
rescue Mysql2::Error => e
|
1368
|
+
if e.message =~ /Unknown system variable/
|
1369
|
+
unless e.message =~ /(binlog_checksum|log_bin_use_v1_row_events)/
|
1370
|
+
errors[sys_var] = false
|
1371
|
+
end
|
1372
|
+
else
|
1373
|
+
raise e
|
1374
|
+
end
|
1375
|
+
end
|
1376
|
+
end
|
1377
|
+
ensure
|
1378
|
+
client.close
|
1379
|
+
end
|
1380
|
+
unless errors.empty?
|
1381
|
+
error_explanation = ""
|
1382
|
+
errors.each_key do |err_key|
|
1383
|
+
error_explanation << "\n * #{err_key} is #{errors[err_key]} but should be #{sys_var_to_check[err_key]}"
|
1384
|
+
end
|
1385
|
+
raise CompatibilityError, "These system variable(s) are not the correct value: #{error_explanation}\n Please change these system variables for FlyData Sync to run correctly"
|
1386
|
+
end
|
1387
|
+
end
|
1388
|
+
|
1389
|
+
def check_writing_permissions
|
1390
|
+
write_errors = []
|
1391
|
+
paths_to_check = ["~/.flydata"]
|
1392
|
+
paths_to_check << @dump_dir unless @dump_dir.to_s.empty?
|
1393
|
+
paths_to_check.each do |path|
|
1394
|
+
full_path = File.expand_path(path)
|
1395
|
+
full_path = File.dirname(full_path) unless File.directory?(full_path)
|
1396
|
+
write_errors << full_path unless File.writable?(full_path)
|
1397
|
+
end
|
1398
|
+
unless write_errors.empty?
|
1399
|
+
error_dir = write_errors.join(", ")
|
1400
|
+
raise CompatibilityError, "We cannot access the directories: #{error_dir}"
|
1401
|
+
end
|
1402
|
+
end
|
1403
|
+
end
|
1268
1404
|
end
|
1269
1405
|
end
|
@@ -3,9 +3,15 @@ module Fluent
|
|
3
3
|
require 'fluent/plugin/in_mysql_binlog'
|
4
4
|
require 'binlog'
|
5
5
|
require 'kodama'
|
6
|
+
|
7
|
+
# Load client library(flydata/cli/lib)
|
8
|
+
lib = File.absolute_path(File.dirname(__FILE__) + '/../..')
|
9
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
10
|
+
require 'flydata'
|
11
|
+
require 'flydata/sync_file_manager'
|
12
|
+
|
6
13
|
require_relative 'preference'
|
7
|
-
require_relative '
|
8
|
-
require_relative '../sync_file_manager'
|
14
|
+
require_relative 'mysql/binlog_position_file'
|
9
15
|
require_relative 'mysql/binlog_record_dispatcher'
|
10
16
|
require_relative 'mysql/context'
|
11
17
|
|
@@ -44,7 +50,8 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
|
|
44
50
|
|
45
51
|
def configure(conf)
|
46
52
|
super
|
47
|
-
|
53
|
+
@binlog_position_file = Mysql::BinLogPositionFile.new(@position_file)
|
54
|
+
unless @binlog_position_file.exists?
|
48
55
|
raise "No position file(#{@position_file}). Initial synchronization is required before starting."
|
49
56
|
end
|
50
57
|
load_custom_conf
|
@@ -111,7 +118,7 @@ EOS
|
|
111
118
|
def event_listener(event)
|
112
119
|
@record_dispatcher.dispatch(event)
|
113
120
|
rescue Exception => e
|
114
|
-
position =
|
121
|
+
position = @binlog_position_file.read
|
115
122
|
$log.error "error occured while processing #{event.event_type} event at #{position}\n#{e.message}\n#{$!.backtrace.join("\n")}"
|
116
123
|
# Not reraising a StandardError because the underlying code can't handle an error well.
|
117
124
|
raise unless e.kind_of?(StandardError)
|
@@ -150,7 +157,7 @@ EOS
|
|
150
157
|
# Creating new thread due to mutex can't lock
|
151
158
|
# in main thread during trap context
|
152
159
|
Thread.new {
|
153
|
-
begin
|
160
|
+
begin
|
154
161
|
Fluent::Engine.shutdown_source
|
155
162
|
Fluent::Engine.flush!
|
156
163
|
$log.debug "flushing thread: flushed"
|
@@ -162,24 +169,6 @@ EOS
|
|
162
169
|
end
|
163
170
|
end
|
164
171
|
|
165
|
-
class BinLogPosition
|
166
|
-
include Comparable
|
167
|
-
attr_accessor :file, :pos
|
168
|
-
|
169
|
-
def initialize(binlog_content)
|
170
|
-
items = binlog_content.split("\t")
|
171
|
-
@file = items[0]
|
172
|
-
@pos = items[1].to_i
|
173
|
-
end
|
174
|
-
|
175
|
-
def <=>(obj)
|
176
|
-
(self.file <=> obj.file) == 0 ? (self.pos <=> obj.pos) : (self.file <=> obj.file)
|
177
|
-
end
|
178
|
-
|
179
|
-
def to_s
|
180
|
-
"#{file}\t#{pos}"
|
181
|
-
end
|
182
|
-
end
|
183
172
|
|
184
173
|
end
|
185
174
|
|
@@ -222,7 +211,7 @@ end
|
|
222
211
|
end
|
223
212
|
|
224
213
|
# HACK
|
225
|
-
# Monkey patch so that we can replace Kodama's logger
|
214
|
+
# Monkey patch so that we can replace Kodama's logger
|
226
215
|
module Kodama
|
227
216
|
|
228
217
|
Client.class_eval do
|