flydata 0.5.17 → 0.5.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/event/event_handler_base.rb +56 -0
- data/flydata-core/lib/flydata-core/event/flydata_event.rb +29 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_handler_registry.rb +11 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_processor.rb +19 -0
- data/flydata-core/lib/flydata-core/event/flydata_event_sender.rb +27 -0
- data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +4 -5
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +12 -2
- data/flydata-core/spec/event/flydata_event_handler_registry_spec.rb +33 -0
- data/flydata-core/spec/event/flydata_event_handler_spec.rb +39 -0
- data/flydata-core/spec/event/flydata_event_processor_spec.rb +42 -0
- data/flydata-core/spec/event/flydata_event_sender_spec.rb +33 -0
- data/flydata-core/spec/event/flydata_event_spec.rb +39 -0
- data/flydata-core/spec/event/shared_event.rb +33 -0
- data/flydata-core/spec/mysql/compatibility_checker_spec.rb +4 -5
- data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +25 -25
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -4
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/start.rb +15 -8
- data/lib/flydata/command/sync.rb +79 -61
- data/lib/flydata/errors.rb +1 -1
- data/lib/flydata/event/api_event_sender.rb +16 -0
- data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +8 -0
- data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +1 -1
- data/lib/flydata/output/forwarder.rb +7 -5
- data/lib/flydata/parser/mysql/dump_parser.rb +76 -12
- data/lib/flydata/queueable_thread.rb +32 -0
- data/lib/flydata/sync_file_manager.rb +45 -0
- data/spec/flydata/command/start_spec.rb +1 -1
- data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +10 -3
- data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +10 -10
- data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +11 -2
- data/spec/flydata/output/forwarder_spec.rb +4 -4
- metadata +15 -2
@@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
46
46
|
) DISTKEY(table_name) SORTKEY(table_name);
|
47
47
|
DROP TABLE IF EXISTS "test_table_all";
|
48
48
|
CREATE TABLE "test_table_all" (
|
49
|
-
"id" int8
|
49
|
+
"id" int8,
|
50
50
|
"col_binary" varchar(202) DEFAULT NULL,
|
51
51
|
"col_blob" varchar(65535),
|
52
52
|
"col_bool" int2 DEFAULT '0',
|
@@ -67,7 +67,7 @@ CREATE TABLE "test_table_all" (
|
|
67
67
|
"col_smallint" int2 DEFAULT NULL,
|
68
68
|
"col_text" varchar(max),
|
69
69
|
"col_time" timestamp DEFAULT NULL,
|
70
|
-
"col_timestamp" timestamp
|
70
|
+
"col_timestamp" timestamp DEFAULT SYSDATE,
|
71
71
|
"col_tinyblob" varchar(255),
|
72
72
|
"col_tinyint" int2 DEFAULT NULL,
|
73
73
|
"col_tinytext" varchar(max),
|
@@ -175,7 +175,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
175
175
|
) DISTKEY(table_name) SORTKEY(table_name);
|
176
176
|
DROP TABLE IF EXISTS "bit_test_def_1";
|
177
177
|
CREATE TABLE "bit_test_def_1" (
|
178
|
-
"id" int4
|
178
|
+
"id" int4,
|
179
179
|
"bit_value" bigint DEFAULT 1,
|
180
180
|
"int_value" int4 DEFAULT 16,
|
181
181
|
PRIMARY KEY ("id")
|
@@ -223,10 +223,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
223
223
|
) DISTKEY(table_name) SORTKEY(table_name);
|
224
224
|
DROP TABLE IF EXISTS "product_order";
|
225
225
|
CREATE TABLE "product_order" (
|
226
|
-
"no" int4
|
227
|
-
"product_category" int4
|
228
|
-
"product_id" int4
|
229
|
-
"customer_id" int4
|
226
|
+
"no" int4,
|
227
|
+
"product_category" int4,
|
228
|
+
"product_id" int4,
|
229
|
+
"customer_id" int4,
|
230
230
|
PRIMARY KEY ("no")
|
231
231
|
) DISTKEY("no") SORTKEY("no");
|
232
232
|
DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
|
@@ -278,7 +278,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
278
278
|
) DISTKEY(table_name) SORTKEY(table_name);
|
279
279
|
DROP TABLE IF EXISTS "test_table_column_comment";
|
280
280
|
CREATE TABLE "test_table_column_comment" (
|
281
|
-
"id" int4
|
281
|
+
"id" int4 DEFAULT '0',
|
282
282
|
"value" varchar(max),
|
283
283
|
PRIMARY KEY ("id")
|
284
284
|
) DISTKEY("id") SORTKEY("id");
|
@@ -325,10 +325,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
325
325
|
) DISTKEY(table_name) SORTKEY(table_name);
|
326
326
|
DROP TABLE IF EXISTS "test_table_enum";
|
327
327
|
CREATE TABLE "test_table_enum" (
|
328
|
-
"id" int4
|
328
|
+
"id" int4,
|
329
329
|
"enum_1" varchar encode bytedict DEFAULT NULL,
|
330
330
|
"enum_2" varchar encode bytedict DEFAULT 'a',
|
331
|
-
"enum_3" varchar encode bytedict
|
331
|
+
"enum_3" varchar encode bytedict,
|
332
332
|
PRIMARY KEY ("id")
|
333
333
|
) DISTKEY("id") SORTKEY("id");
|
334
334
|
DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
|
@@ -376,8 +376,8 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
376
376
|
) DISTKEY(table_name) SORTKEY(table_name);
|
377
377
|
DROP TABLE IF EXISTS "test_table_multi_pk";
|
378
378
|
CREATE TABLE "test_table_multi_pk" (
|
379
|
-
"id1" int4
|
380
|
-
"id2" int4
|
379
|
+
"id1" int4 DEFAULT '0',
|
380
|
+
"id2" int4 DEFAULT '0',
|
381
381
|
"value" varchar(max),
|
382
382
|
PRIMARY KEY ("id1","id2")
|
383
383
|
) DISTKEY("id1") SORTKEY("id1","id2");
|
@@ -432,7 +432,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
432
432
|
) DISTKEY(table_name) SORTKEY(table_name);
|
433
433
|
DROP TABLE IF EXISTS "sample1";
|
434
434
|
CREATE TABLE "sample1" (
|
435
|
-
"id" int4
|
435
|
+
"id" int4,
|
436
436
|
"title" varchar(768) DEFAULT NULL,
|
437
437
|
"name" varchar(max),
|
438
438
|
"num" int4 DEFAULT NULL,
|
@@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
485
485
|
) DISTKEY(table_name) SORTKEY(table_name);
|
486
486
|
DROP TABLE IF EXISTS "sample1";
|
487
487
|
CREATE TABLE "sample1" (
|
488
|
-
"id" int4
|
488
|
+
"id" int4,
|
489
489
|
"title" varchar(768) DEFAULT NULL,
|
490
490
|
"name" varchar(max),
|
491
491
|
"num" int4 DEFAULT NULL,
|
@@ -538,24 +538,24 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
538
538
|
) DISTKEY(table_name) SORTKEY(table_name);
|
539
539
|
DROP TABLE IF EXISTS "invoice_items";
|
540
540
|
CREATE TABLE "invoice_items" (
|
541
|
-
"id" int4
|
542
|
-
"app_id" int4
|
543
|
-
"subscription_id" int4
|
544
|
-
"overage_id" varchar(765)
|
541
|
+
"id" int4,
|
542
|
+
"app_id" int4,
|
543
|
+
"subscription_id" int4,
|
544
|
+
"overage_id" varchar(765),
|
545
545
|
"stripe_invoice_item_id" varchar(765) DEFAULT NULL,
|
546
546
|
"stripe_error" varchar(765) DEFAULT NULL,
|
547
|
-
"synced_to_stripe" int2
|
547
|
+
"synced_to_stripe" int2 DEFAULT '0',
|
548
548
|
"description" varchar(max),
|
549
|
-
"item_name" varchar(765)
|
550
|
-
"item_cost" int4
|
551
|
-
"item_count" int4
|
552
|
-
"total_cost" int4
|
549
|
+
"item_name" varchar(765),
|
550
|
+
"item_cost" int4,
|
551
|
+
"item_count" int4,
|
552
|
+
"total_cost" int4,
|
553
553
|
"created_at" timestamp DEFAULT NULL,
|
554
554
|
"updated_at" timestamp DEFAULT NULL,
|
555
555
|
"bill_ahead" int2 DEFAULT '0',
|
556
556
|
"bill_ahead_resolved_at" timestamp DEFAULT NULL,
|
557
557
|
"stripe_invoice_id" varchar(765) DEFAULT NULL,
|
558
|
-
"is_vat" int2
|
558
|
+
"is_vat" int2 DEFAULT '0',
|
559
559
|
PRIMARY KEY ("id")
|
560
560
|
) DISTKEY("id") SORTKEY("id");
|
561
561
|
DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
|
@@ -632,7 +632,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
|
|
632
632
|
) DISTKEY(table_name) SORTKEY(table_name);
|
633
633
|
DROP TABLE IF EXISTS "zerofill_table";
|
634
634
|
CREATE TABLE "zerofill_table" (
|
635
|
-
"id" int4
|
635
|
+
"id" int4,
|
636
636
|
"value_int" int8 DEFAULT NULL,
|
637
637
|
"value_float" float4 DEFAULT NULL,
|
638
638
|
"value_dec" numeric(10,2) DEFAULT NULL,
|
@@ -31,7 +31,7 @@ describe RedshiftTableDef do
|
|
31
31
|
let(:create_table_queries) { <<EOT.strip }
|
32
32
|
DROP TABLE IF EXISTS #{schema_prefix}"test_table";
|
33
33
|
CREATE TABLE #{schema_prefix}"test_table" (
|
34
|
-
"id" int4
|
34
|
+
"id" int4,
|
35
35
|
"age" int8,
|
36
36
|
"value" varchar(max),
|
37
37
|
PRIMARY KEY ("id")
|
@@ -218,7 +218,7 @@ EOT
|
|
218
218
|
before do
|
219
219
|
column[:not_null] = true
|
220
220
|
end
|
221
|
-
let(:not_null_sql) { "
|
221
|
+
let(:not_null_sql) { "" }
|
222
222
|
it_behaves_like *examples
|
223
223
|
end
|
224
224
|
end
|
@@ -519,11 +519,11 @@ EOT
|
|
519
519
|
end
|
520
520
|
|
521
521
|
context "with not null true" do
|
522
|
-
let(:expected_query) { %Q| #{type_sql}
|
522
|
+
let(:expected_query) { %Q| #{type_sql}#{default_sql}| } # Redshift supports NOT NULL but we do not respect it in order to accept NULL values which may come in future.
|
523
523
|
before do
|
524
524
|
column[:not_null] = true
|
525
525
|
end
|
526
|
-
let(:no_default_sql) {
|
526
|
+
let(:no_default_sql) { "" }
|
527
527
|
|
528
528
|
it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
|
529
529
|
end
|
data/flydata.gemspec
CHANGED
Binary file
|
@@ -11,17 +11,24 @@ module Flydata
|
|
11
11
|
sender_opts
|
12
12
|
end
|
13
13
|
def run(*args)
|
14
|
+
sender_start(args)
|
15
|
+
helper_start(args) unless opts.skip_helper?
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def sender_start(args)
|
14
20
|
sender = Flydata::Command::Sender.new(opts)
|
15
21
|
sender.start
|
16
|
-
unless opts.skip_helper?
|
17
|
-
helper_opts = Flydata::Command::Helper.slop_start
|
18
|
-
helper_opts.parse!(args)
|
19
|
-
helper = Flydata::Command::Helper.new(helper_opts)
|
20
|
-
helper.stop(quiet: true) # kill existing helper process (if any) and start
|
21
|
-
helper.start(quiet: true)
|
22
|
-
end
|
23
22
|
end
|
24
|
-
run_exclusive :
|
23
|
+
run_exclusive :sender_start, command: "start"
|
24
|
+
|
25
|
+
def helper_start(args)
|
26
|
+
helper_opts = Flydata::Command::Helper.slop_start
|
27
|
+
helper_opts.parse!(args)
|
28
|
+
helper = Flydata::Command::Helper.new(helper_opts)
|
29
|
+
helper.stop(quiet: true) # kill existing helper process (if any) and start
|
30
|
+
system "flydata helper:start"
|
31
|
+
end
|
25
32
|
end
|
26
33
|
end
|
27
34
|
end
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -10,6 +10,7 @@ require 'flydata/compatibility_check'
|
|
10
10
|
require 'flydata/errors'
|
11
11
|
require 'flydata/helpers'
|
12
12
|
require 'flydata/json'
|
13
|
+
require 'flydata/queueable_thread'
|
13
14
|
require 'flydata/output/forwarder'
|
14
15
|
require 'flydata/parser/mysql/dump_parser'
|
15
16
|
require 'flydata/preference/data_entry_preference'
|
@@ -18,6 +19,8 @@ require 'flydata-core/table_def'
|
|
18
19
|
require 'flydata-core/mysql/binlog_pos'
|
19
20
|
require 'flydata/mysql/table_ddl'
|
20
21
|
require 'flydata-core/mysql/command_generator'
|
22
|
+
require 'flydata/event/api_event_sender'
|
23
|
+
require 'sigdump/setup'
|
21
24
|
#require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
|
22
25
|
|
23
26
|
module Flydata
|
@@ -158,30 +161,15 @@ EOS
|
|
158
161
|
# Cleanup tables on server
|
159
162
|
de = data_entry
|
160
163
|
cleanup_sync_server(de, @input_tables) unless opts.client?
|
164
|
+
sync_fm = create_sync_file_manager(de)
|
161
165
|
|
162
166
|
# Delete local files
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
sync_fm.dump_pos_path,
|
167
|
-
sync_fm.mysql_table_marshal_dump_path,
|
168
|
-
sync_fm.sync_info_file,
|
169
|
-
sync_fm.stats_path,
|
170
|
-
sync_fm.table_position_file_paths(*@input_tables),
|
171
|
-
sync_fm.table_binlog_pos_paths(*@input_tables),
|
172
|
-
sync_fm.table_binlog_pos_init_paths(*@input_tables),
|
173
|
-
sync_fm.table_rev_file_paths(*@input_tables),
|
174
|
-
sync_fm.table_ddl_file_paths(*@input_tables)
|
175
|
-
]
|
167
|
+
tables_for_deleting = tables.empty? ? sync_fm.tables_from_positions_dir : @input_tables
|
168
|
+
sync_fm.delete_table_control_files(*tables_for_deleting)
|
169
|
+
|
176
170
|
new_tables_after_reset = @unsynced_tables + @input_tables
|
177
|
-
|
178
|
-
|
179
|
-
delete_files << sync_fm.sent_binlog_path
|
180
|
-
delete_files << sync_fm.lock_pid_file
|
181
|
-
end
|
182
|
-
delete_files.flatten.each do |path|
|
183
|
-
FileUtils.rm(path) if File.exists?(path)
|
184
|
-
end
|
171
|
+
delete_binlog =@input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
|
172
|
+
sync_fm.delete_non_table_control_files(delete_binlog)
|
185
173
|
sync_fm.close
|
186
174
|
log_info_stdout("Reset completed successfully.")
|
187
175
|
end
|
@@ -572,6 +560,7 @@ EOS
|
|
572
560
|
|
573
561
|
# Don't call the method unless init sync needs to be run for a table
|
574
562
|
def sync_mysql_to_redshift(de, options = {})
|
563
|
+
FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables"=>target_tables}, de['id'])
|
575
564
|
dp = flydata.data_port.get
|
576
565
|
sync_fm = create_sync_file_manager(de)
|
577
566
|
|
@@ -666,30 +655,31 @@ EOM
|
|
666
655
|
log_info_stdout("Setting binary log position and exporting data from the database.")
|
667
656
|
log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
|
668
657
|
sync_fm.save_sync_info(@full_initial_sync, target_tables)
|
658
|
+
dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
|
669
659
|
if file_dump
|
670
660
|
binlog_pos = nil
|
671
661
|
begin
|
672
|
-
|
673
|
-
new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp) do |_io, _binlog_pos|
|
662
|
+
dump_generator.dump(fp) do |_io, _binlog_pos|
|
674
663
|
binlog_pos = _binlog_pos
|
675
|
-
initialize_binlog_positions_and_call_callback(
|
676
|
-
binlog_pos, options[:binlog_ready_callback], sync_fm)
|
664
|
+
initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
|
677
665
|
end
|
678
666
|
log_info_stdout(" -> Database dump done")
|
679
667
|
rescue Exception => e
|
680
668
|
#Catch all exceptions including SystemExit and Interrupt.
|
681
669
|
log_info_stdout "Quit while running mysqldump, deleting dump file..."
|
682
670
|
sync_fm.delete_dump_file
|
683
|
-
log_info_stdout "Dump file deleted
|
671
|
+
log_info_stdout "Dump file deleted. To restart the FlyData Agent, Following tables may need reset : #{target_tables}"
|
672
|
+
if (target_tables.nil? or target_tables.empty?)
|
673
|
+
log_info_stdout "To do full reset sync use command: flydata sync:reset"
|
674
|
+
else
|
675
|
+
log_info_stdout "To reset these tables, use command: flydata sync:reset #{target_tables.join(' ')}"
|
676
|
+
end
|
684
677
|
raise e
|
685
678
|
end
|
686
679
|
call_block_or_return_io(fp, binlog_pos, &block)
|
687
680
|
else
|
688
|
-
|
689
|
-
|
690
|
-
initialize_binlog_positions_and_call_callback(
|
691
|
-
binlog_pos, options[:binlog_ready_callback], sync_fm)
|
692
|
-
|
681
|
+
dump_generator.dump do |io, binlog_pos|
|
682
|
+
initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
|
693
683
|
block.call(io, binlog_pos, db_bytesize)
|
694
684
|
end
|
695
685
|
end
|
@@ -788,23 +778,33 @@ EOM
|
|
788
778
|
|
789
779
|
bench_start_time = Time.now
|
790
780
|
|
781
|
+
output_thread = QueueableThread.new
|
782
|
+
|
791
783
|
# Start parsing dump file
|
792
784
|
tmp_num_inserted_record = 0
|
785
|
+
skip_checkpoint = false
|
793
786
|
dump_fp = sync_fm.dump_file_path
|
794
787
|
dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
|
795
788
|
|
789
|
+
send_record_counts_threads = []
|
796
790
|
begin
|
797
|
-
threads = []
|
798
791
|
RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
|
799
792
|
Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
|
800
793
|
mysqldump_io,
|
801
794
|
# create table
|
802
795
|
Proc.new { |mysql_table|
|
803
|
-
tmp_num_inserted_record = 0
|
804
|
-
# dump mysql_table for resume
|
805
|
-
#TODO: make it option
|
806
|
-
sync_fm.save_mysql_table_marshal_dump(mysql_table)
|
807
796
|
mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
|
797
|
+
mt = mysql_table.clone
|
798
|
+
mt.value_converters = {}
|
799
|
+
# don't let the closure hold these objects as it causes memory leak
|
800
|
+
mysql_table = nil
|
801
|
+
output_thread.run do
|
802
|
+
tmp_num_inserted_record = 0
|
803
|
+
# dump mysql_table for resume
|
804
|
+
#TODO: make it option
|
805
|
+
sync_fm.save_mysql_table_marshal_dump(mt)
|
806
|
+
log_info_stdout(" -> Started sending data for table '#{mt.table_name}'")
|
807
|
+
end
|
808
808
|
},
|
809
809
|
# insert record
|
810
810
|
Proc.new { |mysql_table, values_set|
|
@@ -812,41 +812,57 @@ EOM
|
|
812
812
|
records = values_set.collect do |values|
|
813
813
|
convert_to_flydata_values(mysql_table, values)
|
814
814
|
json = JSON.generate_kv_pairs(mysql_table.column_names, values)
|
815
|
+
values.clear
|
815
816
|
{table_name: mysql_table_name, log: json}
|
816
817
|
end
|
817
|
-
|
818
|
-
|
819
|
-
|
820
|
-
|
818
|
+
# release resources to prevent the closure from keeping it
|
819
|
+
values_set.clear
|
820
|
+
values_set = nil
|
821
|
+
mysql_table = nil
|
822
|
+
mysql_table_name = nil
|
823
|
+
output_thread.run do
|
824
|
+
forwarder.emit(records)
|
825
|
+
tmp_num_inserted_record += 1
|
826
|
+
skip_checkpoint = !forwarder.buffer_full?
|
827
|
+
end
|
828
|
+
true
|
821
829
|
},
|
822
830
|
# checkpoint
|
823
831
|
Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
824
|
-
# flush if buffer records exist
|
825
832
|
table_name = mysql_table.nil? ? '' : mysql_table.table_name
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
833
|
+
mysql_table = nil
|
834
|
+
output_thread.run do
|
835
|
+
skip = skip_checkpoint
|
836
|
+
skip_checkpoint = false
|
837
|
+
unless skip
|
838
|
+
# flush if buffer records exist
|
839
|
+
if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
|
840
|
+
ret = forwarder.flush # send buffer data to the server before checkpoint
|
841
|
+
sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
|
842
|
+
end
|
830
843
|
|
831
|
-
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
844
|
+
# show the current progress
|
845
|
+
if last_pos.to_i == -1 # stream dump
|
846
|
+
log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
|
847
|
+
else
|
848
|
+
log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
|
849
|
+
end
|
837
850
|
|
838
|
-
|
839
|
-
|
851
|
+
# save check point
|
852
|
+
sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
|
840
853
|
|
841
|
-
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
854
|
+
# send record count for the table
|
855
|
+
if mysql_table &&
|
856
|
+
state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
|
857
|
+
# all records for `mysql_table` have been sent
|
858
|
+
send_record_counts(de, sync_fm, mysql_table.table_name,
|
859
|
+
send_record_counts_threads)
|
860
|
+
log_info_stdout(" -> Finished sending data for table '#{table_name}'...")
|
861
|
+
end
|
862
|
+
end
|
846
863
|
end
|
847
864
|
}
|
848
865
|
)
|
849
|
-
threads.each{|t| t.join}
|
850
866
|
rescue DumpParseError =>e
|
851
867
|
ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n #{e.message}")
|
852
868
|
ee.description = " Please contact support@flydata.com to report the issue."
|
@@ -861,6 +877,8 @@ EOM
|
|
861
877
|
printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
|
862
878
|
end
|
863
879
|
end
|
880
|
+
send_record_counts_threads.each{|t| t.join}
|
881
|
+
output_thread.join
|
864
882
|
forwarder.close
|
865
883
|
log_info_stdout(" -> Done")
|
866
884
|
#log_info_stdout(" -> Records sent to the server")
|
@@ -1037,7 +1055,7 @@ EOM
|
|
1037
1055
|
while (status = check_server_status(tables)) && (status['state'].nil? || status['state'] == 'processing')
|
1038
1056
|
prev_message = status['message']
|
1039
1057
|
if timeout > 0 && Time.now - start_time > timeout
|
1040
|
-
raise ServerDataProcessingTimeout.new
|
1058
|
+
raise ServerDataProcessingTimeout.new("Delayed Data Processing")
|
1041
1059
|
end
|
1042
1060
|
print_progress(status)
|
1043
1061
|
sleep 10
|
@@ -1085,7 +1103,7 @@ EOM
|
|
1085
1103
|
end
|
1086
1104
|
|
1087
1105
|
create_flydata_ctl_table = true
|
1088
|
-
option = {
|
1106
|
+
option = {skip_primary_key_check: opts.skip_primary_key_check?}.merge(mp)
|
1089
1107
|
missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
|
1090
1108
|
if error
|
1091
1109
|
error_list << error.err_hash
|