flydata 0.5.17 → 0.5.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/flydata-core/lib/flydata-core/event/event_handler_base.rb +56 -0
  4. data/flydata-core/lib/flydata-core/event/flydata_event.rb +29 -0
  5. data/flydata-core/lib/flydata-core/event/flydata_event_handler_registry.rb +11 -0
  6. data/flydata-core/lib/flydata-core/event/flydata_event_processor.rb +19 -0
  7. data/flydata-core/lib/flydata-core/event/flydata_event_sender.rb +27 -0
  8. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +4 -5
  9. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +12 -2
  10. data/flydata-core/spec/event/flydata_event_handler_registry_spec.rb +33 -0
  11. data/flydata-core/spec/event/flydata_event_handler_spec.rb +39 -0
  12. data/flydata-core/spec/event/flydata_event_processor_spec.rb +42 -0
  13. data/flydata-core/spec/event/flydata_event_sender_spec.rb +33 -0
  14. data/flydata-core/spec/event/flydata_event_spec.rb +39 -0
  15. data/flydata-core/spec/event/shared_event.rb +33 -0
  16. data/flydata-core/spec/mysql/compatibility_checker_spec.rb +4 -5
  17. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +25 -25
  18. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +4 -4
  19. data/flydata.gemspec +0 -0
  20. data/lib/flydata/command/start.rb +15 -8
  21. data/lib/flydata/command/sync.rb +79 -61
  22. data/lib/flydata/errors.rb +1 -1
  23. data/lib/flydata/event/api_event_sender.rb +16 -0
  24. data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb +8 -0
  25. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +1 -1
  26. data/lib/flydata/output/forwarder.rb +7 -5
  27. data/lib/flydata/parser/mysql/dump_parser.rb +76 -12
  28. data/lib/flydata/queueable_thread.rb +32 -0
  29. data/lib/flydata/sync_file_manager.rb +45 -0
  30. data/spec/flydata/command/start_spec.rb +1 -1
  31. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +10 -3
  32. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +10 -10
  33. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +11 -2
  34. data/spec/flydata/output/forwarder_spec.rb +4 -4
  35. metadata +15 -2
@@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
46
46
  ) DISTKEY(table_name) SORTKEY(table_name);
47
47
  DROP TABLE IF EXISTS "test_table_all";
48
48
  CREATE TABLE "test_table_all" (
49
- "id" int8 NOT NULL,
49
+ "id" int8,
50
50
  "col_binary" varchar(202) DEFAULT NULL,
51
51
  "col_blob" varchar(65535),
52
52
  "col_bool" int2 DEFAULT '0',
@@ -67,7 +67,7 @@ CREATE TABLE "test_table_all" (
67
67
  "col_smallint" int2 DEFAULT NULL,
68
68
  "col_text" varchar(max),
69
69
  "col_time" timestamp DEFAULT NULL,
70
- "col_timestamp" timestamp NOT NULL DEFAULT SYSDATE,
70
+ "col_timestamp" timestamp DEFAULT SYSDATE,
71
71
  "col_tinyblob" varchar(255),
72
72
  "col_tinyint" int2 DEFAULT NULL,
73
73
  "col_tinytext" varchar(max),
@@ -175,7 +175,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
175
175
  ) DISTKEY(table_name) SORTKEY(table_name);
176
176
  DROP TABLE IF EXISTS "bit_test_def_1";
177
177
  CREATE TABLE "bit_test_def_1" (
178
- "id" int4 NOT NULL,
178
+ "id" int4,
179
179
  "bit_value" bigint DEFAULT 1,
180
180
  "int_value" int4 DEFAULT 16,
181
181
  PRIMARY KEY ("id")
@@ -223,10 +223,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
223
223
  ) DISTKEY(table_name) SORTKEY(table_name);
224
224
  DROP TABLE IF EXISTS "product_order";
225
225
  CREATE TABLE "product_order" (
226
- "no" int4 NOT NULL,
227
- "product_category" int4 NOT NULL,
228
- "product_id" int4 NOT NULL,
229
- "customer_id" int4 NOT NULL,
226
+ "no" int4,
227
+ "product_category" int4,
228
+ "product_id" int4,
229
+ "customer_id" int4,
230
230
  PRIMARY KEY ("no")
231
231
  ) DISTKEY("no") SORTKEY("no");
232
232
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
@@ -278,7 +278,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
278
278
  ) DISTKEY(table_name) SORTKEY(table_name);
279
279
  DROP TABLE IF EXISTS "test_table_column_comment";
280
280
  CREATE TABLE "test_table_column_comment" (
281
- "id" int4 NOT NULL DEFAULT '0',
281
+ "id" int4 DEFAULT '0',
282
282
  "value" varchar(max),
283
283
  PRIMARY KEY ("id")
284
284
  ) DISTKEY("id") SORTKEY("id");
@@ -325,10 +325,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
325
325
  ) DISTKEY(table_name) SORTKEY(table_name);
326
326
  DROP TABLE IF EXISTS "test_table_enum";
327
327
  CREATE TABLE "test_table_enum" (
328
- "id" int4 NOT NULL,
328
+ "id" int4,
329
329
  "enum_1" varchar encode bytedict DEFAULT NULL,
330
330
  "enum_2" varchar encode bytedict DEFAULT 'a',
331
- "enum_3" varchar encode bytedict NOT NULL,
331
+ "enum_3" varchar encode bytedict,
332
332
  PRIMARY KEY ("id")
333
333
  ) DISTKEY("id") SORTKEY("id");
334
334
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
@@ -376,8 +376,8 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
376
376
  ) DISTKEY(table_name) SORTKEY(table_name);
377
377
  DROP TABLE IF EXISTS "test_table_multi_pk";
378
378
  CREATE TABLE "test_table_multi_pk" (
379
- "id1" int4 NOT NULL DEFAULT '0',
380
- "id2" int4 NOT NULL DEFAULT '0',
379
+ "id1" int4 DEFAULT '0',
380
+ "id2" int4 DEFAULT '0',
381
381
  "value" varchar(max),
382
382
  PRIMARY KEY ("id1","id2")
383
383
  ) DISTKEY("id1") SORTKEY("id1","id2");
@@ -432,7 +432,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
432
432
  ) DISTKEY(table_name) SORTKEY(table_name);
433
433
  DROP TABLE IF EXISTS "sample1";
434
434
  CREATE TABLE "sample1" (
435
- "id" int4 NOT NULL,
435
+ "id" int4,
436
436
  "title" varchar(768) DEFAULT NULL,
437
437
  "name" varchar(max),
438
438
  "num" int4 DEFAULT NULL,
@@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
485
485
  ) DISTKEY(table_name) SORTKEY(table_name);
486
486
  DROP TABLE IF EXISTS "sample1";
487
487
  CREATE TABLE "sample1" (
488
- "id" int4 NOT NULL,
488
+ "id" int4,
489
489
  "title" varchar(768) DEFAULT NULL,
490
490
  "name" varchar(max),
491
491
  "num" int4 DEFAULT NULL,
@@ -538,24 +538,24 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
538
538
  ) DISTKEY(table_name) SORTKEY(table_name);
539
539
  DROP TABLE IF EXISTS "invoice_items";
540
540
  CREATE TABLE "invoice_items" (
541
- "id" int4 NOT NULL,
542
- "app_id" int4 NOT NULL,
543
- "subscription_id" int4 NOT NULL,
544
- "overage_id" varchar(765) NOT NULL,
541
+ "id" int4,
542
+ "app_id" int4,
543
+ "subscription_id" int4,
544
+ "overage_id" varchar(765),
545
545
  "stripe_invoice_item_id" varchar(765) DEFAULT NULL,
546
546
  "stripe_error" varchar(765) DEFAULT NULL,
547
- "synced_to_stripe" int2 NOT NULL DEFAULT '0',
547
+ "synced_to_stripe" int2 DEFAULT '0',
548
548
  "description" varchar(max),
549
- "item_name" varchar(765) NOT NULL,
550
- "item_cost" int4 NOT NULL,
551
- "item_count" int4 NOT NULL,
552
- "total_cost" int4 NOT NULL,
549
+ "item_name" varchar(765),
550
+ "item_cost" int4,
551
+ "item_count" int4,
552
+ "total_cost" int4,
553
553
  "created_at" timestamp DEFAULT NULL,
554
554
  "updated_at" timestamp DEFAULT NULL,
555
555
  "bill_ahead" int2 DEFAULT '0',
556
556
  "bill_ahead_resolved_at" timestamp DEFAULT NULL,
557
557
  "stripe_invoice_id" varchar(765) DEFAULT NULL,
558
- "is_vat" int2 NOT NULL DEFAULT '0',
558
+ "is_vat" int2 DEFAULT '0',
559
559
  PRIMARY KEY ("id")
560
560
  ) DISTKEY("id") SORTKEY("id");
561
561
  DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
@@ -632,7 +632,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
632
632
  ) DISTKEY(table_name) SORTKEY(table_name);
633
633
  DROP TABLE IF EXISTS "zerofill_table";
634
634
  CREATE TABLE "zerofill_table" (
635
- "id" int4 NOT NULL,
635
+ "id" int4,
636
636
  "value_int" int8 DEFAULT NULL,
637
637
  "value_float" float4 DEFAULT NULL,
638
638
  "value_dec" numeric(10,2) DEFAULT NULL,
@@ -31,7 +31,7 @@ describe RedshiftTableDef do
31
31
  let(:create_table_queries) { <<EOT.strip }
32
32
  DROP TABLE IF EXISTS #{schema_prefix}"test_table";
33
33
  CREATE TABLE #{schema_prefix}"test_table" (
34
- "id" int4 NOT NULL,
34
+ "id" int4,
35
35
  "age" int8,
36
36
  "value" varchar(max),
37
37
  PRIMARY KEY ("id")
@@ -218,7 +218,7 @@ EOT
218
218
  before do
219
219
  column[:not_null] = true
220
220
  end
221
- let(:not_null_sql) { " NOT NULL" }
221
+ let(:not_null_sql) { "" }
222
222
  it_behaves_like *examples
223
223
  end
224
224
  end
@@ -519,11 +519,11 @@ EOT
519
519
  end
520
520
 
521
521
  context "with not null true" do
522
- let(:expected_query) { %Q| #{type_sql} NOT NULL#{default_sql}| }
522
+ let(:expected_query) { %Q| #{type_sql}#{default_sql}| } # Redshift supports NOT NULL but we do not respect it in order to accept NULL values which may come in future.
523
523
  before do
524
524
  column[:not_null] = true
525
525
  end
526
- let(:no_default_sql) { not_null_default_sql }
526
+ let(:no_default_sql) { "" }
527
527
 
528
528
  it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
529
529
  end
data/flydata.gemspec CHANGED
Binary file
@@ -11,17 +11,24 @@ module Flydata
11
11
  sender_opts
12
12
  end
13
13
  def run(*args)
14
+ sender_start(args)
15
+ helper_start(args) unless opts.skip_helper?
16
+ end
17
+
18
+ private
19
+ def sender_start(args)
14
20
  sender = Flydata::Command::Sender.new(opts)
15
21
  sender.start
16
- unless opts.skip_helper?
17
- helper_opts = Flydata::Command::Helper.slop_start
18
- helper_opts.parse!(args)
19
- helper = Flydata::Command::Helper.new(helper_opts)
20
- helper.stop(quiet: true) # kill existing helper process (if any) and start
21
- helper.start(quiet: true)
22
- end
23
22
  end
24
- run_exclusive :run
23
+ run_exclusive :sender_start, command: "start"
24
+
25
+ def helper_start(args)
26
+ helper_opts = Flydata::Command::Helper.slop_start
27
+ helper_opts.parse!(args)
28
+ helper = Flydata::Command::Helper.new(helper_opts)
29
+ helper.stop(quiet: true) # kill existing helper process (if any) and start
30
+ system "flydata helper:start"
31
+ end
25
32
  end
26
33
  end
27
34
  end
@@ -10,6 +10,7 @@ require 'flydata/compatibility_check'
10
10
  require 'flydata/errors'
11
11
  require 'flydata/helpers'
12
12
  require 'flydata/json'
13
+ require 'flydata/queueable_thread'
13
14
  require 'flydata/output/forwarder'
14
15
  require 'flydata/parser/mysql/dump_parser'
15
16
  require 'flydata/preference/data_entry_preference'
@@ -18,6 +19,8 @@ require 'flydata-core/table_def'
18
19
  require 'flydata-core/mysql/binlog_pos'
19
20
  require 'flydata/mysql/table_ddl'
20
21
  require 'flydata-core/mysql/command_generator'
22
+ require 'flydata/event/api_event_sender'
23
+ require 'sigdump/setup'
21
24
  #require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
22
25
 
23
26
  module Flydata
@@ -158,30 +161,15 @@ EOS
158
161
  # Cleanup tables on server
159
162
  de = data_entry
160
163
  cleanup_sync_server(de, @input_tables) unless opts.client?
164
+ sync_fm = create_sync_file_manager(de)
161
165
 
162
166
  # Delete local files
163
- sync_fm = create_sync_file_manager(de)
164
- delete_files = [
165
- sync_fm.dump_file_path,
166
- sync_fm.dump_pos_path,
167
- sync_fm.mysql_table_marshal_dump_path,
168
- sync_fm.sync_info_file,
169
- sync_fm.stats_path,
170
- sync_fm.table_position_file_paths(*@input_tables),
171
- sync_fm.table_binlog_pos_paths(*@input_tables),
172
- sync_fm.table_binlog_pos_init_paths(*@input_tables),
173
- sync_fm.table_rev_file_paths(*@input_tables),
174
- sync_fm.table_ddl_file_paths(*@input_tables)
175
- ]
167
+ tables_for_deleting = tables.empty? ? sync_fm.tables_from_positions_dir : @input_tables
168
+ sync_fm.delete_table_control_files(*tables_for_deleting)
169
+
176
170
  new_tables_after_reset = @unsynced_tables + @input_tables
177
- if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
178
- delete_files << sync_fm.binlog_path
179
- delete_files << sync_fm.sent_binlog_path
180
- delete_files << sync_fm.lock_pid_file
181
- end
182
- delete_files.flatten.each do |path|
183
- FileUtils.rm(path) if File.exists?(path)
184
- end
171
+ delete_binlog =@input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
172
+ sync_fm.delete_non_table_control_files(delete_binlog)
185
173
  sync_fm.close
186
174
  log_info_stdout("Reset completed successfully.")
187
175
  end
@@ -572,6 +560,7 @@ EOS
572
560
 
573
561
  # Don't call the method unless init sync needs to be run for a table
574
562
  def sync_mysql_to_redshift(de, options = {})
563
+ FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables"=>target_tables}, de['id'])
575
564
  dp = flydata.data_port.get
576
565
  sync_fm = create_sync_file_manager(de)
577
566
 
@@ -666,30 +655,31 @@ EOM
666
655
  log_info_stdout("Setting binary log position and exporting data from the database.")
667
656
  log_info_stdout("This process can take hours depending on data size and load on your database. Please be patient...")
668
657
  sync_fm.save_sync_info(@full_initial_sync, target_tables)
658
+ dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
669
659
  if file_dump
670
660
  binlog_pos = nil
671
661
  begin
672
- Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
673
- new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp) do |_io, _binlog_pos|
662
+ dump_generator.dump(fp) do |_io, _binlog_pos|
674
663
  binlog_pos = _binlog_pos
675
- initialize_binlog_positions_and_call_callback(
676
- binlog_pos, options[:binlog_ready_callback], sync_fm)
664
+ initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
677
665
  end
678
666
  log_info_stdout(" -> Database dump done")
679
667
  rescue Exception => e
680
668
  #Catch all exceptions including SystemExit and Interrupt.
681
669
  log_info_stdout "Quit while running mysqldump, deleting dump file..."
682
670
  sync_fm.delete_dump_file
683
- log_info_stdout "Dump file deleted, to restart the FlyData Agent, please run the 'flydata start' command "
671
+ log_info_stdout "Dump file deleted. To restart the FlyData Agent, Following tables may need reset : #{target_tables}"
672
+ if (target_tables.nil? or target_tables.empty?)
673
+ log_info_stdout "To do full reset sync use command: flydata sync:reset"
674
+ else
675
+ log_info_stdout "To reset these tables, use command: flydata sync:reset #{target_tables.join(' ')}"
676
+ end
684
677
  raise e
685
678
  end
686
679
  call_block_or_return_io(fp, binlog_pos, &block)
687
680
  else
688
- Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
689
- new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump do |io, binlog_pos|
690
- initialize_binlog_positions_and_call_callback(
691
- binlog_pos, options[:binlog_ready_callback], sync_fm)
692
-
681
+ dump_generator.dump do |io, binlog_pos|
682
+ initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
693
683
  block.call(io, binlog_pos, db_bytesize)
694
684
  end
695
685
  end
@@ -788,23 +778,33 @@ EOM
788
778
 
789
779
  bench_start_time = Time.now
790
780
 
781
+ output_thread = QueueableThread.new
782
+
791
783
  # Start parsing dump file
792
784
  tmp_num_inserted_record = 0
785
+ skip_checkpoint = false
793
786
  dump_fp = sync_fm.dump_file_path
794
787
  dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
795
788
 
789
+ send_record_counts_threads = []
796
790
  begin
797
- threads = []
798
791
  RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
799
792
  Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
800
793
  mysqldump_io,
801
794
  # create table
802
795
  Proc.new { |mysql_table|
803
- tmp_num_inserted_record = 0
804
- # dump mysql_table for resume
805
- #TODO: make it option
806
- sync_fm.save_mysql_table_marshal_dump(mysql_table)
807
796
  mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
797
+ mt = mysql_table.clone
798
+ mt.value_converters = {}
799
+ # don't let the closure hold these objects as it causes memory leak
800
+ mysql_table = nil
801
+ output_thread.run do
802
+ tmp_num_inserted_record = 0
803
+ # dump mysql_table for resume
804
+ #TODO: make it option
805
+ sync_fm.save_mysql_table_marshal_dump(mt)
806
+ log_info_stdout(" -> Started sending data for table '#{mt.table_name}'")
807
+ end
808
808
  },
809
809
  # insert record
810
810
  Proc.new { |mysql_table, values_set|
@@ -812,41 +812,57 @@ EOM
812
812
  records = values_set.collect do |values|
813
813
  convert_to_flydata_values(mysql_table, values)
814
814
  json = JSON.generate_kv_pairs(mysql_table.column_names, values)
815
+ values.clear
815
816
  {table_name: mysql_table_name, log: json}
816
817
  end
817
- ret = forwarder.emit(records)
818
- sync_fm.save_record_count_stat(mysql_table_name, ret[:record_count]) if ret
819
- tmp_num_inserted_record += 1
820
- ret
818
+ # release resources to prevent the closure from keeping it
819
+ values_set.clear
820
+ values_set = nil
821
+ mysql_table = nil
822
+ mysql_table_name = nil
823
+ output_thread.run do
824
+ forwarder.emit(records)
825
+ tmp_num_inserted_record += 1
826
+ skip_checkpoint = !forwarder.buffer_full?
827
+ end
828
+ true
821
829
  },
822
830
  # checkpoint
823
831
  Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
824
- # flush if buffer records exist
825
832
  table_name = mysql_table.nil? ? '' : mysql_table.table_name
826
- if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
827
- ret = forwarder.flush # send buffer data to the server before checkpoint
828
- sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
829
- end
833
+ mysql_table = nil
834
+ output_thread.run do
835
+ skip = skip_checkpoint
836
+ skip_checkpoint = false
837
+ unless skip
838
+ # flush if buffer records exist
839
+ if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
840
+ ret = forwarder.flush # send buffer data to the server before checkpoint
841
+ sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
842
+ end
830
843
 
831
- # show the current progress
832
- if last_pos.to_i == -1 # stream dump
833
- log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
834
- else
835
- log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
836
- end
844
+ # show the current progress
845
+ if last_pos.to_i == -1 # stream dump
846
+ log_info_stdout(" -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
847
+ else
848
+ log_info_stdout(" -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
849
+ end
837
850
 
838
- # save check point
839
- sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
851
+ # save check point
852
+ sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
840
853
 
841
- # send record count for the table
842
- if mysql_table &&
843
- state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
844
- # all records for `mysql_table` have been sent
845
- send_record_counts(de, sync_fm, mysql_table.table_name, threads)
854
+ # send record count for the table
855
+ if mysql_table &&
856
+ state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
857
+ # all records for `mysql_table` have been sent
858
+ send_record_counts(de, sync_fm, mysql_table.table_name,
859
+ send_record_counts_threads)
860
+ log_info_stdout(" -> Finished sending data for table '#{table_name}'...")
861
+ end
862
+ end
846
863
  end
847
864
  }
848
865
  )
849
- threads.each{|t| t.join}
850
866
  rescue DumpParseError =>e
851
867
  ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n #{e.message}")
852
868
  ee.description = " Please contact support@flydata.com to report the issue."
@@ -861,6 +877,8 @@ EOM
861
877
  printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
862
878
  end
863
879
  end
880
+ send_record_counts_threads.each{|t| t.join}
881
+ output_thread.join
864
882
  forwarder.close
865
883
  log_info_stdout(" -> Done")
866
884
  #log_info_stdout(" -> Records sent to the server")
@@ -1037,7 +1055,7 @@ EOM
1037
1055
  while (status = check_server_status(tables)) && (status['state'].nil? || status['state'] == 'processing')
1038
1056
  prev_message = status['message']
1039
1057
  if timeout > 0 && Time.now - start_time > timeout
1040
- raise ServerDataProcessingTimeout.new
1058
+ raise ServerDataProcessingTimeout.new("Delayed Data Processing")
1041
1059
  end
1042
1060
  print_progress(status)
1043
1061
  sleep 10
@@ -1085,7 +1103,7 @@ EOM
1085
1103
  end
1086
1104
 
1087
1105
  create_flydata_ctl_table = true
1088
- option = {skip_parimay_key_check: opts.skip_primary_key_check?}.merge(mp)
1106
+ option = {skip_primary_key_check: opts.skip_primary_key_check?}.merge(mp)
1089
1107
  missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
1090
1108
  if error
1091
1109
  error_list << error.err_hash
@@ -29,7 +29,7 @@ EOM
29
29
  end
30
30
 
31
31
  class ServerDataProcessingTimeout < AgentError
32
- def initialize(message, options)
32
+ def initialize(message, options = {})
33
33
  super(message)
34
34
  @state = options[:state]
35
35
  end