flydata 0.5.17 → 0.5.20

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_all";
 CREATE TABLE "test_table_all" (
-  "id" int8 NOT NULL,
+  "id" int8,
   "col_binary" varchar(202) DEFAULT NULL,
   "col_blob" varchar(65535),
   "col_bool" int2 DEFAULT '0',
@@ -67,7 +67,7 @@ CREATE TABLE "test_table_all" (
   "col_smallint" int2 DEFAULT NULL,
   "col_text" varchar(max),
   "col_time" timestamp DEFAULT NULL,
-  "col_timestamp" timestamp NOT NULL DEFAULT SYSDATE,
+  "col_timestamp" timestamp DEFAULT SYSDATE,
   "col_tinyblob" varchar(255),
   "col_tinyint" int2 DEFAULT NULL,
   "col_tinytext" varchar(max),
@@ -175,7 +175,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "bit_test_def_1";
 CREATE TABLE "bit_test_def_1" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "bit_value" bigint DEFAULT 1,
   "int_value" int4 DEFAULT 16,
   PRIMARY KEY ("id")
@@ -223,10 +223,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "product_order";
 CREATE TABLE "product_order" (
-  "no" int4 NOT NULL,
-  "product_category" int4 NOT NULL,
-  "product_id" int4 NOT NULL,
-  "customer_id" int4 NOT NULL,
+  "no" int4,
+  "product_category" int4,
+  "product_id" int4,
+  "customer_id" int4,
   PRIMARY KEY ("no")
 ) DISTKEY("no") SORTKEY("no");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'product_order';
@@ -278,7 +278,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_column_comment";
 CREATE TABLE "test_table_column_comment" (
-  "id" int4 NOT NULL DEFAULT '0',
+  "id" int4 DEFAULT '0',
   "value" varchar(max),
   PRIMARY KEY ("id")
 ) DISTKEY("id") SORTKEY("id");
@@ -325,10 +325,10 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_enum";
 CREATE TABLE "test_table_enum" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "enum_1" varchar encode bytedict DEFAULT NULL,
   "enum_2" varchar encode bytedict DEFAULT 'a',
-  "enum_3" varchar encode bytedict NOT NULL,
+  "enum_3" varchar encode bytedict,
   PRIMARY KEY ("id")
 ) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table_enum';
@@ -376,8 +376,8 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "test_table_multi_pk";
 CREATE TABLE "test_table_multi_pk" (
-  "id1" int4 NOT NULL DEFAULT '0',
-  "id2" int4 NOT NULL DEFAULT '0',
+  "id1" int4 DEFAULT '0',
+  "id2" int4 DEFAULT '0',
   "value" varchar(max),
   PRIMARY KEY ("id1","id2")
 ) DISTKEY("id1") SORTKEY("id1","id2");
@@ -432,7 +432,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "title" varchar(768) DEFAULT NULL,
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
@@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "sample1";
 CREATE TABLE "sample1" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "title" varchar(768) DEFAULT NULL,
   "name" varchar(max),
   "num" int4 DEFAULT NULL,
@@ -538,24 +538,24 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "invoice_items";
 CREATE TABLE "invoice_items" (
-  "id" int4 NOT NULL,
-  "app_id" int4 NOT NULL,
-  "subscription_id" int4 NOT NULL,
-  "overage_id" varchar(765) NOT NULL,
+  "id" int4,
+  "app_id" int4,
+  "subscription_id" int4,
+  "overage_id" varchar(765),
   "stripe_invoice_item_id" varchar(765) DEFAULT NULL,
   "stripe_error" varchar(765) DEFAULT NULL,
-  "synced_to_stripe" int2 NOT NULL DEFAULT '0',
+  "synced_to_stripe" int2 DEFAULT '0',
   "description" varchar(max),
-  "item_name" varchar(765) NOT NULL,
-  "item_cost" int4 NOT NULL,
-  "item_count" int4 NOT NULL,
-  "total_cost" int4 NOT NULL,
+  "item_name" varchar(765),
+  "item_cost" int4,
+  "item_count" int4,
+  "total_cost" int4,
   "created_at" timestamp DEFAULT NULL,
   "updated_at" timestamp DEFAULT NULL,
   "bill_ahead" int2 DEFAULT '0',
   "bill_ahead_resolved_at" timestamp DEFAULT NULL,
   "stripe_invoice_id" varchar(765) DEFAULT NULL,
-  "is_vat" int2 NOT NULL DEFAULT '0',
+  "is_vat" int2 DEFAULT '0',
   PRIMARY KEY ("id")
 ) DISTKEY("id") SORTKEY("id");
 DELETE FROM "flydata_ctl_columns" WHERE table_name = 'invoice_items';
@@ -632,7 +632,7 @@ CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
 ) DISTKEY(table_name) SORTKEY(table_name);
 DROP TABLE IF EXISTS "zerofill_table";
 CREATE TABLE "zerofill_table" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "value_int" int8 DEFAULT NULL,
   "value_float" float4 DEFAULT NULL,
   "value_dec" numeric(10,2) DEFAULT NULL,

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -31,7 +31,7 @@ describe RedshiftTableDef do
     let(:create_table_queries) { <<EOT.strip }
 DROP TABLE IF EXISTS #{schema_prefix}"test_table";
 CREATE TABLE #{schema_prefix}"test_table" (
-  "id" int4 NOT NULL,
+  "id" int4,
   "age" int8,
   "value" varchar(max),
   PRIMARY KEY ("id")
@@ -218,7 +218,7 @@ EOT
         before do
           column[:not_null] = true
         end
-        let(:not_null_sql) { " NOT NULL" }
+        let(:not_null_sql) { "" }
         it_behaves_like *examples
       end
     end
@@ -519,11 +519,11 @@ EOT
       end
 
       context "with not null true" do
-        let(:expected_query) { %Q|  #{type_sql} NOT NULL#{default_sql}| }
+        let(:expected_query) { %Q|  #{type_sql}#{default_sql}| } # Redshift supports NOT NULL but we do not respect it in order to accept NULL values which may come in future.
         before do
           column[:not_null] = true
         end
-        let(:no_default_sql) { not_null_default_sql }
+        let(:no_default_sql) { "" }
 
         it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
       end

data/lib/flydata/command/start.rb

@@ -11,17 +11,24 @@ module Flydata
         sender_opts
       end
       def run(*args)
+        sender_start(args)
+        helper_start(args) unless opts.skip_helper?
+      end
+
+      private
+      def sender_start(args)
         sender = Flydata::Command::Sender.new(opts)
         sender.start
-        unless opts.skip_helper?
-          helper_opts = Flydata::Command::Helper.slop_start
-          helper_opts.parse!(args)
-          helper = Flydata::Command::Helper.new(helper_opts)
-          helper.stop(quiet: true) # kill existing helper process (if any) and start
-          helper.start(quiet: true)
-        end
       end
-      run_exclusive :run
+      run_exclusive :sender_start, command: "start"
+
+      def helper_start(args)
+        helper_opts = Flydata::Command::Helper.slop_start
+        helper_opts.parse!(args)
+        helper = Flydata::Command::Helper.new(helper_opts)
+        helper.stop(quiet: true) # kill existing helper process (if any) and start
+        system "flydata helper:start"
+      end
     end
   end
 end

data/lib/flydata/command/sync.rb

@@ -10,6 +10,7 @@ require 'flydata/compatibility_check'
 require 'flydata/errors'
 require 'flydata/helpers'
 require 'flydata/json'
+require 'flydata/queueable_thread'
 require 'flydata/output/forwarder'
 require 'flydata/parser/mysql/dump_parser'
 require 'flydata/preference/data_entry_preference'
@@ -18,6 +19,8 @@ require 'flydata-core/table_def'
 require 'flydata-core/mysql/binlog_pos'
 require 'flydata/mysql/table_ddl'
 require 'flydata-core/mysql/command_generator'
+require 'flydata/event/api_event_sender'
+require 'sigdump/setup'
 #require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
 module Flydata
@@ -158,30 +161,15 @@ EOS
         # Cleanup tables on server
         de = data_entry
         cleanup_sync_server(de, @input_tables) unless opts.client?
+        sync_fm = create_sync_file_manager(de)
 
         # Delete local files
-        sync_fm = create_sync_file_manager(de)
-        delete_files = [
-          sync_fm.dump_file_path,
-          sync_fm.dump_pos_path,
-          sync_fm.mysql_table_marshal_dump_path,
-          sync_fm.sync_info_file,
-          sync_fm.stats_path,
-          sync_fm.table_position_file_paths(*@input_tables),
-          sync_fm.table_binlog_pos_paths(*@input_tables),
-          sync_fm.table_binlog_pos_init_paths(*@input_tables),
-          sync_fm.table_rev_file_paths(*@input_tables),
-          sync_fm.table_ddl_file_paths(*@input_tables)
-        ]
+        tables_for_deleting = tables.empty? ? sync_fm.tables_from_positions_dir : @input_tables
+        sync_fm.delete_table_control_files(*tables_for_deleting)
+
         new_tables_after_reset = @unsynced_tables + @input_tables
-        if @input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
-          delete_files << sync_fm.binlog_path
-          delete_files << sync_fm.sent_binlog_path
-          delete_files << sync_fm.lock_pid_file
-        end
-        delete_files.flatten.each do |path|
-          FileUtils.rm(path) if File.exists?(path)
-        end
+        delete_binlog =@input_tables.empty? or @full_tables.empty? or @full_tables.all?{|ft| new_tables_after_reset.include?(ft)}
+        sync_fm.delete_non_table_control_files(delete_binlog)
         sync_fm.close
         log_info_stdout("Reset completed successfully.")
       end
@@ -572,6 +560,7 @@ EOS
 
       # Don't call the method unless init sync needs to be run for a table
       def sync_mysql_to_redshift(de, options = {})
+        FlydataCore::Event::ApiEventSender.instance.send_event("init_sync", "started", {"tables"=>target_tables}, de['id'])
         dp = flydata.data_port.get
         sync_fm = create_sync_file_manager(de)
 
@@ -666,30 +655,31 @@ EOM
           log_info_stdout("Setting binary log position and exporting data from the database.")
           log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
           sync_fm.save_sync_info(@full_initial_sync, target_tables)
+          dump_generator = Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.new(de['mysql_data_entry_preference'].merge('tables' => target_tables))
           if file_dump
             binlog_pos = nil
             begin
-              Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-                new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump(fp) do |_io, _binlog_pos|
+              dump_generator.dump(fp) do |_io, _binlog_pos|
                 binlog_pos = _binlog_pos
-                initialize_binlog_positions_and_call_callback(
-                  binlog_pos, options[:binlog_ready_callback], sync_fm)
+                initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
               end
               log_info_stdout("  -> Database dump done")
             rescue Exception => e
               #Catch all exceptions including SystemExit and Interrupt.
               log_info_stdout "Quit while running mysqldump, deleting dump file..."
               sync_fm.delete_dump_file
-              log_info_stdout "Dump file deleted, to restart the FlyData Agent, please run the 'flydata start' command "
+              log_info_stdout "Dump file deleted. To restart the FlyData Agent, Following tables may need reset : #{target_tables}"
+              if (target_tables.nil? or target_tables.empty?)
+                log_info_stdout "To do full reset sync use command: flydata sync:reset"
+              else
+                log_info_stdout "To reset these tables, use command: flydata sync:reset #{target_tables.join(' ')}"
+              end
               raise e
             end
             call_block_or_return_io(fp, binlog_pos, &block)
           else
-            Flydata::Parser::Mysql::MysqlDumpGeneratorNoMasterData.
-              new(de['mysql_data_entry_preference'].merge('tables' => target_tables)).dump do |io, binlog_pos|
-              initialize_binlog_positions_and_call_callback(
-                binlog_pos, options[:binlog_ready_callback], sync_fm)
-
+            dump_generator.dump do |io, binlog_pos|
+              initialize_binlog_positions_and_call_callback(binlog_pos, options[:binlog_ready_callback], sync_fm)
               block.call(io, binlog_pos, db_bytesize)
             end
           end
@@ -788,23 +778,33 @@ EOM
 
         bench_start_time = Time.now
 
+        output_thread = QueueableThread.new
+
         # Start parsing dump file
         tmp_num_inserted_record = 0
+        skip_checkpoint = false
         dump_fp = sync_fm.dump_file_path
         dump_file_size = File.exists?(dump_fp) ? File.size(dump_fp) : 1
 
+        send_record_counts_threads = []
         begin
-          threads = []
           RubyProf.start if RUN_PROFILE and defined?(RubyProf) and not RubyProf.running?
           Flydata::Parser::Mysql::MysqlDumpParser.new(option).parse(
             mysqldump_io,
             # create table
             Proc.new { |mysql_table|
-              tmp_num_inserted_record = 0
-              # dump mysql_table for resume
-              #TODO: make it option
-              sync_fm.save_mysql_table_marshal_dump(mysql_table)
               mysql_table.set_value_converters(FlydataCore::TableDef::MysqlTableDef::VALUE_CONVERTERS)
+              mt = mysql_table.clone
+              mt.value_converters = {}
+              # don't let the closure hold these objects as it causes memory leak
+              mysql_table = nil
+              output_thread.run do
+                tmp_num_inserted_record = 0
+                # dump mysql_table for resume
+                #TODO: make it option
+                sync_fm.save_mysql_table_marshal_dump(mt)
+                log_info_stdout("  -> Started sending data for table '#{mt.table_name}'")
+              end
             },
             # insert record
             Proc.new { |mysql_table, values_set|
@@ -812,41 +812,57 @@ EOM
               records = values_set.collect do |values|
                 convert_to_flydata_values(mysql_table, values)
                 json = JSON.generate_kv_pairs(mysql_table.column_names, values)
+                values.clear
                 {table_name: mysql_table_name, log: json}
               end
-              ret = forwarder.emit(records)
-              sync_fm.save_record_count_stat(mysql_table_name, ret[:record_count]) if ret
-              tmp_num_inserted_record += 1
-              ret
+              # release resources to prevent the closure from keeping it
+              values_set.clear
+              values_set = nil
+              mysql_table = nil
+              mysql_table_name = nil
+              output_thread.run do
+                forwarder.emit(records)
+                tmp_num_inserted_record += 1
+                skip_checkpoint = !forwarder.buffer_full?
+              end
+              true
             },
             # checkpoint
             Proc.new { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
-              # flush if buffer records exist
               table_name = mysql_table.nil? ? '' : mysql_table.table_name
-              if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
-                ret = forwarder.flush # send buffer data to the server before checkpoint
-                sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
-              end
+              mysql_table = nil
+              output_thread.run do
+                skip = skip_checkpoint
+                skip_checkpoint = false
+                unless skip
+                  # flush if buffer records exist
+                  if tmp_num_inserted_record > 0 && forwarder.buffer_record_count > 0
+                    ret = forwarder.flush # send buffer data to the server before checkpoint
+                    sync_fm.save_record_count_stat(table_name, ret[:record_count]) if ret
+                  end
 
-              # show the current progress
-              if last_pos.to_i == -1   # stream dump
-                log_info_stdout("  -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
-              else
-                log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
-              end
+                  # show the current progress
+                  if last_pos.to_i == -1   # stream dump
+                    log_info_stdout("  -> #{as_size(bytesize)} (#{bytesize} byte) completed...")
+                  else
+                    log_info_stdout("  -> #{(last_pos.to_f/dump_file_size * 100).round(1)}% (#{last_pos}/#{dump_file_size}) completed...")
+                  end
 
-              # save check point
-              sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
+                  # save check point
+                  sync_fm.save_dump_pos(STATUS_PARSING, table_name, last_pos, binlog_pos, state, substate)
 
-              # send record count for the table
-              if mysql_table &&
-                 state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
-                # all records for `mysql_table` have been sent
-                send_record_counts(de, sync_fm, mysql_table.table_name, threads)
+                  # send record count for the table
+                  if mysql_table &&
+                     state == Flydata::Parser::Mysql::MysqlDumpParser::State::CREATE_TABLE
+                    # all records for `mysql_table` have been sent
+                    send_record_counts(de, sync_fm, mysql_table.table_name,
+                                       send_record_counts_threads)
+                    log_info_stdout("  -> Finished sending data for table '#{table_name}'...")
+                  end
+                end
               end
             }
           )
-          threads.each{|t| t.join}
         rescue DumpParseError =>e
           ee = DumpParseError.new("ERROR: We encountered an error parsing this chunk:\n  #{e.message}")
           ee.description = "  Please contact support@flydata.com to report the issue."
@@ -861,6 +877,8 @@ EOM
             printer.print(File.new("ruby-prof-out-#{Time.now.to_i}.html", "w"), :min_percent => 3)
           end
         end
+        send_record_counts_threads.each{|t| t.join}
+        output_thread.join
         forwarder.close
         log_info_stdout("  -> Done")
         #log_info_stdout("  -> Records sent to the server")
@@ -1037,7 +1055,7 @@ EOM
         while (status = check_server_status(tables)) && (status['state'].nil? || status['state'] == 'processing')
           prev_message = status['message']
           if timeout > 0 && Time.now - start_time > timeout
-            raise ServerDataProcessingTimeout.new
+            raise ServerDataProcessingTimeout.new("Delayed Data Processing")
           end
           print_progress(status)
           sleep 10
@@ -1085,7 +1103,7 @@ EOM
         end
 
         create_flydata_ctl_table = true
-        option = {skip_parimay_key_check: opts.skip_primary_key_check?}.merge(mp)
+        option = {skip_primary_key_check: opts.skip_primary_key_check?}.merge(mp)
         missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, option) do |mysql_tabledef, error|
           if error
             error_list << error.err_hash

data/lib/flydata/errors.rb

@@ -29,7 +29,7 @@ EOM
 end
 
 class ServerDataProcessingTimeout < AgentError
-  def initialize(message, options)
+  def initialize(message, options = {})
     super(message)
     @state = options[:state]
   end