flydata 0.7.17 → 0.7.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 340f31b8701b211b25628d5690237565ff5e46c6
-  data.tar.gz: 25bbb20fa4a67ca4f25f20917a350e649a224b80
+  metadata.gz: 5f6e6673d1001075c0ba38d2c4551aceac38d5b5
+  data.tar.gz: ff03d98fe9137ff5e4a10176a0e9b0c000d68c72
 SHA512:
-  metadata.gz: ae89812c8d23ad80ec1777ec1a6c39764808d7279ff54db865b7bc1cd090045cef87f426af834aaf50d147f1d7c2d9eef1a16150b6844d79701313f84c7b3a17
-  data.tar.gz: 4f6c6c97c6c04ebee65780bf31930fc58a6316cf7830bcac5defd63bc8a9db68d3062577d142e09cd1d86f9cc66b7c4bd207d6b5094ae6c4c89f48b983c88d57
+  metadata.gz: abedde840231a9460445b0983a7ec8d9e1da3d043792352c626d8202b1792787a150571c502ccf39a3d6ea6f3840a0fd617b96922ae55d0c37c81d86cdd73556
+  data.tar.gz: dc1a691a6c514f75339b99ed7c8c46238b8ca6b32bedb3c923e182408382f0bbe13a1e4556c41175876bfa0423a588a9e5af412afb6fe4bd6a7f828f3dedb387

data/VERSION

@@ -1 +1 @@
-0.7.17
+0.7.18

data/flydata-core/lib/flydata-core/postgresql/pg_client.rb

@@ -10,14 +10,21 @@ module Postgresql
 class PGClient
   PG_CONNECT_TIMEOUT = 10.0
 
-  def initialize(dbconf)
+  def initialize(dbconf, options = {})
     @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(dbconf)
+    @options = options
   end
 
   attr_reader :dbconf
 
   def establish_connection
-    @conn = create_connection if @conn.nil?
+    if @conn.nil?
+      @conn = create_connection
+      if @options[:notice_receiver]
+        @conn.set_notice_receiver{|r| @options[:notice_receiver].call(r) }
+      end
+    end
+    @conn
   end
 
   def query(query, params = [])
@@ -42,6 +49,11 @@ class PGClient
       : result
   end
 
+  def exec(query)
+    establish_connection
+    @conn.exec(query)
+  end
+
   def close
     if @conn
       @conn.finish

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -159,13 +159,14 @@ CREATE TABLE IF NOT EXISTS %s (
 EOS
 
   def self.remove_table_sql(flydata_tabledef, schema_name, options)
-    return "" if options[:skip_drop_table]
     table_name = flydata_tabledef[:table_name]
     redshift_tbl = table_name_for_ddl(table_name, schema_name)
     sql = ""
     if options[:backup_postfix]
       # drop backup table if exists
-      sql += drop_backup_table_sql(flydata_tabledef, schema_name, options)
+      unless options[:skip_drop_table]
+        sql += drop_backup_table_sql(flydata_tabledef, schema_name, options)
+      end
       # create an empty table to prevent RENAME TABLE query from failing
       sql += create_table_sql(flydata_tabledef, schema_name,
                        options.merge(create_table_sql: CREATE_TABLE_IF_NOT_EXISTS_SQL))
@@ -173,6 +174,7 @@ EOS
                      "#{table_name}#{options[:backup_postfix]}", nil)
       sql += RENAME_TABLE_SQL % [redshift_tbl, backup_tbl]
     else
+      return "" if options[:skip_drop_table]
       sql += DROP_TABLE_SQL % redshift_tbl
     end
     sql

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -117,7 +117,16 @@ EOT
         option.merge!(skip_drop_table: true)
       end
 
-      let(:drop_table_subquery) { ""}
+      let(:drop_table_subquery) { <<EOT
+CREATE TABLE IF NOT EXISTS #{schema_prefix}"test_table" (
+  "id" int4,
+  "age" int8,
+  "value" varchar(max),
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+ALTER TABLE #{schema_prefix}"test_table" RENAME TO "test_table_flydata20160125232857";
+EOT
+}
       it 'should return ddl' do
         expect(subject).to eq "BEGIN;\n#{create_table_queries}\n#{flydata_ctl_update}\nCOMMIT;\n"
       end

data/lib/flydata/api/redshift_cluster.rb

@@ -9,8 +9,10 @@ module Flydata
         super
       end
 
-      def run_query(sql)
-        @client.post("#{@url_path}/query", nil, {redshift_query: {body: sql}})
+      def show_default
+        # currently one user has one redshift_cluster
+        url_path = "#{@url_path}/show_default?password_required=1"
+        @client.get(url_path)
       end
     end
   end

data/lib/flydata/command/base.rb

@@ -4,6 +4,7 @@ require 'flydata/command_loggable'
 require 'flydata/source'
 require 'flydata/command/exclusive_runnable'
 require 'flydata/preference/data_entry_preference'
+require 'flydata/util/encryptor'
 
 
 module Flydata
@@ -42,14 +43,35 @@ module Flydata
         log_info_stdout("Your current application name is '#{de['purpose_name']}'")
       end
 
-      def data_entry
-        @de ||= retrieve_data_entries.first
+      def data_entry(refresh: false)
+        if @de.nil? || refresh
+          @de = retrieve_data_entries.first
+        end
         raise "No data entry exists.  Please set one up on the FlyData Console (#{dashboard_url})" unless @de
         @de
       end
 
-      def source
-        @source ||= Source.create(data_entry)
+      def data_port
+        return @data_port if @data_port
+        @data_port = flydata.data_port.get
+      end
+
+      def redshift_cluster
+        return @redshift_cluster if @redshift_cluster
+        @redshift_cluster = flydata.redshift_cluster.show_default
+        @redshift_cluster['password'] = Flydata::Util::Encryptor.decrypt(
+          @redshift_cluster['encrypted_password'],
+          data_port['key'],
+          'redshift_cluster password')
+        @redshift_cluster
+      end
+
+      def source(refresh: false)
+        if @source.nil? || refresh
+          @source = nil
+          @source = Source.create(data_entry(refresh: refresh))
+        end
+        @source
       end
 
       def register_crontab

data/lib/flydata/command/sender.rb

@@ -9,7 +9,9 @@ module Flydata
         Slop.new do
           on 'n', 'no-daemon', 'Start FlyData agent as a regular program'
           on 'e', 'no-email', 'Skip sending init-sync-start notification email'
+          on 'y', 'yes', 'Skip command prompt assuming yes to all questions.  Use this for batch operation.'
           on 'force-run', 'Run forcefully, ignoring exclusive run info'
+          on 'auto-create', 'Create tables on Redshift automatically'
         end
       end
       def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward
@@ -66,7 +68,10 @@ module Flydata
         options[:quiet] = true
         Flydata::Command::Sync.new.try_initial_sync(
           source_pos_ready_callback: start_fluentd,
-          no_email: opts.no_email?)
+          no_email: opts.no_email?,
+          auto_create: opts.auto_create?,
+          slop_opts: opts,
+        )
         options[:quiet] = quiet_option
         start_fluentd.call unless fluentd_started
         if options[:show_final_message] && !options[:quiet]

data/lib/flydata/command/sync.rb

@@ -12,6 +12,7 @@ require 'flydata/helpers'
 require 'flydata/json'
 require 'flydata/queueable_thread'
 require 'flydata/output/forwarder'
+require 'flydata/output/ddl_runner'
 require 'flydata/parser'
 require 'flydata/preference/data_entry_preference'
 require 'flydata/sync_file_manager'
@@ -20,6 +21,7 @@ require 'flydata-core/table_def'
 require 'flydata/table_ddl'
 require 'flydata/event/api_event_sender'
 require 'flydata-core/event/event_dictionary'
+require 'flydata-core/record/record'
 require 'sigdump/setup'
 #require 'ruby-prof' # to enable profiling, also set the class' RUN_PROFILE
 
@@ -34,6 +36,11 @@ module Flydata
       INSERT_PROGRESS_INTERVAL = 1000
       SERVER_DATA_PROCESSING_TIMEOUT = 3600 # seconds
 
+      # for sync_info file auto_create_status
+      AUTO_CREATE_STATUS_START = 'START'
+      #AUTO_CREATE_STATUS_SENT_DDL = 'SENT_DDL'
+      AUTO_CREATE_STATUS_CREATED_TABLES = 'CREATED_TABLES'
+
       # for dump.pos file
       STATUS_START = 'START'       # only :source_pos is available at the begining of parse
       STATUS_PARSING = 'PARSING'
@@ -74,9 +81,13 @@ module Flydata
       # Public method
       #  - Called from Sender#start/restart
       def try_initial_sync(options)
+        @opts = options[:slop_opts] if options[:slop_opts]
         handle_initial_sync(options) if source.sync.supported?
       rescue Source::UnsupportedSourceError
         return
+      rescue => e
+        log_error("[error] Unexpcted error happend during inital sync. error:#{e}")
+        raise e
       end
 
       # Command: flydata sync:flush
@@ -130,7 +141,7 @@ EOS
           tables = []
           reset_init = false
         end
-        sync_resumed = set_current_tables(tables, resume: !opts[:all])
+        sync_resumed, auto_create = set_current_tables(tables, resume: !opts[:all])
         target_tables = opts[:all] ? @full_tables : @input_tables
         target_append_only_tables = target_tables & @append_only_tables
         target_full_sync_tables = target_tables - @append_only_tables
@@ -607,7 +618,10 @@ EOS
         end
 
         # Setup instance variables
-        sync_resumed = set_current_tables(nil, resume: true)
+        # Need to try the sync for all tables if auto create mode is on
+        include_all_tables = !!options[:auto_create]
+        sync_resumed, auto_create = set_current_tables(nil, resume: true, include_all_tables: include_all_tables)
+        options[:auto_create] ||= auto_create
 
         if sync_resumed
           # skip confirmation prompts and resume sync right away.
@@ -617,7 +631,7 @@ EOS
         elsif !@unsynced_tables.empty?
           show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@unsynced_tables.join(", ")}\n"
-          unless @no_ddl_generated_tables.empty?
+          if !@no_ddl_generated_tables.empty? && !options[:auto_create]
             unsynced_table_message <<
             "  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
             "    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
@@ -655,7 +669,7 @@ EOS
           _reset(recover_cmd, reset_client_only: false, delete_tbl_ddl: false)
 
           # Setup instance variables again
-          sync_resumed = set_current_tables(nil, resume: true)
+          set_current_tables(nil, resume: true)
         end
 
         begin
@@ -675,6 +689,7 @@ EOS
           perform_initial_sync(de, opt)
         rescue ServerDataProcessingTimeout => e
           ee = ServerDataProcessingTimeout.new("Delayed Data Processing")
+          log_error("[error] Delayed Data Processing. Please check and resume the sync by running 'flydata start'")
           ee.description = <<EOS
   Data processing is taking more than expected.  Please contact support@flydata.com to check the system status.
   Once checked, you can resume sync with the following command.
@@ -684,6 +699,14 @@ EOS
 EOS
           ee.set_backtrace e.backtrace
           raise ee
+        rescue AgentInternalError => e
+          case e.code
+          when AgentInternalError::NO_VALID_TABLE_ERR
+            # Proceed normal restart when no valid table exists for initial sync
+            return
+          else
+            raise e
+          end
         end
         complete(de)
       end
@@ -713,7 +736,7 @@ EOS
         dump_pos_info = sync_fm.load_dump_pos
         if dump_pos_info[:status] == STATUS_PARSED || dump_pos_info[:status] == STATUS_COMPLETE
           initialize_source_positions_and_call_callback(
-                  nil, options[:source_pos_ready_callback], sync_fm)
+                  nil, options[:source_pos_ready_callback], sync_fm, options)
           return
         end
 
@@ -721,7 +744,7 @@ EOS
         fp = sync_fm.dump_file_path
         if file_dump && File.exists?(fp) && File.size(fp) > 0
           initialize_source_positions_and_call_callback(
-                  nil, options[:source_pos_ready_callback], sync_fm)
+                  nil, options[:source_pos_ready_callback], sync_fm, options)
           return call_block_or_return_io(fp, &dump_ready_callback)
         end
 
@@ -770,9 +793,9 @@ EOM
             end
           end
 
-          log_info_stdout("Setting binary log position and exporting data from the database.")
-          log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
-          sync_fm.save_sync_info(@full_initial_sync, target_tables)
+          if sync_fm.load_sync_info.nil?
+            sync_fm.save_sync_info(@full_initial_sync, target_tables, (options[:auto_create] ? AUTO_CREATE_STATUS_START : nil))
+          end
           # This notification will be uncommented after init_sync_finish email integration is released
           unless options[:sync_resumed]
             FlydataCore::Event::ApiEventSender.instance.send_event(
@@ -782,12 +805,19 @@ EOM
               data_entry_id: de['id'],
               data_port_id: de['data_port_id'])
           end
+
+          handle_auto_create(dp, de, sync_fm)
+
+          log_info_stdout("Setting binary log position and exporting data from the database.")
+          log_info_stdout("This process can take hours depending on data size and load on your database.  Please be patient...")
+
           if file_dump
             source_pos = nil
+
             begin
               context.dump(target_tables, fp) do |_io, _source_pos|
                 source_pos = _source_pos
-                initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm)
+                initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm, options)
               end
               log_info_stdout("  -> Database dump done")
             rescue Exception => e
@@ -800,7 +830,7 @@ EOM
             call_block_or_return_io(fp, source_pos, &dump_ready_callback)
           else
             context.dump(target_tables) do |io, source_pos|
-              initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm)
+              initialize_source_positions_and_call_callback(source_pos, options[:source_pos_ready_callback], sync_fm, options)
               dump_ready_callback.call(io, source_pos)
             end
           end
@@ -809,9 +839,41 @@ EOM
         end
       end
 
-      def initialize_source_positions_and_call_callback(source_pos, callback, sync_fm)
+      def handle_auto_create(dp, de, sync_fm)
+        auto_create_status = sync_fm.load_sync_info[:auto_create_status]
+        return if auto_create_status.nil?
+        loop do
+          case auto_create_status
+          when AUTO_CREATE_STATUS_START
+            log_info_stdout("Creating tables on Redshift...")
+            ddl_context = source.sync_generate_table_ddl(dp)
+            flydata_tabledefs = generate_and_run_table_ddl(ddl_context, de, auto_create: true) || []
+            if flydata_tabledefs.empty?
+              log_error_stderr("No valid table for sync. Please check table errors on Dashboard - tables:#{target_tables.join(", ")}")
+              sync_fm.delete_sync_info # No need to keep sync.info
+              raise AgentInternalError.new("No valid tables for sync - tables:#{target_tables.join(", ")}",
+                                           AgentInternalError::NO_VALID_TABLE_ERR)
+            end
+            # no error tables to create ddl
+            table_names = flydata_tabledefs.collect{|d| d[:table_name]}
+            # Refresh source and data_entry to refrect invalid tables
+            # Set tables whose ddl is sent only as input tables because new tables may be added during creating target tables
+            auto_create_status = AUTO_CREATE_STATUS_CREATED_TABLES
+            sync_fm.save_sync_info(@full_initial_sync, table_names, auto_create_status)
+            source(refresh: true)
+            set_current_tables(table_names, resume: true)
+          when AUTO_CREATE_STATUS_CREATED_TABLES
+            log_info_stdout("Tables are created on Redshift... tables:#{target_tables.join(", ")}")
+            break
+          else
+            raise "Invalid auto_create_status in dump/sync.info filedump."
+          end
+        end
+      end
+
+      def initialize_source_positions_and_call_callback(source_pos, callback, sync_fm, options = {})
         if source_pos
-          initialize_positions(sync_fm, source_pos)
+          initialize_positions(sync_fm, source_pos, options)
         else
           # no source_pos was given because dump was completed in the
           # previous init sync attempt.  Position files must be there already
@@ -1152,14 +1214,14 @@ EOM
         end
       end
 
-      def initialize_positions(sync_fm, source_pos)
+      def initialize_positions(sync_fm, source_pos, options)
         sync_fm.save_table_source_pos(target_tables, source_pos)
 
         if @full_initial_sync
           sync_fm.save_source_pos(source_pos)
         end
         sync_fm.install_table_source_pos_files(target_tables)
-        sync_fm.reset_table_position_files(target_tables)
+        sync_fm.reset_table_position_files(target_tables, options)
       end
 
       def convert_to_flydata_values(source_table, values)
@@ -1226,7 +1288,53 @@ EOM
         end
       end
 
-      def generate_flydata_tabledefs(context, de)
+      # Generate and send table ddl to a data server
+      def generate_and_send_table_ddl(context, de)
+        flydata_tabledefs = generate_flydata_tabledefs(context, de)
+        return flydata_tabledefs if flydata_tabledefs.nil? || flydata_tabledefs.empty?
+
+        #TODO: Check per-table position file and raise an error if pos > 0
+
+        base_record = {
+          table_rev: 1,
+          seq: SyncFileManager::INITIAL_SYNC_SEQ,
+          respect_order: true,
+          type: 'initial_sync',
+          src_pos: '-',
+          v: FlydataCore::Record::V2,
+        }
+
+        records = flydata_tabledefs.collect do |flydata_tabledef|
+          table_name = flydata_tabledef[:table_name]
+          ddl_options = flydata_tabledef[:ddl_options] || {}
+          base_record.merge(
+            flydata_tabledef: flydata_tabledef.dup,
+            skip_drop_table: ddl_options[:skip_drop_table],
+            table_name: table_name
+          )
+        end
+
+        forwarder = build_forwarder(context.dp, de)
+        forwarder.emit(records)
+        forwarder.flush
+
+        flydata_tabledefs
+      ensure
+        if forwarder
+          forwarder.close rescue nil
+        end
+      end
+
+      # Generate and run table ddl on Redshift
+      def generate_and_run_table_ddl(context, de, options = {})
+        flydata_tabledefs = generate_flydata_tabledefs(context, de, options)
+        return flydata_tabledefs if flydata_tabledefs.nil? || flydata_tabledefs.empty?
+        ddl_runner = Flydata::Output::RedshiftDDLRunner.new(redshift_cluster, de)
+        ddl_runner.run_ddls(flydata_tabledefs)
+        flydata_tabledefs
+      end
+
+      def generate_flydata_tabledefs(context, de, options = {})
         schema_name = (de['schema_name'] || nil)
         tables = opts.all_tables? ? @full_tables : (@input_tables.empty? ? @unsynced_tables : @input_tables)
 
@@ -1258,25 +1366,32 @@ EOM
         table_validity_hash = Hash.new {|h,k| h[k] = {}}
         tables_without_error = tables
         unless error_list.empty?
-          log_error_stderr("\n\nERROR: FlyData Sync will not sync the following table(s) due to an error.")
+          unless options[:auto_create]
+            log_error_stderr("\n\nERROR: FlyData Sync will not sync the following table(s) due to an error.")
+          end
+
           group_error =  error_list.group_by {|d| d[:error]}
+          error_info_hash = {}
           group_error.each_key do |error|
             group_error[error].each do |hash|
               if table = hash[:table]
+                error_info_hash[table] = error
                 log_error_stderr("  - #{table} (#{error})")
                 table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = error
               end
             end
           end
-          log_error_stderr(<<EOS)
-To sync these table(s), please fix the error(s) and run "flydata sync:generate_table_ddl" again.
-EOS
-          tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
-          unless tables_without_error.empty?
-            log_error_stderr(<<EOS)
 
-The other tables are ready to sync.  To start sync, run the generated script on the Redshift cluster and run "flydata start".
-EOS
+          if options[:auto_create]
+            log_error_stderr("[error] FlyData Sync will not sync the following table(s) due to an error. #{error_info_hash.collect{|k,v| "#{k}(#{v})"}.join(", ")}")
+            log_warn_stderr("To sync these table(s), please fix the error(s) and run \"flydata restart --auto-create\" again.")
+          else
+            log_error_stderr("To sync these table(s), please fix the error(s) and run \"flydata sync:generate_table_ddl\" again.")
+          end
+
+          tables_without_error = tables - error_list.inject([]){|arr, err| arr << err[:table] if err[:table]}
+          unless tables_without_error.empty? || options[:auto_create]
+            log_error_stderr("The other tables are ready to sync.  To start sync, run the generated script on the Redshift cluster and run \"flydata start\".")
           end
         end
 
@@ -1346,7 +1461,11 @@ Thank you for using FlyData!
         sync_fm = create_sync_file_manager
         sync_info = sync_fm.load_sync_info
         sync_resumed = options[:resume] && !!sync_info
-
+        auto_create = if sync_resumed
+                        !!sync_info[:auto_create_status]
+                      else
+                        false
+                      end
         table_lists = source.sync.table_lists
 
         # `full_tables` will either
@@ -1383,13 +1502,14 @@ Thank you for using FlyData!
 
         verify_input_tables(@input_tables, @full_tables)
 
-        sync_resumed
+        [sync_resumed, auto_create]
       end
 
       def validate_initial_sync_status
         sync_fm = create_sync_file_manager
         dump_pos_info = sync_fm.load_dump_pos
-        sync_info_exists = !!sync_fm.load_sync_info
+        sync_info = sync_fm.load_sync_info || {}
+        sync_info_exists = !sync_info.empty?
         dump_file_deleted = !File.exists?(sync_fm.dump_file_path)
         sync_fm.close
 
@@ -1399,7 +1519,7 @@ Thank you for using FlyData!
         end
 
         # check if the previous initial sync was aborted during dump.
-        sync_info_exists && dump_file_deleted
+        sync_info_exists && sync_info[:auto_create_status] == AUTO_CREATE_STATUS_CREATED_TABLES && dump_file_deleted
       end
 
       def target_tables
@@ -1423,8 +1543,8 @@ Thank you for using FlyData!
         end
       end
 
-      def data_entry
-        unless @sync_de
+      def data_entry(refresh: false)
+        if @sync_de.nil? || refresh
           @sync_de = super
           source.sync.setup # updates the data entry contents
         end

data/lib/flydata/errors.rb

@@ -39,4 +39,16 @@ end
 class DumpParseError < AgentError
 end
 
+class AgentInternalError < AgentError
+  NO_VALID_TABLE_ERR = 101
+  UNKNOWN_ERR = 999
+
+  def initialize(message, code = UNKNOWN_ERR)
+    super("#{message} code:#{code}")
+    @code = code
+  end
+
+  attr_reader :code
+end
+
 end

data/lib/flydata/output/ddl_runner.rb

@@ -0,0 +1,70 @@
+require 'flydata-core/postgresql/pg_client'
+require 'flydata-core/table_def/sync_redshift_table_def'
+require 'flydata/command_loggable'
+
+module Flydata
+  module Output
+    class RedshiftDDLRunner
+      include CommandLoggable
+
+      def initialize(dbconf, de)
+        @pg_client = FlydataCore::Postgresql::PGClient.new(dbconf, notice_receiver: Proc.new{|result|
+          log_info_stdout("     #{result.error_message.to_s.strip}")
+        })
+        @schema_name = de['schema_name']
+        @schema_name = nil if @schema_name.to_s.strip.empty?
+      end
+
+      def run_ddls(flydata_tabledefs)
+        @pg_client.establish_connection
+        create_schema
+        create_ctl_tables
+        flydata_tabledefs.each.with_index(1) do |flydata_tabledef, index|
+          run_ddl(flydata_tabledef, index, flydata_tabledefs.size)
+        end
+      ensure
+        @pg_client.close
+      end
+
+      private
+
+      def create_schema
+        return unless @schema_name
+        query = FlydataCore::TableDef::SyncRedshiftTableDef.create_schema_sql(@schema_name)
+        begin
+          @pg_client.exec(query)
+        rescue PG::InsufficientPrivilege
+          # Ignore
+        end
+      end
+
+      def create_ctl_tables
+        query = FlydataCore::TableDef::SyncRedshiftTableDef.create_flydata_ctl_table_sql(@schema_name)
+        log_info_stdout("  -> Creating flydata ctl tables... schema:\"#{@schema_name}\"")
+        log_info("query:\n#{query}")
+        @pg_client.exec(query)
+      end
+
+      def run_ddl(flydata_tabledef, index, total)
+        ddl = FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(
+                flydata_tabledef,
+                flydata_tabledef[:ddl_options].merge(flydata_ctl_table: false))
+        log_info_stdout("  -> Creating \"#{flydata_tabledef[:table_name]}\"... (#{index}/#{total})")
+        log_info("query:\n#{ddl}")
+        @pg_client.exec(ddl)
+      rescue PG::DependentObjectsStillExist => e
+        log_error_stderr("    [error] #{e.to_s.strip}")
+        # ignore this error
+      end
+
+=begin
+      def wrap(query)
+        query = query.strip
+        query = "BEGIN;\n#{query}" unless /^BEGIN;/i.match(query)
+        query = "#{query}\nEND;" unless /END;$/i.match(query)
+        query
+      end
+=end
+    end
+  end
+end

data/lib/flydata/sync_file_manager.rb

@@ -11,6 +11,8 @@ module Flydata
     BACKUP_DIR = ENV['FLYDATA_BACKUP'] || File.join(FLYDATA_HOME, 'backup')
     TABLE_POSITIONS_DIR = FLYDATA_TABLE_POSITIONS_DIR
 
+    INITIAL_SYNC_SEQ = 1
+
     def initialize(data_entry, source = nil)
       @data_entry = data_entry
       @source = source #for Source dependent objects
@@ -171,10 +173,10 @@ module Flydata
     end
 
     # table files
-    def reset_table_position_files(tables)
+    def reset_table_position_files(tables, options = {})
       tables.each do |table_name|
         file = File.join(table_positions_dir_path, table_name + ".pos")
-        File.open(file, "w") {|f| f.write('0') }
+        File.open(file, "w") {|f| f.write("0") }
       end
     end
 
@@ -204,7 +206,7 @@ module Flydata
 
     def increment_table_position(seq)
       seq = seq.to_i + 1
-      seq = FlydataCore::QueryJob::SYNC_FIRST_SEQ if seq == 1
+      seq = "#{seq}.sync" if [1,2].include?(seq)
       seq
     end
 
@@ -223,8 +225,12 @@ module Flydata
       # logical transaction ends
     end
 
+    def table_position_file_path(table_name)
+      File.join(table_positions_dir_path, table_name + ".pos")
+    end
+
     def open_table_position_file(table_name)
-      file = File.join(table_positions_dir_path, table_name + ".pos")
+      file = table_position_file_path(table_name)
       retry_count = 0
       begin
         @table_position_files[table_name] ||= (f = File.open(file, File::RDWR); f.sync = true; f)
@@ -253,6 +259,15 @@ module Flydata
       f.rewind
     end
 
+    # This doen't cache the File object
+    def save_table_positions(table_names, seq)
+      table_names = Array(table_names)
+      table_names.each do |table_name|
+        file_path = table_position_file_path(table_name)
+        File.write(file_path, seq.to_s)
+      end
+    end
+
     def get_table_position(table_name)
       f = open_table_position_file(table_name)
       seq = f.read
@@ -269,21 +284,41 @@ module Flydata
       FLYDATA_LOCK
     end
 
+    # "sync.info" file includes initial sync information for resuming
+    # - initial_sync: True if initial sync is full initial sync
+    # - tables: target tables for initial sync
+    # - auto_create_status: START|SENT_DDL|CREATED_TABLES
+
     def sync_info_file
       File.join(dump_dir, "sync.info")
     end
 
-    def save_sync_info(initial_sync, tables)
+    def delete_sync_info
+      FileUtils.rm(sync_info_file)
+    end
+
+    def save_sync_info(initial_sync, tables, auto_create_status = nil)
       File.open(sync_info_file, "w") do |f|
-        f.write([initial_sync, tables.join(" ")].join("\t"))
+        content = {
+          initial_sync: initial_sync,
+          tables: tables,
+          auto_create_status: auto_create_status,
+        }.to_json
+        f.write(content)
       end
     end
 
     def load_sync_info
       return nil unless File.exists?(sync_info_file)
-      items = File.open(sync_info_file, 'r').readline.split("\t")
-      { initial_sync: (items[0] == 'true'),
-        tables: items[1].split(" ") }
+      line = File.open(sync_info_file, 'r').readline
+      begin
+        JSON.parse(line, symbolize_names: true)
+      rescue
+        # For compatibility
+        items = line.split("\t")
+        { initial_sync: (items[0] == 'true'),
+          tables: items[1].split(" ") }
+      end
     end
 
     def get_table_source_pos_init(table_name)

data/spec/flydata/command/sync_spec.rb

@@ -116,6 +116,7 @@ module Flydata
         context 'with no stream option' do
           before do
             expect(default_sync_fm).to receive(:save_sync_info).once
+            expect(default_sync_fm).to receive(:load_sync_info).and_return(nil, {})
             expect(subject).to receive(:free_disk_space).and_return(disk_byte)
             expect(File).to receive(:dirname)
           end
@@ -139,6 +140,7 @@ module Flydata
         context 'with stream option' do
           it 'will export to io' do
             expect(default_sync_fm).to receive(:save_sync_info).once
+            expect(default_sync_fm).to receive(:load_sync_info).and_return(nil, {})
             expect_any_instance_of(Flydata::SourceMysql::Parser::MysqlDumpGeneratorNoMasterData).to receive(:dump)
 
             subject.send(:generate_source_dump, default_data_entry, default_sync_fm, false)

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -176,8 +176,13 @@ EOT
       records = rows.collect do |row|
         record = row.kind_of?(Hash) && row.keys.include?(:row) ? row : { row: row }
         @seq += 1
+        seq = if [1,2].include?(@seq.to_i)
+                "#{@seq.to_i}.sync"
+              else
+                @seq
+              end
         record.merge({ :type=>type, :table_name=>table, :respect_order=>true,
-          :seq=>@seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1,
+          :seq=>seq, :src_pos=>"#{binlog_file}\t#{position}", :table_rev=>1,
           :v=>2 })
       end
       expect_emitted_records(event, records)
@@ -383,7 +388,7 @@ EOT
               respect_order: true,
               src_pos: "mysql-bin.000048\t689",
               table_rev: 2,  # increment revision
-              seq: 2,
+              seq: "2.sync",
               v: flydata_record_version,
               actions: [{
                 action: :add_column, column: "sum", :type=>'int4(11)', :query=>'add column sum integer'}],
@@ -399,7 +404,7 @@ EOT
               respect_order: true,
               src_pos: "mysql-bin.000048\t800",
               table_rev: 2,  # increment revision
-              seq: 2,
+              seq: "2.sync",
               v: flydata_record_version,
               actions: [{
                 action: :drop_column, column: "sum", :query=>'drop column sum'}],
@@ -416,7 +421,7 @@ EOT
               respect_order: true,
               src_pos: "mysql-bin.000048\t#{337 - 217}",
               table_rev: 1,
-              seq: 2,
+              seq: "2.sync",
               v: flydata_record_version,
               actions: [{
                 action: :add_index,

data/spec/flydata/output/ddl_runner_spec.rb

@@ -0,0 +1,80 @@
+# coding: utf-8
+require 'spec_helper'
+require 'flydata/output/ddl_runner'
+
+module Flydata
+  module Output
+    describe RedshiftDDLRunner do
+      let(:pg_client) { double('pg_client') }
+      let(:dbconf) {
+        {
+          host: 'dummy-host',
+          port: 54399,
+          username: 'dummy-user',
+          password: 'dummy-pass',
+          dbname: 'dummy-db',
+        }
+      }
+      let(:schema_name) { nil }
+      let(:de) {
+        {'schema_name' => schema_name}
+      }
+      let(:subject_object) do
+        described_class.new(dbconf, de)
+      end
+
+      before do
+        allow(FlydataCore::Postgresql::PGClient).to receive(:new).and_return(pg_client)
+        allow(pg_client).to receive(:establish_connection)
+        allow(pg_client).to receive(:exec)
+        allow(pg_client).to receive(:close)
+      end
+
+      describe '.run_ddls' do
+        let(:flydata_tabledef) {
+          {:table_name=>"test_table",
+           :columns=>
+           [{:column=>"id",
+             :type=>"int4(11)",
+             :auto_increment=>true,
+             :not_null=>true,
+             :primary_key=>true},
+             {:column=>"value", :type=>"int4(11)", :default=>nil}],
+           :default_charset=>"UTF_8",
+           :src_ddl=>
+           "CREATE TABLE `test_table` (  `id` int(11) NOT NULL AUTO_INCREMENT,  `value` int(11) DEFAULT NULL,  PRIMARY KEY (`id`)) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8;",
+             :pk_override=>nil,
+             :ddl_options=>
+           {:flydata_ctl_table=>true,
+            :schema_name=>schema_name,
+            :ctl_only=>false,
+            :skip_drop_table=>false,
+            :skip_primary_key_check=>false}}
+        }
+        let(:flydata_tabledefs) { [ flydata_tabledef ] }
+        subject { subject_object.run_ddls(flydata_tabledefs) }
+
+        context 'schema name is nil(not set)' do
+          it 'skips creating schema' do
+            expect(pg_client).to receive(:exec) do |query|
+              expect(query).not_to match(/CREATE SCHEMA/)
+            end
+            subject
+          end
+        end
+
+        context 'schema name is nil(not set)' do
+          let(:schema_name) { 'dummy-schema' }
+          it 'creates schema' do
+            expect(pg_client).to receive(:exec) do |query|
+              expect(query).to match(/CREATE SCHEMA/)
+            end
+            subject
+          end
+        end
+
+      end
+    end
+  end
+
+end

data/spec/flydata/sync_file_manager_spec.rb

@@ -273,7 +273,7 @@ module Flydata
       subject { subject_object }
       context 'when an exception happens in the block' do
         let(:first_pos) { '1.sync' }
-        let(:last_pos) { first_pos.to_i + 1 }
+        let(:last_pos) { "2.sync" }
         before do
           subject.increment_and_save_table_position(test_table) do |seq|
             expect(seq).to eq first_pos

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.17
+  version: 0.7.18
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-17 00:00:00.000000000 Z
+date: 2017-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -700,6 +700,7 @@ files:
 - lib/flydata/json.rb
 - lib/flydata/json/.gitignore
 - lib/flydata/log_monitor.rb
+- lib/flydata/output/ddl_runner.rb
 - lib/flydata/output/forwarder.rb
 - lib/flydata/parser.rb
 - lib/flydata/parser/parser_provider.rb
@@ -848,6 +849,7 @@ files:
 - spec/flydata/helper/worker_spec.rb
 - spec/flydata/heroku_spec.rb
 - spec/flydata/json/json_ext_spec.rb
+- spec/flydata/output/ddl_runner_spec.rb
 - spec/flydata/output/forwarder_spec.rb
 - spec/flydata/plugin_support/context_spec.rb
 - spec/flydata/query_based_sync/client_spec.rb
@@ -910,7 +912,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: FlyData Agent