flydata 0.7.16 → 0.7.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aafcbf0332b985cd94a8fa83302bbccdb5ef86f1
-  data.tar.gz: f6842935284bacbd7cc5046f640c65c376b28bfb
+  metadata.gz: 340f31b8701b211b25628d5690237565ff5e46c6
+  data.tar.gz: 25bbb20fa4a67ca4f25f20917a350e649a224b80
 SHA512:
-  metadata.gz: 8f6ef145d7f026237e35dd351f306b815a901e654a3fa67bc1ddd251fcad5fbc003c5780e2b93260f670f3692efcf78160c8ea688466f70eea91ebe12a3ade8d
-  data.tar.gz: 268449b51660a61f40aa8a731107630c2876c41b656ef4063b2eb41b925162d942c2963017d15a83d91dd78366cacc698d3c8c51d43bc6e2af5fe55b7865d0e4
+  metadata.gz: ae89812c8d23ad80ec1777ec1a6c39764808d7279ff54db865b7bc1cd090045cef87f426af834aaf50d147f1d7c2d9eef1a16150b6844d79701313f84c7b3a17
+  data.tar.gz: 4f6c6c97c6c04ebee65780bf31930fc58a6316cf7830bcac5defd63bc8a9db68d3062577d142e09cd1d86f9cc66b7c4bd207d6b5094ae6c4c89f48b983c88d57

data/VERSION

@@ -1 +1 @@
-0.7.16
+0.7.17

data/lib/flydata/command/sync.rb

@@ -279,13 +279,15 @@ EOS
       def generate_table_ddl(*tables)
         # Compatibility check
         de = data_entry
-        context = source.sync_generate_table_ddl(flydata.data_port.get, opts)
+        context = source.sync_generate_table_ddl(flydata.data_port.get)
         context.run_compatibility_check
 
         # Set instance variables
         set_current_tables(tables, include_all_tables: true)
 
-        do_generate_table_ddl(context, de)
+        unless generate_and_print_table_ddl(context, de)
+          raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`"
+        end
       end
       run_exclusive :generate_table_ddl
 
@@ -355,7 +357,7 @@ EOS
       # Command: flydata sync:compat_check
       #  - Entry method
       def compat_check
-        context = source.sync_generate_table_ddl(flydata.data_port.get, opts)
+        context = source.sync_generate_table_ddl(flydata.data_port.get)
         context.run_compatibility_check
       end
       run_exclusive :compat_check
@@ -561,7 +563,12 @@ EOS
 
           # Delete query queue items for the tables
           log_info_stdout "Deleting data stuck in the server buffer..."
-          cleanup_sync_server(de, broken_tables, queue_only: true) unless broken_tables.empty?
+          unless broken_tables.empty?
+            cleanup_sync_server(de, broken_tables, queue_only: true)
+            # refetch table_status to get the latest seq and source_pos info.  The values
+            # may have been updated since the last fetch called from #_check
+            table_status_hash = get_table_status(broken_tables, context)
+          end
 
           # Save the positions (source_pos and seq)
           log_info_stdout "Fixing table positions..."
@@ -893,15 +900,7 @@ EOM
         end
 
         # Prepare forwarder
-        de_tag_name = de["tag_name#{env_suffix}"]
-        server_port = dp['server_port']
-        data_servers = source.sync.data_servers
-        servers = data_servers ?  data_servers.split(',') :
-                   dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
-        forwarder_type = source.sync.forwarder ||
-          (dp['ssl_enabled'] ? 'sslforwarder' : 'tcpforwarder')
-        forwarder = Flydata::Output::ForwarderFactory.create(forwarder_type, de_tag_name, servers)
-
+        forwarder = build_forwarder(dp, de)
         log_info_stdout("Sending data to FlyData Server...")
 
         # initialize stats
@@ -1220,32 +1219,42 @@ EOM
         log_info_stdout("Done.")
       end
 
-      # Generate table ddl
+      # Generate and print table ddl
+      def generate_and_print_table_ddl(context, de)
+        generate_flydata_tabledefs(context, de) do |flydata_tabledef|
+          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_tabledef[:ddl_options])
+        end
+      end
 
-      def do_generate_table_ddl(context, de)
-        error_list = []
+      def generate_flydata_tabledefs(context, de)
         schema_name = (de['schema_name'] || nil)
-
         tables = opts.all_tables? ? @full_tables : (@input_tables.empty? ? @unsynced_tables : @input_tables)
 
-        raise "There are no valid unsynced tables, if you want to just get ddl for all tables, please run \`flydata sync:generate_table_ddl --all-tables\`" if tables.empty?
-
-        if tables.empty?
-          raise "`tables` (or `tables_append_only`) is neither defined in the data entry nor the local config file"
-        end
+        # Target table checks
+        return nil if tables.empty?
 
+        # Create flydata table definitions with error list
         flydata_tabledefs, error_list, uk_as_pk_override = context.generate_flydata_tabledef(tables,
                         skip_primary_key_check: opts.skip_primary_key_check?)
 
+        # Output DDL for Redshift
         create_flydata_ctl_table = true
         append_only = tables & @append_only_tables
         flydata_tabledefs.each do |flydata_tabledef|
-          skip_drop_table = opts.drop_append_only? ? false : append_only.include?(flydata_tabledef[:table_name])
-          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef,
-            flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?, skip_drop_table: skip_drop_table, skip_primary_key_check: opts.skip_primary_key_check?)
+          ddl_options = {
+            flydata_ctl_table: create_flydata_ctl_table,
+            schema_name: schema_name,
+            ctl_only: opts.ctl_only?,
+            skip_drop_table: opts.drop_append_only? ? false : append_only.include?(flydata_tabledef[:table_name]),
+            skip_primary_key_check: opts.skip_primary_key_check? }
+          flydata_tabledef[:ddl_options] = ddl_options
+          if block_given?
+            yield flydata_tabledef
+          end
           create_flydata_ctl_table = false
         end
 
+        # Set error info to table_validity_hash for error tables
         table_validity_hash = Hash.new {|h,k| h[k] = {}}
         tables_without_error = tables
         unless error_list.empty?
@@ -1270,17 +1279,34 @@ The other tables are ready to sync.  To start sync, run the generated script on
 EOS
           end
         end
+
+        # Reset table error of tables having no error
         tables_without_error.each {|table| table_validity_hash[table][TableAttribute::INVALID_TABLE_REASON] = nil }
 
-        # update table validity and uk_as_pk info on web server
+        # Update table validity and uk_as_pk info on web server
         uk_as_pk_override.each_pair {|table, val| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = val}
         (tables - uk_as_pk_override.keys).each {|table| table_validity_hash[table][TableAttribute::UK_AS_PK_OVERRIDE] = nil }
 
+        # Update table validity on flydata-web
         flydata.data_entry.update_table_validity(de['id'], updated_tables: table_validity_hash) unless table_validity_hash.empty?
 
+        # Create per-table generated_ddl file on postiions directory
         sync_fm = create_sync_file_manager(de)
         sync_fm.save_generated_ddl(tables_without_error, TableDdl::VERSION)
         sync_fm.close
+
+        flydata_tabledefs
+      end
+
+      def build_forwarder(dp, de)
+        de_tag_name = de["tag_name#{env_suffix}"]
+        server_port = dp['server_port']
+        data_servers = source.sync.data_servers
+        servers = data_servers ?  data_servers.split(',') :
+                   dp["servers#{env_suffix}"].collect{|s| "#{s}:#{server_port}"}
+        forwarder_type = source.sync.forwarder ||
+          (dp['ssl_enabled'] ? 'sslforwarder' : 'tcpforwarder')
+        Flydata::Output::ForwarderFactory.create(forwarder_type, de_tag_name, servers)
       end
 
       ALL_DONE_MESSAGE_TEMPLATE =  <<-EOM

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -10,7 +10,7 @@ class SyncGenerateTableDdl < Component
     Source.register(child_class, self)
   end
 
-  def initialize(source, dp, options)
+  def initialize(source, dp, options = {})
     super(source, options)
     @dp = dp
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.16
+  version: 0.7.17
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-11 00:00:00.000000000 Z
+date: 2017-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client