flydata 0.7.15 → 0.7.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5435f4af64a22277e8417bbdd7196ee8aabc6c19
-  data.tar.gz: 7bbbff8d0c4cb13891f35dc8debf891b2aa31953
+  metadata.gz: aafcbf0332b985cd94a8fa83302bbccdb5ef86f1
+  data.tar.gz: f6842935284bacbd7cc5046f640c65c376b28bfb
 SHA512:
-  metadata.gz: 646f5ec5b0f3be8102a80c386291c95b6ad220c3e3401b89acea12ab7b59179b3a806fdfc67c403f9ed44d4a08a7080a49f48528109257faad2c1c3cd9fd84f2
-  data.tar.gz: 179efaee49c60d6573da9b0f6f57476fc8ae26f43ab70bee591f174fec5ba09b90d5a6c4ea2cf95f91417d73f8144d89ae2a568457b28d2034c9479eb629bc17
+  metadata.gz: 8f6ef145d7f026237e35dd351f306b815a901e654a3fa67bc1ddd251fcad5fbc003c5780e2b93260f670f3692efcf78160c8ea688466f70eea91ebe12a3ade8d
+  data.tar.gz: 268449b51660a61f40aa8a731107630c2876c41b656ef4063b2eb41b925162d942c2963017d15a83d91dd78366cacc698d3c8c51d43bc6e2af5fe55b7865d0e4

data/VERSION

@@ -1 +1 @@
-0.7.15
+0.7.16

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -275,6 +275,9 @@ EOS
       # strip outer single-quotes
       default_value = $1 if /^'(.*)'$/m.match(default_value)
     end
+    if default_value.kind_of?(String) &&/^NULL$/i.match(default_value)
+      return NULL_STR
+    end
     if flydata_type.start_with?('year')
       value = convert_year_into_date(remove_single_quote(default_value))
       begin

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -420,6 +420,13 @@ EOT
           let(:default_value_sql) { "'0001-01-01 00:00:00.000000'" }
           it_behaves_like *examples
         end
+
+        context 'when default_value has PostgreSQL-style NULL with cast' do
+          let(:default_value) { "NULL::timestamp without time zone" }
+          let(:default_value_sql) { "NULL" }
+
+          it_behaves_like *examples
+        end
       end
 
       context 'with datetimetz column def' do
@@ -548,6 +555,18 @@ EOT
 
           it_behaves_like *examples
         end
+
+        context 'when default_value has PostgreSQL-style NULL with cast' do
+          let(:default_value) { "NULL::numeric" }
+          let(:default_value_sql) { "NULL" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric(28,2)"
+          end
+          let(:type_sql) { %Q|"value" numeric(28,2)| }
+
+          it_behaves_like *examples
+        end
       end
 
       context 'with bit column def' do

data/lib/flydata/command/sync.rb

@@ -341,7 +341,7 @@ EOS
             message += "\n"
           end
           if pos_mismatch_tables
-            message += " - Incorrect table position(s)\n"
+            message += " - Incorrect table position.  This may not be a real issue, caused by pending upload chunks.  Run sync:flush and try again.\n"
             pos_mismatch_tables.each do |bt|
               message += "     table:#{bt[:table]}, agent position:#{bt[:agent_seq] ? bt[:agent_seq] : '(missing)'}, server position:#{bt[:server_seq]}\n"
             end
@@ -389,7 +389,8 @@ EOS
             # to wait, but there *are* cases where active data processing takes
             # more than 3 minutes between 2 chunk processing if COPY command
             # takes minutes to process.
-            flush_buffer_and_stop(@full_tables, force: false, timeout: 180)
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 180,
+                                  dont_wait_upload: true)
           rescue ServerDataProcessingTimeout => e
             data_stuck_at = e.state
           end
@@ -468,13 +469,28 @@ EOS
           return true
         end
 
-        gt = []
-        pt = []
-        gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
-        pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
-        # position mismatch can be due to query queue items discarded by the copy handler
-        # so it also needs to be in the target table list.
-        tables = gt | pt
+        if corrupt_master_pos_files
+          # TODO This case can be repaied by implementing the following steps.
+          # 1. Flush all pending data (buffer & query queue items) except for
+          #    ones blocked by a gap.
+          # 2. Pick the latest src_pos value from table_status and put as the
+          #    sent.pos.
+          # 3. Rerun sync:repair
+          #
+          log_info_stdout ""
+          log_info_stdout "Master position file(s) is broken.  Full sync reset is necessary to solve the issue."
+          return false
+        end
+
+        broken_tables = []
+        broken_tables = gap_tables.collect{|bt| bt[:table] } if gap_tables
+
+        if broken_tables.empty? && !corrupt_master_pos_files &&
+           !status.include?(:ABNORMAL_SHUTDOWN)
+          log_info_stdout ""
+          log_info_stdout "Sync is in good condition.  Nothing to repair."
+          return true
+        end
 
         # table_status will be created at the end of initial sync.
         # ignore tables that have no table_status, i.e. tables never got init-synced before.
@@ -485,14 +501,9 @@ EOS
 The following issues have been found and will be repaired.
 
 EOS
-        log_info_stdout <<EOS unless tables.empty?
+        log_info_stdout <<EOS unless broken_tables.empty?
  - Sync is broken due to missing data for the following tables
-     #{tables.join("\n     ")}
-
-EOS
-        log_info_stdout <<EOS if corrupt_master_pos_files
- - Sync is broken due to corrupt master position files
-     #{corrupt_master_pos_files.join("\n     ")}
+     #{broken_tables.join("\n     ")}
 
 EOS
         log_info_stdout <<EOS if status.include? :ABNORMAL_SHUTDOWN
@@ -516,25 +527,12 @@ EOS
             # the table status has no src_pos (which is rare.)  Skip the table
             next
           end
-          if tables.empty?
-            # Find the latest source pos to which all table data is up to date
-            if table_status_hash[table]["seq"] == 0
-              # this table's source pos is from the initial sync.  It can't
-              # be used for determining the latest source pos because it can be
-              # newer than the correct cont sync source pos.
-              next
-            end
-            if sent_source_pos.nil? || sent_source_pos < table_source_pos
-              sent_source_pos = table_source_pos
-            end
-          else
-            if tables.include?(table)
-              if sent_source_pos.nil? || sent_source_pos > table_source_pos
-                if oldest_source_pos && table_source_pos < oldest_source_pos
-                  unrepairable_tables << table
-                else
-                  sent_source_pos = table_source_pos
-                end
+          if broken_tables.include?(table)
+            if sent_source_pos.nil? || sent_source_pos > table_source_pos
+              if oldest_source_pos && table_source_pos < oldest_source_pos
+                unrepairable_tables << table
+              else
+                sent_source_pos = table_source_pos
               end
             end
           end
@@ -561,30 +559,26 @@ EOS
           end
           master_source_pos = context.resume_pos(sent_source_pos)
 
-          # Delete agent buffer
-          log_info_stdout "Deleting data in the agent buffer..."
-          files = Flydata::Agent.new(FLYDATA_HOME).delete_buffer_files
-          unless files.empty?
-            $log.debug "Deleted buffer files\n  " + files.join("\n  ")
-          end
-
           # Delete query queue items for the tables
           log_info_stdout "Deleting data stuck in the server buffer..."
-          cleanup_sync_server(de, tables, queue_only: true) unless tables.empty?
+          cleanup_sync_server(de, broken_tables, queue_only: true) unless broken_tables.empty?
 
           # Save the positions (source_pos and seq)
           log_info_stdout "Fixing table positions..."
           synced_tables.each do |table|
-             table_source_pos = table_status_hash[table]["source_pos"]
-             if table_source_pos.nil? || table_source_pos.empty?
-               # no init sync has happened to the table.  No need to set positions.
-               next
-             end
-             pos = table_status_hash[table]["seq"]
-             old_source_pos, old_pos = save_table_positions(table, table_source_pos, pos, context)
-             if pos.to_i != old_pos.to_i && !tables.include?(table)
-               $log.debug "Fixed broken table position.  table:#{table} pos:#{old_pos} -> #{pos}"
-             end
+            if broken_tables.include?(table)
+              table_source_pos = table_status_hash[table]["source_pos"]
+              if table_source_pos.nil? || table_source_pos.empty?
+                # no init sync has happened to the table.  No need to set positions.
+                next
+              end
+              pos = table_status_hash[table]["seq"]
+            else
+              # use "default" positions supplied by #save_table_positions
+              table_source_pos = nil
+              pos = nil
+            end
+            old_source_pos, old_pos = save_table_positions(table, table_source_pos, pos, context)
           end
 
           log_info_stdout "Fixing the master position files..."
@@ -1070,6 +1064,10 @@ EOM
             break
           end
 
+          if state == :UPLOAD && option[:dont_wait_upload]
+            break
+          end
+
           if status['message'] != prev_message
             # making some progress.  Reset timer
             start_time = Time.now
@@ -1081,7 +1079,7 @@ EOM
           print_progress(status)
           sleep 10
         end
-        if (state == :PROCESS)
+        if state == :PROCESS && !option[:dont_wait_upload]
           # :UPLOAD state was skipped due to no data
           log_info_stdout("  -> Done")
           log_info_stdout("Finishing data upload...")
@@ -1312,7 +1310,8 @@ Thank you for using FlyData!
 
         timeout = options.has_key?(:timeout) ? options[:timeout]
                                              : SERVER_DATA_PROCESSING_TIMEOUT
-        wait_for_server_data_processing( timeout: timeout, tables: tables)
+        wait_for_server_data_processing(timeout: timeout, tables: tables,
+          dont_wait_upload: options[:dont_wait_upload])
       end
 
       # Utility methods
@@ -1459,8 +1458,17 @@ Thank you for using FlyData!
       def save_table_positions(table, source_pos, pos, context)
         de = data_entry
         sync_fm = create_sync_file_manager(de)
+
+        master_sent_source_pos = sync_fm.load_sent_source_pos
         old_source_pos = sync_fm.get_table_source_pos(table)
         old_pos = sync_fm.get_table_position(table)
+
+        source_pos ||= old_source_pos
+        source_pos ||= master_sent_source_pos
+        raise "master sent source pos is missing" unless source_pos
+        pos ||= old_pos
+        raise "table pos is missing for table '#{table}'" unless pos
+
         if pos.to_i != old_pos.to_i
           sync_fm.save_table_position(table, pos)
           $log.debug "table pos updated.  table:#{table} pos:#{old_pos} -> #{pos}"

data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb

@@ -32,6 +32,7 @@ class PostgresqlQueryBasedFlydataInput < Input
       tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
       table_revs: @table_revs, dbconf: @dbconf,
       cur_src_pos_file: @source_position_file,
+      cur_sent_pos_file: @sent_position_file,
       table_src_pos_files: @table_src_pos_files,
       table_meta: @table_meta,
       params: {

data/lib/flydata/helper/action/repair.rb

@@ -0,0 +1,13 @@
+require 'flydata/helper/action/agent_action'
+
+module Flydata
+  module Helper
+    module Action
+      class Repair <  AgentAction
+        def command
+          "sync:repair -y"
+        end
+      end
+    end
+  end
+end

data/lib/flydata/helper/action_ownership.rb

@@ -48,6 +48,7 @@ module Flydata
           self.new(:stop_agent, true, Action::StopAgent),
           self.new(:restart_agent, true, Action::RestartAgent),
           self.new(:update_helper_config, false, Action::UpdateHelperConfig),
+          self.new(:repair, true, Action::Repair),
         ].inject({}) do |h, action|
           h[action.action_name] = action
           h

data/lib/flydata/plugin_support/context.rb

@@ -36,6 +36,7 @@ module PluginSupport
                             :table_meta
 
     register_optional_opts  :cur_src_pos_file,
+                            :cur_sent_pos_file,
                             :table_src_pos_files,
                             :params
 

data/lib/flydata/query_based_sync/client.rb

@@ -101,6 +101,9 @@ module QueryBasedSync
 
       # Set the current snapshot to master(resume) binlog.pos file
       context.cur_src_pos_file.save(context.table_meta.current_snapshot)
+      # Also set the same value to the sent.binlog.pos file as the file is
+      # required by the agent commands.
+      context.cur_sent_pos_file.save(context.table_meta.current_snapshot)
 
       log_info("Updated source position -", resume_pos:context.table_meta.current_snapshot)
     end

data/spec/flydata/helper/action/repair_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+require_relative '../helper_shared_context'
+require 'flydata/helper/action/repair'
+
+module Flydata
+  module Helper
+    module Action
+      describe Repair do
+        include_context 'helper context'
+
+        describe "#execute" do
+          subject { described_class.new(config).execute }
+          it "executes the repair action" do
+            expect(Open3).to receive(:capture3).with({}, "flydata sync:repair -y")
+            subject
+          end
+        end
+      end
+    end
+  end
+end

data/spec/flydata/helper/action_ownership_spec.rb

@@ -36,13 +36,13 @@ module Flydata
       describe '.action_ownership_map' do
         subject { described_class.action_ownership_map }
         it do
-          expect(subject.size).to eq(6)
           expect(subject.include?(:check_remote_actions)).to be(true)
           expect(subject.include?(:check_abnormal_shutdown)).to be(true)
           expect(subject.include?(:send_logs)).to be(true)
           expect(subject.include?(:stop_agent)).to be(true)
           expect(subject.include?(:restart_agent)).to be(true)
           expect(subject.include?(:update_helper_config)).to be(true)
+          expect(subject.include?(:repair)).to be(true)
         end
       end
     end

data/spec/flydata/query_based_sync/query_based_sync_context.rb

@@ -69,12 +69,14 @@ shared_context 'query based sync context' do
       ]
   } }
   let(:cur_src_pos_file) { double('cur_src_pos_file') }
+  let(:cur_sent_pos_file) { double('cur_sent_pos_file') }
   let(:table_1_src_pos_file) { double('table_1_src_pos_file') }
   let(:table_2_src_pos_file) { double('table_2_src_pos_file') }
   let(:table_3_src_pos_file) { double('table_3_src_pos_file') }
   let(:context) {
     Flydata::PluginSupport::Context.new(
       cur_src_pos_file: cur_src_pos_file,
+      cur_sent_pos_file: cur_sent_pos_file,
       tables: %w(table_1 table_2 table_3),
       tag: 'test_tag',
       sync_fm: sync_fm,
@@ -107,6 +109,7 @@ shared_context 'query based sync context' do
     allow(table_meta).to receive(:[]).with(:table_1).and_return(table_meta_1)
     allow(table_meta).to receive(:current_snapshot).and_return(current_snapshot)
     allow(cur_src_pos_file).to receive(:save).with(current_snapshot)
+    allow(cur_sent_pos_file).to receive(:save).with(current_snapshot)
     setup_dummy_objects
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.15
+  version: 0.7.16
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-13 00:00:00.000000000 Z
+date: 2017-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -681,6 +681,7 @@ files:
 - lib/flydata/helper/action/agent_action.rb
 - lib/flydata/helper/action/check_abnormal_shutdown.rb
 - lib/flydata/helper/action/check_remote_actions.rb
+- lib/flydata/helper/action/repair.rb
 - lib/flydata/helper/action/restart_agent.rb
 - lib/flydata/helper/action/send_logs.rb
 - lib/flydata/helper/action/stop_agent.rb
@@ -835,6 +836,7 @@ files:
 - spec/flydata/fluent-plugins/sync_source_plugin_context.rb
 - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
 - spec/flydata/helper/action/check_remote_actions_spec.rb
+- spec/flydata/helper/action/repair_spec.rb
 - spec/flydata/helper/action/restart_agent_spec.rb
 - spec/flydata/helper/action/send_logs_spec.rb
 - spec/flydata/helper/action/stop_agent_spec.rb
@@ -908,7 +910,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent