flydata 0.7.11 → 0.7.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5cb99d4fe2e890c02757c744e6c62f0946b2bf4
-  data.tar.gz: 5ca9a5c1618e2775b975adb6702d96399cb57ebf
+  metadata.gz: 77a3df344141ff66884b5deecb9b947c4616b64d
+  data.tar.gz: d3a7bbaa5ba44fb27d24d0d37e6844b4be0b809a
 SHA512:
-  metadata.gz: 634e6cee60de39ac8bdef5fe551095645b24adf5c54f9904499954b7682ee004feedcfa2d7d2a517b75ee2677706d2d9ab45430f17b67d99e0e3970b420d18d8
-  data.tar.gz: c03762c4bb3f013d349499ed50f6b4e9cbf4eb08d9943fdae5d8b6fb77eab53d9ce026813ee72122a03c8e79aa358e7c96e2aaedc49a5361615a6b99a607de22
+  metadata.gz: dcec46188c13aa32bdb56974f81642b71da51cfe1a4de1cd08eade5fcbb8b70d1a5ad329cbc1cd7ddc020571deee70bf346dc1d845362ba262dc79d5cde40a93
+  data.tar.gz: f8d1f84ebcea486f2aa30cafbdef5851c8876b6f9d21ff3220c976fd4b7306d42faa94d53ad7df9f250fd0b6f922484fda867d4bae563db828680c8a50b9d353

data/VERSION

@@ -1 +1 @@
-0.7.11
+0.7.12

data/flydata-core/lib/flydata-core/errors.rb

@@ -33,6 +33,7 @@ module ErrorCode
   REDSHIFT_ACCESS_ERROR = 10009
   BREAKING_ALTER_TABLE_ERROR = 10010
   BREAKING_INFORMATION_SCHEMA_ERROR = 10011
+  BREAKING_TABLE_DDL_ERROR = 10012
   INTERNAL_ERROR = 1200
 end
 
@@ -206,12 +207,18 @@ class StaleTableRevisionError < FlydataError
   def err_code; ErrorCode::TABLE_REVISION_ERROR; end
 end
 
-# Unsupported ALTER TABLE which breaks the tables sync consistency.
+# Unsupported ALTER TABLE which breaks the table's sync consistency.
 class BreakingAlterTableError < RecordDeliveryError
   include BreakingSyncError
   def err_code; ErrorCode::BREAKING_ALTER_TABLE_ERROR; end
 end
 
+# Unsupported DDL which breaks the table's sync consistency.
+class BreakingTableDdlError < RecordDeliveryError
+  include BreakingSyncError
+  def err_code; ErrorCode::BREAKING_TABLE_DDL_ERROR; end
+end
+
 # Unrecoverable information schema error which requires re-synchronize a table
 class BreakingInformationSchemaError < InformationSchemaError
   include BreakingSyncError

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -269,8 +269,10 @@ EOS
     return NULL_STR if default_value.nil?
     # strip type cast
     if default_value.kind_of?(String) &&
-       /^(.+?)(::"?[a-z ]+"?)*$/.match(default_value)
+        /(.+?)(::"?[^':]+?"?)?\z/m.match(default_value)
       default_value = $1
+      # strip outer single-quotes
+      default_value = $1 if /^'(.*)'$/m.match(default_value)
     end
     if flydata_type.start_with?('year')
       value = convert_year_into_date(remove_single_quote(default_value))

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -1215,6 +1215,29 @@ EOS
     subject { subject_object.replace_default_value(flydata_type, redshift_type,
               default_value) }
 
+    context 'with cast expression from PostgreSQL' do
+      context 'with multiple-lines in a default value' do
+        let(:flydata_type) { "text" }
+        let(:redshift_type) { "varchar(max)" }
+        let(:default_value) { "'--- {} \nthe 2nd line'::text" }
+        it { is_expected.to eq "'--- {} \nthe 2nd line'" }
+      end
+
+      context 'with :: in a default value' do
+        let(:flydata_type) { "varchar" }
+        let(:redshift_type) { "varchar" }
+        let(:default_value) { "'FlyDataCore::TableDef'::character varying" }
+        it { is_expected.to eq "'FlyDataCore::TableDef'" }
+      end
+
+      context 'with a single-quote in a default value' do
+        let(:flydata_type) { "varchar" }
+        let(:redshift_type) { "varchar" }
+        let(:default_value) { "'That''s it!'::character varying" }
+        it { is_expected.to eq "'That''s it!'" }
+      end
+    end
+
     context 'with integer type' do
       let(:flydata_type) { "int8" }
       let(:redshift_type) { "int8" }

data/lib/flydata/api/data_entry.rb

@@ -42,7 +42,7 @@ module Flydata
 #   "success": true
 # }
 
-      NUM_TABLES_PER_REQUEST = 110
+      NUM_TABLES_PER_REQUEST = 50
 
       def buffer_stat(data_entry_id, options = {})
         table_array = options[:tables] || ['']
@@ -180,7 +180,11 @@ module Flydata
       # - If value is nil, the setting will be deleted.
       # - If table's attribute hash doesn't have an attribute key, the setting for the attribute for the table will not be changed
       def update_table_validity(data_entry_id, table_update_hash)
-        @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, table_update_hash.to_json)
+        slice_hash(table_update_hash[:updated_tables], NUM_TABLES_PER_REQUEST).each do |table_update_hash_sl|
+          # re-construct sliced argument hash
+          arg_hash = {updated_tables: table_update_hash_sl}
+          @client.post("/#{@model_name.pluralize}/#{data_entry_id}/update_table_validity", {:headers => {:content_type => :json}}, arg_hash.to_json)
+        end
       end
 
       # Tells the server that an initial sync has completed
@@ -210,6 +214,13 @@ module Flydata
 
         message
       end
+
+      # return Array of Hash, sliced in length
+      def slice_hash(original_hash, length)
+        original_hash.each_slice(length).collect do |sl|
+          sl.inject({}) { |h, (k,v)| h[k] = v; h }
+        end
+      end
     end
   end
 end

data/lib/flydata/command/sender.rb

@@ -18,12 +18,18 @@ module Flydata
         else
           options = {show_final_message: options_or_show_final_message}
         end
+
         # Check if process exist
         if process_exist?
           log_info_stdout("Process is still running. Please stop process first.") unless options[:quiet]
           return
         end
 
+        dp = flydata.data_port.get
+        if dp['paused']
+          raise "This application has been stopped. Process cannot be started."
+        end
+
         if agent_locked?
           log_info_stdout("Agent was not shut down properly.  Agent will check the status and fix itself if necessary.")
           repair_opts = Flydata::Command::Sync.slop_repair
@@ -42,7 +48,6 @@ module Flydata
 
         wait_until_server_ready(options)
 
-        dp = flydata.data_port.get
         AgentCompatibilityCheck.new(dp).check
 
         fluentd_started = false

data/lib/flydata/command/sync.rb

@@ -302,8 +302,8 @@ EOS
       # Command: flydata sync:repair
       #  - Entry method
       def repair
-        _repair
-        unless opts.skip_start?
+        need_to_start = _repair
+        if need_to_start && !opts.skip_start?
           Flydata::Command::Sender.new.start
         end
       end
@@ -385,7 +385,11 @@ EOS
         abnormal_shutdown = false
         begin
           begin
-            flush_buffer_and_stop(@full_tables, force: false, timeout: 55) # A short timeout is set.  Otherwise, the check command and the repair command takes forever to complete.
+            # wait until no data gets processed for 3 minutes.  This is long
+            # to wait, but there *are* cases where active data processing takes
+            # more than 3 minutes between 2 chunk processing if COPY command
+            # takes minutes to process.
+            flush_buffer_and_stop(@full_tables, force: false, timeout: 180)
           rescue ServerDataProcessingTimeout => e
             data_stuck_at = e.state
           end
@@ -450,7 +454,7 @@ EOS
         if status.include? :OK
           log_info_stdout ""
           log_info_stdout "Sync is in good condition.  Nothing to repair."
-          return
+          return true
         end
 
         if status.include?(:ABNORMAL_SHUTDOWN) && status.uniq.length == 1
@@ -460,7 +464,7 @@ EOS
 
           # Remove the lock file if exists.
           File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
-          return
+          return true
         end
 
         gt = []
@@ -495,7 +499,10 @@ EOS
 
 EOS
 
-        return unless ask_yes_no("Proceed?")
+        unless ask_yes_no("Proceed?")
+          log_info_stdout "FlyData agent is stopped. Run 'flydata sync:repair' again for retry."
+          return false  # Not restart
+        end
 
         oldest_source_pos = get_oldest_available_source_pos
         unrepairable_tables = []
@@ -577,6 +584,7 @@ EOS
         File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
 
         log_info_stdout "Repair is done.  Restarting."
+        true
       end
 
       # Initial sync

data/lib/flydata/query_based_sync/client.rb

@@ -63,13 +63,22 @@ module QueryBasedSync
           sleep @fetch_interval
         end
       rescue => e
-        #TODO: Introduce retryable error notions
-        log_error_with_backtrace("Unexpected error occured", error: e)
+        handle_error(e, "Unexpected error occured")
         sleep @retry_interval
         retry
       end
     end
 
+    def handle_error(error, message)
+      log_method = if error.kind_of?(FlydataCore::RetryableError)
+                     error = error.original_exception
+                     :log_warn
+                   else
+                     :log_error
+                   end
+      self.send(log_method, message, {error: error}, {backtrace: true})
+    end
+
     def run_once
       resource_requester.start do |req|  # open connection
         context.tables.each do |table_name|

data/lib/flydata/source_postgresql/generate_source_dump.rb

@@ -99,7 +99,7 @@ EOS
     src_pos
   end
 
-  NUM_ROWS = 500000
+  NUM_ROWS = 50000
 
   def dump_table(tabledef, source_pos, io, cli)
     dump_source_table(tabledef, io)

data/lib/flydata/source_postgresql/query_based_sync/resource_requester.rb

@@ -63,7 +63,11 @@ class ResourceRequester < Flydata::QueryBasedSync::ResourceRequester
     # Create and execute query
     query = generate_query(t_meta, from_sid, to_sid, pk_values)
 
-    result = @resource_client.query(query)
+    begin
+      result = @resource_client.query(query)
+    rescue PG::TRSerializationFailure => e
+      raise FlydataCore::RetryableError.new(e)
+    end
 
     # Create response object based on result
     responses = build_responses(table_name, result, from_sid: from_sid, to_sid: to_sid, pk_values: pk_values)

data/spec/flydata/api/data_entry_spec.rb

@@ -175,10 +175,10 @@ describe DataEntry do
         end
       end
     end
-    context "with more than 100 tables" do
-      let(:tables) { 180.times.collect{|i| "tbl#{i}"} }
-      let(:expected_tables1) { tables[0...110].join(",") }
-      let(:expected_tables2) { tables[110..-1].join(",") }
+    context "with more than 50 tables" do
+      let(:tables) { 80.times.collect{|i| "tbl#{i}"} }
+      let(:expected_tables1) { tables[0...50].join(",") }
+      let(:expected_tables2) { tables[50..-1].join(",") }
 
       let(:expected_message) { double('expected_message') }
 
@@ -318,10 +318,10 @@ describe DataEntry do
     let(:status21) { double('status21') }
     let(:status22) { double('status22') }
 
-    context "with more than 100 tables" do
-      let(:tables) { 180.times.collect {|i| "tbl#{i}"} }
-      let(:expected_tables1) { tables[0...110].join(",") }
-      let(:expected_tables2) { tables[110..-1].join(",") }
+    context "with more than 50 tables" do
+      let(:tables) { 80.times.collect {|i| "tbl#{i}"} }
+      let(:expected_tables1) { tables[0...50].join(",") }
+      let(:expected_tables2) { tables[50..-1].join(",") }
 
       it do
         expect(api_client).to receive(:post).

data/spec/flydata/query_based_sync/client_spec.rb

@@ -71,6 +71,40 @@ describe Client do
         expect{subject_object.start}.to raise_error(/Already started/)
       end
     end
+
+    context 'when getting non-retryable error' do
+      before do
+        context.instance_variable_get(:@params)[:retry_interval] = 0.5
+        expect_any_instance_of(DummyResourceRequester).to receive(:start).and_raise("dummy error")
+        expect_any_instance_of(DummyResourceRequester).to receive(:start).and_return(nil)
+      end
+      it 'retries after logging in error level' do
+        expect($log).to receive(:error) do |msg|
+          expect(msg).to include("dummy error")
+        end
+        expect($log).to receive(:warn).never
+        start_client(0.2)
+        sleep 0.6
+      end
+    end
+
+    context 'when getting retryable error' do
+      before do
+        context.instance_variable_get(:@params)[:retry_interval] = 0.5
+        expect_any_instance_of(DummyResourceRequester).to receive(:start).and_raise(
+          FlydataCore::RetryableError.new(RuntimeError.new("dummy retryable error")))
+        expect_any_instance_of(DummyResourceRequester).to receive(:start).and_return(nil)
+      end
+      it 'retries after logging in warn level' do
+        expect($log).to receive(:error).never
+        expect($log).to receive(:warn) do |msg|
+          expect(msg).to include("dummy retryable error")
+        end
+        start_client(0.2)
+        sleep 0.6
+      end
+    end
+
   end
 end
 

data/spec/flydata/source_postgresql/query_based_sync/resource_requester_spec.rb

@@ -62,6 +62,16 @@ describe ResourceRequester do
       let(:table_1_snapshot) { current_snapshot }
       it { is_expected.to be_nil }
     end
+
+    context 'when getting PG::TRSerializationFailure' do
+      before do
+        allow_any_instance_of(FlydataCore::Postgresql::PGClient).to receive(:query).and_raise(
+          PG::TRSerializationFailure.new("ERROR:  canceling statement due to conflict with recovery DETAIL:  User query might have needed to see row versions that must be removed.")
+        )
+      end
+
+      it { expect{subject}.to raise_error(FlydataCore::RetryableError) }
+    end
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.11
+  version: 0.7.12
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client