flydata 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a74c7dd378333e32076bf9d8dfb6f445f2a6b07
-  data.tar.gz: 4fbc074d26fee614f5315b674159543db96da68b
+  metadata.gz: 47e53f1fbb8f9c3312cbac20679d9b389d41994a
+  data.tar.gz: d3ab901fdc3747ab54b82b2a087c3a0aa74fe44f
 SHA512:
-  metadata.gz: 5e88618472bc0641320931b46ab357ba02c0f17d678b1b66e5d1ef467e2ca4b0518414d8c2037f6177c310b18c2dbb7825b090649ad6615ec621e687a9355118
-  data.tar.gz: ad9d6fd284b20b55f36ed25e4c86d54fc323e856d691179e7611a87c424f94c892b33dc1ea09d8e35b7b6e49b861e573ce03611272020d69913982ea9c15473e
+  metadata.gz: 9f92fd99a9411b656ef014e3e64214d50c5549c4eac74b1dbde279f65ccba452dde8698bc26d5cf1dd145b092dcf459b2f839f996a804fbd120d2f5e361de30d
+  data.tar.gz: 9f2181a206ffa141fa7bca9822dcd824ca05e45d65ffe030cc1a0c055c27fe8b7161855e835817813d214ef9ffa78beeeb6e29e6f8c9410bc73368a02017b421

data/VERSION

@@ -1 +1 @@
-0.7.6
+0.7.7

data/lib/flydata/command/sync.rb

@@ -295,10 +295,10 @@ EOS
           message = "\nFollowing errors are found.\n"
 
           if status.include? :STUCK_AT_PROCESS
-            message += " - Data is stuck while processing\n"
+            message += " - Timeout while processing data\n"
           end
           if status.include? :STUCK_AT_UPLOAD
-            message += " - Data is stuck while uploading\n"
+            message += " - Timeout while uploading data\n"
           end
           if status.include? :ABNORMAL_SHUTDOWN
             message += " - Agent was not shut down correctly\n"
@@ -411,13 +411,11 @@ EOS
              _check(context, stop_agent:true)
 
         if status.include? :STUCK_AT_PROCESS
-          e = AgentError.new("Data is stuck while processing")
+          e = AgentError.new("Timeout while processing data")
           e.description = <<EOS
-Data appears to be stuck while processing. Run the following command, and try `flydata sync:repair` again:
+We are currently processing the remaining buffered data. Please wait a few minutes and repeat `flydata sync:repair` command.
 
-flydata sync:flush
-
-If you need help, please contact FlyData Support (support@flydata.com).
+If this issue continues, please contact FlyData Support (support@flydata.com).
 EOS
           raise e
         end
@@ -551,7 +549,7 @@ EOS
         # Remove the lock file if exists.
         File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
 
-        log_info_stdout "Repair is done.  Start Agent with `flydata start` command."
+        log_info_stdout "Repair is done.  Restarting."
       end
 
       # Initial sync
@@ -1270,25 +1268,34 @@ Thank you for using FlyData!
 
         table_lists = source.sync.table_lists
 
-        #full_tables will either include all tables including invalid tables or all valid tables that aren't new tables
+        # `full_tables` will either
+        #  - include all tables (full-sync & append_only) including invalid tables, or
+        #  - all valid tables that aren't new tables
 
         # The 'new_tables' list may include tables which has already been
         # synced for backward compatibility reason.
         # Filter out such tables so that we get a list of 'new' tables with no
         # position file.
         real_new_tables = sync_fm.get_new_table_list(table_lists['new_tables'], "pos")
-        @full_tables = options[:include_all_tables] ? table_lists['tables'] + table_lists['invalid_tables'] :
-                                                      table_lists['tables'] - real_new_tables
+        valid_tables_append_only = table_lists['tables_append_only'] || []
+
+        if options[:include_all_tables]
+          @full_tables = table_lists['tables'] + table_lists['invalid_tables']
+          @append_only_tables = valid_tables_append_only + table_lists['invalid_tables_append_only']
+        else
+          @full_tables = table_lists['tables'] - real_new_tables
+          @append_only_tables = valid_tables_append_only
+        end
 
-        @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
-        @no_ddl_generated_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl") # tables do not have a .generated_ddl file
+        # tables that do not have a .pos file
+        @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos")
+        # tables that do not have a .generated_ddl file
+        @no_ddl_generated_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
 
         @input_tables = sync_resumed ? sync_info[:tables] : input_tables
         @input_tables ||= []
         @full_initial_sync = sync_resumed ? sync_info[:initial_sync] :
                                             (@unsynced_tables == @full_tables)
-        @append_only_tables = table_lists['tables_append_only']
-        @append_only_tables ||= []
 
         sync_fm.close
 

data/lib/flydata/source/sync.rb

@@ -7,6 +7,9 @@ module Flydata
 module Source
 
 class Sync < Component
+  # subclass has to define these constant variables
+  #SOURCE_PREFERENCE_NAME = 'xxxxx_data_entry_preference'
+
   def self.inherited(child_class)
     Source.register(child_class, self)
   end
@@ -18,7 +21,9 @@ class Sync < Component
   #
   # Raises exception when the source does not support sync
   def setup
-    raise UnsupportedSourceError,"subclass must implement"
+    prefs = de[self.class::SOURCE_PREFERENCE_NAME]
+    setup_table_prefs(prefs)
+    setup_pk_override_prefs(prefs)
   end
 
   # Public Interface: Tells if the source support sync or not
@@ -30,47 +35,54 @@ class Sync < Component
 
   # Public Interface: Table lists
   #
-  # Returns lists of tables in a hash.  The following lists will be returned
-  #   "tables" : An array of tables currently in sync
+  # Returns lists of tables in a hash. The following lists will be returned
+  #   "tables" : An array of tables currently in sync (valid tables append-only & valid tables full-sync)
   #   "new_tables" : An array of tables for which no generate_table_ddl has been run yet
   #   "invalid_tables" : An array of tables that had an issue starting sync
+  #   "tables_append_only" : An array of tables whose sync type is append-only
+  #   "invalid_tables_append_only" : An array of append-only tables that had an issue starting sync
+  SOURCE_TABLE_LISTS_KEYS = %w(
+    tables
+    new_tables
+    invalid_tables
+    tables_append_only
+    invalid_tables_append_only
+  )
   def table_lists
-    raise UnsupportedSourceError, "subclass must implement"
+    de[self.class::SOURCE_PREFERENCE_NAME].select {|key, value| SOURCE_TABLE_LISTS_KEYS.include?(key)}
   end
 
   # Public Interface: Data Servers
   #
   # Returns a comma separated list of data servers to which the agent sends data
   def data_servers
-    raise UnsupportedSourceError, "subclass must implement"
+    de[self.class::SOURCE_PREFERENCE_NAME]['data_servers']
   end
 
   # Public Interface: Forwarder
   #
   # Returns a forwarder type in string.  Values are 'tcpforwarder' or 'sslforwarder'.
   def forwarder
-    raise UnsupportedSourceError, "subclass must implement"
+    de[self.class::SOURCE_PREFERENCE_NAME]['forwarder']
   end
 
   private
 
   def setup_table_prefs(prefs)
+    # valid tables
+    prefs['tables_append_only'] = split_pref_string(prefs, 'tables_append_only')
+    prefs['tables'] = split_pref_string(prefs, 'tables')
+
     if prefs['tables_append_only']
-      prefs['tables_append_only'] =
-        prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
-      prefs['tables'] = (prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/) +
-                         prefs['tables_append_only']).uniq
-    else
-      prefs['tables'] = prefs['tables'].to_s.split(/(?:\s*,\s*|\s+)/).uniq
+      prefs['tables'] = (prefs['tables'] + prefs['tables_append_only']).uniq
     end
-    prefs['invalid_tables'] =
-      prefs['invalid_tables'].kind_of?(String) ?
-        prefs['invalid_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
-    prefs['new_tables'] =
-      prefs['new_tables'].kind_of?(String) ?
-        prefs['new_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
 
-    setup_pk_override_prefs(prefs)
+    # invalid tables
+    prefs['invalid_tables'] = split_pref_string(prefs, 'invalid_tables')
+    prefs['invalid_tables_append_only'], prefs['invalid_tables_full_sync'] = categorize_invalid_tables(prefs, prefs['invalid_tables'])
+
+    # new tables
+    prefs['new_tables'] = split_pref_string(prefs, 'new_tables')
   end
 
   def setup_pk_override_prefs(prefs)
@@ -99,6 +111,31 @@ class Sync < Component
       $log.info "Primary key override is updated. Using pk_override: #{prefs['pk_override']}"
     end
   end
+
+  # Utility methods
+  def split_pref_string(prefs, key)
+    return nil unless prefs
+    return [] unless prefs[key].kind_of?(String)
+    prefs[key].split(/(?:\s*,\s*|\s+)/).uniq
+  end
+
+  def categorize_invalid_tables(prefs, invalid_tables)
+    invalid_tables_append_only = []
+    invalid_tables_full_sync = []
+
+    invalid_tables.each do |tbl_name|
+      if append_only_table?(prefs, tbl_name)
+        invalid_tables_append_only << tbl_name
+      else
+        invalid_tables_full_sync << tbl_name
+      end
+    end
+    return [invalid_tables_append_only, invalid_tables_full_sync]
+  end
+
+  def append_only_table?(prefs, table_name)
+    TableAttribute::append_only_table?(prefs['table_attributes'], table_name)
+  end
 end
 
 end

data/lib/flydata/source_mysql/sync.rb

@@ -4,11 +4,11 @@ module Flydata
 module SourceMysql
 
 class Sync < Source::Sync
-  def setup
-    mp = de['mysql_data_entry_preference']
-
-    setup_table_prefs(mp)
+  SOURCE_PREFERENCE_NAME = 'mysql_data_entry_preference'
 
+  def setup
+    super
+    mp = de[SOURCE_PREFERENCE_NAME]
     unless mp['ssl_ca_content'].to_s.strip.empty?
       sync_fm = SyncFileManager.new(de)
       sync_fm.save_ssl_ca(mp['ssl_ca_content'])
@@ -20,18 +20,6 @@ class Sync < Source::Sync
   def supported?
     true
   end
-
-  def table_lists
-    de['mysql_data_entry_preference'].select {|key, value| %w(tables new_tables invalid_tables tables_append_only).include?(key)}
-  end
-
-  def data_servers
-    de['mysql_data_entry_preference']['data_servers']
-  end
-
-  def forwarder
-    de['mysql_data_entry_preference']['forwarder']
-  end
 end
 
 end

data/lib/flydata/source_postgresql/sync.rb

@@ -4,25 +4,11 @@ module Flydata
 module SourcePostgresql
 
 class Sync < Source::Sync
-  def setup
-    setup_table_prefs(de['postgresql_data_entry_preference'])
-  end
+  SOURCE_PREFERENCE_NAME = 'postgresql_data_entry_preference'
 
   def supported?
     true
   end
-
-  def table_lists
-    de['postgresql_data_entry_preference'].select {|key, value| %w(tables new_tables invalid_tables tables_append_only).include?(key)}
-  end
-
-  def data_servers
-    de['postgresql_data_entry_preference']['data_servers']
-  end
-
-  def forwarder
-    de['postgresql_data_entry_preference']['forwarder']
-  end
 end
 
 end

data/lib/flydata/table_attribute.rb

@@ -54,5 +54,15 @@ module TableAttribute
       tbl_attr[key.to_s] = val
     end
   end
+
+  def self.find_table(table_attributes, tbl_name)
+    table_attributes.find {|tbl_attr| tbl_attr['table_name'] == tbl_name}
+  end
+
+  def self.append_only_table?(table_attributes, tbl_name)
+    attr = find_table(table_attributes, tbl_name)
+    raise "Sync type for invalid table `#{tbl_name}` not known" unless attr
+    attr.has_key?('omit_events')
+  end
 end
 end

data/spec/flydata/command/sync_spec.rb

@@ -74,6 +74,7 @@ module Flydata
           end
         end
       end
+
       describe '#generate_source_dump' do
         let (:flydata) { double('flydata') }
         let (:dp) { double('dp') }
@@ -88,6 +89,7 @@ module Flydata
         let (:target_tables) { ["test_table_1"] }
         let (:db_byte) { 1 }
         let (:disk_byte) { 100 }
+
         before do
           require 'flydata/source_mysql/generate_source_dump'
           allow_any_instance_of(Flydata::SourceMysql::GenerateSourceDump).to receive(:dump_size).and_return(db_byte)
@@ -109,6 +111,7 @@ module Flydata
           expect(subject).to receive(:ask_yes_no).and_return(true).at_least(:once)
           expect_any_instance_of(FlydataCore::Event::ApiEventSender).to receive(:send_event).once
         end
+
         context 'with no stream option' do
           before do
             expect(default_sync_fm).to receive(:save_sync_info).once
@@ -131,6 +134,7 @@ module Flydata
             }.to raise_error
           end
         end
+
         context 'with stream option' do
           it 'will export to io' do
             expect(default_sync_fm).to receive(:save_sync_info).once
@@ -140,6 +144,7 @@ module Flydata
           end
         end
       end
+
       describe '#convert_to_flydata_values' do
         subject { subject_object.send(:convert_to_flydata_values, source_table, values) }
         let(:values) { [4, 'John', nil, col4_value, nil, nil] }
@@ -175,14 +180,34 @@ module Flydata
         subject { subject_object.send(:data_entry) }
 
         let(:de) { { 'mysql_data_entry_preference' => mp } }
-        let(:mp) { { 'tables' => 'Users,Addresses',
-                     'table_attributes' => [
-                       {"table_name"=>"Users", "status"=>"init_sync_pending"},
-                       {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
-                     ],
-                     'pk_override' => {"Users"=>["id"]}
+        let(:mp) { { 'tables' => default_tables_str,
+                     'table_attributes' => default_table_attributes,
+                     'pk_override' => pk_override_hash
                   } }
+        let(:default_tables_str) { 'Users,Addresses' }
+        let(:default_table_attributes) {[
+          {"table_name"=>"Users", "status"=>"init_sync_pending"},
+          {"table_name"=>"Addresses", "status"=>"init_sync_pending"}
+        ]}
 
+        let(:append_only_tables_list) { %w|Invoices Sessions Addresses| }
+        let(:append_only_tables_str) { 'Invoices,Sessions,Addresses' }
+        let(:tbl_attrs_for_append_only_tables) { [
+          {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+          {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
+          {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
+        ] }
+
+        let(:invalid_tables_list) { %w|error_fullsync_1 error_append_2| }
+        let(:invalid_tables_str) { 'error_fullsync_1,error_append_2' }
+        let(:tbl_attrs_for_invalid_tables) {[
+          {"table_name"=>"error_fullsync_1", "status"=>"init_sync_pending",
+           "invalid_table_reason"=>"no primary key defined"},
+          {"table_name"=>"error_append_2", "omit_events"=>["delete"], "status"=>"init_sync_pending",
+           "invalid_table_reason"=>"table does not exist in the MySQL database"},
+        ]}
+
+        let(:pk_override_hash) { {"Users"=>["id"]} }
         let(:sfm) { double('sfm') }
         let(:ssl_ca_content) { double('ssl_ca_content') }
         let(:ssl_ca_path) { double('ssl_ca_path') }
@@ -199,26 +224,74 @@ module Flydata
               expect(subject_object).to receive(:retrieve_data_entries).
                 and_return([de])
             end
+
             context 'without tables_append_only' do
               it "expands a table list string to an array of tables" do
                 subject
                 expect(mp['tables']).to eq %w(Users Addresses)
               end
             end
+
             context 'with tables_append_only' do
               before do
-                mp['tables_append_only'] = 'Invoices,Sessions,Addresses'
-                mp['table_attributes'] += [
-                    {"table_name"=>"Invoices", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
-                    {"table_name"=>"Sessions", "omit_events"=>["delete"], "status"=>"init_sync_pending"},
-                    {"table_name"=>"Addresses", "omit_events"=>["delete"], "status"=>"init_sync_pending"}
-                ]
+                mp['tables_append_only'] = append_only_tables_str
+                mp['table_attributes'] = default_table_attributes + tbl_attrs_for_append_only_tables
               end
               it "creates an array of tables from 'tables' and 'tables_append_only' combined" do
                 subject
                 expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
               end
+
+              it 'creates an array of append-only tables' do
+                subject
+                expect(mp['tables_append_only']).to eq append_only_tables_list
+              end
+            end
+
+            context 'with invalid tables and invalid append-only tables' do
+              before do
+                mp['tables_append_only'] = append_only_tables_str
+                mp['table_attributes'] = default_table_attributes + tbl_attrs_for_append_only_tables
+
+                mp['invalid_tables'] = invalid_tables_str
+                mp['table_attributes'] = default_table_attributes + 
+                                         tbl_attrs_for_append_only_tables +
+                                         tbl_attrs_for_invalid_tables
+              end
+
+              it 'does not change value for `tables`' do
+                subject
+                expect(mp['tables']).to eq %w(Users Addresses Invoices Sessions)
+              end
+
+              it 'does not change value for `tables_append_only`' do
+                subject
+                expect(mp['tables_append_only']).to eq append_only_tables_list
+              end
+
+              it 'creates an array of invalid tables' do
+                subject
+                expect(mp['invalid_tables']).to eq invalid_tables_list
+              end
+
+              it 'categorize invalid tables by sync type' do
+                subject
+                expect(mp['invalid_tables_append_only']).to eq %w(error_append_2)
+                expect(mp['invalid_tables_full_sync']).to eq %w(error_fullsync_1)
+              end
+            end
+
+            context 'with an invalid table which do not have table_attributes entry' do
+              before do
+                mp['invalid_tables'] = invalid_tables_str + ',table_from_conf'
+                mp['table_attributes'] += default_table_attributes + tbl_attrs_for_invalid_tables
+              end
+
+              it 'cannot determine sync type for the table and raise an error' do
+                expect { subject }.to raise_error /Sync type for invalid table `table_from_conf` not known/
+              end
             end
+
             context 'with ssl_ca_content' do
               before { mp["ssl_ca_content"] = ssl_ca_content }
               it "saves the content to a local file via SyncFileManager" do
@@ -233,6 +306,7 @@ module Flydata
               end
             end
           end
+
           context 'called twice' do
             before { subject }
             it "repurposes the saved de" do
@@ -242,6 +316,7 @@ module Flydata
             end
           end
         end
+
         context 'type RedshiftFileDataEntry' do
           before { de['type'] = 'RedshiftFileDataEntry' }
           it "raises an error about unsupported data entry" do
@@ -249,6 +324,101 @@ module Flydata
           end
         end
       end
+
+      describe '#set_current_tables' do
+        subject { subject_object.send(:set_current_tables, input_tables, options) }
+        let(:input_tables) { nil }
+        let(:table_lists) {{
+            "tables" => tables,
+            "invalid_tables" => invalid_tables,
+            "new_tables" => new_tables,
+            "tables_append_only" => tables_append_only,
+            "invalid_tables_append_only" => invalid_tables_append_only,
+        }}
+        let(:real_new_tables) { [] }
+        let(:tables) { ["table1", "table2", "table4"] }
+        let(:invalid_tables) { ["table3_invalid","append3_invalid"] }
+        let(:invalid_tables_append_only) { ["append3_invalid"] }
+        let(:new_tables) { ["table4"] }
+       
+        let(:data_entry_with_table_lists) do
+          new_de = default_data_entry.dup
+          new_de['mysql_data_entry_preference'].merge!(table_lists) 
+          new_de
+        end
+        before do
+          allow(subject_object).to receive(:data_entry).and_return(data_entry_with_table_lists)
+        end
+
+        context 'when include_all_tables option is true' do
+          let(:options) { { include_all_tables: true } }
+
+          context 'when there are append-only tables (valid & invalid)' do
+            let(:tables_append_only) { ["append1","append2"] }
+            let(:invalid_tables_append_only) { ["append3_invalid"] }
+            it do
+              subject
+              expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+                tables + invalid_tables)
+              expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(
+                tables_append_only + invalid_tables_append_only)
+            end
+          end
+
+          context "when table_lists['tables_append_only'] is nil" do
+            let(:tables_append_only) { nil }
+            it do
+              subject
+              expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+                tables + invalid_tables)
+              expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(
+                invalid_tables_append_only)
+            end
+          end
+
+          context 'when there is no append-only tables' do
+          let(:tables_append_only) { [] }
+          let(:invalid_tables_append_only) { [] }
+            it do
+              subject
+              expect(subject_object.instance_variable_get(:@full_tables)).to eq(
+                tables + invalid_tables)
+              expect(subject_object.instance_variable_get(:@append_only_tables)).to eq([])
+            end
+          end
+        end
+
+        context 'when include_all_tables option is false' do
+          let(:options) { { include_all_tables: false } }
+          context 'when there is no real_new_tables' do
+            let(:real_new_tables) { [] }
+            let(:tables_append_only) { ["append1","append2"] }
+            let(:invalid_tables_append_only) { ["append3_invalid"] }
+            before do
+              allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).and_return(real_new_tables)
+            end
+            it do
+              subject
+              expect(subject_object.instance_variable_get(:@full_tables)).to eq(tables - real_new_tables)
+              expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(tables_append_only)
+            end
+          end
+
+          context 'there is real_new_tables' do
+            let(:real_new_tables) { ['table1'] }
+            let(:tables_append_only) { ["append1","append2"] }
+            let(:invalid_tables_append_only) { ["append3_invalid"] }
+            before do
+              allow_any_instance_of(SyncFileManager).to receive(:get_new_table_list).and_return(real_new_tables)
+            end
+            it do
+              subject
+              expect(subject_object.instance_variable_get(:@full_tables)).to eq(tables - real_new_tables)
+              expect(subject_object.instance_variable_get(:@append_only_tables)).to eq(tables_append_only)
+            end
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-15 00:00:00.000000000 Z
+date: 2016-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client