flydata 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c1e7d064581a7d76441fa77bc814c1ebba986cd
-  data.tar.gz: bb14ff1bbf83246092ddbc44332dadc42725e1f7
+  metadata.gz: b4388aaa10ad1cba39ffe66fa6b6db2fed3e5268
+  data.tar.gz: 91dfd8a833cc5f3b4b8e47d72e9276e58fc8212a
 SHA512:
-  metadata.gz: b3fee3b20cdb7ef01f45a05d3485e637fcb794c15c4bb31e2027535241db6ee31de9c7b97e8b5a4ab1892b415a1a26a336c525d8a9a19b960c2d4b5e5677006e
-  data.tar.gz: 477339f6ca63c80d13a9d499e062c5e42cae23a99a24d058f37fdbea01829084c9f137e7c00baa88bc56ff5389fd858119fc2c4ce1583358505e9dd7bfac4d05
+  metadata.gz: 0f88c6786e5ab53a363c486d2d43def69b4b861f8da830de2cad0dd884f27f9b328792c8633c32b76fedc245979cc660977b59840434e3ef3ec3ddafa4c10547
+  data.tar.gz: 4db0d551a501458a49c7386919f827ed6961dddd737ac82c20eb096f6c3d192c125e7045e027f462d8caf1ea106c84db8711a66abae024929382bb8888cba627

data/VERSION

@@ -1 +1 @@
-0.7.1
+0.7.2

data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb

@@ -160,7 +160,9 @@ class PostgresqlTableDef < Base
     column[:column] = information_schema_column["column_name"]
     column[:type] = convert_to_flydata_type(information_schema_column)
     column[:not_null] = true if information_schema_column["is_nullable"] == "NO"
-    column[:primary_key] = true if information_schema_column["is_primary"]
+    # index column: information_schema_column["is_primary"] == "f"
+    # regular column: information_schema_column["is_primary"] == nil
+    column[:primary_key] = true if information_schema_column["is_primary"] == "t"
     column[:default] = case column[:type]
       when 'boolean'
         to_boolean(information_schema_column["column_default"])

data/flydata-core/lib/flydata-core/table_def/value_conv.rb

@@ -8,7 +8,7 @@ module ValueConv
     # It's impossible to determine a decimal point of a currency sting without
     # knowing its format.  Here, we're making a best effort guess to support
     # as many currency formats as we can without knowing the format.
-    value = value.gsub(/[^0-9\.,]/, '') # remove all chars but numbers and possible decimal point chars
+    value = value.gsub(/[^0-9\.,-]/, '') # remove all chars but numbers, minus sign, and possible decimal point chars
 
     whole_part = value
     fractional_part = nil
@@ -19,7 +19,7 @@ module ValueConv
         fractional_part = value[-2..-1]
       end
     end
-    value = whole_part.gsub(/[^0-9]/, '')
+    value = whole_part.gsub(/[^0-9-]/, '')
     value += ".#{fractional_part}" if fractional_part
     value
   end

data/flydata-core/spec/table_def/value_conv_spec.rb

@@ -8,8 +8,15 @@ describe ValueConv do
     subject { described_class.strip_currency_format(value) }
 
     context 'in dollar format' do
-      let(:value) { "$2,392.40" }
-      it { is_expected.to eq "2392.40" }
+      context 'with positive value' do
+        let(:value) { "$2,392.40" }
+        it { is_expected.to eq "2392.40" }
+      end
+
+      context 'with negative value' do
+        let(:value) { "-$2,392.40" }
+        it { is_expected.to eq "-2392.40" }
+      end
     end
 
     context 'in JPY format' do

data/lib/flydata/command/sync.rb

@@ -360,7 +360,9 @@ EOS
 
           # Agent is stopped but locked.  There was an abnormal shutdown.
           abnormal_shutdown = sender.agent_locked?
+
           table_status_hash = get_table_status(@full_tables, source_pos_context)
+
           corrupt_master_pos_files = check_master_position_files
           pos_mismatch_tables = check_position_files(table_status_hash)
           gap_tables = check_gaps(table_status_hash)
@@ -400,12 +402,17 @@ EOS
         set_current_tables
         # Stop agent.  Check sync and make sure the state is :STUCK_AT_UPLOAD
         # Get table status for the tables.
-        status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash = _check(context, stop_agent:true)
+        status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash =
+             _check(context, stop_agent:true)
 
         if status.include? :STUCK_AT_PROCESS
           e = AgentError.new("Data is stuck while processing")
           e.description = <<EOS
-Data appears to be stuck while processing.  Contact FlyData Support (support@flydata.com) to solve the issue.
+Data appears to be stuck while processing. Run the following command, and try `flydata sync:repair` again:
+
+flydata sync:flush
+
+If you need help, please contact FlyData Support (support@flydata.com).
 EOS
           raise e
         end
@@ -419,8 +426,7 @@ EOS
         if status.include?(:ABNORMAL_SHUTDOWN) && status.uniq.length == 1
           # abnormal shutdown occurred, but all other conditions are healthy.
           log_info_stdout ""
-          log_info_stdout "Agent process was shut down unexpectedly"
-          log_info_stdout "but sync is in good condition.  Nothing to repair."
+          log_info_stdout "Agent process was shut down unexpectedly, but sync is in good condition. Nothing to repair."
 
           # Remove the lock file if exists.
           File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
@@ -431,7 +437,13 @@ EOS
         pt = []
         gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
         pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
-        tables = gt | pt # position mismatch can be due to query queue items discarded by the copy handler so it also needs to be in the target table list.
+        # position mismatch can be due to query queue items discarded by the copy handler
+        # so it also needs to be in the target table list.
+        tables = gt | pt
+
+        # table_status will be created at the end of initial sync.
+        # ignore tables that have no table_status, i.e. tables never got init-synced before.
+        synced_tables = @full_tables & table_status_hash.keys
 
         log_info_stdout <<EOS
 
@@ -459,7 +471,7 @@ EOS
         unrepairable_tables = []
         # Determine the master source positions
         sent_source_pos = nil
-        @full_tables.each do |table|
+        synced_tables.each do |table|
           table_source_pos = table_status_hash[table]["source_pos"]
 
           if table_source_pos.nil? || table_source_pos.empty?
@@ -495,7 +507,7 @@ EOS
           if oldest_source_pos && sent_source_pos < oldest_source_pos
             e = AgentError.new("Repair failed due to expired source position")
             e.description = <<EOS
-  Repair failed because the starting source position `#{sent_source_pos} no longer exists.  Run full initial sync instead.
+  Repair failed because the starting source position `#{sent_source_pos}` no longer exists.  Run full initial sync instead.
 EOS
             raise e
           end
@@ -514,7 +526,7 @@ EOS
 
           # Save the positions (source_pos and seq)
           log_info_stdout "Fixing table positions..."
-          @full_tables.each do |table|
+          synced_tables.each do |table|
              table_source_pos = table_status_hash[table]["source_pos"]
              if table_source_pos.nil? || table_source_pos.empty?
                # no init sync has happened to the table.  No need to set positions.
@@ -555,7 +567,7 @@ EOS
         elsif !@unsynced_tables.empty?
           show_purpose_name
           unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@unsynced_tables.join(", ")}\n"
-          unless @ddl_tables.empty?
+          unless @no_ddl_generated_tables.empty?
             unsynced_table_message <<
             "  WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
             "    We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
@@ -567,7 +579,7 @@ EOS
             initial_sync(options.merge(sync_resumed: false))
           else
             #If generate_table_ddl has not been run for these tables, warn user
-            unless @ddl_tables.empty?
+            unless @no_ddl_generated_tables.empty?
               say("  You can generate DDL SQL for your new tables by running this command")
               say("    $> flydata sync:generate_table_ddl > create_table.sql")
             end
@@ -1258,7 +1270,7 @@ Thank you for using FlyData!
                                                       table_lists['tables'] - real_new_tables
 
         @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
-        @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
+        @no_ddl_generated_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl") # tables do not have a .generated_ddl file
 
         @input_tables = sync_resumed ? sync_info[:tables] : input_tables
         @input_tables ||= []
@@ -1414,10 +1426,6 @@ Thank you for using FlyData!
           source_pos =  src_pos_str ? source_pos_context.create_source_pos(src_pos_str) : nil
           table_status_hash[table].merge!( { "source_pos" => source_pos } )
         end
-        missing_tables = tables - table_status_hash.keys
-        unless missing_tables.empty?
-          raise "table status is not available for these table(s): #{missing_tables.join(",")}"
-        end
         populate_initial_source_positions(table_status_hash, sync_fm)
         table_status_hash
       end

data/lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb

@@ -118,7 +118,7 @@ EOS
       case k
       when "bit", "varbit"; -> (v) { v ? '%d' % v.to_i(2) : nil }
       when "boolean"; -> (v) { FlydataCore::TableDef::PostgresqlTableDef::to_boolean(v) }
-      when "money"; -> (v) { FlydataCore::TableDef::ValueConv.strip_currency_format(v) }
+      when "money"; -> (v) { v ? FlydataCore::TableDef::ValueConv.strip_currency_format(v) : nil }
       else
         nil
       end

data/spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb

@@ -157,10 +157,18 @@ EOS
     end
 
     context 'when type is varbit' do
-      let(:type) { 'varbit' }
-      let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
+      context 'when value is not nil' do
+        let(:type) { 'varbit' }
+        let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
+
+        it { is_expected.to eq "18446744073709551613" }
+      end
 
-      it { is_expected.to eq "18446744073709551613" }
+      context 'when value is nil' do
+        let(:type) { 'varbit' }
+        let(:value) { nil }
+        it { is_expected.to eq nil }
+      end
     end
 
     context 'when type is boolean' do
@@ -182,6 +190,33 @@ EOS
         it { is_expected.to eq nil }
       end
     end
+
+    context 'when type is money' do
+      context 'when value is positive' do
+        let(:type) { 'money' }
+        let(:value) { "$25.83" }
+        it { is_expected.to eq "25.83" }
+      end
+
+      context 'when value is zero' do
+        let(:type) { 'money' }
+        let(:value) { "$0.00" }
+        it { is_expected.to eq "0.00" }
+      end
+=begin
+      #TODO fix bug and enable this test case
+      context 'when value is negative' do
+        let(:type) { 'money' }
+        let(:value) { "-$10.00" }
+        it { is_expected.to eq "-10.00" }
+      end
+=end
+      context 'when value is nil' do
+        let(:type) { 'money' }
+        let(:value) { nil }
+        it { is_expected.to eq nil }
+      end
+    end
   end
 
   describe '#pk_conditions' do

data/spec/flydata/source_postgresql/table_meta_spec.rb

@@ -26,10 +26,12 @@ module Flydata::SourcePostgresql
     end
 
     let(:raw_columns) { [
-      build_column('table_a', 'id', nil, 'NO', true, 'int', nil, 4, 0),
-      build_column('table_a', 'val', nil, 'YES', false, 'character varying', 24, nil, nil),
-      build_column('table_b', 'name', nil, 'NO', true, 'character varying', 36, nil, nil),
-      build_column('table_b', 'count', nil, 'YES', false, 'int', nil, 4, 0),
+      build_column('table_a', 'id', nil, 'NO', 't', 'integer', nil, 32, 0),
+      build_column('table_a', 'val', nil, 'YES', nil, 'character varying', 24, nil, nil),
+      build_column('table_b', 'name', nil, 'NO', 't', 'character varying', 36, nil, nil),#primary key (name, prod_id)
+      build_column('table_b', 'prod_id', nil, 'NO', 't', 'integer', nil, 32, 0),
+      build_column('table_b', 'serial_no', nil, 'YES', 'f', 'integer', nil, 32, 0),#index key
+      build_column('table_b', 'count', nil, 'YES', nil, 'integer', nil, 32, 0),
     ] }
     let(:current_snapshots) { [{'txid_current_snapshot'=>'1010:1010:'}] }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-05 00:00:00.000000000 Z
+date: 2016-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -869,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: FlyData Agent