flydata 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4c1e7d064581a7d76441fa77bc814c1ebba986cd
4
- data.tar.gz: bb14ff1bbf83246092ddbc44332dadc42725e1f7
3
+ metadata.gz: b4388aaa10ad1cba39ffe66fa6b6db2fed3e5268
4
+ data.tar.gz: 91dfd8a833cc5f3b4b8e47d72e9276e58fc8212a
5
5
  SHA512:
6
- metadata.gz: b3fee3b20cdb7ef01f45a05d3485e637fcb794c15c4bb31e2027535241db6ee31de9c7b97e8b5a4ab1892b415a1a26a336c525d8a9a19b960c2d4b5e5677006e
7
- data.tar.gz: 477339f6ca63c80d13a9d499e062c5e42cae23a99a24d058f37fdbea01829084c9f137e7c00baa88bc56ff5389fd858119fc2c4ce1583358505e9dd7bfac4d05
6
+ metadata.gz: 0f88c6786e5ab53a363c486d2d43def69b4b861f8da830de2cad0dd884f27f9b328792c8633c32b76fedc245979cc660977b59840434e3ef3ec3ddafa4c10547
7
+ data.tar.gz: 4db0d551a501458a49c7386919f827ed6961dddd737ac82c20eb096f6c3d192c125e7045e027f462d8caf1ea106c84db8711a66abae024929382bb8888cba627
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.7.1
1
+ 0.7.2
@@ -160,7 +160,9 @@ class PostgresqlTableDef < Base
160
160
  column[:column] = information_schema_column["column_name"]
161
161
  column[:type] = convert_to_flydata_type(information_schema_column)
162
162
  column[:not_null] = true if information_schema_column["is_nullable"] == "NO"
163
- column[:primary_key] = true if information_schema_column["is_primary"]
163
+ # index column: information_schema_column["is_primary"] == "f"
164
+ # regular column: information_schema_column["is_primary"] == nil
165
+ column[:primary_key] = true if information_schema_column["is_primary"] == "t"
164
166
  column[:default] = case column[:type]
165
167
  when 'boolean'
166
168
  to_boolean(information_schema_column["column_default"])
@@ -8,7 +8,7 @@ module ValueConv
8
8
  # It's impossible to determine a decimal point of a currency sting without
9
9
  # knowing its format. Here, we're making a best effort guess to support
10
10
  # as many currency formats as we can without knowing the format.
11
- value = value.gsub(/[^0-9\.,]/, '') # remove all chars but numbers and possible decimal point chars
11
+ value = value.gsub(/[^0-9\.,-]/, '') # remove all chars but numbers, minus sign, and possible decimal point chars
12
12
 
13
13
  whole_part = value
14
14
  fractional_part = nil
@@ -19,7 +19,7 @@ module ValueConv
19
19
  fractional_part = value[-2..-1]
20
20
  end
21
21
  end
22
- value = whole_part.gsub(/[^0-9]/, '')
22
+ value = whole_part.gsub(/[^0-9-]/, '')
23
23
  value += ".#{fractional_part}" if fractional_part
24
24
  value
25
25
  end
@@ -8,8 +8,15 @@ describe ValueConv do
8
8
  subject { described_class.strip_currency_format(value) }
9
9
 
10
10
  context 'in dollar format' do
11
- let(:value) { "$2,392.40" }
12
- it { is_expected.to eq "2392.40" }
11
+ context 'with positive value' do
12
+ let(:value) { "$2,392.40" }
13
+ it { is_expected.to eq "2392.40" }
14
+ end
15
+
16
+ context 'with negative value' do
17
+ let(:value) { "-$2,392.40" }
18
+ it { is_expected.to eq "-2392.40" }
19
+ end
13
20
  end
14
21
 
15
22
  context 'in JPY format' do
Binary file
@@ -360,7 +360,9 @@ EOS
360
360
 
361
361
  # Agent is stopped but locked. There was an abnormal shutdown.
362
362
  abnormal_shutdown = sender.agent_locked?
363
+
363
364
  table_status_hash = get_table_status(@full_tables, source_pos_context)
365
+
364
366
  corrupt_master_pos_files = check_master_position_files
365
367
  pos_mismatch_tables = check_position_files(table_status_hash)
366
368
  gap_tables = check_gaps(table_status_hash)
@@ -400,12 +402,17 @@ EOS
400
402
  set_current_tables
401
403
  # Stop agent. Check sync and make sure the state is :STUCK_AT_UPLOAD
402
404
  # Get table status for the tables.
403
- status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash = _check(context, stop_agent:true)
405
+ status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash =
406
+ _check(context, stop_agent:true)
404
407
 
405
408
  if status.include? :STUCK_AT_PROCESS
406
409
  e = AgentError.new("Data is stuck while processing")
407
410
  e.description = <<EOS
408
- Data appears to be stuck while processing. Contact FlyData Support (support@flydata.com) to solve the issue.
411
+ Data appears to be stuck while processing. Run the following command, and try `flydata sync:repair` again:
412
+
413
+ flydata sync:flush
414
+
415
+ If you need help, please contact FlyData Support (support@flydata.com).
409
416
  EOS
410
417
  raise e
411
418
  end
@@ -419,8 +426,7 @@ EOS
419
426
  if status.include?(:ABNORMAL_SHUTDOWN) && status.uniq.length == 1
420
427
  # abnormal shutdown occurred, but all other conditions are healthy.
421
428
  log_info_stdout ""
422
- log_info_stdout "Agent process was shut down unexpectedly"
423
- log_info_stdout "but sync is in good condition. Nothing to repair."
429
+ log_info_stdout "Agent process was shut down unexpectedly, but sync is in good condition. Nothing to repair."
424
430
 
425
431
  # Remove the lock file if exists.
426
432
  File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
@@ -431,7 +437,13 @@ EOS
431
437
  pt = []
432
438
  gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
433
439
  pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
434
- tables = gt | pt # position mismatch can be due to query queue items discarded by the copy handler so it also needs to be in the target table list.
440
+ # position mismatch can be due to query queue items discarded by the copy handler
441
+ # so it also needs to be in the target table list.
442
+ tables = gt | pt
443
+
444
+ # table_status will be created at the end of initial sync.
445
+ # ignore tables that have no table_status, i.e. tables never got init-synced before.
446
+ synced_tables = @full_tables & table_status_hash.keys
435
447
 
436
448
  log_info_stdout <<EOS
437
449
 
@@ -459,7 +471,7 @@ EOS
459
471
  unrepairable_tables = []
460
472
  # Determine the master source positions
461
473
  sent_source_pos = nil
462
- @full_tables.each do |table|
474
+ synced_tables.each do |table|
463
475
  table_source_pos = table_status_hash[table]["source_pos"]
464
476
 
465
477
  if table_source_pos.nil? || table_source_pos.empty?
@@ -495,7 +507,7 @@ EOS
495
507
  if oldest_source_pos && sent_source_pos < oldest_source_pos
496
508
  e = AgentError.new("Repair failed due to expired source position")
497
509
  e.description = <<EOS
498
- Repair failed because the starting source position `#{sent_source_pos} no longer exists. Run full initial sync instead.
510
+ Repair failed because the starting source position `#{sent_source_pos}` no longer exists. Run full initial sync instead.
499
511
  EOS
500
512
  raise e
501
513
  end
@@ -514,7 +526,7 @@ EOS
514
526
 
515
527
  # Save the positions (source_pos and seq)
516
528
  log_info_stdout "Fixing table positions..."
517
- @full_tables.each do |table|
529
+ synced_tables.each do |table|
518
530
  table_source_pos = table_status_hash[table]["source_pos"]
519
531
  if table_source_pos.nil? || table_source_pos.empty?
520
532
  # no init sync has happened to the table. No need to set positions.
@@ -555,7 +567,7 @@ EOS
555
567
  elsif !@unsynced_tables.empty?
556
568
  show_purpose_name
557
569
  unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@unsynced_tables.join(", ")}\n"
558
- unless @ddl_tables.empty?
570
+ unless @no_ddl_generated_tables.empty?
559
571
  unsynced_table_message <<
560
572
  " WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
561
573
  " We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
@@ -567,7 +579,7 @@ EOS
567
579
  initial_sync(options.merge(sync_resumed: false))
568
580
  else
569
581
  #If generate_table_ddl has not been run for these tables, warn user
570
- unless @ddl_tables.empty?
582
+ unless @no_ddl_generated_tables.empty?
571
583
  say(" You can generate DDL SQL for your new tables by running this command")
572
584
  say(" $> flydata sync:generate_table_ddl > create_table.sql")
573
585
  end
@@ -1258,7 +1270,7 @@ Thank you for using FlyData!
1258
1270
  table_lists['tables'] - real_new_tables
1259
1271
 
1260
1272
  @unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
1261
- @ddl_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl")
1273
+ @no_ddl_generated_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl") # tables do not have a .generated_ddl file
1262
1274
 
1263
1275
  @input_tables = sync_resumed ? sync_info[:tables] : input_tables
1264
1276
  @input_tables ||= []
@@ -1414,10 +1426,6 @@ Thank you for using FlyData!
1414
1426
  source_pos = src_pos_str ? source_pos_context.create_source_pos(src_pos_str) : nil
1415
1427
  table_status_hash[table].merge!( { "source_pos" => source_pos } )
1416
1428
  end
1417
- missing_tables = tables - table_status_hash.keys
1418
- unless missing_tables.empty?
1419
- raise "table status is not available for these table(s): #{missing_tables.join(",")}"
1420
- end
1421
1429
  populate_initial_source_positions(table_status_hash, sync_fm)
1422
1430
  table_status_hash
1423
1431
  end
@@ -118,7 +118,7 @@ EOS
118
118
  case k
119
119
  when "bit", "varbit"; -> (v) { v ? '%d' % v.to_i(2) : nil }
120
120
  when "boolean"; -> (v) { FlydataCore::TableDef::PostgresqlTableDef::to_boolean(v) }
121
- when "money"; -> (v) { FlydataCore::TableDef::ValueConv.strip_currency_format(v) }
121
+ when "money"; -> (v) { v ? FlydataCore::TableDef::ValueConv.strip_currency_format(v) : nil }
122
122
  else
123
123
  nil
124
124
  end
@@ -157,10 +157,18 @@ EOS
157
157
  end
158
158
 
159
159
  context 'when type is varbit' do
160
- let(:type) { 'varbit' }
161
- let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
160
+ context 'when value is not nil' do
161
+ let(:type) { 'varbit' }
162
+ let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
163
+
164
+ it { is_expected.to eq "18446744073709551613" }
165
+ end
162
166
 
163
- it { is_expected.to eq "18446744073709551613" }
167
+ context 'when value is nil' do
168
+ let(:type) { 'varbit' }
169
+ let(:value) { nil }
170
+ it { is_expected.to eq nil }
171
+ end
164
172
  end
165
173
 
166
174
  context 'when type is boolean' do
@@ -182,6 +190,33 @@ EOS
182
190
  it { is_expected.to eq nil }
183
191
  end
184
192
  end
193
+
194
+ context 'when type is money' do
195
+ context 'when value is positive' do
196
+ let(:type) { 'money' }
197
+ let(:value) { "$25.83" }
198
+ it { is_expected.to eq "25.83" }
199
+ end
200
+
201
+ context 'when value is zero' do
202
+ let(:type) { 'money' }
203
+ let(:value) { "$0.00" }
204
+ it { is_expected.to eq "0.00" }
205
+ end
206
+ =begin
207
+ #TODO fix bug and enable this test case
208
+ context 'when value is negative' do
209
+ let(:type) { 'money' }
210
+ let(:value) { "-$10.00" }
211
+ it { is_expected.to eq "-10.00" }
212
+ end
213
+ =end
214
+ context 'when value is nil' do
215
+ let(:type) { 'money' }
216
+ let(:value) { nil }
217
+ it { is_expected.to eq nil }
218
+ end
219
+ end
185
220
  end
186
221
 
187
222
  describe '#pk_conditions' do
@@ -26,10 +26,12 @@ module Flydata::SourcePostgresql
26
26
  end
27
27
 
28
28
  let(:raw_columns) { [
29
- build_column('table_a', 'id', nil, 'NO', true, 'int', nil, 4, 0),
30
- build_column('table_a', 'val', nil, 'YES', false, 'character varying', 24, nil, nil),
31
- build_column('table_b', 'name', nil, 'NO', true, 'character varying', 36, nil, nil),
32
- build_column('table_b', 'count', nil, 'YES', false, 'int', nil, 4, 0),
29
+ build_column('table_a', 'id', nil, 'NO', 't', 'integer', nil, 32, 0),
30
+ build_column('table_a', 'val', nil, 'YES', nil, 'character varying', 24, nil, nil),
31
+ build_column('table_b', 'name', nil, 'NO', 't', 'character varying', 36, nil, nil),#primary key (name, prod_id)
32
+ build_column('table_b', 'prod_id', nil, 'NO', 't', 'integer', nil, 32, 0),
33
+ build_column('table_b', 'serial_no', nil, 'YES', 'f', 'integer', nil, 32, 0),#index key
34
+ build_column('table_b', 'count', nil, 'YES', nil, 'integer', nil, 32, 0),
33
35
  ] }
34
36
  let(:current_snapshots) { [{'txid_current_snapshot'=>'1010:1010:'}] }
35
37
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.7.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-04-05 00:00:00.000000000 Z
15
+ date: 2016-04-12 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -869,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
869
869
  version: '0'
870
870
  requirements: []
871
871
  rubyforge_project:
872
- rubygems_version: 2.4.3
872
+ rubygems_version: 2.2.2
873
873
  signing_key:
874
874
  specification_version: 4
875
875
  summary: FlyData Agent