flydata 0.7.1 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +3 -1
- data/flydata-core/lib/flydata-core/table_def/value_conv.rb +2 -2
- data/flydata-core/spec/table_def/value_conv_spec.rb +9 -2
- data/flydata.gemspec +0 -0
- data/lib/flydata/command/sync.rb +23 -15
- data/lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb +1 -1
- data/spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb +38 -3
- data/spec/flydata/source_postgresql/table_meta_spec.rb +6 -4
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4388aaa10ad1cba39ffe66fa6b6db2fed3e5268
|
4
|
+
data.tar.gz: 91dfd8a833cc5f3b4b8e47d72e9276e58fc8212a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0f88c6786e5ab53a363c486d2d43def69b4b861f8da830de2cad0dd884f27f9b328792c8633c32b76fedc245979cc660977b59840434e3ef3ec3ddafa4c10547
|
7
|
+
data.tar.gz: 4db0d551a501458a49c7386919f827ed6961dddd737ac82c20eb096f6c3d192c125e7045e027f462d8caf1ea106c84db8711a66abae024929382bb8888cba627
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.2
|
@@ -160,7 +160,9 @@ class PostgresqlTableDef < Base
|
|
160
160
|
column[:column] = information_schema_column["column_name"]
|
161
161
|
column[:type] = convert_to_flydata_type(information_schema_column)
|
162
162
|
column[:not_null] = true if information_schema_column["is_nullable"] == "NO"
|
163
|
-
column
|
163
|
+
# index column: information_schema_column["is_primary"] == "f"
|
164
|
+
# regular column: information_schema_column["is_primary"] == nil
|
165
|
+
column[:primary_key] = true if information_schema_column["is_primary"] == "t"
|
164
166
|
column[:default] = case column[:type]
|
165
167
|
when 'boolean'
|
166
168
|
to_boolean(information_schema_column["column_default"])
|
@@ -8,7 +8,7 @@ module ValueConv
|
|
8
8
|
# It's impossible to determine a decimal point of a currency sting without
|
9
9
|
# knowing its format. Here, we're making a best effort guess to support
|
10
10
|
# as many currency formats as we can without knowing the format.
|
11
|
-
value = value.gsub(/[^0-9
|
11
|
+
value = value.gsub(/[^0-9\.,-]/, '') # remove all chars but numbers, minus sign, and possible decimal point chars
|
12
12
|
|
13
13
|
whole_part = value
|
14
14
|
fractional_part = nil
|
@@ -19,7 +19,7 @@ module ValueConv
|
|
19
19
|
fractional_part = value[-2..-1]
|
20
20
|
end
|
21
21
|
end
|
22
|
-
value = whole_part.gsub(/[^0-9]/, '')
|
22
|
+
value = whole_part.gsub(/[^0-9-]/, '')
|
23
23
|
value += ".#{fractional_part}" if fractional_part
|
24
24
|
value
|
25
25
|
end
|
@@ -8,8 +8,15 @@ describe ValueConv do
|
|
8
8
|
subject { described_class.strip_currency_format(value) }
|
9
9
|
|
10
10
|
context 'in dollar format' do
|
11
|
-
|
12
|
-
|
11
|
+
context 'with positive value' do
|
12
|
+
let(:value) { "$2,392.40" }
|
13
|
+
it { is_expected.to eq "2392.40" }
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'with negative value' do
|
17
|
+
let(:value) { "-$2,392.40" }
|
18
|
+
it { is_expected.to eq "-2392.40" }
|
19
|
+
end
|
13
20
|
end
|
14
21
|
|
15
22
|
context 'in JPY format' do
|
data/flydata.gemspec
CHANGED
Binary file
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -360,7 +360,9 @@ EOS
|
|
360
360
|
|
361
361
|
# Agent is stopped but locked. There was an abnormal shutdown.
|
362
362
|
abnormal_shutdown = sender.agent_locked?
|
363
|
+
|
363
364
|
table_status_hash = get_table_status(@full_tables, source_pos_context)
|
365
|
+
|
364
366
|
corrupt_master_pos_files = check_master_position_files
|
365
367
|
pos_mismatch_tables = check_position_files(table_status_hash)
|
366
368
|
gap_tables = check_gaps(table_status_hash)
|
@@ -400,12 +402,17 @@ EOS
|
|
400
402
|
set_current_tables
|
401
403
|
# Stop agent. Check sync and make sure the state is :STUCK_AT_UPLOAD
|
402
404
|
# Get table status for the tables.
|
403
|
-
status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash =
|
405
|
+
status, corrupt_master_pos_files, pos_mismatch_tables, gap_tables, table_status_hash =
|
406
|
+
_check(context, stop_agent:true)
|
404
407
|
|
405
408
|
if status.include? :STUCK_AT_PROCESS
|
406
409
|
e = AgentError.new("Data is stuck while processing")
|
407
410
|
e.description = <<EOS
|
408
|
-
Data appears to be stuck while processing.
|
411
|
+
Data appears to be stuck while processing. Run the following command, and try `flydata sync:repair` again:
|
412
|
+
|
413
|
+
flydata sync:flush
|
414
|
+
|
415
|
+
If you need help, please contact FlyData Support (support@flydata.com).
|
409
416
|
EOS
|
410
417
|
raise e
|
411
418
|
end
|
@@ -419,8 +426,7 @@ EOS
|
|
419
426
|
if status.include?(:ABNORMAL_SHUTDOWN) && status.uniq.length == 1
|
420
427
|
# abnormal shutdown occurred, but all other conditions are healthy.
|
421
428
|
log_info_stdout ""
|
422
|
-
log_info_stdout "Agent process was shut down unexpectedly"
|
423
|
-
log_info_stdout "but sync is in good condition. Nothing to repair."
|
429
|
+
log_info_stdout "Agent process was shut down unexpectedly, but sync is in good condition. Nothing to repair."
|
424
430
|
|
425
431
|
# Remove the lock file if exists.
|
426
432
|
File.delete(FLYDATA_LOCK) if File.exists?(FLYDATA_LOCK)
|
@@ -431,7 +437,13 @@ EOS
|
|
431
437
|
pt = []
|
432
438
|
gt = gap_tables.collect{|bt| bt[:table] } if gap_tables
|
433
439
|
pt = pos_mismatch_tables.collect{|bt| bt[:table] } if pos_mismatch_tables
|
434
|
-
|
440
|
+
# position mismatch can be due to query queue items discarded by the copy handler
|
441
|
+
# so it also needs to be in the target table list.
|
442
|
+
tables = gt | pt
|
443
|
+
|
444
|
+
# table_status will be created at the end of initial sync.
|
445
|
+
# ignore tables that have no table_status, i.e. tables never got init-synced before.
|
446
|
+
synced_tables = @full_tables & table_status_hash.keys
|
435
447
|
|
436
448
|
log_info_stdout <<EOS
|
437
449
|
|
@@ -459,7 +471,7 @@ EOS
|
|
459
471
|
unrepairable_tables = []
|
460
472
|
# Determine the master source positions
|
461
473
|
sent_source_pos = nil
|
462
|
-
|
474
|
+
synced_tables.each do |table|
|
463
475
|
table_source_pos = table_status_hash[table]["source_pos"]
|
464
476
|
|
465
477
|
if table_source_pos.nil? || table_source_pos.empty?
|
@@ -495,7 +507,7 @@ EOS
|
|
495
507
|
if oldest_source_pos && sent_source_pos < oldest_source_pos
|
496
508
|
e = AgentError.new("Repair failed due to expired source position")
|
497
509
|
e.description = <<EOS
|
498
|
-
Repair failed because the starting source position `#{sent_source_pos} no longer exists. Run full initial sync instead.
|
510
|
+
Repair failed because the starting source position `#{sent_source_pos}` no longer exists. Run full initial sync instead.
|
499
511
|
EOS
|
500
512
|
raise e
|
501
513
|
end
|
@@ -514,7 +526,7 @@ EOS
|
|
514
526
|
|
515
527
|
# Save the positions (source_pos and seq)
|
516
528
|
log_info_stdout "Fixing table positions..."
|
517
|
-
|
529
|
+
synced_tables.each do |table|
|
518
530
|
table_source_pos = table_status_hash[table]["source_pos"]
|
519
531
|
if table_source_pos.nil? || table_source_pos.empty?
|
520
532
|
# no init sync has happened to the table. No need to set positions.
|
@@ -555,7 +567,7 @@ EOS
|
|
555
567
|
elsif !@unsynced_tables.empty?
|
556
568
|
show_purpose_name
|
557
569
|
unsynced_table_message = "We've noticed that these tables have not been synced yet: #{@unsynced_tables.join(", ")}\n"
|
558
|
-
unless @
|
570
|
+
unless @no_ddl_generated_tables.empty?
|
559
571
|
unsynced_table_message <<
|
560
572
|
" WARNING: We've noticed that at least one of these tables have not had their DDL generated yet.\n" +
|
561
573
|
" We recommend you run our 'flydata sync:generate_table_ddl > create_table.sql'\n" +
|
@@ -567,7 +579,7 @@ EOS
|
|
567
579
|
initial_sync(options.merge(sync_resumed: false))
|
568
580
|
else
|
569
581
|
#If generate_table_ddl has not been run for these tables, warn user
|
570
|
-
unless @
|
582
|
+
unless @no_ddl_generated_tables.empty?
|
571
583
|
say(" You can generate DDL SQL for your new tables by running this command")
|
572
584
|
say(" $> flydata sync:generate_table_ddl > create_table.sql")
|
573
585
|
end
|
@@ -1258,7 +1270,7 @@ Thank you for using FlyData!
|
|
1258
1270
|
table_lists['tables'] - real_new_tables
|
1259
1271
|
|
1260
1272
|
@unsynced_tables = sync_fm.get_new_table_list(@full_tables, "pos") # Get list of tables that do not have a .pos file
|
1261
|
-
@
|
1273
|
+
@no_ddl_generated_tables = sync_fm.get_new_table_list(@full_tables, "generated_ddl") # tables do not have a .generated_ddl file
|
1262
1274
|
|
1263
1275
|
@input_tables = sync_resumed ? sync_info[:tables] : input_tables
|
1264
1276
|
@input_tables ||= []
|
@@ -1414,10 +1426,6 @@ Thank you for using FlyData!
|
|
1414
1426
|
source_pos = src_pos_str ? source_pos_context.create_source_pos(src_pos_str) : nil
|
1415
1427
|
table_status_hash[table].merge!( { "source_pos" => source_pos } )
|
1416
1428
|
end
|
1417
|
-
missing_tables = tables - table_status_hash.keys
|
1418
|
-
unless missing_tables.empty?
|
1419
|
-
raise "table status is not available for these table(s): #{missing_tables.join(",")}"
|
1420
|
-
end
|
1421
1429
|
populate_initial_source_positions(table_status_hash, sync_fm)
|
1422
1430
|
table_status_hash
|
1423
1431
|
end
|
@@ -118,7 +118,7 @@ EOS
|
|
118
118
|
case k
|
119
119
|
when "bit", "varbit"; -> (v) { v ? '%d' % v.to_i(2) : nil }
|
120
120
|
when "boolean"; -> (v) { FlydataCore::TableDef::PostgresqlTableDef::to_boolean(v) }
|
121
|
-
when "money"; -> (v) { FlydataCore::TableDef::ValueConv.strip_currency_format(v) }
|
121
|
+
when "money"; -> (v) { v ? FlydataCore::TableDef::ValueConv.strip_currency_format(v) : nil }
|
122
122
|
else
|
123
123
|
nil
|
124
124
|
end
|
@@ -157,10 +157,18 @@ EOS
|
|
157
157
|
end
|
158
158
|
|
159
159
|
context 'when type is varbit' do
|
160
|
-
|
161
|
-
|
160
|
+
context 'when value is not nil' do
|
161
|
+
let(:type) { 'varbit' }
|
162
|
+
let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
|
163
|
+
|
164
|
+
it { is_expected.to eq "18446744073709551613" }
|
165
|
+
end
|
162
166
|
|
163
|
-
|
167
|
+
context 'when value is nil' do
|
168
|
+
let(:type) { 'varbit' }
|
169
|
+
let(:value) { nil }
|
170
|
+
it { is_expected.to eq nil }
|
171
|
+
end
|
164
172
|
end
|
165
173
|
|
166
174
|
context 'when type is boolean' do
|
@@ -182,6 +190,33 @@ EOS
|
|
182
190
|
it { is_expected.to eq nil }
|
183
191
|
end
|
184
192
|
end
|
193
|
+
|
194
|
+
context 'when type is money' do
|
195
|
+
context 'when value is positive' do
|
196
|
+
let(:type) { 'money' }
|
197
|
+
let(:value) { "$25.83" }
|
198
|
+
it { is_expected.to eq "25.83" }
|
199
|
+
end
|
200
|
+
|
201
|
+
context 'when value is zero' do
|
202
|
+
let(:type) { 'money' }
|
203
|
+
let(:value) { "$0.00" }
|
204
|
+
it { is_expected.to eq "0.00" }
|
205
|
+
end
|
206
|
+
=begin
|
207
|
+
#TODO fix bug and enable this test case
|
208
|
+
context 'when value is negative' do
|
209
|
+
let(:type) { 'money' }
|
210
|
+
let(:value) { "-$10.00" }
|
211
|
+
it { is_expected.to eq "-10.00" }
|
212
|
+
end
|
213
|
+
=end
|
214
|
+
context 'when value is nil' do
|
215
|
+
let(:type) { 'money' }
|
216
|
+
let(:value) { nil }
|
217
|
+
it { is_expected.to eq nil }
|
218
|
+
end
|
219
|
+
end
|
185
220
|
end
|
186
221
|
|
187
222
|
describe '#pk_conditions' do
|
@@ -26,10 +26,12 @@ module Flydata::SourcePostgresql
|
|
26
26
|
end
|
27
27
|
|
28
28
|
let(:raw_columns) { [
|
29
|
-
build_column('table_a', 'id', nil, 'NO',
|
30
|
-
build_column('table_a', 'val', nil, 'YES',
|
31
|
-
build_column('table_b', 'name', nil, 'NO',
|
32
|
-
build_column('table_b', '
|
29
|
+
build_column('table_a', 'id', nil, 'NO', 't', 'integer', nil, 32, 0),
|
30
|
+
build_column('table_a', 'val', nil, 'YES', nil, 'character varying', 24, nil, nil),
|
31
|
+
build_column('table_b', 'name', nil, 'NO', 't', 'character varying', 36, nil, nil),#primary key (name, prod_id)
|
32
|
+
build_column('table_b', 'prod_id', nil, 'NO', 't', 'integer', nil, 32, 0),
|
33
|
+
build_column('table_b', 'serial_no', nil, 'YES', 'f', 'integer', nil, 32, 0),#index key
|
34
|
+
build_column('table_b', 'count', nil, 'YES', nil, 'integer', nil, 32, 0),
|
33
35
|
] }
|
34
36
|
let(:current_snapshots) { [{'txid_current_snapshot'=>'1010:1010:'}] }
|
35
37
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-04-
|
15
|
+
date: 2016-04-12 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -869,7 +869,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
869
869
|
version: '0'
|
870
870
|
requirements: []
|
871
871
|
rubyforge_project:
|
872
|
-
rubygems_version: 2.
|
872
|
+
rubygems_version: 2.2.2
|
873
873
|
signing_key:
|
874
874
|
specification_version: 4
|
875
875
|
summary: FlyData Agent
|