flydata 0.6.11 → 0.6.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +1 -1
  3. data/Gemfile.lock +4 -4
  4. data/VERSION +1 -1
  5. data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +34 -0
  6. data/flydata-core/lib/flydata-core/table_def/base.rb +10 -0
  7. data/flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb +20 -4
  8. data/flydata-core/spec/postgresql/source_pos_spec.rb +43 -0
  9. data/flydata-core/spec/table_def/base_spec.rb +51 -0
  10. data/flydata.gemspec +0 -0
  11. data/lib/flydata/command/sender.rb +9 -6
  12. data/lib/flydata/command/setup.rb +6 -12
  13. data/lib/flydata/command/sync.rb +31 -17
  14. data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb +2 -3
  15. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +15 -14
  16. data/lib/flydata/parser/source_table.rb +4 -3
  17. data/lib/flydata/plugin_support/context.rb +46 -0
  18. data/lib/flydata/plugin_support/sync_record_emittable.rb +69 -0
  19. data/lib/flydata/source/component.rb +1 -1
  20. data/lib/flydata/source/generate_source_dump.rb +3 -2
  21. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +12 -11
  22. data/lib/flydata/source_mysql/parser/dump_parser.rb +0 -4
  23. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler.rb +8 -2
  24. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_position_file.rb +7 -1
  25. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher.rb +10 -4
  26. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_handler.rb +8 -2
  27. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_dispatcher.rb +9 -3
  28. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_record_handler.rb +16 -34
  29. data/lib/flydata/source_mysql/plugin_support/context.rb +7 -0
  30. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler.rb +11 -19
  31. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler.rb +8 -2
  32. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler.rb +8 -2
  33. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta.rb +5 -1
  34. data/lib/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_table_query_handler.rb +8 -2
  35. data/lib/flydata/source_postgresql/generate_source_dump.rb +175 -0
  36. data/lib/flydata/source_postgresql/parse_dump_and_send.rb +126 -0
  37. data/lib/flydata/source_postgresql/pg_client.rb +43 -0
  38. data/lib/flydata/source_postgresql/postgresql_component.rb +12 -0
  39. data/lib/flydata/source_postgresql/setup.rb +24 -0
  40. data/lib/flydata/source_postgresql/source_pos.rb +18 -0
  41. data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb +7 -15
  42. data/lib/flydata/sync_file_manager.rb +39 -28
  43. data/spec/flydata/command/setup_spec.rb +0 -1
  44. data/spec/flydata/command/sync_spec.rb +2 -2
  45. data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +5 -6
  46. data/spec/flydata/plugin_support/context_spec.rb +27 -0
  47. data/spec/flydata/source_mysql/parser/dump_parser_spec.rb +4 -4
  48. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/alter_table_query_handler_spec.rb +3 -3
  49. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/binlog_query_dispatcher_spec.rb +5 -5
  50. data/spec/flydata/source_mysql/plugin_support/context_spec.rb +26 -0
  51. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/ddl_query_handler_spec.rb +3 -3
  52. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/dml_record_handler_spec.rb +2 -2
  53. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/drop_database_query_handler_spec.rb +3 -3
  54. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/shared_query_handler_context.rb +3 -1
  55. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/table_meta_spec.rb +3 -3
  56. data/spec/flydata/{fluent-plugins/mysql → source_mysql/plugin_support}/truncate_query_handler_spec.rb +7 -4
  57. data/spec/flydata/source_postgresql/generate_source_dump_spec.rb +144 -0
  58. data/spec/flydata/sync_file_manager_spec.rb +1 -1
  59. metadata +38 -24
  60. data/lib/flydata/fluent-plugins/mysql/context.rb +0 -25
@@ -0,0 +1,144 @@
1
+ require 'flydata/source_postgresql/generate_source_dump'
2
+
3
+ module Flydata
4
+ module SourcePostgresql
5
+
6
+ describe GenerateSourceDump do
7
+ let(:subject_object) { described_class.new(source, dp, options) }
8
+
9
+ let(:source) { Source.create(de) }
10
+ let(:de) { { "type" => "RedshiftPostgresqlDataEntry",
11
+ "postgresql_data_entry_preference" => de_prefs } }
12
+ let(:de_prefs) {
13
+ { "host" => "localhost", "port" => 5432,
14
+ "username" => "testuser", "password" => "testpass",
15
+ "database" => "testdb", "schema" => "testschema",
16
+ }
17
+ }
18
+ let(:dp) { double('dp') }
19
+ let(:options) { double('options') }
20
+
21
+ describe '#dump' do
22
+ subject { subject_object.dump(tables, file_path, &src_pos_callback) }
23
+
24
+ let(:tables) { %w( test1 test2 ) }
25
+ let(:src_pos_callback) { Proc.new{} }
26
+
27
+ context 'when file_path is given' do
28
+ let(:file_path) { double('file_path') }
29
+ before do
30
+ allow(File).to receive(:open).with(file_path, "w").
31
+ and_return(io)
32
+ allow(PGClient).to receive(:new).with(de_prefs).and_return(cli)
33
+ allow(subject_object).to receive(:get_source_pos).
34
+ with(cli, &src_pos_callback).and_return(source_pos)
35
+ allow(source).to receive(:sync_generate_table_ddl).with(dp, nil).
36
+ and_return context
37
+ allow(subject_object).to receive(:dump_table).
38
+ with(tabledef1, source_pos, io, cli)
39
+ allow(subject_object).to receive(:dump_table).
40
+ with(tabledef2, source_pos, io, cli)
41
+ end
42
+ let(:io) { double('io') }
43
+ before do
44
+ allow(io).to receive(:close)
45
+ end
46
+
47
+ let(:cli) { double('cli') }
48
+ let(:source_pos) { double('source_pos') }
49
+ let(:context) { double('context') }
50
+ let(:tabledef1) { double('tabledef1') }
51
+ let(:tabledef2) { double('tabledef2') }
52
+ let(:missing_tables) { [] }
53
+ let(:error) { double('error') }
54
+
55
+ it 'does expected things' do
56
+ expect(File).to receive(:open).with(file_path, "w").
57
+ and_return(io)
58
+ expect(PGClient).to receive(:new).with(de_prefs).and_return(cli)
59
+ expect(subject_object).to receive(:get_source_pos).
60
+ with(cli, &src_pos_callback).and_return(source_pos)
61
+ expect(source).to receive(:sync_generate_table_ddl).with(dp, nil).
62
+ and_return context
63
+ expect(context).to receive(:each_source_tabledef).
64
+ with(tables, de_prefs).
65
+ and_yield(tabledef1, nil).and_yield(tabledef2, nil).
66
+ and_return missing_tables
67
+ expect(subject_object).to receive(:dump_table).
68
+ with(tabledef1, source_pos, io, cli)
69
+ expect(subject_object).to receive(:dump_table).
70
+ with(tabledef2, source_pos, io, cli)
71
+
72
+ subject
73
+ end
74
+
75
+ context 'when a table is missing' do
76
+ let(:missing_tables) { [ "test2" ] }
77
+ before do
78
+ allow(context).to receive(:each_source_tabledef).
79
+ with(tables, de_prefs).
80
+ and_yield(tabledef1, nil).
81
+ and_return missing_tables
82
+ end
83
+ it "does nothing for the missing table. we might need to add an error
84
+ handling" do
85
+ expect(subject_object).to receive(:dump_table).
86
+ with(tabledef1, source_pos, io, cli)
87
+ expect(subject_object).not_to receive(:dump_table).
88
+ with(tabledef2, source_pos, io, cli)
89
+
90
+ subject
91
+ end
92
+ end
93
+
94
+ context 'when a table has an error' do
95
+ before do
96
+ allow(context).to receive(:each_source_tabledef).
97
+ with(tables, de_prefs).
98
+ and_yield(nil, error).
99
+ and_yield(tabledef2, nil).
100
+ and_return missing_tables
101
+ end
102
+ it "does nothing for the error table. we might need to add an error
103
+ handling" do
104
+ expect(subject_object).not_to receive(:dump_table).
105
+ with(tabledef1, source_pos, io, cli)
106
+ expect(subject_object).to receive(:dump_table).
107
+ with(tabledef2, source_pos, io, cli)
108
+
109
+ subject
110
+ end
111
+ end
112
+ end
113
+ context 'when file_path is nil' do
114
+ let(:file_path) { nil }
115
+ it 'raises error because the feature is not supported yet' do
116
+ expect{subject}.to raise_error /not been implemented yet/
117
+ end
118
+ end
119
+ end
120
+
121
+ describe '#pk_conditions' do
122
+ subject { subject_object.send(:pk_conditions, pk_columns) }
123
+
124
+ context 'with a single primary key' do
125
+ let(:pk_columns) { [ 'id' ] }
126
+
127
+ it { is_expected.to eq %Q| AND ("id" > $1)| }
128
+ end
129
+
130
+ context 'with two primary keys' do
131
+ let(:pk_columns) { [ 'pk1', 'pk2' ] }
132
+
133
+ it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2)))| }
134
+ end
135
+ context 'with three primary keys' do
136
+ let(:pk_columns) { [ 'pk1', 'pk2', 'pk3' ] }
137
+
138
+ it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2 OR ("pk2" = $2 AND ("pk3" > $3)))))| }
139
+ end
140
+ end
141
+ end
142
+
143
+ end
144
+ end
@@ -165,7 +165,7 @@ module Flydata
165
165
  end
166
166
  it do
167
167
  expect(subject.table_name).to eq(table_name)
168
- expect(subject.columns).to eq(columns_hash)
168
+ expect(subject.instance_variable_get(:@columns)).to eq(columns_hash)
169
169
  end
170
170
  end
171
171
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.11
4
+ version: 0.6.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2016-02-25 00:00:00.000000000 Z
15
+ date: 2016-03-09 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -262,7 +262,7 @@ dependencies:
262
262
  requirements:
263
263
  - - '>='
264
264
  - !ruby/object:Gem::Version
265
- version: 0.1.10
265
+ version: 0.1.11
266
266
  - - ~>
267
267
  - !ruby/object:Gem::Version
268
268
  version: '0.1'
@@ -272,7 +272,7 @@ dependencies:
272
272
  requirements:
273
273
  - - '>='
274
274
  - !ruby/object:Gem::Version
275
- version: 0.1.10
275
+ version: 0.1.11
276
276
  - - ~>
277
277
  - !ruby/object:Gem::Version
278
278
  version: '0.1'
@@ -544,6 +544,7 @@ files:
544
544
  - flydata-core/lib/flydata-core/mysql/config.rb
545
545
  - flydata-core/lib/flydata-core/mysql/ssl.rb
546
546
  - flydata-core/lib/flydata-core/option_validator.rb
547
+ - flydata-core/lib/flydata-core/postgresql/source_pos.rb
547
548
  - flydata-core/lib/flydata-core/query_job.rb
548
549
  - flydata-core/lib/flydata-core/record/record.rb
549
550
  - flydata-core/lib/flydata-core/redshift/string.rb
@@ -571,10 +572,12 @@ files:
571
572
  - flydata-core/spec/mysql/compatibility_checker_spec.rb
572
573
  - flydata-core/spec/mysql/config_spec.rb
573
574
  - flydata-core/spec/option_validator_spec.rb
575
+ - flydata-core/spec/postgresql/source_pos_spec.rb
574
576
  - flydata-core/spec/redshift/string_spec.rb
575
577
  - flydata-core/spec/spec_helper.rb
576
578
  - flydata-core/spec/string_utils_spec.rb
577
579
  - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
580
+ - flydata-core/spec/table_def/base_spec.rb
578
581
  - flydata-core/spec/table_def/mysql_table_def_spec.rb
579
582
  - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
580
583
  - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
@@ -640,18 +643,6 @@ files:
640
643
  - lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb
641
644
  - lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb
642
645
  - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
643
- - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
644
- - lib/flydata/fluent-plugins/mysql/binlog_position_file.rb
645
- - lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb
646
- - lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb
647
- - lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb
648
- - lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb
649
- - lib/flydata/fluent-plugins/mysql/context.rb
650
- - lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb
651
- - lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
652
- - lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb
653
- - lib/flydata/fluent-plugins/mysql/table_meta.rb
654
- - lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb
655
646
  - lib/flydata/fluent-plugins/out_forward_ssl.rb
656
647
  - lib/flydata/flydata_crontab.sh
657
648
  - lib/flydata/helper/action/agent_action.rb
@@ -679,6 +670,8 @@ files:
679
670
  - lib/flydata/parser.rb
680
671
  - lib/flydata/parser/parser_provider.rb
681
672
  - lib/flydata/parser/source_table.rb
673
+ - lib/flydata/plugin_support/context.rb
674
+ - lib/flydata/plugin_support/sync_record_emittable.rb
682
675
  - lib/flydata/preference/data_entry_preference.rb
683
676
  - lib/flydata/proxy.rb
684
677
  - lib/flydata/queueable_thread.rb
@@ -706,12 +699,30 @@ files:
706
699
  - lib/flydata/source_mysql/parser/.gitignore
707
700
  - lib/flydata/source_mysql/parser/dump_parser.rb
708
701
  - lib/flydata/source_mysql/parser/mysql_alter_table.treetop
702
+ - lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb
703
+ - lib/flydata/source_mysql/plugin_support/binlog_position_file.rb
704
+ - lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb
705
+ - lib/flydata/source_mysql/plugin_support/binlog_query_handler.rb
706
+ - lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb
707
+ - lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb
708
+ - lib/flydata/source_mysql/plugin_support/context.rb
709
+ - lib/flydata/source_mysql/plugin_support/ddl_query_handler.rb
710
+ - lib/flydata/source_mysql/plugin_support/dml_record_handler.rb
711
+ - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
712
+ - lib/flydata/source_mysql/plugin_support/table_meta.rb
713
+ - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
709
714
  - lib/flydata/source_mysql/setup.rb
710
715
  - lib/flydata/source_mysql/source_pos.rb
711
716
  - lib/flydata/source_mysql/sync.rb
712
717
  - lib/flydata/source_mysql/sync_generate_table_ddl.rb
713
718
  - lib/flydata/source_mysql/table_ddl.rb
714
719
  - lib/flydata/source_postgresql/data_entry.rb
720
+ - lib/flydata/source_postgresql/generate_source_dump.rb
721
+ - lib/flydata/source_postgresql/parse_dump_and_send.rb
722
+ - lib/flydata/source_postgresql/pg_client.rb
723
+ - lib/flydata/source_postgresql/postgresql_component.rb
724
+ - lib/flydata/source_postgresql/setup.rb
725
+ - lib/flydata/source_postgresql/source_pos.rb
715
726
  - lib/flydata/source_postgresql/sync.rb
716
727
  - lib/flydata/source_postgresql/sync_generate_table_ddl.rb
717
728
  - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
@@ -747,14 +758,6 @@ files:
747
758
  - spec/flydata/compatibility_check_spec.rb
748
759
  - spec/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector_spec.rb
749
760
  - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
750
- - spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
751
- - spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
752
- - spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb
753
- - spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb
754
- - spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb
755
- - spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
756
- - spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
757
- - spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
758
761
  - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
759
762
  - spec/flydata/helper/action/check_remote_actions_spec.rb
760
763
  - spec/flydata/helper/action/restart_agent_spec.rb
@@ -769,12 +772,23 @@ files:
769
772
  - spec/flydata/heroku_spec.rb
770
773
  - spec/flydata/json/json_ext_spec.rb
771
774
  - spec/flydata/output/forwarder_spec.rb
775
+ - spec/flydata/plugin_support/context_spec.rb
772
776
  - spec/flydata/source_mysql/generate_source_dump_spec.rb
773
777
  - spec/flydata/source_mysql/mysql_compatibility_check_spec.rb
774
778
  - spec/flydata/source_mysql/parser/alter_table_parser_spec.rb
775
779
  - spec/flydata/source_mysql/parser/dump_parser_spec.rb
780
+ - spec/flydata/source_mysql/plugin_support/alter_table_query_handler_spec.rb
781
+ - spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb
782
+ - spec/flydata/source_mysql/plugin_support/context_spec.rb
783
+ - spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb
784
+ - spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb
785
+ - spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb
786
+ - spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
787
+ - spec/flydata/source_mysql/plugin_support/table_meta_spec.rb
788
+ - spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
776
789
  - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
777
790
  - spec/flydata/source_mysql/table_ddl_spec.rb
791
+ - spec/flydata/source_postgresql/generate_source_dump_spec.rb
778
792
  - spec/flydata/source_spec.rb
779
793
  - spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb
780
794
  - spec/flydata/sync_file_manager_spec.rb
@@ -1,25 +0,0 @@
1
- module Mysql
2
- class Context
3
- MANDATORY_OPTS = [
4
- :database, :tables, :tag, :sync_fm, :omit_events, :table_meta, :table_revs,
5
- ]
6
- OPTIONAL_OPTS = [
7
- :current_binlog_file
8
- ]
9
-
10
- (MANDATORY_OPTS + OPTIONAL_OPTS).each do |opt|
11
- attr_accessor opt
12
- end
13
-
14
- def initialize(opts)
15
- missing_opts = MANDATORY_OPTS - opts.keys
16
- unless (missing_opts.empty?)
17
- raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}"
18
- end
19
-
20
- opts.each do |k, v|
21
- self.instance_variable_set(:"@#{k}", v)
22
- end
23
- end
24
- end
25
- end