flydata 0.7.14 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +23 -38
- data/flydata-core/lib/flydata-core/postgresql/snapshot.rb +7 -10
- data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +9 -5
- data/flydata-core/lib/flydata-core/source_pos_base.rb +77 -0
- data/flydata-core/spec/mysql/binlog_pos_spec.rb +65 -2
- data/flydata-core/spec/postgresql/snapshot_spec.rb +31 -0
- data/flydata-core/spec/postgresql/source_pos_spec.rb +35 -0
- data/flydata.gemspec +4 -2
- data/lib/flydata/command/sync.rb +7 -2
- data/lib/flydata/plugin_support/sync_record_emittable.rb +6 -1
- data/lib/flydata/source_mysql/plugin_support/context.rb +6 -0
- data/lib/flydata/source_mysql/plugin_support/drop_table_query_handler.rb +3 -0
- data/spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb +34 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5435f4af64a22277e8417bbdd7196ee8aabc6c19
|
4
|
+
data.tar.gz: 7bbbff8d0c4cb13891f35dc8debf891b2aa31953
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 646f5ec5b0f3be8102a80c386291c95b6ad220c3e3401b89acea12ab7b59179b3a806fdfc67c403f9ed44d4a08a7080a49f48528109257faad2c1c3cd9fd84f2
|
7
|
+
data.tar.gz: 179efaee49c60d6573da9b0f6f57476fc8ae26f43ab70bee591f174fec5ba09b90d5a6c4ea2cf95f91417d73f8144d89ae2a568457b28d2034c9479eb629bc17
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.15
|
@@ -1,28 +1,28 @@
|
|
1
|
+
require 'flydata-core/source_pos_base'
|
2
|
+
|
1
3
|
module FlydataCore
|
2
4
|
module Mysql
|
3
5
|
|
4
6
|
class BinlogPos
|
5
|
-
include
|
7
|
+
include SourcePosBase
|
8
|
+
|
6
9
|
def initialize(binlog_str_or_filename_or_hash, binlog_pos = nil)
|
7
10
|
arg = binlog_str_or_filename_or_hash
|
8
11
|
if binlog_pos
|
9
12
|
@pos = binlog_pos
|
10
13
|
@filename = arg
|
11
14
|
elsif arg.kind_of?(String)
|
12
|
-
|
13
|
-
set_empty
|
14
|
-
else
|
15
|
-
@filename, @pos = arg.split("\t")
|
16
|
-
end
|
15
|
+
@filename, @pos = arg.split("\t")
|
17
16
|
elsif arg.kind_of?(Hash)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
@pos = arg[:pos]
|
18
|
+
@filename = arg[:filename] ||arg[:binfile]
|
19
|
+
end
|
20
|
+
|
21
|
+
# Set -1 to pos if filename is predefined value
|
22
|
+
if set_predefined_flags(@filename)
|
23
|
+
@pos = -1
|
25
24
|
end
|
25
|
+
|
26
26
|
if @filename.nil? || @pos.nil?
|
27
27
|
raise "Invalid initialize argument (#{arg}, #{binlog_pos})"
|
28
28
|
end
|
@@ -35,43 +35,28 @@ class BinlogPos
|
|
35
35
|
|
36
36
|
attr_reader :filename, :pos
|
37
37
|
|
38
|
-
|
39
|
-
@filename && @filename == "-"
|
40
|
-
end
|
41
|
-
|
38
|
+
# Override (Comparable)
|
42
39
|
def <=>(other_pos)
|
43
|
-
|
44
|
-
|
45
|
-
end
|
40
|
+
s = super
|
41
|
+
return s if s
|
46
42
|
|
47
|
-
other_pos =
|
48
|
-
|
49
|
-
return 0
|
50
|
-
elsif empty? || other_pos.empty?
|
51
|
-
raise ArgumentError.new("comparison with empty BinlogPos failed")
|
52
|
-
else
|
53
|
-
(@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
|
54
|
-
end
|
43
|
+
other_pos = self.class.new(other_pos) unless other_pos.kind_of?(self.class)
|
44
|
+
(@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
|
55
45
|
end
|
56
46
|
|
47
|
+
# Override (Comparable)
|
57
48
|
def ==(other_pos)
|
58
49
|
return false if other_pos.nil?
|
59
50
|
return @filename == other_pos.filename if empty? || other_pos.empty?
|
60
51
|
self.<=>(other_pos) == 0
|
61
52
|
end
|
62
53
|
|
54
|
+
# Override
|
63
55
|
def to_s
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
private
|
68
|
-
def set_empty
|
69
|
-
# @filename == "-" is the definition of `empty`. @pos could be anything but nul
|
70
|
-
@filename = "-"
|
71
|
-
@pos = -1
|
56
|
+
s = super
|
57
|
+
return s if s
|
58
|
+
"#{@filename}\t#{@pos}"
|
72
59
|
end
|
73
|
-
|
74
|
-
|
75
60
|
end
|
76
61
|
|
77
62
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'flydata-core/source_pos_base'
|
2
|
+
|
1
3
|
# Snapshot components
|
2
4
|
# http://www.postgresql.org/docs/8.3/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
|
3
5
|
#
|
@@ -7,12 +9,12 @@ module FlydataCore
|
|
7
9
|
module Postgresql
|
8
10
|
|
9
11
|
class Snapshot
|
10
|
-
include
|
12
|
+
include SourcePosBase
|
11
13
|
|
12
14
|
def initialize(txid_snapshot)
|
13
15
|
@txid_snapshot_str = txid_snapshot.to_s
|
14
16
|
|
15
|
-
return if
|
17
|
+
return if set_predefined_flags(@txid_snapshot_str)
|
16
18
|
xmin_str, xmax_str, xip_list_str = @txid_snapshot_str.split(':')
|
17
19
|
|
18
20
|
raise ArgumentError, "Invalid snapshot - xmin is empty." if xmin_str.to_s.empty?
|
@@ -29,15 +31,10 @@ class Snapshot
|
|
29
31
|
@txid_snapshot_str
|
30
32
|
end
|
31
33
|
|
32
|
-
|
33
|
-
@txid_snapshot_str == '-'
|
34
|
-
end
|
35
|
-
|
34
|
+
# Override (Comparable)
|
36
35
|
def <=>(other)
|
37
|
-
|
38
|
-
|
39
|
-
raise ArgumentError.new("comparison with empty source pos failed")
|
40
|
-
end
|
36
|
+
s = super
|
37
|
+
return s if s
|
41
38
|
|
42
39
|
if @xmin == other.xmin
|
43
40
|
if @xmax == other.xmax
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'flydata-core/postgresql/snapshot'
|
3
|
+
require 'flydata-core/source_pos_base'
|
3
4
|
|
4
5
|
module FlydataCore
|
5
6
|
module Postgresql
|
6
7
|
|
7
8
|
class SourcePos
|
8
|
-
include
|
9
|
+
include SourcePosBase
|
9
10
|
|
10
11
|
# Source Position data for PostgreSQL
|
11
12
|
#
|
@@ -53,17 +54,17 @@ class SourcePos
|
|
53
54
|
end
|
54
55
|
@pk_values = pk_values # must be array or nil
|
55
56
|
end
|
57
|
+
|
58
|
+
set_predefined_flags(@snapshot)
|
56
59
|
end
|
57
60
|
|
58
61
|
attr_reader :snapshot
|
59
62
|
attr_reader :to_snapshot
|
60
63
|
attr_reader :pk_values
|
61
64
|
|
62
|
-
def empty?
|
63
|
-
@snapshot.empty?
|
64
|
-
end
|
65
|
-
|
66
65
|
def to_s
|
66
|
+
s = super
|
67
|
+
return s if s
|
67
68
|
pk_values = @pk_values ? @pk_values.to_json : ''
|
68
69
|
"#{@snapshot}\t#{@to_snapshot}\t#{pk_values}"
|
69
70
|
end
|
@@ -71,6 +72,9 @@ class SourcePos
|
|
71
72
|
#TODO: Need to revisit the logic along with the spec
|
72
73
|
# http://www.postgresql.org/docs/9.5/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
|
73
74
|
def <=>(other)
|
75
|
+
s = super
|
76
|
+
return s if s
|
77
|
+
|
74
78
|
if @snapshot != other.snapshot
|
75
79
|
return @snapshot <=> other.snapshot
|
76
80
|
elsif @pk_values.nil? && !other.pk_values.nil?
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module SourcePosBase
|
3
|
+
EMPTY = "-"
|
4
|
+
INFINITY = 'infinity'
|
5
|
+
PREDEFINED_SRC_LIST = [EMPTY, INFINITY]
|
6
|
+
|
7
|
+
def self.included(base)
|
8
|
+
base.class_eval do
|
9
|
+
include Comparable
|
10
|
+
attr_reader :src_pos_empty, :src_pos_infinity
|
11
|
+
|
12
|
+
def self.empty_src_pos
|
13
|
+
self.new(EMPTY)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.infinity_src_pos
|
17
|
+
self.new(INFINITY)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_s
|
23
|
+
if empty?
|
24
|
+
EMPTY
|
25
|
+
elsif infinity?
|
26
|
+
INFINITY
|
27
|
+
else
|
28
|
+
nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# Return true if one of flags is set
|
33
|
+
def set_predefined_flags(str)
|
34
|
+
case str.to_s
|
35
|
+
when EMPTY
|
36
|
+
@src_pos_empty = true
|
37
|
+
when INFINITY
|
38
|
+
@src_pos_infinity = true
|
39
|
+
else
|
40
|
+
false
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def empty?
|
45
|
+
@src_pos_empty == true
|
46
|
+
end
|
47
|
+
|
48
|
+
def infinity?
|
49
|
+
@src_pos_infinity == true
|
50
|
+
end
|
51
|
+
|
52
|
+
# Override (Comparable)
|
53
|
+
def <=>(other_pos)
|
54
|
+
if other_pos.nil?
|
55
|
+
raise ArgumentError.new("comparison with nil failed")
|
56
|
+
end
|
57
|
+
|
58
|
+
other_pos = self.class.new(other_pos) unless other_pos.kind_of?(self.class)
|
59
|
+
|
60
|
+
if infinity? && other_pos.infinity?
|
61
|
+
return 0
|
62
|
+
elsif infinity?
|
63
|
+
return 1
|
64
|
+
elsif other_pos.infinity?
|
65
|
+
return -1
|
66
|
+
end
|
67
|
+
|
68
|
+
if empty? && other_pos.empty?
|
69
|
+
return 0
|
70
|
+
elsif empty? || other_pos.empty?
|
71
|
+
raise ArgumentError.new("comparison with empty source pos failed")
|
72
|
+
end
|
73
|
+
|
74
|
+
nil
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -20,6 +20,16 @@ module Mysql
|
|
20
20
|
describe '#to_s' do
|
21
21
|
subject { subject_object.to_s }
|
22
22
|
it { is_expected.to eq(binlog_str) }
|
23
|
+
|
24
|
+
context 'when filename is "-"' do
|
25
|
+
let(:binlog_str) { "-" }
|
26
|
+
it { is_expected.to eq("-") }
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'when filename is "infinity"' do
|
30
|
+
let(:binlog_str) { "infinity" }
|
31
|
+
it { is_expected.to eq("infinity") }
|
32
|
+
end
|
23
33
|
end
|
24
34
|
|
25
35
|
describe '.new' do
|
@@ -46,6 +56,32 @@ module Mysql
|
|
46
56
|
let(:arg1) { { filename: filename, pos: pos } }
|
47
57
|
it_behaves_like "properly constructed"
|
48
58
|
end
|
59
|
+
|
60
|
+
context 'when given a empty("-") string' do
|
61
|
+
let(:arg1) { { filename: "-", pos: nil} }
|
62
|
+
let(:filename) { '-' }
|
63
|
+
let(:pos) { -1 }
|
64
|
+
it_behaves_like "properly constructed"
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'when given a "infinity" string' do
|
68
|
+
let(:arg1) { { filename: "infinity", pos: 10} }
|
69
|
+
let(:filename) { 'infinity' }
|
70
|
+
let(:pos) { -1 }
|
71
|
+
it_behaves_like "properly constructed"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe '.empty_src_pos' do
|
76
|
+
subject { described_class.empty_src_pos }
|
77
|
+
it { expect(subject.empty?).to be(true) }
|
78
|
+
it { expect(subject.to_s).to eq('-') }
|
79
|
+
end
|
80
|
+
|
81
|
+
describe '.infinity_src_pos' do
|
82
|
+
subject { described_class.infinity_src_pos }
|
83
|
+
it { expect(subject.infinity?).to be(true) }
|
84
|
+
it { expect(subject.to_s).to eq('infinity') }
|
49
85
|
end
|
50
86
|
|
51
87
|
describe '.load' do
|
@@ -105,7 +141,7 @@ module Mysql
|
|
105
141
|
it { is_expected.to eq -1 }
|
106
142
|
end
|
107
143
|
end
|
108
|
-
|
144
|
+
|
109
145
|
context 'when the target binlog filename is older' do
|
110
146
|
let(:arg_filename) { older_filename }
|
111
147
|
let(:expected_result) { 1 }
|
@@ -121,21 +157,48 @@ module Mysql
|
|
121
157
|
let(:arg_str) { "-" }
|
122
158
|
it { expect { subject }.to raise_error(ArgumentError) }
|
123
159
|
end
|
160
|
+
|
161
|
+
context 'when the target is infinity' do
|
162
|
+
let(:arg_str) { "infinity" }
|
163
|
+
it { is_expected.to eq -1 }
|
164
|
+
end
|
165
|
+
|
124
166
|
context 'when self is empty' do
|
125
167
|
let(:binlog_str) { "-" }
|
126
168
|
context 'when target is empty' do
|
127
169
|
let(:arg_str) { "-" }
|
128
170
|
it { is_expected.to eq 0 }
|
129
171
|
end
|
130
|
-
context 'when target is
|
172
|
+
context 'when target is infinity' do
|
173
|
+
let(:arg_str) { "infinity" }
|
174
|
+
it { is_expected.to eq -1 }
|
175
|
+
end
|
176
|
+
context 'when target is normal value' do
|
131
177
|
let(:arg) { double('arg') }
|
132
178
|
before do
|
133
179
|
allow(arg).to receive(:kind_of?).with(BinlogPos).and_return(true)
|
134
180
|
allow(arg).to receive(:empty?).and_return(false)
|
181
|
+
allow(arg).to receive(:infinity?).and_return(false)
|
135
182
|
end
|
136
183
|
it { expect { subject }.to raise_error(ArgumentError) }
|
137
184
|
end
|
138
185
|
end
|
186
|
+
|
187
|
+
context 'when self is infinity' do
|
188
|
+
let(:binlog_str) { "infinity" }
|
189
|
+
context 'when target is infinity' do
|
190
|
+
let(:arg_str) { "infinity" }
|
191
|
+
it { is_expected.to eq 0 }
|
192
|
+
end
|
193
|
+
context 'when target is empty' do
|
194
|
+
let(:arg_str) { "-" }
|
195
|
+
it { is_expected.to eq 1 }
|
196
|
+
end
|
197
|
+
context 'when target is normal value' do
|
198
|
+
let(:arg_str) { "mysql-bin.000064\t104" }
|
199
|
+
it { is_expected.to eq 1 }
|
200
|
+
end
|
201
|
+
end
|
139
202
|
end
|
140
203
|
|
141
204
|
describe '#<' do
|
@@ -19,6 +19,16 @@ module FlydataCore
|
|
19
19
|
it { expect(subject).to be_empty }
|
20
20
|
end
|
21
21
|
|
22
|
+
context 'when text is infinity' do
|
23
|
+
let(:snapshot_text) { 'infinity' }
|
24
|
+
it { expect(subject.xmin).to be_nil }
|
25
|
+
it { expect(subject.xmax).to be_nil }
|
26
|
+
it { expect(subject.xip_list).to be_nil }
|
27
|
+
it { expect(subject.to_s).to eq('infinity') }
|
28
|
+
it { expect(subject).not_to be_empty }
|
29
|
+
it { expect(subject.infinity?).to be(true)}
|
30
|
+
end
|
31
|
+
|
22
32
|
context 'when xip_list is empty' do
|
23
33
|
let(:snapshot_text) { '10:18:' }
|
24
34
|
it { expect(subject.xmin).to eq(10) }
|
@@ -50,6 +60,20 @@ module FlydataCore
|
|
50
60
|
end
|
51
61
|
end
|
52
62
|
|
63
|
+
describe '#infinity?' do
|
64
|
+
subject { subject_object.infinity? }
|
65
|
+
|
66
|
+
context 'when snapshot_txt is infinity' do
|
67
|
+
let(:snapshot_text) { 'infinity' }
|
68
|
+
it { is_expected.to be(true) }
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'when snapshot_txt is normal value' do
|
72
|
+
let(:snapshot_text) { '10:10:' }
|
73
|
+
it { is_expected.to be(false) }
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
53
77
|
describe '#<=>' do
|
54
78
|
def new_ss(txt)
|
55
79
|
Snapshot.new(txt)
|
@@ -64,6 +88,10 @@ module FlydataCore
|
|
64
88
|
it { expect(new_ss("10:18:10,11,12") == new_ss("10:18:11,12")).to be(false) }
|
65
89
|
it { expect(new_ss("-") == new_ss("-")).to be(true) }
|
66
90
|
it { expect(new_ss("-") == new_ss("10:10:")).to be(false) }
|
91
|
+
it { expect(new_ss("-") == new_ss("infinity")).to be(false) }
|
92
|
+
it { expect(new_ss("infinity") == new_ss("infinity")).to be(true) }
|
93
|
+
it { expect(new_ss("infinity") == new_ss("-")).to be(false) }
|
94
|
+
it { expect(new_ss("infinity") == new_ss("10:10:")).to be(false) }
|
67
95
|
|
68
96
|
it { expect(new_ss("10:10:") > new_ss("10:10:")).to be(false) }
|
69
97
|
it { expect(new_ss("10:10:") > new_ss("11:11:")).to be(false) }
|
@@ -74,6 +102,9 @@ module FlydataCore
|
|
74
102
|
it { expect(new_ss("10:18:") > new_ss("10:18:10,11,12")).to be(true) }
|
75
103
|
it { expect(new_ss("10:18:10,11,12") > new_ss("10:18:10,11,12")).to be(false) }
|
76
104
|
it { expect(new_ss("10:18:10,11,12") > new_ss("10:18:11,12")).to be(false) }
|
105
|
+
it { expect(new_ss("10:18:10,11,12") > new_ss("infinity")).to be(false) }
|
106
|
+
it { expect(new_ss("infinity") > new_ss("10:18:")).to be(true) }
|
107
|
+
|
77
108
|
end
|
78
109
|
end
|
79
110
|
end
|
@@ -27,6 +27,16 @@ describe SourcePos do
|
|
27
27
|
it { expect(subject.to_snapshot).to eq(Snapshot.new(to_snapshot)) }
|
28
28
|
it { expect(subject.pk_values).to eq(pk_values) }
|
29
29
|
end
|
30
|
+
|
31
|
+
context 'with empty string' do
|
32
|
+
let(:snapshot) { '-' }
|
33
|
+
it { expect(subject_object.empty?).to be(true) }
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'with infinity string' do
|
37
|
+
let(:snapshot) { 'infinity' }
|
38
|
+
it { expect(subject_object.snapshot.infinity?).to be(true) }
|
39
|
+
end
|
30
40
|
end
|
31
41
|
|
32
42
|
describe '#to_s' do
|
@@ -36,6 +46,21 @@ describe SourcePos do
|
|
36
46
|
let(:pk_values) { nil }
|
37
47
|
it { is_expected.to eq %Q|#{snapshot}\t\t| }
|
38
48
|
end
|
49
|
+
|
50
|
+
context 'when empty ' do
|
51
|
+
let(:snapshot) { '-' }
|
52
|
+
let(:to_snapshot) { nil }
|
53
|
+
let(:pk_values) { nil }
|
54
|
+
it { is_expected.to eq '-' }
|
55
|
+
end
|
56
|
+
|
57
|
+
context 'when infinity' do
|
58
|
+
let(:snapshot) { 'infinity' }
|
59
|
+
let(:to_snapshot) { nil }
|
60
|
+
let(:pk_values) { nil }
|
61
|
+
it { is_expected.to eq 'infinity' }
|
62
|
+
end
|
63
|
+
|
39
64
|
context 'when pk_values is not nil' do
|
40
65
|
let(:to_snapshot) { '1234:1234:' }
|
41
66
|
let(:pk_values) { { "user_id" => 1, "address_id" => 3 } }
|
@@ -63,6 +88,7 @@ describe SourcePos do
|
|
63
88
|
def src_pos(snapshot, pk_values = nil)
|
64
89
|
described_class.new(snapshot, '2222:2222:', pk_values)
|
65
90
|
end
|
91
|
+
|
66
92
|
context 'when both have no pk_values' do
|
67
93
|
it { expect(src_pos('1111:1111:') == src_pos('1111:1111:')).to be(true) }
|
68
94
|
it { expect(src_pos('1111:1111:') == src_pos('1112:1112:')).to be(false) }
|
@@ -98,6 +124,15 @@ describe SourcePos do
|
|
98
124
|
it { expect(src_pos('1111:1111:',pkv1) > src_pos('1112:1112:',pkv2)).to be(false) }
|
99
125
|
it { expect(src_pos('1111:1111:',pkv1) > src_pos('1111:1111:',pkv2)).to be(false) }
|
100
126
|
end
|
127
|
+
|
128
|
+
context 'when one of pos is infinity' do
|
129
|
+
it { expect(src_pos('infinity') == src_pos('1111:1111:')).to be(false) }
|
130
|
+
it { expect(src_pos('infinity') == src_pos('infinity')).to be(true) }
|
131
|
+
it { expect(src_pos('infinity') < src_pos('1112:1112:')).to be(false) }
|
132
|
+
it { expect(src_pos('1111:1111:') < src_pos('infinity')).to be(true) }
|
133
|
+
it { expect(src_pos('infinity') > src_pos('1110:1110:')).to be(true) }
|
134
|
+
it { expect(src_pos('1111:1111:') > src_pos('infinity')).to be(false) }
|
135
|
+
end
|
101
136
|
end
|
102
137
|
end
|
103
138
|
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.7.
|
8
|
+
s.version = "0.7.15"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2016-12-
|
12
|
+
s.date = "2016-12-13"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
|
@@ -84,6 +84,7 @@ Gem::Specification.new do |s|
|
|
84
84
|
"flydata-core/lib/flydata-core/query_job.rb",
|
85
85
|
"flydata-core/lib/flydata-core/record/record.rb",
|
86
86
|
"flydata-core/lib/flydata-core/redshift/string.rb",
|
87
|
+
"flydata-core/lib/flydata-core/source_pos_base.rb",
|
87
88
|
"flydata-core/lib/flydata-core/string_utils.rb",
|
88
89
|
"flydata-core/lib/flydata-core/table_def.rb",
|
89
90
|
"flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
|
@@ -388,6 +389,7 @@ Gem::Specification.new do |s|
|
|
388
389
|
"spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb",
|
389
390
|
"spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb",
|
390
391
|
"spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb",
|
392
|
+
"spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb",
|
391
393
|
"spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb",
|
392
394
|
"spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb",
|
393
395
|
"spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb",
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -1466,8 +1466,13 @@ Thank you for using FlyData!
|
|
1466
1466
|
$log.debug "table pos updated. table:#{table} pos:#{old_pos} -> #{pos}"
|
1467
1467
|
end
|
1468
1468
|
if source_pos != old_source_pos
|
1469
|
-
|
1470
|
-
|
1469
|
+
# keep binlog pos file if source pos is infinity until reset happenes
|
1470
|
+
if old_source_pos && old_source_pos.infinity?
|
1471
|
+
$log.debug "table source_pos update skipped. table:#{table} source_pos:`#{old_source_pos}` -> `#{old_source_pos}`"
|
1472
|
+
else
|
1473
|
+
sync_fm.save_table_source_pos(table, source_pos, destination: :positions)
|
1474
|
+
$log.debug "table source_pos updated. table:#{table} source_pos:`#{old_source_pos}` -> `#{source_pos}`"
|
1475
|
+
end
|
1471
1476
|
end
|
1472
1477
|
[old_source_pos, old_pos]
|
1473
1478
|
end
|
@@ -62,7 +62,12 @@ module PluginSupport
|
|
62
62
|
[timestamp, r]
|
63
63
|
end
|
64
64
|
Fluent::Engine.emit_array(tag, array)
|
65
|
-
|
65
|
+
if table
|
66
|
+
@context.sync_fm.save_table_position(table, seq)
|
67
|
+
if options[:set_infinity_to_table_binlog_pos]
|
68
|
+
@context.set_infinity_to_table_binlog_pos(table)
|
69
|
+
end
|
70
|
+
end
|
66
71
|
end
|
67
72
|
|
68
73
|
end
|
@@ -24,6 +24,12 @@ module PluginSupport
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
27
|
+
|
28
|
+
def set_infinity_to_table_binlog_pos(table_name)
|
29
|
+
new_src_pos = FlydataCore::Mysql::BinlogPos.infinity_src_pos
|
30
|
+
sync_fm.save_table_source_pos(table_name, new_src_pos, destination: :positions)
|
31
|
+
@table_binlog_pos[table_name] = new_src_pos
|
32
|
+
end
|
27
33
|
end
|
28
34
|
|
29
35
|
end
|
@@ -22,6 +22,9 @@ module PluginSupport
|
|
22
22
|
$log.info "type:drop_table table_name:'#{table_name}' query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
|
23
23
|
|
24
24
|
opt[:increment_table_rev] = true
|
25
|
+
|
26
|
+
# Set infinity to table binlog pos to ignore following events
|
27
|
+
opt[:set_infinity_to_table_binlog_pos] = true
|
25
28
|
{ table_name: table_name, query: record["query"] }
|
26
29
|
end
|
27
30
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/source_mysql/plugin_support/drop_table_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
|
+
describe DropTableQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
let(:query) { "DROP TABLE #{table}" }
|
10
|
+
let(:normalized_query) { "DROP TABLE #{table}" }
|
11
|
+
let(:subject_object) { described_class.new(context) }
|
12
|
+
|
13
|
+
describe '#process' do
|
14
|
+
subject { subject_object.process(record) }
|
15
|
+
|
16
|
+
it 'handles a drop table event' do
|
17
|
+
expect(Fluent::Engine).to receive(:emit_array).once
|
18
|
+
expect(sync_fm).to receive(:increment_table_rev).with(table, 1)
|
19
|
+
expect(sync_fm).to receive(:save_table_position).with(table, 200)
|
20
|
+
expect(sync_fm).to receive(:save_table_source_pos).with(table, FlydataCore::Mysql::BinlogPos.infinity_src_pos, destination: :positions)
|
21
|
+
subject
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'when unsupported table is dropped' do
|
25
|
+
let(:normalized_query) { "DROP TABLE bar" }
|
26
|
+
it 'ignore a drop table event' do
|
27
|
+
expect(Fluent::Engine).to receive(:emit_array).never
|
28
|
+
expect(sync_fm).to receive(:save_table_source_pos).never
|
29
|
+
subject
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-12-
|
15
|
+
date: 2016-12-13 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -559,6 +559,7 @@ files:
|
|
559
559
|
- flydata-core/lib/flydata-core/query_job.rb
|
560
560
|
- flydata-core/lib/flydata-core/record/record.rb
|
561
561
|
- flydata-core/lib/flydata-core/redshift/string.rb
|
562
|
+
- flydata-core/lib/flydata-core/source_pos_base.rb
|
562
563
|
- flydata-core/lib/flydata-core/string_utils.rb
|
563
564
|
- flydata-core/lib/flydata-core/table_def.rb
|
564
565
|
- flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
|
@@ -863,6 +864,7 @@ files:
|
|
863
864
|
- spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb
|
864
865
|
- spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb
|
865
866
|
- spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb
|
867
|
+
- spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb
|
866
868
|
- spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
|
867
869
|
- spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
|
868
870
|
- spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
|