flydata 0.7.14 → 0.7.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +23 -38
- data/flydata-core/lib/flydata-core/postgresql/snapshot.rb +7 -10
- data/flydata-core/lib/flydata-core/postgresql/source_pos.rb +9 -5
- data/flydata-core/lib/flydata-core/source_pos_base.rb +77 -0
- data/flydata-core/spec/mysql/binlog_pos_spec.rb +65 -2
- data/flydata-core/spec/postgresql/snapshot_spec.rb +31 -0
- data/flydata-core/spec/postgresql/source_pos_spec.rb +35 -0
- data/flydata.gemspec +4 -2
- data/lib/flydata/command/sync.rb +7 -2
- data/lib/flydata/plugin_support/sync_record_emittable.rb +6 -1
- data/lib/flydata/source_mysql/plugin_support/context.rb +6 -0
- data/lib/flydata/source_mysql/plugin_support/drop_table_query_handler.rb +3 -0
- data/spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb +34 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5435f4af64a22277e8417bbdd7196ee8aabc6c19
|
4
|
+
data.tar.gz: 7bbbff8d0c4cb13891f35dc8debf891b2aa31953
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 646f5ec5b0f3be8102a80c386291c95b6ad220c3e3401b89acea12ab7b59179b3a806fdfc67c403f9ed44d4a08a7080a49f48528109257faad2c1c3cd9fd84f2
|
7
|
+
data.tar.gz: 179efaee49c60d6573da9b0f6f57476fc8ae26f43ab70bee591f174fec5ba09b90d5a6c4ea2cf95f91417d73f8144d89ae2a568457b28d2034c9479eb629bc17
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.7.
|
1
|
+
0.7.15
|
@@ -1,28 +1,28 @@
|
|
1
|
+
require 'flydata-core/source_pos_base'
|
2
|
+
|
1
3
|
module FlydataCore
|
2
4
|
module Mysql
|
3
5
|
|
4
6
|
class BinlogPos
|
5
|
-
include
|
7
|
+
include SourcePosBase
|
8
|
+
|
6
9
|
def initialize(binlog_str_or_filename_or_hash, binlog_pos = nil)
|
7
10
|
arg = binlog_str_or_filename_or_hash
|
8
11
|
if binlog_pos
|
9
12
|
@pos = binlog_pos
|
10
13
|
@filename = arg
|
11
14
|
elsif arg.kind_of?(String)
|
12
|
-
|
13
|
-
set_empty
|
14
|
-
else
|
15
|
-
@filename, @pos = arg.split("\t")
|
16
|
-
end
|
15
|
+
@filename, @pos = arg.split("\t")
|
17
16
|
elsif arg.kind_of?(Hash)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
@pos = arg[:pos]
|
18
|
+
@filename = arg[:filename] ||arg[:binfile]
|
19
|
+
end
|
20
|
+
|
21
|
+
# Set -1 to pos if filename is predefined value
|
22
|
+
if set_predefined_flags(@filename)
|
23
|
+
@pos = -1
|
25
24
|
end
|
25
|
+
|
26
26
|
if @filename.nil? || @pos.nil?
|
27
27
|
raise "Invalid initialize argument (#{arg}, #{binlog_pos})"
|
28
28
|
end
|
@@ -35,43 +35,28 @@ class BinlogPos
|
|
35
35
|
|
36
36
|
attr_reader :filename, :pos
|
37
37
|
|
38
|
-
|
39
|
-
@filename && @filename == "-"
|
40
|
-
end
|
41
|
-
|
38
|
+
# Override (Comparable)
|
42
39
|
def <=>(other_pos)
|
43
|
-
|
44
|
-
|
45
|
-
end
|
40
|
+
s = super
|
41
|
+
return s if s
|
46
42
|
|
47
|
-
other_pos =
|
48
|
-
|
49
|
-
return 0
|
50
|
-
elsif empty? || other_pos.empty?
|
51
|
-
raise ArgumentError.new("comparison with empty BinlogPos failed")
|
52
|
-
else
|
53
|
-
(@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
|
54
|
-
end
|
43
|
+
other_pos = self.class.new(other_pos) unless other_pos.kind_of?(self.class)
|
44
|
+
(@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
|
55
45
|
end
|
56
46
|
|
47
|
+
# Override (Comparable)
|
57
48
|
def ==(other_pos)
|
58
49
|
return false if other_pos.nil?
|
59
50
|
return @filename == other_pos.filename if empty? || other_pos.empty?
|
60
51
|
self.<=>(other_pos) == 0
|
61
52
|
end
|
62
53
|
|
54
|
+
# Override
|
63
55
|
def to_s
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
private
|
68
|
-
def set_empty
|
69
|
-
# @filename == "-" is the definition of `empty`. @pos could be anything but nul
|
70
|
-
@filename = "-"
|
71
|
-
@pos = -1
|
56
|
+
s = super
|
57
|
+
return s if s
|
58
|
+
"#{@filename}\t#{@pos}"
|
72
59
|
end
|
73
|
-
|
74
|
-
|
75
60
|
end
|
76
61
|
|
77
62
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'flydata-core/source_pos_base'
|
2
|
+
|
1
3
|
# Snapshot components
|
2
4
|
# http://www.postgresql.org/docs/8.3/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
|
3
5
|
#
|
@@ -7,12 +9,12 @@ module FlydataCore
|
|
7
9
|
module Postgresql
|
8
10
|
|
9
11
|
class Snapshot
|
10
|
-
include
|
12
|
+
include SourcePosBase
|
11
13
|
|
12
14
|
def initialize(txid_snapshot)
|
13
15
|
@txid_snapshot_str = txid_snapshot.to_s
|
14
16
|
|
15
|
-
return if
|
17
|
+
return if set_predefined_flags(@txid_snapshot_str)
|
16
18
|
xmin_str, xmax_str, xip_list_str = @txid_snapshot_str.split(':')
|
17
19
|
|
18
20
|
raise ArgumentError, "Invalid snapshot - xmin is empty." if xmin_str.to_s.empty?
|
@@ -29,15 +31,10 @@ class Snapshot
|
|
29
31
|
@txid_snapshot_str
|
30
32
|
end
|
31
33
|
|
32
|
-
|
33
|
-
@txid_snapshot_str == '-'
|
34
|
-
end
|
35
|
-
|
34
|
+
# Override (Comparable)
|
36
35
|
def <=>(other)
|
37
|
-
|
38
|
-
|
39
|
-
raise ArgumentError.new("comparison with empty source pos failed")
|
40
|
-
end
|
36
|
+
s = super
|
37
|
+
return s if s
|
41
38
|
|
42
39
|
if @xmin == other.xmin
|
43
40
|
if @xmax == other.xmax
|
@@ -1,11 +1,12 @@
|
|
1
1
|
require 'json'
|
2
2
|
require 'flydata-core/postgresql/snapshot'
|
3
|
+
require 'flydata-core/source_pos_base'
|
3
4
|
|
4
5
|
module FlydataCore
|
5
6
|
module Postgresql
|
6
7
|
|
7
8
|
class SourcePos
|
8
|
-
include
|
9
|
+
include SourcePosBase
|
9
10
|
|
10
11
|
# Source Position data for PostgreSQL
|
11
12
|
#
|
@@ -53,17 +54,17 @@ class SourcePos
|
|
53
54
|
end
|
54
55
|
@pk_values = pk_values # must be array or nil
|
55
56
|
end
|
57
|
+
|
58
|
+
set_predefined_flags(@snapshot)
|
56
59
|
end
|
57
60
|
|
58
61
|
attr_reader :snapshot
|
59
62
|
attr_reader :to_snapshot
|
60
63
|
attr_reader :pk_values
|
61
64
|
|
62
|
-
def empty?
|
63
|
-
@snapshot.empty?
|
64
|
-
end
|
65
|
-
|
66
65
|
def to_s
|
66
|
+
s = super
|
67
|
+
return s if s
|
67
68
|
pk_values = @pk_values ? @pk_values.to_json : ''
|
68
69
|
"#{@snapshot}\t#{@to_snapshot}\t#{pk_values}"
|
69
70
|
end
|
@@ -71,6 +72,9 @@ class SourcePos
|
|
71
72
|
#TODO: Need to revisit the logic along with the spec
|
72
73
|
# http://www.postgresql.org/docs/9.5/static/functions-info.html#FUNCTIONS-TXID-SNAPSHOT-PARTS
|
73
74
|
def <=>(other)
|
75
|
+
s = super
|
76
|
+
return s if s
|
77
|
+
|
74
78
|
if @snapshot != other.snapshot
|
75
79
|
return @snapshot <=> other.snapshot
|
76
80
|
elsif @pk_values.nil? && !other.pk_values.nil?
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module FlydataCore
|
2
|
+
module SourcePosBase
|
3
|
+
EMPTY = "-"
|
4
|
+
INFINITY = 'infinity'
|
5
|
+
PREDEFINED_SRC_LIST = [EMPTY, INFINITY]
|
6
|
+
|
7
|
+
def self.included(base)
|
8
|
+
base.class_eval do
|
9
|
+
include Comparable
|
10
|
+
attr_reader :src_pos_empty, :src_pos_infinity
|
11
|
+
|
12
|
+
def self.empty_src_pos
|
13
|
+
self.new(EMPTY)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.infinity_src_pos
|
17
|
+
self.new(INFINITY)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def to_s
|
23
|
+
if empty?
|
24
|
+
EMPTY
|
25
|
+
elsif infinity?
|
26
|
+
INFINITY
|
27
|
+
else
|
28
|
+
nil
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# Return true if one of flags is set
|
33
|
+
def set_predefined_flags(str)
|
34
|
+
case str.to_s
|
35
|
+
when EMPTY
|
36
|
+
@src_pos_empty = true
|
37
|
+
when INFINITY
|
38
|
+
@src_pos_infinity = true
|
39
|
+
else
|
40
|
+
false
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def empty?
|
45
|
+
@src_pos_empty == true
|
46
|
+
end
|
47
|
+
|
48
|
+
def infinity?
|
49
|
+
@src_pos_infinity == true
|
50
|
+
end
|
51
|
+
|
52
|
+
# Override (Comparable)
|
53
|
+
def <=>(other_pos)
|
54
|
+
if other_pos.nil?
|
55
|
+
raise ArgumentError.new("comparison with nil failed")
|
56
|
+
end
|
57
|
+
|
58
|
+
other_pos = self.class.new(other_pos) unless other_pos.kind_of?(self.class)
|
59
|
+
|
60
|
+
if infinity? && other_pos.infinity?
|
61
|
+
return 0
|
62
|
+
elsif infinity?
|
63
|
+
return 1
|
64
|
+
elsif other_pos.infinity?
|
65
|
+
return -1
|
66
|
+
end
|
67
|
+
|
68
|
+
if empty? && other_pos.empty?
|
69
|
+
return 0
|
70
|
+
elsif empty? || other_pos.empty?
|
71
|
+
raise ArgumentError.new("comparison with empty source pos failed")
|
72
|
+
end
|
73
|
+
|
74
|
+
nil
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -20,6 +20,16 @@ module Mysql
|
|
20
20
|
describe '#to_s' do
|
21
21
|
subject { subject_object.to_s }
|
22
22
|
it { is_expected.to eq(binlog_str) }
|
23
|
+
|
24
|
+
context 'when filename is "-"' do
|
25
|
+
let(:binlog_str) { "-" }
|
26
|
+
it { is_expected.to eq("-") }
|
27
|
+
end
|
28
|
+
|
29
|
+
context 'when filename is "infinity"' do
|
30
|
+
let(:binlog_str) { "infinity" }
|
31
|
+
it { is_expected.to eq("infinity") }
|
32
|
+
end
|
23
33
|
end
|
24
34
|
|
25
35
|
describe '.new' do
|
@@ -46,6 +56,32 @@ module Mysql
|
|
46
56
|
let(:arg1) { { filename: filename, pos: pos } }
|
47
57
|
it_behaves_like "properly constructed"
|
48
58
|
end
|
59
|
+
|
60
|
+
context 'when given a empty("-") string' do
|
61
|
+
let(:arg1) { { filename: "-", pos: nil} }
|
62
|
+
let(:filename) { '-' }
|
63
|
+
let(:pos) { -1 }
|
64
|
+
it_behaves_like "properly constructed"
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'when given a "infinity" string' do
|
68
|
+
let(:arg1) { { filename: "infinity", pos: 10} }
|
69
|
+
let(:filename) { 'infinity' }
|
70
|
+
let(:pos) { -1 }
|
71
|
+
it_behaves_like "properly constructed"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe '.empty_src_pos' do
|
76
|
+
subject { described_class.empty_src_pos }
|
77
|
+
it { expect(subject.empty?).to be(true) }
|
78
|
+
it { expect(subject.to_s).to eq('-') }
|
79
|
+
end
|
80
|
+
|
81
|
+
describe '.infinity_src_pos' do
|
82
|
+
subject { described_class.infinity_src_pos }
|
83
|
+
it { expect(subject.infinity?).to be(true) }
|
84
|
+
it { expect(subject.to_s).to eq('infinity') }
|
49
85
|
end
|
50
86
|
|
51
87
|
describe '.load' do
|
@@ -105,7 +141,7 @@ module Mysql
|
|
105
141
|
it { is_expected.to eq -1 }
|
106
142
|
end
|
107
143
|
end
|
108
|
-
|
144
|
+
|
109
145
|
context 'when the target binlog filename is older' do
|
110
146
|
let(:arg_filename) { older_filename }
|
111
147
|
let(:expected_result) { 1 }
|
@@ -121,21 +157,48 @@ module Mysql
|
|
121
157
|
let(:arg_str) { "-" }
|
122
158
|
it { expect { subject }.to raise_error(ArgumentError) }
|
123
159
|
end
|
160
|
+
|
161
|
+
context 'when the target is infinity' do
|
162
|
+
let(:arg_str) { "infinity" }
|
163
|
+
it { is_expected.to eq -1 }
|
164
|
+
end
|
165
|
+
|
124
166
|
context 'when self is empty' do
|
125
167
|
let(:binlog_str) { "-" }
|
126
168
|
context 'when target is empty' do
|
127
169
|
let(:arg_str) { "-" }
|
128
170
|
it { is_expected.to eq 0 }
|
129
171
|
end
|
130
|
-
context 'when target is
|
172
|
+
context 'when target is infinity' do
|
173
|
+
let(:arg_str) { "infinity" }
|
174
|
+
it { is_expected.to eq -1 }
|
175
|
+
end
|
176
|
+
context 'when target is normal value' do
|
131
177
|
let(:arg) { double('arg') }
|
132
178
|
before do
|
133
179
|
allow(arg).to receive(:kind_of?).with(BinlogPos).and_return(true)
|
134
180
|
allow(arg).to receive(:empty?).and_return(false)
|
181
|
+
allow(arg).to receive(:infinity?).and_return(false)
|
135
182
|
end
|
136
183
|
it { expect { subject }.to raise_error(ArgumentError) }
|
137
184
|
end
|
138
185
|
end
|
186
|
+
|
187
|
+
context 'when self is infinity' do
|
188
|
+
let(:binlog_str) { "infinity" }
|
189
|
+
context 'when target is infinity' do
|
190
|
+
let(:arg_str) { "infinity" }
|
191
|
+
it { is_expected.to eq 0 }
|
192
|
+
end
|
193
|
+
context 'when target is empty' do
|
194
|
+
let(:arg_str) { "-" }
|
195
|
+
it { is_expected.to eq 1 }
|
196
|
+
end
|
197
|
+
context 'when target is normal value' do
|
198
|
+
let(:arg_str) { "mysql-bin.000064\t104" }
|
199
|
+
it { is_expected.to eq 1 }
|
200
|
+
end
|
201
|
+
end
|
139
202
|
end
|
140
203
|
|
141
204
|
describe '#<' do
|
@@ -19,6 +19,16 @@ module FlydataCore
|
|
19
19
|
it { expect(subject).to be_empty }
|
20
20
|
end
|
21
21
|
|
22
|
+
context 'when text is infinity' do
|
23
|
+
let(:snapshot_text) { 'infinity' }
|
24
|
+
it { expect(subject.xmin).to be_nil }
|
25
|
+
it { expect(subject.xmax).to be_nil }
|
26
|
+
it { expect(subject.xip_list).to be_nil }
|
27
|
+
it { expect(subject.to_s).to eq('infinity') }
|
28
|
+
it { expect(subject).not_to be_empty }
|
29
|
+
it { expect(subject.infinity?).to be(true)}
|
30
|
+
end
|
31
|
+
|
22
32
|
context 'when xip_list is empty' do
|
23
33
|
let(:snapshot_text) { '10:18:' }
|
24
34
|
it { expect(subject.xmin).to eq(10) }
|
@@ -50,6 +60,20 @@ module FlydataCore
|
|
50
60
|
end
|
51
61
|
end
|
52
62
|
|
63
|
+
describe '#infinity?' do
|
64
|
+
subject { subject_object.infinity? }
|
65
|
+
|
66
|
+
context 'when snapshot_txt is infinity' do
|
67
|
+
let(:snapshot_text) { 'infinity' }
|
68
|
+
it { is_expected.to be(true) }
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'when snapshot_txt is normal value' do
|
72
|
+
let(:snapshot_text) { '10:10:' }
|
73
|
+
it { is_expected.to be(false) }
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
53
77
|
describe '#<=>' do
|
54
78
|
def new_ss(txt)
|
55
79
|
Snapshot.new(txt)
|
@@ -64,6 +88,10 @@ module FlydataCore
|
|
64
88
|
it { expect(new_ss("10:18:10,11,12") == new_ss("10:18:11,12")).to be(false) }
|
65
89
|
it { expect(new_ss("-") == new_ss("-")).to be(true) }
|
66
90
|
it { expect(new_ss("-") == new_ss("10:10:")).to be(false) }
|
91
|
+
it { expect(new_ss("-") == new_ss("infinity")).to be(false) }
|
92
|
+
it { expect(new_ss("infinity") == new_ss("infinity")).to be(true) }
|
93
|
+
it { expect(new_ss("infinity") == new_ss("-")).to be(false) }
|
94
|
+
it { expect(new_ss("infinity") == new_ss("10:10:")).to be(false) }
|
67
95
|
|
68
96
|
it { expect(new_ss("10:10:") > new_ss("10:10:")).to be(false) }
|
69
97
|
it { expect(new_ss("10:10:") > new_ss("11:11:")).to be(false) }
|
@@ -74,6 +102,9 @@ module FlydataCore
|
|
74
102
|
it { expect(new_ss("10:18:") > new_ss("10:18:10,11,12")).to be(true) }
|
75
103
|
it { expect(new_ss("10:18:10,11,12") > new_ss("10:18:10,11,12")).to be(false) }
|
76
104
|
it { expect(new_ss("10:18:10,11,12") > new_ss("10:18:11,12")).to be(false) }
|
105
|
+
it { expect(new_ss("10:18:10,11,12") > new_ss("infinity")).to be(false) }
|
106
|
+
it { expect(new_ss("infinity") > new_ss("10:18:")).to be(true) }
|
107
|
+
|
77
108
|
end
|
78
109
|
end
|
79
110
|
end
|
@@ -27,6 +27,16 @@ describe SourcePos do
|
|
27
27
|
it { expect(subject.to_snapshot).to eq(Snapshot.new(to_snapshot)) }
|
28
28
|
it { expect(subject.pk_values).to eq(pk_values) }
|
29
29
|
end
|
30
|
+
|
31
|
+
context 'with empty string' do
|
32
|
+
let(:snapshot) { '-' }
|
33
|
+
it { expect(subject_object.empty?).to be(true) }
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'with infinity string' do
|
37
|
+
let(:snapshot) { 'infinity' }
|
38
|
+
it { expect(subject_object.snapshot.infinity?).to be(true) }
|
39
|
+
end
|
30
40
|
end
|
31
41
|
|
32
42
|
describe '#to_s' do
|
@@ -36,6 +46,21 @@ describe SourcePos do
|
|
36
46
|
let(:pk_values) { nil }
|
37
47
|
it { is_expected.to eq %Q|#{snapshot}\t\t| }
|
38
48
|
end
|
49
|
+
|
50
|
+
context 'when empty ' do
|
51
|
+
let(:snapshot) { '-' }
|
52
|
+
let(:to_snapshot) { nil }
|
53
|
+
let(:pk_values) { nil }
|
54
|
+
it { is_expected.to eq '-' }
|
55
|
+
end
|
56
|
+
|
57
|
+
context 'when infinity' do
|
58
|
+
let(:snapshot) { 'infinity' }
|
59
|
+
let(:to_snapshot) { nil }
|
60
|
+
let(:pk_values) { nil }
|
61
|
+
it { is_expected.to eq 'infinity' }
|
62
|
+
end
|
63
|
+
|
39
64
|
context 'when pk_values is not nil' do
|
40
65
|
let(:to_snapshot) { '1234:1234:' }
|
41
66
|
let(:pk_values) { { "user_id" => 1, "address_id" => 3 } }
|
@@ -63,6 +88,7 @@ describe SourcePos do
|
|
63
88
|
def src_pos(snapshot, pk_values = nil)
|
64
89
|
described_class.new(snapshot, '2222:2222:', pk_values)
|
65
90
|
end
|
91
|
+
|
66
92
|
context 'when both have no pk_values' do
|
67
93
|
it { expect(src_pos('1111:1111:') == src_pos('1111:1111:')).to be(true) }
|
68
94
|
it { expect(src_pos('1111:1111:') == src_pos('1112:1112:')).to be(false) }
|
@@ -98,6 +124,15 @@ describe SourcePos do
|
|
98
124
|
it { expect(src_pos('1111:1111:',pkv1) > src_pos('1112:1112:',pkv2)).to be(false) }
|
99
125
|
it { expect(src_pos('1111:1111:',pkv1) > src_pos('1111:1111:',pkv2)).to be(false) }
|
100
126
|
end
|
127
|
+
|
128
|
+
context 'when one of pos is infinity' do
|
129
|
+
it { expect(src_pos('infinity') == src_pos('1111:1111:')).to be(false) }
|
130
|
+
it { expect(src_pos('infinity') == src_pos('infinity')).to be(true) }
|
131
|
+
it { expect(src_pos('infinity') < src_pos('1112:1112:')).to be(false) }
|
132
|
+
it { expect(src_pos('1111:1111:') < src_pos('infinity')).to be(true) }
|
133
|
+
it { expect(src_pos('infinity') > src_pos('1110:1110:')).to be(true) }
|
134
|
+
it { expect(src_pos('1111:1111:') > src_pos('infinity')).to be(false) }
|
135
|
+
end
|
101
136
|
end
|
102
137
|
end
|
103
138
|
|
data/flydata.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "flydata"
|
8
|
-
s.version = "0.7.
|
8
|
+
s.version = "0.7.15"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
12
|
-
s.date = "2016-12-
|
12
|
+
s.date = "2016-12-13"
|
13
13
|
s.description = "FlyData Agent"
|
14
14
|
s.email = "sysadmin@flydata.com"
|
15
15
|
s.executables = ["fdredshift", "flydata", "serverinfo", "split_sync_ddl.rb"]
|
@@ -84,6 +84,7 @@ Gem::Specification.new do |s|
|
|
84
84
|
"flydata-core/lib/flydata-core/query_job.rb",
|
85
85
|
"flydata-core/lib/flydata-core/record/record.rb",
|
86
86
|
"flydata-core/lib/flydata-core/redshift/string.rb",
|
87
|
+
"flydata-core/lib/flydata-core/source_pos_base.rb",
|
87
88
|
"flydata-core/lib/flydata-core/string_utils.rb",
|
88
89
|
"flydata-core/lib/flydata-core/table_def.rb",
|
89
90
|
"flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
|
@@ -388,6 +389,7 @@ Gem::Specification.new do |s|
|
|
388
389
|
"spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb",
|
389
390
|
"spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb",
|
390
391
|
"spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb",
|
392
|
+
"spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb",
|
391
393
|
"spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb",
|
392
394
|
"spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb",
|
393
395
|
"spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb",
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -1466,8 +1466,13 @@ Thank you for using FlyData!
|
|
1466
1466
|
$log.debug "table pos updated. table:#{table} pos:#{old_pos} -> #{pos}"
|
1467
1467
|
end
|
1468
1468
|
if source_pos != old_source_pos
|
1469
|
-
|
1470
|
-
|
1469
|
+
# keep binlog pos file if source pos is infinity until reset happenes
|
1470
|
+
if old_source_pos && old_source_pos.infinity?
|
1471
|
+
$log.debug "table source_pos update skipped. table:#{table} source_pos:`#{old_source_pos}` -> `#{old_source_pos}`"
|
1472
|
+
else
|
1473
|
+
sync_fm.save_table_source_pos(table, source_pos, destination: :positions)
|
1474
|
+
$log.debug "table source_pos updated. table:#{table} source_pos:`#{old_source_pos}` -> `#{source_pos}`"
|
1475
|
+
end
|
1471
1476
|
end
|
1472
1477
|
[old_source_pos, old_pos]
|
1473
1478
|
end
|
@@ -62,7 +62,12 @@ module PluginSupport
|
|
62
62
|
[timestamp, r]
|
63
63
|
end
|
64
64
|
Fluent::Engine.emit_array(tag, array)
|
65
|
-
|
65
|
+
if table
|
66
|
+
@context.sync_fm.save_table_position(table, seq)
|
67
|
+
if options[:set_infinity_to_table_binlog_pos]
|
68
|
+
@context.set_infinity_to_table_binlog_pos(table)
|
69
|
+
end
|
70
|
+
end
|
66
71
|
end
|
67
72
|
|
68
73
|
end
|
@@ -24,6 +24,12 @@ module PluginSupport
|
|
24
24
|
end
|
25
25
|
end
|
26
26
|
end
|
27
|
+
|
28
|
+
def set_infinity_to_table_binlog_pos(table_name)
|
29
|
+
new_src_pos = FlydataCore::Mysql::BinlogPos.infinity_src_pos
|
30
|
+
sync_fm.save_table_source_pos(table_name, new_src_pos, destination: :positions)
|
31
|
+
@table_binlog_pos[table_name] = new_src_pos
|
32
|
+
end
|
27
33
|
end
|
28
34
|
|
29
35
|
end
|
@@ -22,6 +22,9 @@ module PluginSupport
|
|
22
22
|
$log.info "type:drop_table table_name:'#{table_name}' query:'#{record["query"]}' binlog_pos:'#{binlog_pos(record)}'"
|
23
23
|
|
24
24
|
opt[:increment_table_rev] = true
|
25
|
+
|
26
|
+
# Set infinity to table binlog pos to ignore following events
|
27
|
+
opt[:set_infinity_to_table_binlog_pos] = true
|
25
28
|
{ table_name: table_name, query: record["query"] }
|
26
29
|
end
|
27
30
|
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'fluent_plugins_spec_helper'
|
2
|
+
require 'flydata/source_mysql/plugin_support/drop_table_query_handler'
|
3
|
+
require 'flydata/source_mysql/plugin_support/shared_query_handler_context'
|
4
|
+
|
5
|
+
module Flydata::SourceMysql::PluginSupport
|
6
|
+
describe DropTableQueryHandler do
|
7
|
+
include_context "query handler context"
|
8
|
+
|
9
|
+
let(:query) { "DROP TABLE #{table}" }
|
10
|
+
let(:normalized_query) { "DROP TABLE #{table}" }
|
11
|
+
let(:subject_object) { described_class.new(context) }
|
12
|
+
|
13
|
+
describe '#process' do
|
14
|
+
subject { subject_object.process(record) }
|
15
|
+
|
16
|
+
it 'handles a drop table event' do
|
17
|
+
expect(Fluent::Engine).to receive(:emit_array).once
|
18
|
+
expect(sync_fm).to receive(:increment_table_rev).with(table, 1)
|
19
|
+
expect(sync_fm).to receive(:save_table_position).with(table, 200)
|
20
|
+
expect(sync_fm).to receive(:save_table_source_pos).with(table, FlydataCore::Mysql::BinlogPos.infinity_src_pos, destination: :positions)
|
21
|
+
subject
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'when unsupported table is dropped' do
|
25
|
+
let(:normalized_query) { "DROP TABLE bar" }
|
26
|
+
it 'ignore a drop table event' do
|
27
|
+
expect(Fluent::Engine).to receive(:emit_array).never
|
28
|
+
expect(sync_fm).to receive(:save_table_source_pos).never
|
29
|
+
subject
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2016-12-
|
15
|
+
date: 2016-12-13 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -559,6 +559,7 @@ files:
|
|
559
559
|
- flydata-core/lib/flydata-core/query_job.rb
|
560
560
|
- flydata-core/lib/flydata-core/record/record.rb
|
561
561
|
- flydata-core/lib/flydata-core/redshift/string.rb
|
562
|
+
- flydata-core/lib/flydata-core/source_pos_base.rb
|
562
563
|
- flydata-core/lib/flydata-core/string_utils.rb
|
563
564
|
- flydata-core/lib/flydata-core/table_def.rb
|
564
565
|
- flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
|
@@ -863,6 +864,7 @@ files:
|
|
863
864
|
- spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb
|
864
865
|
- spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb
|
865
866
|
- spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb
|
867
|
+
- spec/flydata/source_mysql/plugin_support/drop_table_query_handler_spec.rb
|
866
868
|
- spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
|
867
869
|
- spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
|
868
870
|
- spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
|