flydata 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata.gemspec +9 -5
- data/lib/flydata/command/sync.rb +89 -988
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +6 -1
- data/lib/flydata/helpers.rb +11 -0
- data/lib/flydata/output/forwarder.rb +166 -0
- data/lib/flydata/parser/mysql/dump_parser.rb +729 -0
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +214 -2
- data/lib/flydata/sync_file_manager.rb +1 -1
- data/lib/flydata/table_def/mysql_table_def.rb +61 -47
- data/lib/flydata/table_def/redshift_table_def.rb +30 -26
- data/spec/flydata/command/sync_spec.rb +0 -1160
- data/spec/flydata/output/forwarder_spec.rb +105 -0
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +224 -23
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +900 -0
- data/spec/flydata/sync_file_manager_spec.rb +159 -0
- data/spec/flydata/table_def/mysql_table_def_spec.rb +2 -2
- data/spec/flydata/table_def/redshift_table_def_spec.rb +199 -44
- metadata +8 -3
@@ -0,0 +1,159 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require 'spec_helper'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module FileUtil
|
6
|
+
describe SyncFileManager do
|
7
|
+
let(:default_mysqldump_dir) do
|
8
|
+
File.join('/tmp', "sync_dump_#{Time.now.to_i}")
|
9
|
+
end
|
10
|
+
let(:default_data_entry) do
|
11
|
+
{"id"=>93,
|
12
|
+
"name"=>"flydata_sync_mysql",
|
13
|
+
"data_port_id"=>52,
|
14
|
+
"display_name"=>"flydata_sync_mysql",
|
15
|
+
"enabled"=>true,
|
16
|
+
"heroku_log_type"=>nil,
|
17
|
+
"heroku_resource_id"=>nil,
|
18
|
+
"log_deletion"=>nil,
|
19
|
+
"log_file_delimiter"=>nil,
|
20
|
+
"log_file_type"=>nil,
|
21
|
+
"log_path"=>nil,
|
22
|
+
"redshift_schema_name"=>"",
|
23
|
+
"redshift_table_name"=>nil,
|
24
|
+
"created_at"=>"2014-01-22T18:58:43Z",
|
25
|
+
"updated_at"=>"2014-01-30T02:42:26Z",
|
26
|
+
"type"=>"RedshiftMysqlDataEntry",
|
27
|
+
"tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
|
28
|
+
"tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
|
29
|
+
"data_port_key"=>"a458c641",
|
30
|
+
"mysql_data_entry_preference" =>
|
31
|
+
{ "host"=>"localhost", "port"=>3306, "username"=>"masashi",
|
32
|
+
"password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
|
33
|
+
"mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
|
34
|
+
"data_servers"=>"localhost:9905" }
|
35
|
+
}
|
36
|
+
end
|
37
|
+
let(:default_sfm) { SyncFileManager.new(default_data_entry) }
|
38
|
+
|
39
|
+
let (:status) { 'PARSING' }
|
40
|
+
let (:table_name) { 'test_table' }
|
41
|
+
let (:last_pos) { 9999 }
|
42
|
+
let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
|
43
|
+
let (:state) { 'CREATE_TABLE' }
|
44
|
+
let (:substate) { nil }
|
45
|
+
|
46
|
+
after :each do
|
47
|
+
if Dir.exists?(default_mysqldump_dir)
|
48
|
+
Dir.delete(default_mysqldump_dir) rescue nil
|
49
|
+
end
|
50
|
+
if File.exists?(default_mysqldump_dir)
|
51
|
+
File.delete(default_mysqldump_dir) rescue nil
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
describe '#dump_file_path' do
|
56
|
+
context 'when mysqldump_dir param is nil' do
|
57
|
+
before do
|
58
|
+
default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
|
59
|
+
end
|
60
|
+
it do
|
61
|
+
expect(default_sfm.dump_file_path).to eq(
|
62
|
+
File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
|
63
|
+
end
|
64
|
+
end
|
65
|
+
context 'when file exists' do
|
66
|
+
before { `touch #{default_mysqldump_dir}`}
|
67
|
+
it do
|
68
|
+
expect{default_sfm.dump_file_path}.to raise_error
|
69
|
+
end
|
70
|
+
end
|
71
|
+
context 'when directory exists' do
|
72
|
+
before { `mkdir -p #{default_mysqldump_dir}`}
|
73
|
+
it do
|
74
|
+
expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
|
75
|
+
expect(default_sfm.dump_file_path).to eq(
|
76
|
+
File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
|
77
|
+
end
|
78
|
+
end
|
79
|
+
context 'when directory or file does not exist' do
|
80
|
+
it do
|
81
|
+
expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
|
82
|
+
expect(default_sfm.dump_file_path).to eq(
|
83
|
+
File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
|
84
|
+
end
|
85
|
+
end
|
86
|
+
context 'when file name includes "~"' do
|
87
|
+
let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
|
88
|
+
it do
|
89
|
+
expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
|
90
|
+
expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
|
91
|
+
expect(default_sfm.dump_file_path).to eq(
|
92
|
+
File.join(expected_dir, 'flydata_sync_mysql.dump'))
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
describe '#dump_pos_path' do
|
98
|
+
it { expect(default_sfm.dump_pos_path).to eq(
|
99
|
+
File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
|
100
|
+
end
|
101
|
+
|
102
|
+
describe '#save_dump_pos' do
|
103
|
+
context 'without mysql marshal data' do
|
104
|
+
it do
|
105
|
+
expect{default_sfm.save_dump_pos(
|
106
|
+
status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
|
107
|
+
expect(default_sfm.load_dump_pos).to eq({
|
108
|
+
status: status, table_name: table_name, last_pos: last_pos,
|
109
|
+
binlog_pos: binlog_pos, state: state, substate: substate,
|
110
|
+
mysql_table: nil
|
111
|
+
})
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe '#load_dump_pos' do
|
117
|
+
let (:mysql_table) do
|
118
|
+
Flydata::Parser::Mysql::MysqlTable.new(
|
119
|
+
table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
|
120
|
+
)
|
121
|
+
end
|
122
|
+
|
123
|
+
context 'with mysql marshal data' do
|
124
|
+
before do
|
125
|
+
default_sfm.save_mysql_table_marshal_dump(mysql_table)
|
126
|
+
default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
|
127
|
+
end
|
128
|
+
it do
|
129
|
+
ret = default_sfm.load_dump_pos
|
130
|
+
mt = ret.delete(:mysql_table)
|
131
|
+
expect(ret).to eq({
|
132
|
+
status: status, table_name: table_name, last_pos: last_pos,
|
133
|
+
binlog_pos: binlog_pos, state: state, substate: substate,
|
134
|
+
})
|
135
|
+
expect(mt.table_name).to eq(table_name)
|
136
|
+
expect(mt.columns).to eq({
|
137
|
+
'id' => { format_type: 'int' },
|
138
|
+
'value' => { format_type: 'text' },
|
139
|
+
})
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
describe '#save_binlog' do
|
145
|
+
let(:binfile) { 'mysqlbinlog.000001' }
|
146
|
+
let(:pos) { 107 }
|
147
|
+
let(:binlog_pos) { {binfile: binfile, pos: pos} }
|
148
|
+
it do
|
149
|
+
default_sfm.save_binlog(binlog_pos)
|
150
|
+
expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
describe '#binlog_path' do
|
155
|
+
it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
@@ -31,7 +31,7 @@ describe MysqlTableDef do
|
|
31
31
|
expect(subject[:columns]).to eq(
|
32
32
|
[
|
33
33
|
{:column=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
|
34
|
-
{:column=>"col_binary", :type=>"binary(
|
34
|
+
{:column=>"col_binary", :type=>"binary(202)", :default=>nil},
|
35
35
|
{:column=>"col_blob", :type=>"varbinary(65535)"},
|
36
36
|
{:column=>"col_bool", :type=>"int1(1)", :default=>"0"},
|
37
37
|
{:column=>"col_char", :type=>"varchar(18)", :default=>nil},
|
@@ -55,7 +55,7 @@ describe MysqlTableDef do
|
|
55
55
|
{:column=>"col_tinyblob", :type=>"varbinary(255)"},
|
56
56
|
{:column=>"col_tinyint", :type=>"int1(4)", :default=>nil},
|
57
57
|
{:column=>"col_tinytext", :type=>"text"},
|
58
|
-
{:column=>"col_varbinary", :type=>"varbinary(
|
58
|
+
{:column=>"col_varbinary", :type=>"varbinary(512)", :default=>nil},
|
59
59
|
{:column=>"col_varchar", :type=>"varchar(372)", :default=>nil}
|
60
60
|
]
|
61
61
|
)
|
@@ -140,74 +140,229 @@ EOT
|
|
140
140
|
end
|
141
141
|
|
142
142
|
describe '.column_def_sql' do
|
143
|
+
subject { described_class.column_def_sql(column, opt) }
|
143
144
|
let(:column) { {} }
|
144
|
-
subject { described_class.column_def_sql(column) }
|
145
145
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
146
|
+
let(:no_default_sql) { "" }
|
147
|
+
shared_examples "default values" do |*examples|
|
148
|
+
context "default nil" do
|
149
|
+
before do
|
150
|
+
column[:default] = nil
|
151
151
|
end
|
152
|
+
let(:default_sql) { " DEFAULT NULL" }
|
153
|
+
it_behaves_like *examples
|
152
154
|
end
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
155
|
+
context "no default" do
|
156
|
+
let(:default_sql) { no_default_sql }
|
157
|
+
it_behaves_like *examples
|
158
|
+
end
|
159
|
+
context "defalut value" do
|
160
|
+
before do
|
161
|
+
column[:default] = default_value
|
158
162
|
end
|
163
|
+
let(:default_sql) { " DEFAULT #{default_value_sql}" }
|
164
|
+
it_behaves_like *examples
|
159
165
|
end
|
166
|
+
end
|
160
167
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
168
|
+
shared_examples "not null values" do |*examples|
|
169
|
+
context "without not null" do
|
170
|
+
let(:not_null_sql) { "" }
|
171
|
+
it_behaves_like *examples
|
172
|
+
end
|
173
|
+
context "with not null false" do
|
174
|
+
before do
|
175
|
+
column[:not_null] = false
|
165
176
|
end
|
177
|
+
let(:not_null_sql) { "" }
|
178
|
+
it_behaves_like *examples
|
166
179
|
end
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
context 'when size exceeds max size' do
|
171
|
-
let(:column) { {:column=>"col_medium_blob", :type=>"varbinary(16777215)", :default=>nil} }
|
172
|
-
it do
|
173
|
-
expect(subject).to eq(' "col_medium_blob" varchar(65535) DEFAULT NULL')
|
180
|
+
context "with not null true" do
|
181
|
+
before do
|
182
|
+
column[:not_null] = true
|
174
183
|
end
|
175
|
-
|
184
|
+
let(:not_null_sql) { " NOT NULL" }
|
185
|
+
it_behaves_like *examples
|
186
|
+
end
|
176
187
|
end
|
177
|
-
|
178
|
-
|
179
|
-
let(:column) { {:column=>"value_small_int", :type=>"int2(5) unsigned", :default=>nil} }
|
188
|
+
|
189
|
+
shared_examples "test generated Redshift SQL for column" do
|
180
190
|
it do
|
181
|
-
|
191
|
+
expect(subject).to eq(expected_query)
|
182
192
|
end
|
183
193
|
end
|
184
194
|
|
185
|
-
|
186
|
-
context '
|
187
|
-
|
188
|
-
|
189
|
-
|
195
|
+
shared_examples "test column types" do |*examples|
|
196
|
+
context 'with varchar column def' do
|
197
|
+
context 'when size is smaller than max size' do
|
198
|
+
let(:default_value) { "something" }
|
199
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
200
|
+
let(:not_null_default_sql) { " DEFAULT ''" }
|
201
|
+
before do
|
202
|
+
column[:column] = "col_char"
|
203
|
+
column[:type] = "varchar(18)"
|
204
|
+
end
|
205
|
+
let(:type_sql) { %Q|"col_char" varchar(18)| }
|
206
|
+
|
207
|
+
it_behaves_like *examples
|
208
|
+
end
|
209
|
+
|
210
|
+
context 'when size exceeds varchar max size' do
|
211
|
+
let(:default_value) { "something" }
|
212
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
213
|
+
let(:not_null_default_sql) { " DEFAULT ''" }
|
214
|
+
before do
|
215
|
+
column[:column] = "col_char"
|
216
|
+
column[:type] = "varchar(1000000000)"
|
217
|
+
end
|
218
|
+
let(:type_sql) { %Q|"col_char" varchar(65535)| }
|
219
|
+
|
220
|
+
it_behaves_like *examples
|
221
|
+
end
|
222
|
+
|
223
|
+
context 'when size is max size' do
|
224
|
+
let(:default_value) { "something" }
|
225
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
226
|
+
let(:not_null_default_sql) { " DEFAULT ''" }
|
227
|
+
before do
|
228
|
+
column[:column] = "col_char"
|
229
|
+
column[:type] = "varchar(max)"
|
230
|
+
end
|
231
|
+
let(:type_sql) { %Q|"col_char" varchar(max)| }
|
232
|
+
|
233
|
+
it_behaves_like *examples
|
190
234
|
end
|
191
235
|
end
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
236
|
+
|
237
|
+
context 'with medium blob column def' do
|
238
|
+
context 'when size exceeds max size' do
|
239
|
+
let(:default_value) { "something" }
|
240
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
241
|
+
let(:not_null_default_sql) { " DEFAULT ''" }
|
242
|
+
before do
|
243
|
+
column[:column] = "col_medium_blob"
|
244
|
+
column[:type] = "varbinary(16777215)"
|
245
|
+
end
|
246
|
+
let(:type_sql) { %Q|"col_medium_blob" varchar(65535)| }
|
247
|
+
|
248
|
+
it_behaves_like *examples
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
context 'with unsigned column def' do
|
253
|
+
let(:default_value) { 4 }
|
254
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
255
|
+
let(:not_null_default_sql) { " DEFAULT '0'" }
|
256
|
+
before do
|
257
|
+
column[:column] = "value_small_int"
|
258
|
+
column[:type] = "int2(5) unsigned"
|
196
259
|
end
|
260
|
+
let(:type_sql) { %Q|"value_small_int" int4| }
|
261
|
+
|
262
|
+
it_behaves_like *examples
|
197
263
|
end
|
198
264
|
|
199
|
-
context '
|
200
|
-
let(:
|
201
|
-
|
202
|
-
|
265
|
+
context 'with datetime column def' do
|
266
|
+
let(:default_value) { '2014-10-17 22:22:22' }
|
267
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
268
|
+
let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
|
269
|
+
before do
|
270
|
+
column[:column] = "value_datetime"
|
271
|
+
column[:type] = "datetime"
|
203
272
|
end
|
273
|
+
let(:type_sql) { %Q|"value_datetime" timestamp| }
|
274
|
+
|
275
|
+
it_behaves_like *examples
|
204
276
|
end
|
205
277
|
|
206
|
-
context '
|
207
|
-
|
208
|
-
|
209
|
-
|
278
|
+
context 'with decimal column def' do
|
279
|
+
context 'when precision exceeds max allowed' do
|
280
|
+
let(:default_value) { 4 }
|
281
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
282
|
+
let(:not_null_default_sql) { " DEFAULT '0'" }
|
283
|
+
before do
|
284
|
+
column[:column] = "value"
|
285
|
+
column[:type] = "numeric(65,30)"
|
286
|
+
end
|
287
|
+
let(:type_sql) { %Q|"value" numeric(38,30)| }
|
288
|
+
|
289
|
+
it_behaves_like *examples
|
210
290
|
end
|
291
|
+
context 'when scale exceeds max allowed' do #not possible
|
292
|
+
let(:default_value) { 4 }
|
293
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
294
|
+
let(:not_null_default_sql) { " DEFAULT '0'" }
|
295
|
+
before do
|
296
|
+
column[:column] = "value"
|
297
|
+
column[:type] = "numeric(45,44)"
|
298
|
+
end
|
299
|
+
let(:type_sql) { %Q|"value" numeric(38,37)| }
|
300
|
+
|
301
|
+
it_behaves_like *examples
|
302
|
+
end
|
303
|
+
|
304
|
+
context 'when precision and scale exceeds max allowed (for unsigned)' do
|
305
|
+
let(:default_value) { 4 }
|
306
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
307
|
+
let(:not_null_default_sql) { " DEFAULT '0'" }
|
308
|
+
before do
|
309
|
+
column[:column] = "value"
|
310
|
+
column[:type] = "numeric(65,44) unsigned"
|
311
|
+
end
|
312
|
+
let(:type_sql) {%Q|"value" numeric(38,37)| }
|
313
|
+
|
314
|
+
it_behaves_like *examples
|
315
|
+
end
|
316
|
+
|
317
|
+
context 'when precision and scale are within limit (for unsigned)' do
|
318
|
+
let(:default_value) { 4 }
|
319
|
+
let(:default_value_sql) { "'#{default_value}'" }
|
320
|
+
let(:not_null_default_sql) { " DEFAULT '0'" }
|
321
|
+
before do
|
322
|
+
column[:column] = "value"
|
323
|
+
column[:type] = "numeric(37,29) unsigned"
|
324
|
+
end
|
325
|
+
let(:type_sql) { %Q|"value" numeric(37,29)| }
|
326
|
+
|
327
|
+
it_behaves_like *examples
|
328
|
+
end
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
context 'for create table' do
|
333
|
+
let(:opt) { {} }
|
334
|
+
let(:expected_query) { %Q| #{type_sql}#{not_null_sql}#{default_sql}| }
|
335
|
+
let(:no_default_sql) { "" }
|
336
|
+
|
337
|
+
it_behaves_like "test column types", "not null values", "default values", "test generated Redshift SQL for column"
|
338
|
+
end
|
339
|
+
|
340
|
+
context 'for alter table' do
|
341
|
+
let(:opt) { { for: :alter_table } }
|
342
|
+
|
343
|
+
context "without not null" do
|
344
|
+
let(:expected_query) { %Q| #{type_sql}#{default_sql}| }
|
345
|
+
let(:no_default_sql) { "" }
|
346
|
+
|
347
|
+
it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
|
348
|
+
end
|
349
|
+
context "with not null false" do
|
350
|
+
let(:expected_query) { %Q| #{type_sql}#{default_sql}| }
|
351
|
+
before do
|
352
|
+
column[:not_null] = false
|
353
|
+
end
|
354
|
+
let(:no_default_sql) { "" }
|
355
|
+
|
356
|
+
it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
|
357
|
+
end
|
358
|
+
context "with not null true" do
|
359
|
+
let(:expected_query) { %Q| #{type_sql} NOT NULL#{default_sql}| }
|
360
|
+
before do
|
361
|
+
column[:not_null] = true
|
362
|
+
end
|
363
|
+
let(:no_default_sql) { not_null_default_sql }
|
364
|
+
|
365
|
+
it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
|
211
366
|
end
|
212
367
|
end
|
213
368
|
end
|