flydata 0.2.8 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,159 @@
1
+ # coding: utf-8
2
+ require 'spec_helper'
3
+
4
+ module Flydata
5
+ module FileUtil
6
+ describe SyncFileManager do
7
+ let(:default_mysqldump_dir) do
8
+ File.join('/tmp', "sync_dump_#{Time.now.to_i}")
9
+ end
10
+ let(:default_data_entry) do
11
+ {"id"=>93,
12
+ "name"=>"flydata_sync_mysql",
13
+ "data_port_id"=>52,
14
+ "display_name"=>"flydata_sync_mysql",
15
+ "enabled"=>true,
16
+ "heroku_log_type"=>nil,
17
+ "heroku_resource_id"=>nil,
18
+ "log_deletion"=>nil,
19
+ "log_file_delimiter"=>nil,
20
+ "log_file_type"=>nil,
21
+ "log_path"=>nil,
22
+ "redshift_schema_name"=>"",
23
+ "redshift_table_name"=>nil,
24
+ "created_at"=>"2014-01-22T18:58:43Z",
25
+ "updated_at"=>"2014-01-30T02:42:26Z",
26
+ "type"=>"RedshiftMysqlDataEntry",
27
+ "tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
28
+ "tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
29
+ "data_port_key"=>"a458c641",
30
+ "mysql_data_entry_preference" =>
31
+ { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
32
+ "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
33
+ "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
34
+ "data_servers"=>"localhost:9905" }
35
+ }
36
+ end
37
+ let(:default_sfm) { SyncFileManager.new(default_data_entry) }
38
+
39
+ let (:status) { 'PARSING' }
40
+ let (:table_name) { 'test_table' }
41
+ let (:last_pos) { 9999 }
42
+ let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111} }
43
+ let (:state) { 'CREATE_TABLE' }
44
+ let (:substate) { nil }
45
+
46
+ after :each do
47
+ if Dir.exists?(default_mysqldump_dir)
48
+ Dir.delete(default_mysqldump_dir) rescue nil
49
+ end
50
+ if File.exists?(default_mysqldump_dir)
51
+ File.delete(default_mysqldump_dir) rescue nil
52
+ end
53
+ end
54
+
55
+ describe '#dump_file_path' do
56
+ context 'when mysqldump_dir param is nil' do
57
+ before do
58
+ default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
59
+ end
60
+ it do
61
+ expect(default_sfm.dump_file_path).to eq(
62
+ File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
63
+ end
64
+ end
65
+ context 'when file exists' do
66
+ before { `touch #{default_mysqldump_dir}`}
67
+ it do
68
+ expect{default_sfm.dump_file_path}.to raise_error
69
+ end
70
+ end
71
+ context 'when directory exists' do
72
+ before { `mkdir -p #{default_mysqldump_dir}`}
73
+ it do
74
+ expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
75
+ expect(default_sfm.dump_file_path).to eq(
76
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
77
+ end
78
+ end
79
+ context 'when directory or file does not exist' do
80
+ it do
81
+ expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
82
+ expect(default_sfm.dump_file_path).to eq(
83
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
84
+ end
85
+ end
86
+ context 'when file name includes "~"' do
87
+ let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
88
+ it do
89
+ expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
90
+ expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
91
+ expect(default_sfm.dump_file_path).to eq(
92
+ File.join(expected_dir, 'flydata_sync_mysql.dump'))
93
+ end
94
+ end
95
+ end
96
+
97
+ describe '#dump_pos_path' do
98
+ it { expect(default_sfm.dump_pos_path).to eq(
99
+ File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
100
+ end
101
+
102
+ describe '#save_dump_pos' do
103
+ context 'without mysql marshal data' do
104
+ it do
105
+ expect{default_sfm.save_dump_pos(
106
+ status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
107
+ expect(default_sfm.load_dump_pos).to eq({
108
+ status: status, table_name: table_name, last_pos: last_pos,
109
+ binlog_pos: binlog_pos, state: state, substate: substate,
110
+ mysql_table: nil
111
+ })
112
+ end
113
+ end
114
+ end
115
+
116
+ describe '#load_dump_pos' do
117
+ let (:mysql_table) do
118
+ Flydata::Parser::Mysql::MysqlTable.new(
119
+ table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
120
+ )
121
+ end
122
+
123
+ context 'with mysql marshal data' do
124
+ before do
125
+ default_sfm.save_mysql_table_marshal_dump(mysql_table)
126
+ default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
127
+ end
128
+ it do
129
+ ret = default_sfm.load_dump_pos
130
+ mt = ret.delete(:mysql_table)
131
+ expect(ret).to eq({
132
+ status: status, table_name: table_name, last_pos: last_pos,
133
+ binlog_pos: binlog_pos, state: state, substate: substate,
134
+ })
135
+ expect(mt.table_name).to eq(table_name)
136
+ expect(mt.columns).to eq({
137
+ 'id' => { format_type: 'int' },
138
+ 'value' => { format_type: 'text' },
139
+ })
140
+ end
141
+ end
142
+ end
143
+
144
+ describe '#save_binlog' do
145
+ let(:binfile) { 'mysqlbinlog.000001' }
146
+ let(:pos) { 107 }
147
+ let(:binlog_pos) { {binfile: binfile, pos: pos} }
148
+ it do
149
+ default_sfm.save_binlog(binlog_pos)
150
+ expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
151
+ end
152
+ end
153
+
154
+ describe '#binlog_path' do
155
+ it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
156
+ end
157
+ end
158
+ end
159
+ end
@@ -31,7 +31,7 @@ describe MysqlTableDef do
31
31
  expect(subject[:columns]).to eq(
32
32
  [
33
33
  {:column=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
34
- {:column=>"col_binary", :type=>"binary(100)", :default=>nil},
34
+ {:column=>"col_binary", :type=>"binary(202)", :default=>nil},
35
35
  {:column=>"col_blob", :type=>"varbinary(65535)"},
36
36
  {:column=>"col_bool", :type=>"int1(1)", :default=>"0"},
37
37
  {:column=>"col_char", :type=>"varchar(18)", :default=>nil},
@@ -55,7 +55,7 @@ describe MysqlTableDef do
55
55
  {:column=>"col_tinyblob", :type=>"varbinary(255)"},
56
56
  {:column=>"col_tinyint", :type=>"int1(4)", :default=>nil},
57
57
  {:column=>"col_tinytext", :type=>"text"},
58
- {:column=>"col_varbinary", :type=>"varbinary(255)", :default=>nil},
58
+ {:column=>"col_varbinary", :type=>"varbinary(512)", :default=>nil},
59
59
  {:column=>"col_varchar", :type=>"varchar(372)", :default=>nil}
60
60
  ]
61
61
  )
@@ -140,74 +140,229 @@ EOT
140
140
  end
141
141
 
142
142
  describe '.column_def_sql' do
143
+ subject { described_class.column_def_sql(column, opt) }
143
144
  let(:column) { {} }
144
- subject { described_class.column_def_sql(column) }
145
145
 
146
- context 'with varchar column def' do
147
- context 'when size is smaller than max size' do
148
- let(:column) { {:column=>"col_char", :type=>"varchar(18)", :default=>nil} }
149
- it do
150
- expect(subject).to eq(' "col_char" varchar(18) DEFAULT NULL')
146
+ let(:no_default_sql) { "" }
147
+ shared_examples "default values" do |*examples|
148
+ context "default nil" do
149
+ before do
150
+ column[:default] = nil
151
151
  end
152
+ let(:default_sql) { " DEFAULT NULL" }
153
+ it_behaves_like *examples
152
154
  end
153
-
154
- context 'when size exceeds varchar max size' do
155
- let(:column) { {:column=>"col_char", :type=>"varchar(1000000000)", :default=>nil} }
156
- it do
157
- expect(subject).to eq(' "col_char" varchar(65535) DEFAULT NULL')
155
+ context "no default" do
156
+ let(:default_sql) { no_default_sql }
157
+ it_behaves_like *examples
158
+ end
159
+ context "defalut value" do
160
+ before do
161
+ column[:default] = default_value
158
162
  end
163
+ let(:default_sql) { " DEFAULT #{default_value_sql}" }
164
+ it_behaves_like *examples
159
165
  end
166
+ end
160
167
 
161
- context 'when size is max size' do
162
- let(:column) { {:column=>"col_char", :type=>"varchar(max)", :default=>nil} }
163
- it do
164
- expect(subject).to eq(' "col_char" varchar(max) DEFAULT NULL')
168
+ shared_examples "not null values" do |*examples|
169
+ context "without not null" do
170
+ let(:not_null_sql) { "" }
171
+ it_behaves_like *examples
172
+ end
173
+ context "with not null false" do
174
+ before do
175
+ column[:not_null] = false
165
176
  end
177
+ let(:not_null_sql) { "" }
178
+ it_behaves_like *examples
166
179
  end
167
- end
168
-
169
- context 'with medium blob column def' do
170
- context 'when size exceeds max size' do
171
- let(:column) { {:column=>"col_medium_blob", :type=>"varbinary(16777215)", :default=>nil} }
172
- it do
173
- expect(subject).to eq(' "col_medium_blob" varchar(65535) DEFAULT NULL')
180
+ context "with not null true" do
181
+ before do
182
+ column[:not_null] = true
174
183
  end
175
- end
184
+ let(:not_null_sql) { " NOT NULL" }
185
+ it_behaves_like *examples
186
+ end
176
187
  end
177
-
178
- context 'with unsigned column def' do
179
- let(:column) { {:column=>"value_small_int", :type=>"int2(5) unsigned", :default=>nil} }
188
+
189
+ shared_examples "test generated Redshift SQL for column" do
180
190
  it do
181
- expect(subject).to eq(' "value_small_int" int4 DEFAULT NULL')
191
+ expect(subject).to eq(expected_query)
182
192
  end
183
193
  end
184
194
 
185
- context 'with decimal column def' do
186
- context 'when precision exceeds max allowed' do
187
- let(:column) { {:column=>"value", :type=>"numeric(65,30)", :default=>nil} }
188
- it 'should replace precision with max value' do
189
- expect(subject).to eq(' "value" numeric(38,30) DEFAULT NULL')
195
+ shared_examples "test column types" do |*examples|
196
+ context 'with varchar column def' do
197
+ context 'when size is smaller than max size' do
198
+ let(:default_value) { "something" }
199
+ let(:default_value_sql) { "'#{default_value}'" }
200
+ let(:not_null_default_sql) { " DEFAULT ''" }
201
+ before do
202
+ column[:column] = "col_char"
203
+ column[:type] = "varchar(18)"
204
+ end
205
+ let(:type_sql) { %Q|"col_char" varchar(18)| }
206
+
207
+ it_behaves_like *examples
208
+ end
209
+
210
+ context 'when size exceeds varchar max size' do
211
+ let(:default_value) { "something" }
212
+ let(:default_value_sql) { "'#{default_value}'" }
213
+ let(:not_null_default_sql) { " DEFAULT ''" }
214
+ before do
215
+ column[:column] = "col_char"
216
+ column[:type] = "varchar(1000000000)"
217
+ end
218
+ let(:type_sql) { %Q|"col_char" varchar(65535)| }
219
+
220
+ it_behaves_like *examples
221
+ end
222
+
223
+ context 'when size is max size' do
224
+ let(:default_value) { "something" }
225
+ let(:default_value_sql) { "'#{default_value}'" }
226
+ let(:not_null_default_sql) { " DEFAULT ''" }
227
+ before do
228
+ column[:column] = "col_char"
229
+ column[:type] = "varchar(max)"
230
+ end
231
+ let(:type_sql) { %Q|"col_char" varchar(max)| }
232
+
233
+ it_behaves_like *examples
190
234
  end
191
235
  end
192
- context 'when scale exceeds max allowed' do #not possible
193
- let(:column) { {:column=>"value", :type=>"numeric(45,44)", :default=>nil} }
194
- it 'should replace scale with max value' do
195
- expect(subject).to eq(' "value" numeric(38,37) DEFAULT NULL')
236
+
237
+ context 'with medium blob column def' do
238
+ context 'when size exceeds max size' do
239
+ let(:default_value) { "something" }
240
+ let(:default_value_sql) { "'#{default_value}'" }
241
+ let(:not_null_default_sql) { " DEFAULT ''" }
242
+ before do
243
+ column[:column] = "col_medium_blob"
244
+ column[:type] = "varbinary(16777215)"
245
+ end
246
+ let(:type_sql) { %Q|"col_medium_blob" varchar(65535)| }
247
+
248
+ it_behaves_like *examples
249
+ end
250
+ end
251
+
252
+ context 'with unsigned column def' do
253
+ let(:default_value) { 4 }
254
+ let(:default_value_sql) { "'#{default_value}'" }
255
+ let(:not_null_default_sql) { " DEFAULT '0'" }
256
+ before do
257
+ column[:column] = "value_small_int"
258
+ column[:type] = "int2(5) unsigned"
196
259
  end
260
+ let(:type_sql) { %Q|"value_small_int" int4| }
261
+
262
+ it_behaves_like *examples
197
263
  end
198
264
 
199
- context 'when precision and scale exceeds max allowed (for unsigned)' do
200
- let(:column) { {:column=>"value", :type=>"numeric(65,44) unsigned", :default=>nil} }
201
- it 'should replace precision with max value' do
202
- expect(subject).to eq(' "value" numeric(38,37) DEFAULT NULL')
265
+ context 'with datetime column def' do
266
+ let(:default_value) { '2014-10-17 22:22:22' }
267
+ let(:default_value_sql) { "'#{default_value}'" }
268
+ let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
269
+ before do
270
+ column[:column] = "value_datetime"
271
+ column[:type] = "datetime"
203
272
  end
273
+ let(:type_sql) { %Q|"value_datetime" timestamp| }
274
+
275
+ it_behaves_like *examples
204
276
  end
205
277
 
206
- context 'when precision and scale are within limit (for unsigned)' do
207
- let(:column) { {:column=>"value", :type=>"numeric(37,29) unsigned", :default=>nil} }
208
- it 'should do nothing' do
209
- expect(subject).to eq(' "value" numeric(37,29) DEFAULT NULL')
278
+ context 'with decimal column def' do
279
+ context 'when precision exceeds max allowed' do
280
+ let(:default_value) { 4 }
281
+ let(:default_value_sql) { "'#{default_value}'" }
282
+ let(:not_null_default_sql) { " DEFAULT '0'" }
283
+ before do
284
+ column[:column] = "value"
285
+ column[:type] = "numeric(65,30)"
286
+ end
287
+ let(:type_sql) { %Q|"value" numeric(38,30)| }
288
+
289
+ it_behaves_like *examples
210
290
  end
291
+ context 'when scale exceeds max allowed' do #not possible
292
+ let(:default_value) { 4 }
293
+ let(:default_value_sql) { "'#{default_value}'" }
294
+ let(:not_null_default_sql) { " DEFAULT '0'" }
295
+ before do
296
+ column[:column] = "value"
297
+ column[:type] = "numeric(45,44)"
298
+ end
299
+ let(:type_sql) { %Q|"value" numeric(38,37)| }
300
+
301
+ it_behaves_like *examples
302
+ end
303
+
304
+ context 'when precision and scale exceeds max allowed (for unsigned)' do
305
+ let(:default_value) { 4 }
306
+ let(:default_value_sql) { "'#{default_value}'" }
307
+ let(:not_null_default_sql) { " DEFAULT '0'" }
308
+ before do
309
+ column[:column] = "value"
310
+ column[:type] = "numeric(65,44) unsigned"
311
+ end
312
+ let(:type_sql) {%Q|"value" numeric(38,37)| }
313
+
314
+ it_behaves_like *examples
315
+ end
316
+
317
+ context 'when precision and scale are within limit (for unsigned)' do
318
+ let(:default_value) { 4 }
319
+ let(:default_value_sql) { "'#{default_value}'" }
320
+ let(:not_null_default_sql) { " DEFAULT '0'" }
321
+ before do
322
+ column[:column] = "value"
323
+ column[:type] = "numeric(37,29) unsigned"
324
+ end
325
+ let(:type_sql) { %Q|"value" numeric(37,29)| }
326
+
327
+ it_behaves_like *examples
328
+ end
329
+ end
330
+ end
331
+
332
+ context 'for create table' do
333
+ let(:opt) { {} }
334
+ let(:expected_query) { %Q| #{type_sql}#{not_null_sql}#{default_sql}| }
335
+ let(:no_default_sql) { "" }
336
+
337
+ it_behaves_like "test column types", "not null values", "default values", "test generated Redshift SQL for column"
338
+ end
339
+
340
+ context 'for alter table' do
341
+ let(:opt) { { for: :alter_table } }
342
+
343
+ context "without not null" do
344
+ let(:expected_query) { %Q| #{type_sql}#{default_sql}| }
345
+ let(:no_default_sql) { "" }
346
+
347
+ it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
348
+ end
349
+ context "with not null false" do
350
+ let(:expected_query) { %Q| #{type_sql}#{default_sql}| }
351
+ before do
352
+ column[:not_null] = false
353
+ end
354
+ let(:no_default_sql) { "" }
355
+
356
+ it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
357
+ end
358
+ context "with not null true" do
359
+ let(:expected_query) { %Q| #{type_sql} NOT NULL#{default_sql}| }
360
+ before do
361
+ column[:not_null] = true
362
+ end
363
+ let(:no_default_sql) { not_null_default_sql }
364
+
365
+ it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
211
366
  end
212
367
  end
213
368
  end