flydata 0.3.23 → 0.3.24
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/flydata-core/lib/flydata-core/errors.rb +1 -0
- data/flydata-core/lib/flydata-core/table_def.rb +2 -0
- data/flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb +13 -0
- data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +1 -1
- data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +8 -5
- data/flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb +13 -0
- data/flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb +30 -0
- data/flydata-core/spec/table_def/mysql_table_def_spec.rb +53 -0
- data/flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump +44 -0
- data/flydata-core/spec/table_def/redshift_table_def_spec.rb +42 -0
- data/flydata-core/spec/table_def/sync_redshift_table_def_spec.rb +22 -0
- data/flydata.gemspec +9 -4
- data/lib/flydata/command/sync.rb +1 -1
- data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb +19 -18
- data/lib/flydata/parser/mysql/dump_parser.rb +3 -1
- data/lib/flydata/parser/mysql/mysql_alter_table.treetop +13 -2
- data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb +32 -1
- data/spec/flydata/parser/mysql/alter_table_parser_spec.rb +86 -14
- data/spec/flydata/parser/mysql/dump_parser_spec.rb +28 -5
- metadata +8 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2c6d22e3fb1b51c996297d48f7bd886c250d5d11
|
4
|
+
data.tar.gz: 23ac7e6b2b97a76bf27d538370fb9c73e018e842
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f5455a1b18dcbccb9edae44ec15ac2332710410107687251cba290a99e671b83567c84dde29b7428696750b1732ebae7410093921170ad04ab1f4c938c1da67c
|
7
|
+
data.tar.gz: d72631b42f67a3ae378ed215663149414cf1cd80d02eac97fe97c0517b7813a5341ee93a05daf02774f094daf90c19381d3b1709a428729a99fb2101b6f80468
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.24
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'flydata-core/table_def/redshift_table_def'
|
2
|
+
|
3
|
+
module FlydataCore
|
4
|
+
module TableDef
|
5
|
+
|
6
|
+
class AutoloadRedshiftTableDef < RedshiftTableDef
|
7
|
+
# Of the valid identifier charaters of RedShift table/column name,
|
8
|
+
# Autoload only supports '$' as the valid special character.
|
9
|
+
VALID_IDENTIFIER_CHARACTERS = %w"0-9 a-z $ _"
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
13
|
+
end
|
@@ -182,7 +182,7 @@ class MysqlTableDef
|
|
182
182
|
while pos < line.length
|
183
183
|
case cond
|
184
184
|
when :column_name #`column_name` ...
|
185
|
-
pos = line.index(' ', 1)
|
185
|
+
pos = line[0] == '`' ? line.index('`', 1) + 1 : line.index(' ', 1)
|
186
186
|
column[:column] = if line[0] == '`'
|
187
187
|
line[1..pos-2]
|
188
188
|
else
|
@@ -231,7 +231,7 @@ EOS
|
|
231
231
|
|
232
232
|
sql += <<EOS
|
233
233
|
COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{convert_to_valid_column_name(col[:column])}"
|
234
|
-
IS '#{col[:comment]}';
|
234
|
+
IS '#{escape(col[:comment])}';
|
235
235
|
EOS
|
236
236
|
end
|
237
237
|
sql
|
@@ -247,11 +247,11 @@ INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
|
|
247
247
|
EOS
|
248
248
|
def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
|
249
249
|
flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
|
250
|
-
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
|
250
|
+
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, escape(flydata_tabledef[:table_name]), flydata_ctl_tbl ]
|
251
251
|
values = []
|
252
252
|
flydata_tabledef[:columns].each.with_index(1) do |col, i|
|
253
253
|
charset = col[:charset] ? " cs:#{col[:charset]}" : ""
|
254
|
-
values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
|
254
|
+
values << "('#{escape(flydata_tabledef[:table_name])}', '#{escape(col[:column])}', '#{escape(col[:type])}#{charset}', #{i})"
|
255
255
|
end
|
256
256
|
sql += values.join(",\n") + ';'
|
257
257
|
sql
|
@@ -286,8 +286,11 @@ EOS
|
|
286
286
|
|
287
287
|
MAX_COLUMNNAME_LENGTH = 127
|
288
288
|
|
289
|
+
VALID_IDENTIFIER_CHARACTERS = [" "] + %w"a-z 0-9 ! # $ % & ' ( ) * + , . \\- \\/ : ; < = > ? @ \\[ \\\\ \\] ^ { \\| } ~ `"
|
290
|
+
|
289
291
|
def self.convert_to_valid_name(key)
|
290
|
-
|
292
|
+
match = /[^#{self::VALID_IDENTIFIER_CHARACTERS.join}]/
|
293
|
+
name = key.downcase.gsub(match, '_')
|
291
294
|
do_convert = yield name if block_given?
|
292
295
|
name = "_#{name}" if do_convert
|
293
296
|
if name.length > MAX_COLUMNNAME_LENGTH
|
@@ -305,7 +308,7 @@ EOS
|
|
305
308
|
end
|
306
309
|
|
307
310
|
def self.escape(text)
|
308
|
-
text.gsub(
|
311
|
+
text.gsub(/(['\\])/, "\\\\\\1")
|
309
312
|
end
|
310
313
|
|
311
314
|
def self.check_and_replace_max(params, max_size_a)
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'flydata-core/table_def/redshift_table_def'
|
2
|
+
|
3
|
+
module FlydataCore
|
4
|
+
module TableDef
|
5
|
+
|
6
|
+
class SyncRedshiftTableDef < RedshiftTableDef
|
7
|
+
# For now, of the valid identifier charaters of RedShift table/column name,
|
8
|
+
# Sync does not support the period character (.)
|
9
|
+
VALID_IDENTIFIER_CHARACTERS = superclass::VALID_IDENTIFIER_CHARACTERS - ["."]
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/table_def/autoload_redshift_table_def'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module TableDef
|
6
|
+
|
7
|
+
describe AutoloadRedshiftTableDef do
|
8
|
+
describe '.convert_to_valid_name' do
|
9
|
+
subject { described_class.convert_to_valid_name(key) }
|
10
|
+
|
11
|
+
context "with a key that has special chars" do
|
12
|
+
let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
|
13
|
+
let(:expected_value) { %q|____$___________________________| }
|
14
|
+
|
15
|
+
it "Autoload only supports $ in a table/column name. Other special characters are replaced with underscore (_)" do
|
16
|
+
is_expected.to eq expected_value
|
17
|
+
end
|
18
|
+
end
|
19
|
+
context "with a key that has a period in it" do
|
20
|
+
let(:key) { %q|my.table| }
|
21
|
+
let(:expected_value) { %q|my_table| }
|
22
|
+
it "Autoload does not support period (.) in a table/column name. It's replaced with an underscore (_)" do
|
23
|
+
is_expected.to eq expected_value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -245,6 +245,14 @@ describe MysqlTableDef do
|
|
245
245
|
expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
|
246
246
|
end
|
247
247
|
end
|
248
|
+
|
249
|
+
context 'when a column name has space in it' do
|
250
|
+
let(:dump_file_io) { file_io('mysqldump_test_col_name_with_space.dump')}
|
251
|
+
it 'should extract col name properly' do
|
252
|
+
expect(subject[:columns][4][:column]).to eq("space test")
|
253
|
+
expect(subject[:columns][4][:type]).to eq("varchar(300)")
|
254
|
+
end
|
255
|
+
end
|
248
256
|
end
|
249
257
|
|
250
258
|
describe '.convert_to_flydata_type' do
|
@@ -411,6 +419,51 @@ describe MysqlTableDef do
|
|
411
419
|
end
|
412
420
|
end
|
413
421
|
end
|
422
|
+
|
423
|
+
describe '.parse_one_column_def' do
|
424
|
+
subject { described_class.parse_one_column_def(query) }
|
425
|
+
|
426
|
+
let(:query) { " `#{colname}` text," }
|
427
|
+
let(:expected_result) {
|
428
|
+
{ column: colname,
|
429
|
+
type: 'text',
|
430
|
+
}
|
431
|
+
}
|
432
|
+
|
433
|
+
context 'with an ASCII column name' do
|
434
|
+
let(:colname) { "name" }
|
435
|
+
|
436
|
+
it { is_expected.to eq expected_result }
|
437
|
+
end
|
438
|
+
|
439
|
+
context 'with an non-ASCII column name' do
|
440
|
+
let(:colname) { "在庫ID" }
|
441
|
+
|
442
|
+
it { is_expected.to eq expected_result }
|
443
|
+
end
|
444
|
+
|
445
|
+
context 'with a column name including space' do
|
446
|
+
let(:colname) { "full name" }
|
447
|
+
|
448
|
+
it { is_expected.to eq expected_result }
|
449
|
+
end
|
450
|
+
|
451
|
+
context 'with a column name including all supported ASCII special characters' do
|
452
|
+
# all special characters but '.' and '`'.
|
453
|
+
# '.' has its own spec. '`' should not come because MySQL doesn't allow it
|
454
|
+
let(:colname) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
|
455
|
+
|
456
|
+
it { is_expected.to eq expected_result }
|
457
|
+
end
|
458
|
+
|
459
|
+
context 'with a column name including comma' do
|
460
|
+
let(:colname) { %q|my.column| }
|
461
|
+
|
462
|
+
it "FlyData does not support a column name with a period in it. However, the agent accepts it as is. Necessary conversion happens at the server side." do
|
463
|
+
is_expected.to eq expected_result
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
414
467
|
end
|
415
468
|
|
416
469
|
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
-- MySQL dump 10.13 Distrib 5.5.43, for debian-linux-gnu (x86_64)
|
2
|
+
--
|
3
|
+
-- Host: localhost Database: flydata_sync
|
4
|
+
-- ------------------------------------------------------
|
5
|
+
-- Server version 5.5.43-0ubuntu0.12.04.1-log
|
6
|
+
|
7
|
+
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
|
8
|
+
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
|
9
|
+
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
|
10
|
+
/*!40101 SET NAMES utf8 */;
|
11
|
+
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
|
12
|
+
/*!40103 SET TIME_ZONE='+00:00' */;
|
13
|
+
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
|
14
|
+
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
|
15
|
+
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
|
16
|
+
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
|
17
|
+
|
18
|
+
--
|
19
|
+
-- Table structure for table `sample`
|
20
|
+
--
|
21
|
+
|
22
|
+
DROP TABLE IF EXISTS `sample`;
|
23
|
+
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
24
|
+
/*!40101 SET character_set_client = utf8 */;
|
25
|
+
CREATE TABLE `sample` (
|
26
|
+
`id` int(11) NOT NULL AUTO_INCREMENT,
|
27
|
+
`name` text,
|
28
|
+
`created_at` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
|
29
|
+
`test_enum` enum('1','2') DEFAULT NULL,
|
30
|
+
`space test` varchar(100) DEFAULT NULL,
|
31
|
+
PRIMARY KEY (`id`)
|
32
|
+
) ENGINE=InnoDB AUTO_INCREMENT=192 DEFAULT CHARSET=latin1;
|
33
|
+
/*!40101 SET character_set_client = @saved_cs_client */;
|
34
|
+
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
|
35
|
+
|
36
|
+
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
|
37
|
+
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
|
38
|
+
/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
|
39
|
+
/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
|
40
|
+
/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
|
41
|
+
/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
|
42
|
+
/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
|
43
|
+
|
44
|
+
-- Dump completed on 2015-05-28 11:29:57
|
@@ -568,6 +568,20 @@ EOS
|
|
568
568
|
|
569
569
|
it { is_expected.to eq expected_sql }
|
570
570
|
end
|
571
|
+
|
572
|
+
context "column name with a single quote" do
|
573
|
+
let(:column_name) { "90's hits" }
|
574
|
+
let(:expected_column_name) { "90\\'s hits" }
|
575
|
+
|
576
|
+
it { is_expected.to eq expected_sql }
|
577
|
+
end
|
578
|
+
|
579
|
+
context "column name with a backslash" do
|
580
|
+
let(:column_name) { "\\^_^/" }
|
581
|
+
let(:expected_column_name) { "\\\\^_^/" }
|
582
|
+
|
583
|
+
it { is_expected.to eq expected_sql }
|
584
|
+
end
|
571
585
|
end
|
572
586
|
end
|
573
587
|
|
@@ -826,6 +840,34 @@ EOS
|
|
826
840
|
subject { subject_object.convert_to_valid_name(key) }
|
827
841
|
|
828
842
|
it_behaves_like "expecting no underscore in front of numbers"
|
843
|
+
|
844
|
+
context "with a key that has spaces" do
|
845
|
+
let(:key) { "space test" }
|
846
|
+
let(:expected_value) { key }
|
847
|
+
it { is_expected.to eq expected_value }
|
848
|
+
end
|
849
|
+
|
850
|
+
context "with a key that has non-ASCII chars" do
|
851
|
+
let(:key) { "行列123" }
|
852
|
+
let(:expected_value) { "__123" }
|
853
|
+
it "Redshift does not support non-ASCII table/column name. Those characters will be replaced with '_'" do
|
854
|
+
is_expected.to eq expected_value
|
855
|
+
end
|
856
|
+
end
|
857
|
+
context "with a key that has special chars" do
|
858
|
+
let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
|
859
|
+
let(:expected_value) { %q| !_#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
|
860
|
+
it "Redshift supports all these special characters but double quote (\") which is replaced with underscore (_)" do
|
861
|
+
is_expected.to eq expected_value
|
862
|
+
end
|
863
|
+
end
|
864
|
+
context "with a key that has a period in it" do
|
865
|
+
let(:key) { %q|my.table| }
|
866
|
+
let(:expected_value) { key }
|
867
|
+
it "Redshift supports period (.) in a table/column name" do
|
868
|
+
is_expected.to eq expected_value
|
869
|
+
end
|
870
|
+
end
|
829
871
|
end
|
830
872
|
|
831
873
|
describe ".convert_to_valid_column_name" do
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'flydata-core/table_def/sync_redshift_table_def'
|
3
|
+
|
4
|
+
module FlydataCore
|
5
|
+
module TableDef
|
6
|
+
|
7
|
+
describe SyncRedshiftTableDef do
|
8
|
+
describe '.convert_to_valid_name' do
|
9
|
+
subject { described_class.convert_to_valid_name(key) }
|
10
|
+
|
11
|
+
context "with a key that has a period in it" do
|
12
|
+
let(:key) { %q|my.table| }
|
13
|
+
let(:expected_value) { %q|my_table| }
|
14
|
+
it "Sync does not support period (.) in a table/column name. It's replaced with an underscore (_)" do
|
15
|
+
is_expected.to eq expected_value
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
data/flydata.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: flydata 0.3.
|
5
|
+
# stub: flydata 0.3.24 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "flydata"
|
9
|
-
s.version = "0.3.
|
9
|
+
s.version = "0.3.24"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
|
14
|
-
s.date = "2015-
|
14
|
+
s.date = "2015-06-01"
|
15
15
|
s.description = "FlyData Agent"
|
16
16
|
s.email = "sysadmin@flydata.com"
|
17
17
|
s.executables = ["fdmysqldump", "flydata", "serverinfo"]
|
@@ -50,17 +50,21 @@ Gem::Specification.new do |s|
|
|
50
50
|
"flydata-core/lib/flydata-core/record/record.rb",
|
51
51
|
"flydata-core/lib/flydata-core/redshift/string.rb",
|
52
52
|
"flydata-core/lib/flydata-core/table_def.rb",
|
53
|
+
"flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
|
53
54
|
"flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
|
54
55
|
"flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
|
56
|
+
"flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb",
|
55
57
|
"flydata-core/lib/flydata-core/thread_context.rb",
|
56
58
|
"flydata-core/spec/config/user_maintenance_spec.rb",
|
57
59
|
"flydata-core/spec/fluent/config_helper_spec.rb",
|
58
60
|
"flydata-core/spec/logger_spec.rb",
|
59
61
|
"flydata-core/spec/redshift/string_spec.rb",
|
60
62
|
"flydata-core/spec/spec_helper.rb",
|
63
|
+
"flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
|
61
64
|
"flydata-core/spec/table_def/mysql_table_def_spec.rb",
|
62
65
|
"flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
|
63
66
|
"flydata-core/spec/table_def/mysqldump_test_bit_table.dump",
|
67
|
+
"flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump",
|
64
68
|
"flydata-core/spec/table_def/mysqldump_test_column_charset.dump",
|
65
69
|
"flydata-core/spec/table_def/mysqldump_test_foreign_key.dump",
|
66
70
|
"flydata-core/spec/table_def/mysqldump_test_table_all.dump",
|
@@ -73,6 +77,7 @@ Gem::Specification.new do |s|
|
|
73
77
|
"flydata-core/spec/table_def/mysqldump_test_unique_key3.dump",
|
74
78
|
"flydata-core/spec/table_def/mysqldump_test_unsigned.dump",
|
75
79
|
"flydata-core/spec/table_def/redshift_table_def_spec.rb",
|
80
|
+
"flydata-core/spec/table_def/sync_redshift_table_def_spec.rb",
|
76
81
|
"flydata.gemspec",
|
77
82
|
"lib/fly_data_model.rb",
|
78
83
|
"lib/flydata.rb",
|
@@ -181,7 +186,7 @@ Gem::Specification.new do |s|
|
|
181
186
|
]
|
182
187
|
s.homepage = "http://flydata.com/"
|
183
188
|
s.licenses = ["All right reserved."]
|
184
|
-
s.rubygems_version = "2.4.
|
189
|
+
s.rubygems_version = "2.4.3"
|
185
190
|
s.summary = "FlyData Agent"
|
186
191
|
|
187
192
|
if s.respond_to? :specification_version then
|
data/lib/flydata/command/sync.rb
CHANGED
@@ -760,7 +760,7 @@ EOM
|
|
760
760
|
next
|
761
761
|
end
|
762
762
|
flydata_tabledef = mysql_tabledef.to_flydata_tabledef
|
763
|
-
puts FlydataCore::TableDef::
|
763
|
+
puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
|
764
764
|
create_flydata_ctl_table = false
|
765
765
|
end
|
766
766
|
unless error_list.empty?
|
@@ -20,35 +20,36 @@ module Mysql
|
|
20
20
|
|
21
21
|
private
|
22
22
|
|
23
|
+
PLACEMENT = '{}'
|
23
24
|
def normalize_query(queries)
|
24
|
-
|
25
|
-
|
26
|
-
queries.split(';').each do |query|
|
27
|
-
query = query.lstrip
|
28
|
-
yield(query + ";") if block_given? && !query.empty?
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def strip_comments(query)
|
33
|
-
q = query.dup
|
25
|
+
q = queries.dup
|
26
|
+
strings = []
|
34
27
|
# \/\*.*?\*\/ /* */ style comment
|
35
28
|
# `.*?` `resource_name`
|
36
29
|
# '(?:\\.|.)*?' 'string'
|
37
30
|
# "(?:\\.|.)*?" "string"
|
38
31
|
# --\s+.*?(?:\n|$) -- style comment
|
39
32
|
# #\s+.*?(?:\n|$) # style comment
|
40
|
-
|
41
|
-
comment_or_quoted =
|
33
|
+
q.gsub!(/(\/\*.*?\*\/|`.*?`|'(?:\\.|.)*?'|"(?:\\.|.)*?"|--\s+.*?(?:\n|$)|#.*?(?:\n|$))/m) do |m|
|
34
|
+
comment_or_quoted = $&
|
42
35
|
if comment_or_quoted.start_with?("/*") || comment_or_quoted.start_with?("--") ||
|
43
36
|
comment_or_quoted.start_with?("#")
|
44
|
-
# comment. replace with
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
37
|
+
# comment. replace with a space
|
38
|
+
' '
|
39
|
+
else
|
40
|
+
# string. save the original and replace with an temporary placement
|
41
|
+
strings << comment_or_quoted
|
42
|
+
PLACEMENT
|
49
43
|
end
|
50
44
|
end
|
51
|
-
q
|
45
|
+
q = q.gsub(/\s+/, ' ') # replace multiple spaces with a space
|
46
|
+
q.split(';').each do |query|
|
47
|
+
query = query.lstrip
|
48
|
+
query.gsub!(PLACEMENT) do |m|
|
49
|
+
strings.shift
|
50
|
+
end
|
51
|
+
yield(query + ";") if block_given? && !query.empty?
|
52
|
+
end
|
52
53
|
end
|
53
54
|
end
|
54
55
|
|
@@ -323,8 +323,10 @@ EOS
|
|
323
323
|
|
324
324
|
# parse column line
|
325
325
|
line = line[0..-2] if line.end_with?(',')
|
326
|
+
colname_end_index = line.index('`', 1) - 1
|
327
|
+
column[:column_name] = line[1..colname_end_index]
|
328
|
+
line = line[colname_end_index + 3..-1]
|
326
329
|
items = line.split
|
327
|
-
column[:column_name] = items.shift[1..-2]
|
328
330
|
column[:format_type_str] = format_type_str = items.shift
|
329
331
|
pos = format_type_str.index('(')
|
330
332
|
if pos
|
@@ -1205,12 +1205,23 @@ grammar MysqlAlterTable
|
|
1205
1205
|
}
|
1206
1206
|
end
|
1207
1207
|
|
1208
|
+
# This rules does not exist in sql_yacc.yy, but should be close enough.
|
1209
|
+
rule ident_in_quote
|
1210
|
+
# 1..1 prevents Treetop from embedding the SyntaxNode to an upper node
|
1211
|
+
# without the custom method.
|
1212
|
+
( [^`]* ) 1..1 {
|
1213
|
+
def value
|
1214
|
+
text_value
|
1215
|
+
end
|
1216
|
+
}
|
1217
|
+
end
|
1218
|
+
|
1208
1219
|
rule ident_quoted
|
1209
|
-
'`'
|
1220
|
+
'`' ident_in_quote '`' {
|
1210
1221
|
# #value returns a normalized value of the element while #text_value
|
1211
1222
|
# returns its text as is
|
1212
1223
|
def value
|
1213
|
-
|
1224
|
+
ident_in_quote.value
|
1214
1225
|
end
|
1215
1226
|
}
|
1216
1227
|
end
|
@@ -60,7 +60,8 @@ EOT
|
|
60
60
|
r
|
61
61
|
end
|
62
62
|
let(:context) { double('context') }
|
63
|
-
|
63
|
+
let(:subject_object) { described_class.new(context) }
|
64
|
+
subject { subject_object }
|
64
65
|
|
65
66
|
shared_examples "a dispatcher that calls query handler with correct query" do
|
66
67
|
it do
|
@@ -233,5 +234,35 @@ EOS
|
|
233
234
|
end
|
234
235
|
end
|
235
236
|
end
|
237
|
+
describe '#normalize_query' do
|
238
|
+
context 'straightforwad query' do
|
239
|
+
let(:query) { "ALTER TABLE test_table DROP COLUMN test_column" }
|
240
|
+
let(:expected_normalized) { ["#{query};"] }
|
241
|
+
|
242
|
+
it do
|
243
|
+
expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
context 'two queries' do
|
247
|
+
let(:subquery1) { "ALTER TABLE test_table DROP COLUMN test_column" }
|
248
|
+
let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
|
249
|
+
let(:query) { "#{subquery1}; #{subquery2};" }
|
250
|
+
let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
|
251
|
+
|
252
|
+
it do
|
253
|
+
expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
|
254
|
+
end
|
255
|
+
end
|
256
|
+
context 'a column name includes a semicolon' do
|
257
|
+
let(:subquery1) { %Q|ALTER TABLE test_table DROP COLUMN "test_;column"| }
|
258
|
+
let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
|
259
|
+
let(:query) { "#{subquery1}; #{subquery2};" }
|
260
|
+
let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
|
261
|
+
|
262
|
+
it do
|
263
|
+
expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
236
267
|
end
|
237
268
|
end
|
@@ -10,7 +10,7 @@ describe 'MysqlAlterTableParser' do
|
|
10
10
|
let(:query) { "" }
|
11
11
|
|
12
12
|
let(:parser) { @parser_class.new }
|
13
|
-
subject { parser.parse(query).tree }
|
13
|
+
subject { res = parser.parse(query); raise RuntimeError, parser.inspect unless res; res.tree }
|
14
14
|
|
15
15
|
|
16
16
|
describe '#parse' do
|
@@ -33,6 +33,22 @@ describe 'MysqlAlterTableParser' do
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
+
context 'with a non-ascii table name' do
|
37
|
+
let(:query) { "alter table `テスト` add column value varchar(26)" }
|
38
|
+
it do
|
39
|
+
expect(subject).to eq({
|
40
|
+
type: :alter_table,
|
41
|
+
table_name: "テスト",
|
42
|
+
actions: [{
|
43
|
+
action: :add_column,
|
44
|
+
column: "value",
|
45
|
+
type: "varchar(78)",
|
46
|
+
query: "add column value varchar(26)"
|
47
|
+
}]
|
48
|
+
})
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
36
52
|
context 'with different column types' do
|
37
53
|
context 'with longblob' do
|
38
54
|
let(:query) { "alter table test_table add column value longblob" }
|
@@ -1547,11 +1563,23 @@ describe 'MysqlAlterTableParser' do
|
|
1547
1563
|
end
|
1548
1564
|
shared_examples "test position" do |*examples|
|
1549
1565
|
context "with after" do
|
1550
|
-
|
1551
|
-
let(:
|
1552
|
-
|
1553
|
-
{ after
|
1554
|
-
|
1566
|
+
context 'an ordinary column name' do
|
1567
|
+
let(:pos_column) { 'test_name' }
|
1568
|
+
it_behaves_like *examples do
|
1569
|
+
let(:position) {" after #{pos_column}"}
|
1570
|
+
let(:position_hash){
|
1571
|
+
{ after: pos_column.gsub(/(^`|`$)/, '') }
|
1572
|
+
}
|
1573
|
+
end
|
1574
|
+
end
|
1575
|
+
context 'an extrordinary column name' do
|
1576
|
+
let(:pos_column) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
|
1577
|
+
it_behaves_like *examples do
|
1578
|
+
let(:position) {" after #{pos_column}"}
|
1579
|
+
let(:position_hash){
|
1580
|
+
{ after: pos_column.gsub(/(^`|`$)/, '') }
|
1581
|
+
}
|
1582
|
+
end
|
1555
1583
|
end
|
1556
1584
|
end
|
1557
1585
|
context "with first" do
|
@@ -1757,14 +1785,58 @@ describe 'MysqlAlterTableParser' do
|
|
1757
1785
|
end
|
1758
1786
|
context 'change column' do
|
1759
1787
|
context 'change column name' do
|
1760
|
-
|
1761
|
-
|
1762
|
-
{
|
1763
|
-
|
1764
|
-
|
1765
|
-
|
1766
|
-
|
1767
|
-
|
1788
|
+
shared_examples "renaming a column name without an issue" do
|
1789
|
+
let(:alter_table_action) { "change#{column} #{colname_old} #{colname_new} #{col_def}#{position}" }
|
1790
|
+
let(:change_action_hash) {
|
1791
|
+
{
|
1792
|
+
old_column: colname_old.gsub(/^`|`$/, ''),
|
1793
|
+
column: colname_new.gsub(/^`|`$/, ''),
|
1794
|
+
}.merge!(position_hash).merge!(col_def_hash)
|
1795
|
+
}
|
1796
|
+
it_behaves_like "test optional column", "test column definition", "test position", "test parser parsing a change column"
|
1797
|
+
end
|
1798
|
+
context 'ordinary column names' do
|
1799
|
+
let(:colname_old) { 'name' }
|
1800
|
+
let(:colname_new) { 'name1' }
|
1801
|
+
|
1802
|
+
it_behaves_like "renaming a column name without an issue"
|
1803
|
+
end
|
1804
|
+
context 'renaming a name including space' do
|
1805
|
+
let(:colname_old) { '`name including spaces`' }
|
1806
|
+
let(:colname_new) { 'name1' }
|
1807
|
+
|
1808
|
+
it_behaves_like "renaming a column name without an issue"
|
1809
|
+
end
|
1810
|
+
context 'renaming to a name including space' do
|
1811
|
+
let(:colname_old) { 'name' }
|
1812
|
+
let(:colname_new) { '`name including spaces`' }
|
1813
|
+
|
1814
|
+
it_behaves_like "renaming a column name without an issue"
|
1815
|
+
end
|
1816
|
+
context 'renaming a non-ascii name' do
|
1817
|
+
let(:colname_old) { '`コラム`' }
|
1818
|
+
let(:colname_new) { '`name including spaces`' }
|
1819
|
+
|
1820
|
+
it_behaves_like "renaming a column name without an issue"
|
1821
|
+
end
|
1822
|
+
context 'renaming a column with special characters' do
|
1823
|
+
let(:colname_old) { %q|` !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
|
1824
|
+
let(:colname_new) { '`name including spaces`' }
|
1825
|
+
|
1826
|
+
it_behaves_like "renaming a column name without an issue"
|
1827
|
+
end
|
1828
|
+
context 'adding a single quote to a column with special characters' do
|
1829
|
+
let(:colname_old) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
|
1830
|
+
let(:colname_new) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.'`,}
|
1831
|
+
|
1832
|
+
it_behaves_like "renaming a column name without an issue"
|
1833
|
+
end
|
1834
|
+
context 'renaming a non-ascii name' do
|
1835
|
+
let(:colname_old) { '`my.table`' }
|
1836
|
+
let(:colname_new) { '`name including spaces`' }
|
1837
|
+
|
1838
|
+
it_behaves_like "renaming a column name without an issue"
|
1839
|
+
end
|
1768
1840
|
end
|
1769
1841
|
context 'no change to column name' do
|
1770
1842
|
let(:alter_table_action) { "change#{column} name1 name1 #{col_def}#{position}" }
|
@@ -147,7 +147,7 @@ module Flydata
|
|
147
147
|
EOT
|
148
148
|
|
149
149
|
def index_after(content, string)
|
150
|
-
content.index(string) + string.bytesize + 1
|
150
|
+
content[0...content.index(string)].bytesize + string.bytesize + 1
|
151
151
|
end
|
152
152
|
|
153
153
|
let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
|
@@ -178,7 +178,7 @@ EOT
|
|
178
178
|
end
|
179
179
|
end
|
180
180
|
|
181
|
-
|
181
|
+
shared_examples 'dump contains create table without records' do
|
182
182
|
let(:dump_content) { <<EOT
|
183
183
|
#{DUMP_HEADER}
|
184
184
|
|
@@ -186,7 +186,7 @@ DROP TABLE IF EXISTS `users_login`;
|
|
186
186
|
/*!40101 SET @saved_cs_client = @@character_set_client */;
|
187
187
|
/*!40101 SET character_set_client = utf8 */;
|
188
188
|
CREATE TABLE `users_login` (
|
189
|
-
`
|
189
|
+
`#{col_name}` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
|
190
190
|
`login_count` int(10) unsigned DEFAULT '0' COMMENT 'login count',
|
191
191
|
`create_time` datetime NOT NULL COMMENT 'create time',
|
192
192
|
`update_time` datetime NOT NULL COMMENT 'update time',
|
@@ -220,7 +220,7 @@ EOT
|
|
220
220
|
}
|
221
221
|
expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
222
222
|
expect(mysql_table.table_name).to eq('users_login')
|
223
|
-
expect(mysql_table.columns.keys).to eq([
|
223
|
+
expect(mysql_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
|
224
224
|
expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
|
225
225
|
expect(binlog_pos).to eq(default_binlog_pos)
|
226
226
|
expect(state).to eq(MysqlDumpParser::State::INSERT_RECORD)
|
@@ -242,6 +242,29 @@ EOT
|
|
242
242
|
end
|
243
243
|
end
|
244
244
|
|
245
|
+
context 'when dump contains create table without records' do
|
246
|
+
context 'when one of the column name has space' do
|
247
|
+
let(:col_name) { "user id" }
|
248
|
+
include_examples 'dump contains create table without records'
|
249
|
+
end
|
250
|
+
context 'when the column name does not have space' do
|
251
|
+
let(:col_name) { "user_id" }
|
252
|
+
include_examples 'dump contains create table without records'
|
253
|
+
end
|
254
|
+
context 'when the column name is non-ascii' do
|
255
|
+
let(:col_name) { "行列" }
|
256
|
+
include_examples 'dump contains create table without records'
|
257
|
+
end
|
258
|
+
context 'when the column name includes special characters' do
|
259
|
+
let(:col_name) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
|
260
|
+
include_examples 'dump contains create table without records'
|
261
|
+
end
|
262
|
+
context 'when the column name includes a dot' do
|
263
|
+
let(:col_name) { %q|my.column| }
|
264
|
+
include_examples 'dump contains create table without records'
|
265
|
+
end
|
266
|
+
end
|
267
|
+
|
245
268
|
context 'when dump contains create table with multi inserts and invalid utf8 byte sequences' do
|
246
269
|
let(:dump_content) { <<EOT
|
247
270
|
#{DUMP_HEADER}
|
@@ -303,7 +326,7 @@ EOT
|
|
303
326
|
{state: MysqlDumpParser::State::CREATE_TABLE},
|
304
327
|
{state: MysqlDumpParser::State::INSERT_RECORD},
|
305
328
|
{state: MysqlDumpParser::State::CREATE_TABLE,
|
306
|
-
last_pos: index_after(dump_content, 'UNLOCK TABLES;')
|
329
|
+
last_pos: index_after(dump_content, 'UNLOCK TABLES;')}
|
307
330
|
].each do |expected_params|
|
308
331
|
expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
|
309
332
|
expect(mysql_table.table_name).to eq('users_login') if mysql_table
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: flydata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Koichi Fujikawa
|
@@ -12,7 +12,7 @@ authors:
|
|
12
12
|
autorequire:
|
13
13
|
bindir: bin
|
14
14
|
cert_chain: []
|
15
|
-
date: 2015-
|
15
|
+
date: 2015-06-01 00:00:00.000000000 Z
|
16
16
|
dependencies:
|
17
17
|
- !ruby/object:Gem::Dependency
|
18
18
|
name: rest-client
|
@@ -473,17 +473,21 @@ files:
|
|
473
473
|
- flydata-core/lib/flydata-core/record/record.rb
|
474
474
|
- flydata-core/lib/flydata-core/redshift/string.rb
|
475
475
|
- flydata-core/lib/flydata-core/table_def.rb
|
476
|
+
- flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
|
476
477
|
- flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
|
477
478
|
- flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
|
479
|
+
- flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb
|
478
480
|
- flydata-core/lib/flydata-core/thread_context.rb
|
479
481
|
- flydata-core/spec/config/user_maintenance_spec.rb
|
480
482
|
- flydata-core/spec/fluent/config_helper_spec.rb
|
481
483
|
- flydata-core/spec/logger_spec.rb
|
482
484
|
- flydata-core/spec/redshift/string_spec.rb
|
483
485
|
- flydata-core/spec/spec_helper.rb
|
486
|
+
- flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
|
484
487
|
- flydata-core/spec/table_def/mysql_table_def_spec.rb
|
485
488
|
- flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
|
486
489
|
- flydata-core/spec/table_def/mysqldump_test_bit_table.dump
|
490
|
+
- flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump
|
487
491
|
- flydata-core/spec/table_def/mysqldump_test_column_charset.dump
|
488
492
|
- flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
|
489
493
|
- flydata-core/spec/table_def/mysqldump_test_table_all.dump
|
@@ -496,6 +500,7 @@ files:
|
|
496
500
|
- flydata-core/spec/table_def/mysqldump_test_unique_key3.dump
|
497
501
|
- flydata-core/spec/table_def/mysqldump_test_unsigned.dump
|
498
502
|
- flydata-core/spec/table_def/redshift_table_def_spec.rb
|
503
|
+
- flydata-core/spec/table_def/sync_redshift_table_def_spec.rb
|
499
504
|
- flydata.gemspec
|
500
505
|
- lib/fly_data_model.rb
|
501
506
|
- lib/flydata.rb
|
@@ -621,7 +626,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
621
626
|
version: '0'
|
622
627
|
requirements: []
|
623
628
|
rubyforge_project:
|
624
|
-
rubygems_version: 2.4.
|
629
|
+
rubygems_version: 2.4.3
|
625
630
|
signing_key:
|
626
631
|
specification_version: 4
|
627
632
|
summary: FlyData Agent
|