flydata 0.3.23 → 0.3.24

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 65c746356a222838bb65756fb206ac897ccc236d
4
- data.tar.gz: 8a057f92fa831366d156ffd3c1a2b295ace4e682
3
+ metadata.gz: 2c6d22e3fb1b51c996297d48f7bd886c250d5d11
4
+ data.tar.gz: 23ac7e6b2b97a76bf27d538370fb9c73e018e842
5
5
  SHA512:
6
- metadata.gz: 9b61f370d5295a8249f8b4485c0ba4e3011b0aa2c2fb7b79d1215c022a6fba9dd503d8e5f2a22c86c42b38da1953b6feb7ff72a90e7a1cdfa6bfdcd3943ac1d0
7
- data.tar.gz: d2593d38159c408fad8c84842163ad579fe3251a71875a3c0eee172d65be00599c9f1658f24eb113e8e52f61a063665b059b6a9e585e8705377087b3c801e9cb
6
+ metadata.gz: f5455a1b18dcbccb9edae44ec15ac2332710410107687251cba290a99e671b83567c84dde29b7428696750b1732ebae7410093921170ad04ab1f4c938c1da67c
7
+ data.tar.gz: d72631b42f67a3ae378ed215663149414cf1cd80d02eac97fe97c0517b7813a5341ee93a05daf02774f094daf90c19381d3b1709a428729a99fb2101b6f80468
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.3.23
1
+ 0.3.24
@@ -299,6 +299,7 @@ class UnsupportedRecordFormat < StandardError; end
299
299
  class TableDefError < StandardError
300
300
  attr_reader :err_hash
301
301
  def initialize(err_hash)
302
+ super(err_hash[:error])
302
303
  @err_hash = err_hash
303
304
  end
304
305
  end
@@ -1,2 +1,4 @@
1
1
  require 'flydata-core/table_def/redshift_table_def'
2
+ require 'flydata-core/table_def/sync_redshift_table_def'
3
+ require 'flydata-core/table_def/autoload_redshift_table_def'
2
4
  require 'flydata-core/table_def/mysql_table_def'
@@ -0,0 +1,13 @@
1
+ require 'flydata-core/table_def/redshift_table_def'
2
+
3
+ module FlydataCore
4
+ module TableDef
5
+
6
+ class AutoloadRedshiftTableDef < RedshiftTableDef
7
+ # Of the valid identifier charaters of RedShift table/column name,
8
+ # Autoload only supports '$' as the valid special character.
9
+ VALID_IDENTIFIER_CHARACTERS = %w"0-9 a-z $ _"
10
+ end
11
+
12
+ end
13
+ end
@@ -182,7 +182,7 @@ class MysqlTableDef
182
182
  while pos < line.length
183
183
  case cond
184
184
  when :column_name #`column_name` ...
185
- pos = line.index(' ', 1)
185
+ pos = line[0] == '`' ? line.index('`', 1) + 1 : line.index(' ', 1)
186
186
  column[:column] = if line[0] == '`'
187
187
  line[1..pos-2]
188
188
  else
@@ -231,7 +231,7 @@ EOS
231
231
 
232
232
  sql += <<EOS
233
233
  COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{convert_to_valid_column_name(col[:column])}"
234
- IS '#{col[:comment]}';
234
+ IS '#{escape(col[:comment])}';
235
235
  EOS
236
236
  end
237
237
  sql
@@ -247,11 +247,11 @@ INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
247
247
  EOS
248
248
  def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
249
249
  flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
250
- sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
250
+ sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, escape(flydata_tabledef[:table_name]), flydata_ctl_tbl ]
251
251
  values = []
252
252
  flydata_tabledef[:columns].each.with_index(1) do |col, i|
253
253
  charset = col[:charset] ? " cs:#{col[:charset]}" : ""
254
- values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
254
+ values << "('#{escape(flydata_tabledef[:table_name])}', '#{escape(col[:column])}', '#{escape(col[:type])}#{charset}', #{i})"
255
255
  end
256
256
  sql += values.join(",\n") + ';'
257
257
  sql
@@ -286,8 +286,11 @@ EOS
286
286
 
287
287
  MAX_COLUMNNAME_LENGTH = 127
288
288
 
289
+ VALID_IDENTIFIER_CHARACTERS = [" "] + %w"a-z 0-9 ! # $ % & ' ( ) * + , . \\- \\/ : ; < = > ? @ \\[ \\\\ \\] ^ { \\| } ~ `"
290
+
289
291
  def self.convert_to_valid_name(key)
290
- name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
292
+ match = /[^#{self::VALID_IDENTIFIER_CHARACTERS.join}]/
293
+ name = key.downcase.gsub(match, '_')
291
294
  do_convert = yield name if block_given?
292
295
  name = "_#{name}" if do_convert
293
296
  if name.length > MAX_COLUMNNAME_LENGTH
@@ -305,7 +308,7 @@ EOS
305
308
  end
306
309
 
307
310
  def self.escape(text)
308
- text.gsub("'", "\\\\'")
311
+ text.gsub(/(['\\])/, "\\\\\\1")
309
312
  end
310
313
 
311
314
  def self.check_and_replace_max(params, max_size_a)
@@ -0,0 +1,13 @@
1
+ require 'flydata-core/table_def/redshift_table_def'
2
+
3
+ module FlydataCore
4
+ module TableDef
5
+
6
+ class SyncRedshiftTableDef < RedshiftTableDef
7
+ # For now, of the valid identifier charaters of RedShift table/column name,
8
+ # Sync does not support the period character (.)
9
+ VALID_IDENTIFIER_CHARACTERS = superclass::VALID_IDENTIFIER_CHARACTERS - ["."]
10
+ end
11
+
12
+ end
13
+ end
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/table_def/autoload_redshift_table_def'
3
+
4
+ module FlydataCore
5
+ module TableDef
6
+
7
+ describe AutoloadRedshiftTableDef do
8
+ describe '.convert_to_valid_name' do
9
+ subject { described_class.convert_to_valid_name(key) }
10
+
11
+ context "with a key that has special chars" do
12
+ let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
13
+ let(:expected_value) { %q|____$___________________________| }
14
+
15
+ it "Autoload only supports $ in a table/column name. Other special characters are replaced with underscore (_)" do
16
+ is_expected.to eq expected_value
17
+ end
18
+ end
19
+ context "with a key that has a period in it" do
20
+ let(:key) { %q|my.table| }
21
+ let(:expected_value) { %q|my_table| }
22
+ it "Autoload does not support period (.) in a table/column name. It's replaced with an underscore (_)" do
23
+ is_expected.to eq expected_value
24
+ end
25
+ end
26
+ end
27
+ end
28
+
29
+ end
30
+ end
@@ -245,6 +245,14 @@ describe MysqlTableDef do
245
245
  expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
246
246
  end
247
247
  end
248
+
249
+ context 'when a column name has space in it' do
250
+ let(:dump_file_io) { file_io('mysqldump_test_col_name_with_space.dump')}
251
+ it 'should extract col name properly' do
252
+ expect(subject[:columns][4][:column]).to eq("space test")
253
+ expect(subject[:columns][4][:type]).to eq("varchar(300)")
254
+ end
255
+ end
248
256
  end
249
257
 
250
258
  describe '.convert_to_flydata_type' do
@@ -411,6 +419,51 @@ describe MysqlTableDef do
411
419
  end
412
420
  end
413
421
  end
422
+
423
+ describe '.parse_one_column_def' do
424
+ subject { described_class.parse_one_column_def(query) }
425
+
426
+ let(:query) { " `#{colname}` text," }
427
+ let(:expected_result) {
428
+ { column: colname,
429
+ type: 'text',
430
+ }
431
+ }
432
+
433
+ context 'with an ASCII column name' do
434
+ let(:colname) { "name" }
435
+
436
+ it { is_expected.to eq expected_result }
437
+ end
438
+
439
+ context 'with an non-ASCII column name' do
440
+ let(:colname) { "在庫ID" }
441
+
442
+ it { is_expected.to eq expected_result }
443
+ end
444
+
445
+ context 'with a column name including space' do
446
+ let(:colname) { "full name" }
447
+
448
+ it { is_expected.to eq expected_result }
449
+ end
450
+
451
+ context 'with a column name including all supported ASCII special characters' do
452
+ # all special characters but '.' and '`'.
453
+ # '.' has its own spec. '`' should not come because MySQL doesn't allow it
454
+ let(:colname) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
455
+
456
+ it { is_expected.to eq expected_result }
457
+ end
458
+
459
+ context 'with a column name including comma' do
460
+ let(:colname) { %q|my.column| }
461
+
462
+ it "FlyData does not support a column name with a period in it. However, the agent accepts it as is. Necessary conversion happens at the server side." do
463
+ is_expected.to eq expected_result
464
+ end
465
+ end
466
+ end
414
467
  end
415
468
 
416
469
  end
@@ -0,0 +1,44 @@
1
+ -- MySQL dump 10.13 Distrib 5.5.43, for debian-linux-gnu (x86_64)
2
+ --
3
+ -- Host: localhost Database: flydata_sync
4
+ -- ------------------------------------------------------
5
+ -- Server version 5.5.43-0ubuntu0.12.04.1-log
6
+
7
+ /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
8
+ /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
9
+ /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
10
+ /*!40101 SET NAMES utf8 */;
11
+ /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
12
+ /*!40103 SET TIME_ZONE='+00:00' */;
13
+ /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
14
+ /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
15
+ /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
16
+ /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
17
+
18
+ --
19
+ -- Table structure for table `sample`
20
+ --
21
+
22
+ DROP TABLE IF EXISTS `sample`;
23
+ /*!40101 SET @saved_cs_client = @@character_set_client */;
24
+ /*!40101 SET character_set_client = utf8 */;
25
+ CREATE TABLE `sample` (
26
+ `id` int(11) NOT NULL AUTO_INCREMENT,
27
+ `name` text,
28
+ `created_at` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
29
+ `test_enum` enum('1','2') DEFAULT NULL,
30
+ `space test` varchar(100) DEFAULT NULL,
31
+ PRIMARY KEY (`id`)
32
+ ) ENGINE=InnoDB AUTO_INCREMENT=192 DEFAULT CHARSET=latin1;
33
+ /*!40101 SET character_set_client = @saved_cs_client */;
34
+ /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
35
+
36
+ /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
37
+ /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
38
+ /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
39
+ /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
40
+ /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
41
+ /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
42
+ /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
43
+
44
+ -- Dump completed on 2015-05-28 11:29:57
@@ -568,6 +568,20 @@ EOS
568
568
 
569
569
  it { is_expected.to eq expected_sql }
570
570
  end
571
+
572
+ context "column name with a single quote" do
573
+ let(:column_name) { "90's hits" }
574
+ let(:expected_column_name) { "90\\'s hits" }
575
+
576
+ it { is_expected.to eq expected_sql }
577
+ end
578
+
579
+ context "column name with a backslash" do
580
+ let(:column_name) { "\\^_^/" }
581
+ let(:expected_column_name) { "\\\\^_^/" }
582
+
583
+ it { is_expected.to eq expected_sql }
584
+ end
571
585
  end
572
586
  end
573
587
 
@@ -826,6 +840,34 @@ EOS
826
840
  subject { subject_object.convert_to_valid_name(key) }
827
841
 
828
842
  it_behaves_like "expecting no underscore in front of numbers"
843
+
844
+ context "with a key that has spaces" do
845
+ let(:key) { "space test" }
846
+ let(:expected_value) { key }
847
+ it { is_expected.to eq expected_value }
848
+ end
849
+
850
+ context "with a key that has non-ASCII chars" do
851
+ let(:key) { "行列123" }
852
+ let(:expected_value) { "__123" }
853
+ it "Redshift does not support non-ASCII table/column name. Those characters will be replaced with '_'" do
854
+ is_expected.to eq expected_value
855
+ end
856
+ end
857
+ context "with a key that has special chars" do
858
+ let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
859
+ let(:expected_value) { %q| !_#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
860
+ it "Redshift supports all these special characters but double quote (\") which is replaced with underscore (_)" do
861
+ is_expected.to eq expected_value
862
+ end
863
+ end
864
+ context "with a key that has a period in it" do
865
+ let(:key) { %q|my.table| }
866
+ let(:expected_value) { key }
867
+ it "Redshift supports period (.) in a table/column name" do
868
+ is_expected.to eq expected_value
869
+ end
870
+ end
829
871
  end
830
872
 
831
873
  describe ".convert_to_valid_column_name" do
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/table_def/sync_redshift_table_def'
3
+
4
+ module FlydataCore
5
+ module TableDef
6
+
7
+ describe SyncRedshiftTableDef do
8
+ describe '.convert_to_valid_name' do
9
+ subject { described_class.convert_to_valid_name(key) }
10
+
11
+ context "with a key that has a period in it" do
12
+ let(:key) { %q|my.table| }
13
+ let(:expected_value) { %q|my_table| }
14
+ it "Sync does not support period (.) in a table/column name. It's replaced with an underscore (_)" do
15
+ is_expected.to eq expected_value
16
+ end
17
+ end
18
+ end
19
+ end
20
+
21
+ end
22
+ end
@@ -2,16 +2,16 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: flydata 0.3.23 ruby lib
5
+ # stub: flydata 0.3.24 ruby lib
6
6
 
7
7
  Gem::Specification.new do |s|
8
8
  s.name = "flydata"
9
- s.version = "0.3.23"
9
+ s.version = "0.3.24"
10
10
 
11
11
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
12
  s.require_paths = ["lib"]
13
13
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
14
- s.date = "2015-05-16"
14
+ s.date = "2015-06-01"
15
15
  s.description = "FlyData Agent"
16
16
  s.email = "sysadmin@flydata.com"
17
17
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -50,17 +50,21 @@ Gem::Specification.new do |s|
50
50
  "flydata-core/lib/flydata-core/record/record.rb",
51
51
  "flydata-core/lib/flydata-core/redshift/string.rb",
52
52
  "flydata-core/lib/flydata-core/table_def.rb",
53
+ "flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
53
54
  "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
54
55
  "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
56
+ "flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb",
55
57
  "flydata-core/lib/flydata-core/thread_context.rb",
56
58
  "flydata-core/spec/config/user_maintenance_spec.rb",
57
59
  "flydata-core/spec/fluent/config_helper_spec.rb",
58
60
  "flydata-core/spec/logger_spec.rb",
59
61
  "flydata-core/spec/redshift/string_spec.rb",
60
62
  "flydata-core/spec/spec_helper.rb",
63
+ "flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
61
64
  "flydata-core/spec/table_def/mysql_table_def_spec.rb",
62
65
  "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
63
66
  "flydata-core/spec/table_def/mysqldump_test_bit_table.dump",
67
+ "flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump",
64
68
  "flydata-core/spec/table_def/mysqldump_test_column_charset.dump",
65
69
  "flydata-core/spec/table_def/mysqldump_test_foreign_key.dump",
66
70
  "flydata-core/spec/table_def/mysqldump_test_table_all.dump",
@@ -73,6 +77,7 @@ Gem::Specification.new do |s|
73
77
  "flydata-core/spec/table_def/mysqldump_test_unique_key3.dump",
74
78
  "flydata-core/spec/table_def/mysqldump_test_unsigned.dump",
75
79
  "flydata-core/spec/table_def/redshift_table_def_spec.rb",
80
+ "flydata-core/spec/table_def/sync_redshift_table_def_spec.rb",
76
81
  "flydata.gemspec",
77
82
  "lib/fly_data_model.rb",
78
83
  "lib/flydata.rb",
@@ -181,7 +186,7 @@ Gem::Specification.new do |s|
181
186
  ]
182
187
  s.homepage = "http://flydata.com/"
183
188
  s.licenses = ["All right reserved."]
184
- s.rubygems_version = "2.4.6"
189
+ s.rubygems_version = "2.4.3"
185
190
  s.summary = "FlyData Agent"
186
191
 
187
192
  if s.respond_to? :specification_version then
@@ -760,7 +760,7 @@ EOM
760
760
  next
761
761
  end
762
762
  flydata_tabledef = mysql_tabledef.to_flydata_tabledef
763
- puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
763
+ puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
764
764
  create_flydata_ctl_table = false
765
765
  end
766
766
  unless error_list.empty?
@@ -20,35 +20,36 @@ module Mysql
20
20
 
21
21
  private
22
22
 
23
+ PLACEMENT = '{}'
23
24
  def normalize_query(queries)
24
- queries = strip_comments(queries)
25
- queries = queries.gsub(/\s+/, ' ') # replace multiple spaces with a space
26
- queries.split(';').each do |query|
27
- query = query.lstrip
28
- yield(query + ";") if block_given? && !query.empty?
29
- end
30
- end
31
-
32
- def strip_comments(query)
33
- q = query.dup
25
+ q = queries.dup
26
+ strings = []
34
27
  # \/\*.*?\*\/ /* */ style comment
35
28
  # `.*?` `resource_name`
36
29
  # '(?:\\.|.)*?' 'string'
37
30
  # "(?:\\.|.)*?" "string"
38
31
  # --\s+.*?(?:\n|$) -- style comment
39
32
  # #\s+.*?(?:\n|$) # style comment
40
- query.scan(/(\/\*.*?\*\/|`.*?`|'(?:\\.|.)*?'|"(?:\\.|.)*?"|--\s+.*?(?:\n|$)|#.*?(?:\n|$))/m) do |m|
41
- comment_or_quoted = m.first
33
+ q.gsub!(/(\/\*.*?\*\/|`.*?`|'(?:\\.|.)*?'|"(?:\\.|.)*?"|--\s+.*?(?:\n|$)|#.*?(?:\n|$))/m) do |m|
34
+ comment_or_quoted = $&
42
35
  if comment_or_quoted.start_with?("/*") || comment_or_quoted.start_with?("--") ||
43
36
  comment_or_quoted.start_with?("#")
44
- # comment. replace with spaces of the same length
45
- idx_from = $~.offset(0)[0]
46
- idx_to = $~.offset(0)[1]
47
- len = idx_to - idx_from
48
- q[idx_from...idx_to] = ' ' * len
37
+ # comment. replace with a space
38
+ ' '
39
+ else
40
+ # string. save the original and replace with an temporary placement
41
+ strings << comment_or_quoted
42
+ PLACEMENT
49
43
  end
50
44
  end
51
- q
45
+ q = q.gsub(/\s+/, ' ') # replace multiple spaces with a space
46
+ q.split(';').each do |query|
47
+ query = query.lstrip
48
+ query.gsub!(PLACEMENT) do |m|
49
+ strings.shift
50
+ end
51
+ yield(query + ";") if block_given? && !query.empty?
52
+ end
52
53
  end
53
54
  end
54
55
 
@@ -323,8 +323,10 @@ EOS
323
323
 
324
324
  # parse column line
325
325
  line = line[0..-2] if line.end_with?(',')
326
+ colname_end_index = line.index('`', 1) - 1
327
+ column[:column_name] = line[1..colname_end_index]
328
+ line = line[colname_end_index + 3..-1]
326
329
  items = line.split
327
- column[:column_name] = items.shift[1..-2]
328
330
  column[:format_type_str] = format_type_str = items.shift
329
331
  pos = format_type_str.index('(')
330
332
  if pos
@@ -1205,12 +1205,23 @@ grammar MysqlAlterTable
1205
1205
  }
1206
1206
  end
1207
1207
 
1208
+ # This rules does not exist in sql_yacc.yy, but should be close enough.
1209
+ rule ident_in_quote
1210
+ # 1..1 prevents Treetop from embedding the SyntaxNode to an upper node
1211
+ # without the custom method.
1212
+ ( [^`]* ) 1..1 {
1213
+ def value
1214
+ text_value
1215
+ end
1216
+ }
1217
+ end
1218
+
1208
1219
  rule ident_quoted
1209
- '`' ident_sym '`' {
1220
+ '`' ident_in_quote '`' {
1210
1221
  # #value returns a normalized value of the element while #text_value
1211
1222
  # returns its text as is
1212
1223
  def value
1213
- ident_sym.value
1224
+ ident_in_quote.value
1214
1225
  end
1215
1226
  }
1216
1227
  end
@@ -60,7 +60,8 @@ EOT
60
60
  r
61
61
  end
62
62
  let(:context) { double('context') }
63
- subject { described_class.new(context) }
63
+ let(:subject_object) { described_class.new(context) }
64
+ subject { subject_object }
64
65
 
65
66
  shared_examples "a dispatcher that calls query handler with correct query" do
66
67
  it do
@@ -233,5 +234,35 @@ EOS
233
234
  end
234
235
  end
235
236
  end
237
+ describe '#normalize_query' do
238
+ context 'straightforwad query' do
239
+ let(:query) { "ALTER TABLE test_table DROP COLUMN test_column" }
240
+ let(:expected_normalized) { ["#{query};"] }
241
+
242
+ it do
243
+ expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
244
+ end
245
+ end
246
+ context 'two queries' do
247
+ let(:subquery1) { "ALTER TABLE test_table DROP COLUMN test_column" }
248
+ let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
249
+ let(:query) { "#{subquery1}; #{subquery2};" }
250
+ let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
251
+
252
+ it do
253
+ expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
254
+ end
255
+ end
256
+ context 'a column name includes a semicolon' do
257
+ let(:subquery1) { %Q|ALTER TABLE test_table DROP COLUMN "test_;column"| }
258
+ let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
259
+ let(:query) { "#{subquery1}; #{subquery2};" }
260
+ let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
261
+
262
+ it do
263
+ expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
264
+ end
265
+ end
266
+ end
236
267
  end
237
268
  end
@@ -10,7 +10,7 @@ describe 'MysqlAlterTableParser' do
10
10
  let(:query) { "" }
11
11
 
12
12
  let(:parser) { @parser_class.new }
13
- subject { parser.parse(query).tree }
13
+ subject { res = parser.parse(query); raise RuntimeError, parser.inspect unless res; res.tree }
14
14
 
15
15
 
16
16
  describe '#parse' do
@@ -33,6 +33,22 @@ describe 'MysqlAlterTableParser' do
33
33
  end
34
34
  end
35
35
 
36
+ context 'with a non-ascii table name' do
37
+ let(:query) { "alter table `テスト` add column value varchar(26)" }
38
+ it do
39
+ expect(subject).to eq({
40
+ type: :alter_table,
41
+ table_name: "テスト",
42
+ actions: [{
43
+ action: :add_column,
44
+ column: "value",
45
+ type: "varchar(78)",
46
+ query: "add column value varchar(26)"
47
+ }]
48
+ })
49
+ end
50
+ end
51
+
36
52
  context 'with different column types' do
37
53
  context 'with longblob' do
38
54
  let(:query) { "alter table test_table add column value longblob" }
@@ -1547,11 +1563,23 @@ describe 'MysqlAlterTableParser' do
1547
1563
  end
1548
1564
  shared_examples "test position" do |*examples|
1549
1565
  context "with after" do
1550
- it_behaves_like *examples do
1551
- let(:position) {" after test_name"}
1552
- let(:position_hash){
1553
- { after: "test_name" }
1554
- }
1566
+ context 'an ordinary column name' do
1567
+ let(:pos_column) { 'test_name' }
1568
+ it_behaves_like *examples do
1569
+ let(:position) {" after #{pos_column}"}
1570
+ let(:position_hash){
1571
+ { after: pos_column.gsub(/(^`|`$)/, '') }
1572
+ }
1573
+ end
1574
+ end
1575
+ context 'an extrordinary column name' do
1576
+ let(:pos_column) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
1577
+ it_behaves_like *examples do
1578
+ let(:position) {" after #{pos_column}"}
1579
+ let(:position_hash){
1580
+ { after: pos_column.gsub(/(^`|`$)/, '') }
1581
+ }
1582
+ end
1555
1583
  end
1556
1584
  end
1557
1585
  context "with first" do
@@ -1757,14 +1785,58 @@ describe 'MysqlAlterTableParser' do
1757
1785
  end
1758
1786
  context 'change column' do
1759
1787
  context 'change column name' do
1760
- let(:alter_table_action) { "change#{column} name name1 #{col_def}#{position}" }
1761
- let(:change_action_hash) {
1762
- {
1763
- old_column: "name",
1764
- column: "name1",
1765
- }.merge!(position_hash).merge!(col_def_hash)
1766
- }
1767
- it_behaves_like "test optional column", "test column definition", "test position", "test parser parsing a change column"
1788
+ shared_examples "renaming a column name without an issue" do
1789
+ let(:alter_table_action) { "change#{column} #{colname_old} #{colname_new} #{col_def}#{position}" }
1790
+ let(:change_action_hash) {
1791
+ {
1792
+ old_column: colname_old.gsub(/^`|`$/, ''),
1793
+ column: colname_new.gsub(/^`|`$/, ''),
1794
+ }.merge!(position_hash).merge!(col_def_hash)
1795
+ }
1796
+ it_behaves_like "test optional column", "test column definition", "test position", "test parser parsing a change column"
1797
+ end
1798
+ context 'ordinary column names' do
1799
+ let(:colname_old) { 'name' }
1800
+ let(:colname_new) { 'name1' }
1801
+
1802
+ it_behaves_like "renaming a column name without an issue"
1803
+ end
1804
+ context 'renaming a name including space' do
1805
+ let(:colname_old) { '`name including spaces`' }
1806
+ let(:colname_new) { 'name1' }
1807
+
1808
+ it_behaves_like "renaming a column name without an issue"
1809
+ end
1810
+ context 'renaming to a name including space' do
1811
+ let(:colname_old) { 'name' }
1812
+ let(:colname_new) { '`name including spaces`' }
1813
+
1814
+ it_behaves_like "renaming a column name without an issue"
1815
+ end
1816
+ context 'renaming a non-ascii name' do
1817
+ let(:colname_old) { '`コラム`' }
1818
+ let(:colname_new) { '`name including spaces`' }
1819
+
1820
+ it_behaves_like "renaming a column name without an issue"
1821
+ end
1822
+ context 'renaming a column with special characters' do
1823
+ let(:colname_old) { %q|` !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
1824
+ let(:colname_new) { '`name including spaces`' }
1825
+
1826
+ it_behaves_like "renaming a column name without an issue"
1827
+ end
1828
+ context 'adding a single quote to a column with special characters' do
1829
+ let(:colname_old) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
1830
+ let(:colname_new) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.'`,}
1831
+
1832
+ it_behaves_like "renaming a column name without an issue"
1833
+ end
1834
+ context 'renaming a non-ascii name' do
1835
+ let(:colname_old) { '`my.table`' }
1836
+ let(:colname_new) { '`name including spaces`' }
1837
+
1838
+ it_behaves_like "renaming a column name without an issue"
1839
+ end
1768
1840
  end
1769
1841
  context 'no change to column name' do
1770
1842
  let(:alter_table_action) { "change#{column} name1 name1 #{col_def}#{position}" }
@@ -147,7 +147,7 @@ module Flydata
147
147
  EOT
148
148
 
149
149
  def index_after(content, string)
150
- content.index(string) + string.bytesize + 1
150
+ content[0...content.index(string)].bytesize + string.bytesize + 1
151
151
  end
152
152
 
153
153
  let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
@@ -178,7 +178,7 @@ EOT
178
178
  end
179
179
  end
180
180
 
181
- context 'when dump contains create table without records' do
181
+ shared_examples 'dump contains create table without records' do
182
182
  let(:dump_content) { <<EOT
183
183
  #{DUMP_HEADER}
184
184
 
@@ -186,7 +186,7 @@ DROP TABLE IF EXISTS `users_login`;
186
186
  /*!40101 SET @saved_cs_client = @@character_set_client */;
187
187
  /*!40101 SET character_set_client = utf8 */;
188
188
  CREATE TABLE `users_login` (
189
- `user_id` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
189
+ `#{col_name}` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
190
190
  `login_count` int(10) unsigned DEFAULT '0' COMMENT 'login count',
191
191
  `create_time` datetime NOT NULL COMMENT 'create time',
192
192
  `update_time` datetime NOT NULL COMMENT 'update time',
@@ -220,7 +220,7 @@ EOT
220
220
  }
221
221
  expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
222
222
  expect(mysql_table.table_name).to eq('users_login')
223
- expect(mysql_table.columns.keys).to eq(['user_id', 'login_count', 'create_time', 'update_time'])
223
+ expect(mysql_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
224
224
  expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
225
225
  expect(binlog_pos).to eq(default_binlog_pos)
226
226
  expect(state).to eq(MysqlDumpParser::State::INSERT_RECORD)
@@ -242,6 +242,29 @@ EOT
242
242
  end
243
243
  end
244
244
 
245
+ context 'when dump contains create table without records' do
246
+ context 'when one of the column name has space' do
247
+ let(:col_name) { "user id" }
248
+ include_examples 'dump contains create table without records'
249
+ end
250
+ context 'when the column name does not have space' do
251
+ let(:col_name) { "user_id" }
252
+ include_examples 'dump contains create table without records'
253
+ end
254
+ context 'when the column name is non-ascii' do
255
+ let(:col_name) { "行列" }
256
+ include_examples 'dump contains create table without records'
257
+ end
258
+ context 'when the column name includes special characters' do
259
+ let(:col_name) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
260
+ include_examples 'dump contains create table without records'
261
+ end
262
+ context 'when the column name includes a dot' do
263
+ let(:col_name) { %q|my.column| }
264
+ include_examples 'dump contains create table without records'
265
+ end
266
+ end
267
+
245
268
  context 'when dump contains create table with multi inserts and invalid utf8 byte sequences' do
246
269
  let(:dump_content) { <<EOT
247
270
  #{DUMP_HEADER}
@@ -303,7 +326,7 @@ EOT
303
326
  {state: MysqlDumpParser::State::CREATE_TABLE},
304
327
  {state: MysqlDumpParser::State::INSERT_RECORD},
305
328
  {state: MysqlDumpParser::State::CREATE_TABLE,
306
- last_pos: index_after(dump_content, 'UNLOCK TABLES;') + 10}
329
+ last_pos: index_after(dump_content, 'UNLOCK TABLES;')}
307
330
  ].each do |expected_params|
308
331
  expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
309
332
  expect(mysql_table.table_name).to eq('users_login') if mysql_table
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.23
4
+ version: 0.3.24
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2015-05-16 00:00:00.000000000 Z
15
+ date: 2015-06-01 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -473,17 +473,21 @@ files:
473
473
  - flydata-core/lib/flydata-core/record/record.rb
474
474
  - flydata-core/lib/flydata-core/redshift/string.rb
475
475
  - flydata-core/lib/flydata-core/table_def.rb
476
+ - flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
476
477
  - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
477
478
  - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
479
+ - flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb
478
480
  - flydata-core/lib/flydata-core/thread_context.rb
479
481
  - flydata-core/spec/config/user_maintenance_spec.rb
480
482
  - flydata-core/spec/fluent/config_helper_spec.rb
481
483
  - flydata-core/spec/logger_spec.rb
482
484
  - flydata-core/spec/redshift/string_spec.rb
483
485
  - flydata-core/spec/spec_helper.rb
486
+ - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
484
487
  - flydata-core/spec/table_def/mysql_table_def_spec.rb
485
488
  - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
486
489
  - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
490
+ - flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump
487
491
  - flydata-core/spec/table_def/mysqldump_test_column_charset.dump
488
492
  - flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
489
493
  - flydata-core/spec/table_def/mysqldump_test_table_all.dump
@@ -496,6 +500,7 @@ files:
496
500
  - flydata-core/spec/table_def/mysqldump_test_unique_key3.dump
497
501
  - flydata-core/spec/table_def/mysqldump_test_unsigned.dump
498
502
  - flydata-core/spec/table_def/redshift_table_def_spec.rb
503
+ - flydata-core/spec/table_def/sync_redshift_table_def_spec.rb
499
504
  - flydata.gemspec
500
505
  - lib/fly_data_model.rb
501
506
  - lib/flydata.rb
@@ -621,7 +626,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
621
626
  version: '0'
622
627
  requirements: []
623
628
  rubyforge_project:
624
- rubygems_version: 2.4.6
629
+ rubygems_version: 2.4.3
625
630
  signing_key:
626
631
  specification_version: 4
627
632
  summary: FlyData Agent