flydata 0.3.23 → 0.3.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65c746356a222838bb65756fb206ac897ccc236d
-  data.tar.gz: 8a057f92fa831366d156ffd3c1a2b295ace4e682
+  metadata.gz: 2c6d22e3fb1b51c996297d48f7bd886c250d5d11
+  data.tar.gz: 23ac7e6b2b97a76bf27d538370fb9c73e018e842
 SHA512:
-  metadata.gz: 9b61f370d5295a8249f8b4485c0ba4e3011b0aa2c2fb7b79d1215c022a6fba9dd503d8e5f2a22c86c42b38da1953b6feb7ff72a90e7a1cdfa6bfdcd3943ac1d0
-  data.tar.gz: d2593d38159c408fad8c84842163ad579fe3251a71875a3c0eee172d65be00599c9f1658f24eb113e8e52f61a063665b059b6a9e585e8705377087b3c801e9cb
+  metadata.gz: f5455a1b18dcbccb9edae44ec15ac2332710410107687251cba290a99e671b83567c84dde29b7428696750b1732ebae7410093921170ad04ab1f4c938c1da67c
+  data.tar.gz: d72631b42f67a3ae378ed215663149414cf1cd80d02eac97fe97c0517b7813a5341ee93a05daf02774f094daf90c19381d3b1709a428729a99fb2101b6f80468

data/VERSION

@@ -1 +1 @@
-0.3.23
+0.3.24

data/flydata-core/lib/flydata-core/errors.rb

@@ -299,6 +299,7 @@ class UnsupportedRecordFormat < StandardError; end
 class TableDefError < StandardError
   attr_reader :err_hash
   def initialize(err_hash)
+    super(err_hash[:error])
     @err_hash = err_hash
   end
 end

data/flydata-core/lib/flydata-core/table_def.rb

@@ -1,2 +1,4 @@
 require 'flydata-core/table_def/redshift_table_def'
+require 'flydata-core/table_def/sync_redshift_table_def'
+require 'flydata-core/table_def/autoload_redshift_table_def'
 require 'flydata-core/table_def/mysql_table_def'

data/flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb

@@ -0,0 +1,13 @@
+require 'flydata-core/table_def/redshift_table_def'
+
+module FlydataCore
+module TableDef
+
+class AutoloadRedshiftTableDef < RedshiftTableDef
+  # Of the valid identifier charaters of RedShift table/column name,
+  # Autoload only supports '$' as the valid special character.
+  VALID_IDENTIFIER_CHARACTERS = %w"0-9 a-z $ _"
+end
+
+end
+end

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -182,7 +182,7 @@ class MysqlTableDef
     while pos < line.length
       case cond
       when :column_name  #`column_name` ...
-        pos = line.index(' ', 1)
+        pos =  line[0] == '`' ?  line.index('`', 1) + 1 : line.index(' ', 1)
         column[:column] = if line[0] == '`'
                           line[1..pos-2]
                         else

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -231,7 +231,7 @@ EOS
 
       sql += <<EOS
 COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{convert_to_valid_column_name(col[:column])}"
-  IS '#{col[:comment]}';
+  IS '#{escape(col[:comment])}';
 EOS
     end
     sql
@@ -247,11 +247,11 @@ INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
 EOS
   def self.flydata_ctl_columns_sql(flydata_tabledef, schema_name)
     flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name, :columns)
-    sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
+    sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, escape(flydata_tabledef[:table_name]), flydata_ctl_tbl ]
     values = []
     flydata_tabledef[:columns].each.with_index(1) do |col, i|
       charset = col[:charset] ? " cs:#{col[:charset]}" : ""
-      values << "('#{flydata_tabledef[:table_name]}', '#{col[:column]}', '#{escape(col[:type])}#{charset}', #{i})"
+      values << "('#{escape(flydata_tabledef[:table_name])}', '#{escape(col[:column])}', '#{escape(col[:type])}#{charset}', #{i})"
     end
     sql += values.join(",\n") + ';'
     sql
@@ -286,8 +286,11 @@ EOS
 
   MAX_COLUMNNAME_LENGTH = 127
 
+  VALID_IDENTIFIER_CHARACTERS = [" "] + %w"a-z 0-9 ! # $ % & ' ( ) * + , . \\- \\/ : ; < = > ? @ \\[ \\\\ \\] ^ { \\| } ~ `"
+
   def self.convert_to_valid_name(key)
-    name = key.downcase.gsub(/[^a-z0-9_$]/, '_')
+    match = /[^#{self::VALID_IDENTIFIER_CHARACTERS.join}]/
+    name = key.downcase.gsub(match, '_')
     do_convert = yield name if block_given?
     name = "_#{name}" if do_convert
     if name.length > MAX_COLUMNNAME_LENGTH
@@ -305,7 +308,7 @@ EOS
   end
 
   def self.escape(text)
-    text.gsub("'", "\\\\'")
+    text.gsub(/(['\\])/, "\\\\\\1")
   end
 
   def self.check_and_replace_max(params, max_size_a)

data/flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb

@@ -0,0 +1,13 @@
+require 'flydata-core/table_def/redshift_table_def'
+
+module FlydataCore
+module TableDef
+
+class SyncRedshiftTableDef < RedshiftTableDef
+  # For now, of the valid identifier charaters of RedShift table/column name,
+  # Sync does not support the period character (.)
+  VALID_IDENTIFIER_CHARACTERS = superclass::VALID_IDENTIFIER_CHARACTERS - ["."]
+end
+
+end
+end

data/flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require 'flydata-core/table_def/autoload_redshift_table_def'
+
+module FlydataCore
+module TableDef
+
+describe AutoloadRedshiftTableDef do
+  describe '.convert_to_valid_name' do
+    subject { described_class.convert_to_valid_name(key) }
+
+    context "with a key that has special chars" do
+      let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
+      let(:expected_value) { %q|____$___________________________| }
+
+      it "Autoload only supports $ in a table/column name.  Other special characters are replaced with underscore (_)" do
+        is_expected.to eq expected_value
+      end
+    end
+    context "with a key that has a period in it" do
+      let(:key) { %q|my.table| }
+      let(:expected_value) { %q|my_table| }
+      it "Autoload does not support period (.) in a table/column name.  It's replaced with an underscore (_)" do
+        is_expected.to eq expected_value
+      end
+    end
+  end
+end
+
+end
+end

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -245,6 +245,14 @@ describe MysqlTableDef do
         expect(subject[:columns][1][:charset]).to eq("ISO_8859_1")
       end
     end
+
+    context 'when a column name has space in it' do
+      let(:dump_file_io) { file_io('mysqldump_test_col_name_with_space.dump')}
+      it 'should extract col name properly' do
+        expect(subject[:columns][4][:column]).to eq("space test")
+        expect(subject[:columns][4][:type]).to eq("varchar(300)")
+      end
+    end
   end
 
   describe '.convert_to_flydata_type' do
@@ -411,6 +419,51 @@ describe MysqlTableDef do
       end
     end
   end
+
+  describe '.parse_one_column_def' do
+    subject { described_class.parse_one_column_def(query) }
+
+    let(:query) { "  `#{colname}` text," }
+    let(:expected_result) {
+      { column: colname,
+        type: 'text',
+      }
+    }
+
+    context 'with an ASCII column name' do
+      let(:colname) { "name" }
+
+      it { is_expected.to eq expected_result }
+    end
+
+    context 'with an non-ASCII column name' do
+      let(:colname) { "在庫ID" }
+
+      it { is_expected.to eq expected_result }
+    end
+
+    context 'with a column name including space' do
+      let(:colname) { "full name" }
+
+      it { is_expected.to eq expected_result }
+    end
+
+    context 'with a column name including all supported ASCII special characters' do
+      # all special characters but '.' and '`'.
+      # '.' has its own spec.  '`' should not come because MySQL doesn't allow it
+      let(:colname) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
+
+      it { is_expected.to eq expected_result }
+    end
+
+    context 'with a column name including comma' do
+      let(:colname) { %q|my.column| }
+
+      it "FlyData does not support a column name with a period in it.  However, the agent accepts it as is.  Necessary conversion happens at the server side." do
+        is_expected.to eq expected_result
+      end
+    end
+  end
 end
 
 end

data/flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump

@@ -0,0 +1,44 @@
+-- MySQL dump 10.13  Distrib 5.5.43, for debian-linux-gnu (x86_64)
+--
+-- Host: localhost    Database: flydata_sync
+-- ------------------------------------------------------
+-- Server version	5.5.43-0ubuntu0.12.04.1-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `sample`
+--
+
+DROP TABLE IF EXISTS `sample`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `sample` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` text,
+  `created_at` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `test_enum` enum('1','2') DEFAULT NULL,
+  `space test` varchar(100) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=192 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2015-05-28 11:29:57

data/flydata-core/spec/table_def/redshift_table_def_spec.rb

@@ -568,6 +568,20 @@ EOS
 
         it { is_expected.to eq expected_sql }
       end
+
+      context "column name with a single quote" do
+        let(:column_name) { "90's hits" }
+        let(:expected_column_name) { "90\\'s hits" }
+
+        it { is_expected.to eq expected_sql }
+      end
+
+      context "column name with a backslash" do
+        let(:column_name) { "\\^_^/" }
+        let(:expected_column_name) { "\\\\^_^/" }
+
+        it { is_expected.to eq expected_sql }
+      end
     end
   end
 
@@ -826,6 +840,34 @@ EOS
     subject { subject_object.convert_to_valid_name(key) }
 
     it_behaves_like "expecting no underscore in front of numbers"
+
+    context "with a key that has spaces" do
+      let(:key) { "space test" }
+      let(:expected_value) { key }
+      it { is_expected.to eq expected_value }
+    end
+
+    context "with a key that has non-ASCII chars" do
+      let(:key) { "行列123" }
+      let(:expected_value) { "__123" }
+      it "Redshift does not support non-ASCII table/column name.  Those characters will be replaced with '_'" do
+        is_expected.to eq expected_value
+      end
+    end
+    context "with a key that has special chars" do
+      let(:key) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
+      let(:expected_value) { %q| !_#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
+      it "Redshift supports all these special characters but double quote (\") which is replaced with underscore (_)" do
+        is_expected.to eq expected_value
+      end
+    end
+    context "with a key that has a period in it" do
+      let(:key) { %q|my.table| }
+      let(:expected_value) { key }
+      it "Redshift supports period (.) in a table/column name" do
+        is_expected.to eq expected_value
+      end
+    end
   end
 
   describe ".convert_to_valid_column_name" do

data/flydata-core/spec/table_def/sync_redshift_table_def_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+require 'flydata-core/table_def/sync_redshift_table_def'
+
+module FlydataCore
+module TableDef
+
+describe SyncRedshiftTableDef do
+  describe '.convert_to_valid_name' do
+    subject { described_class.convert_to_valid_name(key) }
+
+    context "with a key that has a period in it" do
+      let(:key) { %q|my.table| }
+      let(:expected_value) { %q|my_table| }
+      it "Sync does not support period (.) in a table/column name.  It's replaced with an underscore (_)" do
+        is_expected.to eq expected_value
+      end
+    end
+  end
+end
+
+end
+end

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.3.23 ruby lib
+# stub: flydata 0.3.24 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.3.23"
+  s.version = "0.3.24"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-05-16"
+  s.date = "2015-06-01"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -50,17 +50,21 @@ Gem::Specification.new do |s|
     "flydata-core/lib/flydata-core/record/record.rb",
     "flydata-core/lib/flydata-core/redshift/string.rb",
     "flydata-core/lib/flydata-core/table_def.rb",
+    "flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/mysql_table_def.rb",
     "flydata-core/lib/flydata-core/table_def/redshift_table_def.rb",
+    "flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb",
     "flydata-core/lib/flydata-core/thread_context.rb",
     "flydata-core/spec/config/user_maintenance_spec.rb",
     "flydata-core/spec/fluent/config_helper_spec.rb",
     "flydata-core/spec/logger_spec.rb",
     "flydata-core/spec/redshift/string_spec.rb",
     "flydata-core/spec/spec_helper.rb",
+    "flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_table_def_spec.rb",
     "flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb",
     "flydata-core/spec/table_def/mysqldump_test_bit_table.dump",
+    "flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump",
     "flydata-core/spec/table_def/mysqldump_test_column_charset.dump",
     "flydata-core/spec/table_def/mysqldump_test_foreign_key.dump",
     "flydata-core/spec/table_def/mysqldump_test_table_all.dump",
@@ -73,6 +77,7 @@ Gem::Specification.new do |s|
     "flydata-core/spec/table_def/mysqldump_test_unique_key3.dump",
     "flydata-core/spec/table_def/mysqldump_test_unsigned.dump",
     "flydata-core/spec/table_def/redshift_table_def_spec.rb",
+    "flydata-core/spec/table_def/sync_redshift_table_def_spec.rb",
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",
@@ -181,7 +186,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://flydata.com/"
   s.licenses = ["All right reserved."]
-  s.rubygems_version = "2.4.6"
+  s.rubygems_version = "2.4.3"
   s.summary = "FlyData Agent"
 
   if s.respond_to? :specification_version then

data/lib/flydata/command/sync.rb

@@ -760,7 +760,7 @@ EOM
             next
           end
           flydata_tabledef = mysql_tabledef.to_flydata_tabledef
-          puts FlydataCore::TableDef::RedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
+          puts FlydataCore::TableDef::SyncRedshiftTableDef.from_flydata_tabledef(flydata_tabledef, flydata_ctl_table: create_flydata_ctl_table, schema_name: schema_name, ctl_only: opts.ctl_only?)
           create_flydata_ctl_table = false
         end
         unless error_list.empty?

data/lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb

@@ -20,35 +20,36 @@ module Mysql
 
     private
 
+    PLACEMENT = '{}'
     def normalize_query(queries)
-      queries = strip_comments(queries)
-      queries = queries.gsub(/\s+/, ' ') # replace multiple spaces with a space
-      queries.split(';').each do |query|
-        query = query.lstrip
-        yield(query + ";") if block_given? && !query.empty?
-      end
-    end
-
-    def strip_comments(query)
-      q = query.dup
+      q = queries.dup
+      strings = []
       # \/\*.*?\*\/        /* */ style comment
       # `.*?`              `resource_name`
       # '(?:\\.|.)*?'      'string'
       # "(?:\\.|.)*?"      "string"
       # --\s+.*?(?:\n|$)   -- style comment
       # #\s+.*?(?:\n|$)    # style comment
-      query.scan(/(\/\*.*?\*\/|`.*?`|'(?:\\.|.)*?'|"(?:\\.|.)*?"|--\s+.*?(?:\n|$)|#.*?(?:\n|$))/m) do |m|
-        comment_or_quoted = m.first
+      q.gsub!(/(\/\*.*?\*\/|`.*?`|'(?:\\.|.)*?'|"(?:\\.|.)*?"|--\s+.*?(?:\n|$)|#.*?(?:\n|$))/m) do |m|
+        comment_or_quoted = $&
         if comment_or_quoted.start_with?("/*") || comment_or_quoted.start_with?("--") ||
            comment_or_quoted.start_with?("#")
-          # comment. replace with spaces of the same length
-          idx_from = $~.offset(0)[0]
-          idx_to = $~.offset(0)[1]
-          len = idx_to - idx_from
-          q[idx_from...idx_to] = ' ' * len
+          # comment. replace with a space
+          ' '
+        else
+          # string.  save the original and replace with an temporary placement
+          strings << comment_or_quoted
+          PLACEMENT
         end
       end
-      q
+      q = q.gsub(/\s+/, ' ') # replace multiple spaces with a space
+      q.split(';').each do |query|
+        query = query.lstrip
+        query.gsub!(PLACEMENT) do |m|
+          strings.shift
+        end
+        yield(query + ";") if block_given? && !query.empty?
+      end
     end
   end
 

data/lib/flydata/parser/mysql/dump_parser.rb

@@ -323,8 +323,10 @@ EOS
 
               # parse column line
               line = line[0..-2] if line.end_with?(',')
+              colname_end_index = line.index('`', 1) - 1
+              column[:column_name] = line[1..colname_end_index]
+              line = line[colname_end_index + 3..-1]
               items = line.split
-              column[:column_name] = items.shift[1..-2]
               column[:format_type_str] = format_type_str = items.shift
               pos = format_type_str.index('(')
               if pos

data/lib/flydata/parser/mysql/mysql_alter_table.treetop

@@ -1205,12 +1205,23 @@ grammar MysqlAlterTable
     }
   end
 
+  # This rules does not exist in sql_yacc.yy, but should be close enough.
+  rule ident_in_quote
+    # 1..1 prevents Treetop from embedding the SyntaxNode to an upper node
+    # without the custom method.
+    ( [^`]* ) 1..1 {
+      def value
+        text_value
+      end
+    }
+  end
+
   rule ident_quoted
-    '`' ident_sym '`' {
+    '`' ident_in_quote '`' {
       # #value returns a normalized value of the element while #text_value
       # returns its text as is
       def value
-        ident_sym.value
+        ident_in_quote.value
       end
     }
   end

data/spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb

@@ -60,7 +60,8 @@ EOT
       r
     end
     let(:context) { double('context') }
-    subject { described_class.new(context) }
+    let(:subject_object) { described_class.new(context) }
+    subject { subject_object }
 
     shared_examples "a dispatcher that calls query handler with correct query" do
       it do
@@ -233,5 +234,35 @@ EOS
         end
       end
     end
+    describe '#normalize_query' do
+      context 'straightforwad query' do
+        let(:query) { "ALTER TABLE test_table DROP COLUMN test_column" }
+        let(:expected_normalized) { ["#{query};"] }
+
+        it do
+          expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
+        end
+      end
+      context 'two queries' do
+        let(:subquery1) { "ALTER TABLE test_table DROP COLUMN test_column" }
+        let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
+        let(:query) { "#{subquery1}; #{subquery2};" }
+        let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
+
+        it do
+          expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
+        end
+      end
+      context 'a column name includes a semicolon' do
+        let(:subquery1) { %Q|ALTER TABLE test_table DROP COLUMN "test_;column"| }
+        let(:subquery2) { "ALTER TABLE test_table2 DROP COLUMN test_column2" }
+        let(:query) { "#{subquery1}; #{subquery2};" }
+        let(:expected_normalized) { ["#{subquery1};", "#{subquery2};"] }
+
+        it do
+          expect {|b| subject_object.send(:normalize_query, query, &b) }.to yield_successive_args(*expected_normalized)
+        end
+      end
+    end
   end
 end

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -10,7 +10,7 @@ describe 'MysqlAlterTableParser' do
   let(:query) { "" }
 
   let(:parser) { @parser_class.new }
-  subject { parser.parse(query).tree }
+  subject { res = parser.parse(query); raise RuntimeError, parser.inspect unless res; res.tree }
 
 
   describe '#parse' do
@@ -33,6 +33,22 @@ describe 'MysqlAlterTableParser' do
         end
       end
 
+      context 'with a non-ascii table name' do
+        let(:query) { "alter table `テスト` add column value varchar(26)" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "テスト",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar(26)"
+            }]
+          })
+        end
+      end
+
       context 'with different column types' do
         context 'with longblob' do
           let(:query) { "alter table test_table add column value longblob" }
@@ -1547,11 +1563,23 @@ describe 'MysqlAlterTableParser' do
     end
     shared_examples "test position" do |*examples|
       context "with after" do
-        it_behaves_like *examples do
-          let(:position) {" after test_name"}
-          let(:position_hash){
-            { after: "test_name" }
-          }
+        context 'an ordinary column name' do
+          let(:pos_column) { 'test_name' }
+          it_behaves_like *examples do
+            let(:position) {" after #{pos_column}"}
+            let(:position_hash){
+              { after: pos_column.gsub(/(^`|`$)/, '')  }
+            }
+          end
+        end
+        context 'an extrordinary column name' do
+          let(:pos_column) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
+          it_behaves_like *examples do
+            let(:position) {" after #{pos_column}"}
+            let(:position_hash){
+              { after: pos_column.gsub(/(^`|`$)/, '')  }
+            }
+          end
         end
       end
       context "with first" do
@@ -1757,14 +1785,58 @@ describe 'MysqlAlterTableParser' do
     end
     context 'change column' do
       context 'change column name' do
-        let(:alter_table_action) { "change#{column} name name1 #{col_def}#{position}" }
-        let(:change_action_hash) {
-          {
-            old_column: "name",
-            column: "name1",
-          }.merge!(position_hash).merge!(col_def_hash)
-        }
-        it_behaves_like "test optional column", "test column definition", "test position", "test parser parsing a change column"
+        shared_examples "renaming a column name without an issue" do
+          let(:alter_table_action) { "change#{column} #{colname_old} #{colname_new} #{col_def}#{position}" }
+          let(:change_action_hash) {
+            {
+              old_column: colname_old.gsub(/^`|`$/, ''),
+              column: colname_new.gsub(/^`|`$/, ''),
+            }.merge!(position_hash).merge!(col_def_hash)
+          }
+          it_behaves_like "test optional column", "test column definition", "test position", "test parser parsing a change column"
+        end
+        context 'ordinary column names' do
+          let(:colname_old) { 'name' }
+          let(:colname_new) { 'name1' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'renaming a name including space' do
+          let(:colname_old) { '`name including spaces`' }
+          let(:colname_new) { 'name1' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'renaming to a name including space' do
+          let(:colname_old) { 'name' }
+          let(:colname_new) { '`name including spaces`' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'renaming a non-ascii name' do
+          let(:colname_old) { '`コラム`' }
+          let(:colname_new) { '`name including spaces`' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'renaming a column with special characters' do
+          let(:colname_old) { %q|` !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~`| }
+          let(:colname_new) { '`name including spaces`' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'adding a single quote to a column with special characters' do
+          let(:colname_old) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.`, }
+          let(:colname_new) { %q,` !"#$%&'()*+'-/:;<=>?@[\]^_{|}~.'`,}
+
+          it_behaves_like "renaming a column name without an issue"
+        end
+        context 'renaming a non-ascii name' do
+          let(:colname_old) { '`my.table`' }
+          let(:colname_new) { '`name including spaces`' }
+
+          it_behaves_like "renaming a column name without an issue"
+        end
       end
       context 'no change to column name' do
         let(:alter_table_action) { "change#{column} name1 name1 #{col_def}#{position}" }

data/spec/flydata/parser/mysql/dump_parser_spec.rb

@@ -147,7 +147,7 @@ module Flydata
 EOT
 
           def index_after(content, string)
-            content.index(string) + string.bytesize + 1
+            content[0...content.index(string)].bytesize + string.bytesize + 1
           end
 
           let(:default_parser) { MysqlDumpParser.new(binlog_pos: default_binlog_pos) }
@@ -178,7 +178,7 @@ EOT
             end
           end
 
-          context 'when dump contains create table without records' do
+          shared_examples 'dump contains create table without records' do
             let(:dump_content) { <<EOT
 #{DUMP_HEADER}
 
@@ -186,7 +186,7 @@ DROP TABLE IF EXISTS `users_login`;
 /*!40101 SET @saved_cs_client     = @@character_set_client */;
 /*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `users_login` (
-  `user_id` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
+  `#{col_name}` int(10) unsigned NOT NULL DEFAULT '0' COMMENT 'users id',
   `login_count` int(10) unsigned DEFAULT '0' COMMENT 'login count',
   `create_time` datetime NOT NULL COMMENT 'create time',
   `update_time` datetime NOT NULL COMMENT 'update time',
@@ -220,7 +220,7 @@ EOT
               }
               expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
                 expect(mysql_table.table_name).to eq('users_login')
-                expect(mysql_table.columns.keys).to eq(['user_id', 'login_count', 'create_time', 'update_time'])
+                expect(mysql_table.columns.keys).to eq([col_name, 'login_count', 'create_time', 'update_time'])
                 expect(last_pos).to eq(index_after(dump_content, 'ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT=\'\';'))
                 expect(binlog_pos).to eq(default_binlog_pos)
                 expect(state).to eq(MysqlDumpParser::State::INSERT_RECORD)
@@ -242,6 +242,29 @@ EOT
             end
           end
 
+          context 'when dump contains create table without records' do
+            context 'when one of the column name has space' do
+              let(:col_name) { "user id" }
+              include_examples 'dump contains create table without records'
+            end
+            context 'when the column name does not have space' do
+              let(:col_name) { "user_id" }
+              include_examples 'dump contains create table without records'
+            end
+            context 'when the column name is non-ascii' do
+              let(:col_name) { "行列" }
+              include_examples 'dump contains create table without records'
+            end
+            context 'when the column name includes special characters' do
+              let(:col_name) { %q| !"#$%&'()*+,-/:;<=>?@[\\]^_{\|}~| }
+              include_examples 'dump contains create table without records'
+            end
+            context 'when the column name includes a dot' do
+              let(:col_name) { %q|my.column| }
+              include_examples 'dump contains create table without records'
+            end
+          end
+
           context 'when dump contains create table with multi inserts and invalid utf8 byte sequences' do
             let(:dump_content) { <<EOT
 #{DUMP_HEADER}
@@ -303,7 +326,7 @@ EOT
                 {state: MysqlDumpParser::State::CREATE_TABLE},
                 {state: MysqlDumpParser::State::INSERT_RECORD},
                 {state: MysqlDumpParser::State::CREATE_TABLE,
-                 last_pos: index_after(dump_content, 'UNLOCK TABLES;') + 10}
+                 last_pos: index_after(dump_content, 'UNLOCK TABLES;')}
               ].each do |expected_params|
                 expect(check_point_block).to receive(:call) { |mysql_table, last_pos, bytesize, binlog_pos, state, substate|
                   expect(mysql_table.table_name).to eq('users_login') if mysql_table

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.23
+  version: 0.3.24
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-16 00:00:00.000000000 Z
+date: 2015-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -473,17 +473,21 @@ files:
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
 - flydata-core/lib/flydata-core/table_def.rb
+- flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
+- flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
 - flydata-core/spec/config/user_maintenance_spec.rb
 - flydata-core/spec/fluent/config_helper_spec.rb
 - flydata-core/spec/logger_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
+- flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
+- flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump
 - flydata-core/spec/table_def/mysqldump_test_column_charset.dump
 - flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
 - flydata-core/spec/table_def/mysqldump_test_table_all.dump
@@ -496,6 +500,7 @@ files:
 - flydata-core/spec/table_def/mysqldump_test_unique_key3.dump
 - flydata-core/spec/table_def/mysqldump_test_unsigned.dump
 - flydata-core/spec/table_def/redshift_table_def_spec.rb
+- flydata-core/spec/table_def/sync_redshift_table_def_spec.rb
 - flydata.gemspec
 - lib/fly_data_model.rb
 - lib/flydata.rb
@@ -621,7 +626,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent