flydata 0.7.7 → 0.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47e53f1fbb8f9c3312cbac20679d9b389d41994a
-  data.tar.gz: d3ab901fdc3747ab54b82b2a087c3a0aa74fe44f
+  metadata.gz: 3d9438eb32d8d567ad63149916961bde54cdf52c
+  data.tar.gz: dd2f138fd9f823e1cbee1d2d091262a6a008f336
 SHA512:
-  metadata.gz: 9f92fd99a9411b656ef014e3e64214d50c5549c4eac74b1dbde279f65ccba452dde8698bc26d5cf1dd145b092dcf459b2f839f996a804fbd120d2f5e361de30d
-  data.tar.gz: 9f2181a206ffa141fa7bca9822dcd824ca05e45d65ffe030cc1a0c055c27fe8b7161855e835817813d214ef9ffa78beeeb6e29e6f8c9410bc73368a02017b421
+  metadata.gz: af5eb23ccfa7691ff8a28c4767d256f988cb72377073b994bcaa7d45f94e38223253151e6702a10b5184ed28c37e22feac3c8e1eebd6792680341cddb24457e4
+  data.tar.gz: 5385e752f2531d6bea7ad8c739c1b0a485e20e1e600872319902238e4cef0b96096288192ddc9463b53afc363617e0c23a02fe0814cb33b84d01f0d8c5db891c

data/VERSION

@@ -1 +1 @@
-0.7.7
+0.7.8

data/flydata-core/lib/flydata-core/table_def/base.rb

@@ -135,7 +135,7 @@ class Base
 
   def self.check_pk(table_name, columns, options)
     unless pk_exist?(columns) || options[:skip_primary_key_check]
-      raise TableDefError, {error: "no primary key defined", table: table_name}
+      raise TableDefError, {error: "No primary key defined", table: table_name}
     end
   end
 

data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb

@@ -94,6 +94,7 @@ class MysqlTableDef < Base
     default_charset = nil
     default_source_charset = nil
     comment = nil
+    tabledef_error = nil
 
     position = :before_create_table
 
@@ -106,6 +107,22 @@ class MysqlTableDef < Base
           table_name = $1
           table_def += line.chomp
           next
+        elsif line =~ /CREATE ALGORITHM/
+          # /*!50001 CREATE ALGORITHM=UNDEFINED */
+          # /*!50013 DEFINER=`admin`@`%` SQL SECURITY DEFINER */
+          # /*!50001 VIEW `sample_view` AS select distinct `sample`.`name` AS `name` from `sample` */;
+          if line =~ /VIEW `(.*?)`/
+            # VIEW definition is given in the same line
+            table_name = $1
+            tabledef_error = {error: "VIEW is not supported", table: table_name}
+            position = :tabledef_error
+            next
+          elsif line =~ /;/
+            next
+          else
+            position = :in_create_algorithm
+            next
+          end
         end
 
       when :in_create_table
@@ -123,7 +140,11 @@ class MysqlTableDef < Base
           parse_key(line, columns)
         #) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='test table';
         elsif stripped_line.start_with?(')')
-          if line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
+          if line =~ /ENGINE\s*=\s*(BLACKHOLE|MEMORY)\s/
+            tabledef_error = {error: "#{$1} STORATE ENGINE table is not supported", table: table_name}
+            position = :tabledef_error
+            next
+          elsif line =~ /DEFAULT CHARSET\s*=\s*([a-z0-9]+)/
             default_source_charset = $1
             default_charset = flydata_charset(default_source_charset)
           end
@@ -140,10 +161,28 @@ class MysqlTableDef < Base
           # "Unknown table definition. Skip. (#{line})"
         end
 
+      when :in_create_algorithm
+        #/*!50001 VIEW `sample_view` AS select distinct `sample`.`name` AS `name` from `sample` */;
+        if line =~ /VIEW `(.*?)`/
+          table_name = $1
+          tabledef_error = {error: "VIEW is not supported", table: table_name}
+          position = :tabledef_error
+          next
+        elsif line =~ /;/
+          position = :before_create_table
+          next
+        else
+          # Skip.
+        end
+
+      when :tabledef_error
+        raise TableDefError, tabledef_error
+
       when :after_create_table
         break
       end
     end
+
     position == :after_create_table ? [table_def, table_name, columns, column_def, unique_keys, default_charset, default_source_charset, comment] : nil
   end
 

data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb

@@ -63,7 +63,7 @@ class RedshiftTableDef
       tabledef += drop_backup_table_sql(flydata_tabledef, schema_name, options) unless options[:ctl_only]
     rescue => e
       # Catch errors from generating schema. Generally an unsupported data type
-      raise TableDefError, {error: "errors generating schema. Please contact us for further instructions", table: flydata_tabledef[:table_name]}
+      raise TableDefError, {error: "Errors generating schema. Please contact us for further instructions", table: flydata_tabledef[:table_name]}
     end
   end
 
@@ -197,8 +197,8 @@ EOS
 
     unless options[:skip_primary_key_check]
       unless pk_def
-        $log.error "no primary key defined in table: #{flydata_tabledef[:table_name]}"
-        raise "no primary key defined"
+        $log.error "No primary key defined in table: #{flydata_tabledef[:table_name]}"
+        raise "No primary key defined"
       end
     end
     lines << pk_def if pk_def

data/flydata-core/spec/table_def/mysql_table_def_spec.rb

@@ -1,10 +1,11 @@
 require 'spec_helper'
 require 'flydata-core/table_def'
+require 'logger'
+$log = Logger.new(STDOUT)
 
 module FlydataCore
 module TableDef
 
-
 describe MysqlTableDef do
 
   # file full path which exists in same directory.
@@ -110,13 +111,52 @@ describe MysqlTableDef do
       end
     end
 
-    context 'when table does not have primary key' do
+    context 'when table does not have either primary key or unique key' do
       let(:dump_file_io) { file_io('mysqldump_test_table_no_pk.dump') }
       it 'should raise an error' do
         expect{subject}.to raise_error(FlydataCore::TableDefError)
       end
     end
 
+    context 'when table does not have primary key but has unique key' do
+      let(:dump_file_io) { file_io('mysqldump_test_table_only_uk.dump') }
+      it 'should not raise an error' do
+        expect{subject}.not_to raise_error
+      end
+
+      it 'should create pk_override from uk' do
+        expect(subject[:pk_override]).to eq( ['id'] )
+      end
+    end
+
+    context 'when table which has a unique key using NULL column' do
+      let(:dump_file_io) { file_io('mysqldump_test_null_uk.dump') }
+      it 'should raise an error' do
+        expect{subject}.to raise_error(FlydataCore::TableDefError)
+      end
+    end
+
+    context 'when table is VIEW (unsupported)' do
+      let(:dump_file_io) { file_io('mysqldump_test_view1.dump') }
+      it 'should raise an error' do
+        expect{subject}.to raise_error(FlydataCore::TableDefError)
+      end
+    end
+
+    context 'when table is MEMORY ENGINE (unsupported)' do
+      let(:dump_file_io) { file_io('mysqldump_test_memory_engine.dump') }
+      it 'should raise an error' do
+        expect{subject}.to raise_error(FlydataCore::TableDefError)
+      end
+    end
+
+    context 'when table is BLACKHOLE ENGINE (unsupported)' do
+      let(:dump_file_io) { file_io('mysqldump_test_blackhole_engine.dump') }
+      it 'should raise an error' do
+        expect{subject}.to raise_error(FlydataCore::TableDefError)
+      end
+    end
+
     context 'when column has comment' do
       let(:dump_file_io) { file_io('mysqldump_test_table_column_comment.dump') }
       it 'comment should be set' do
@@ -143,7 +183,7 @@ describe MysqlTableDef do
       end
     end
 
-    context 'when table has multiple pk' do
+    context 'when table has multiple-column pk' do
       let(:dump_file_io) { file_io('mysqldump_test_table_multi_pk.dump') }
       it 'multi pk should be set' do
         expect(subject[:columns]).to eq(

data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
 require 'flydata-core/table_def'
 require 'timecop'
+require 'logger'
+$log = Logger.new(STDOUT)
 
 module FlydataCore
   module TableDef
@@ -748,6 +750,60 @@ EOT
         end
       end
 
+      context 'with mysqldump_test_table_only_uk.dump' do
+        let(:dump_file_name) { 'mysqldump_test_table_only_uk.dump' }
+
+        it 'should return ddl using unique key as primary key' do
+          expect(subject).to eq( <<EOT )
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  column_name varchar(128) NOT NULL,
+  src_data_type varchar(1024) NOT NULL,
+  revision int NOT NULL DEFAULT 1,
+  ordinal_position int NOT NULL,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+BEGIN;
+DROP TABLE IF EXISTS "only_uk_flydata20160125232857";
+CREATE TABLE IF NOT EXISTS "only_uk" (
+  "id" numeric(20,0),
+  "tag" varchar(129) DEFAULT NULL,
+  "comment" varchar(72),
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+ALTER TABLE "only_uk" RENAME TO "only_uk_flydata20160125232857";
+CREATE TABLE "only_uk" (
+  "id" numeric(20,0),
+  "tag" varchar(129) DEFAULT NULL,
+  "comment" varchar(72),
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+DELETE FROM "flydata_ctl_columns" WHERE table_name = 'only_uk';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('only_uk', 'id', 'int8(20) unsigned', 1),
+('only_uk', 'tag', 'varchar(129)', 2),
+('only_uk', 'comment', 'varchar(72)', 3);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'only_uk';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('only_uk', 'cs', 'ISO_8859_1'),
+('only_uk', 'revision', 1),
+('only_uk', 'src_ddl', 'CREATE TABLE `only_uk` (  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,  `tag` varchar(43) DEFAULT NULL,  `comment` varchar(24) NOT NULL,  UNIQUE KEY `id` (`id`)) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1;'),
+('only_uk', 'pk_override', 'id');
+COMMIT;
+DROP TABLE IF EXISTS "only_uk_flydata20160125232857";
+EOT
+        end
+      end
+
       context 'with mysqldump_test_unsigned.dump' do
         let(:dump_file_name) { 'mysqldump_test_unsigned.dump' }
 
@@ -818,6 +874,56 @@ INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
 ) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1;".split("\n").join}');
 COMMIT;
 DROP TABLE IF EXISTS "zerofill_table_flydata20160125232857";
+EOT
+        end
+      end
+
+      context 'with mysqldump_test_view_and_table.dump' do
+        let(:dump_file_name) { 'mysqldump_test_view_and_table.dump' }
+
+        it 'should return ddl including queries only for a table, and no queries for a view' do
+          expect(subject).to eq( <<EOT )
+CREATE TABLE IF NOT EXISTS "flydata_ctl_columns"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  column_name varchar(128) NOT NULL,
+  src_data_type varchar(1024) NOT NULL,
+  revision int NOT NULL DEFAULT 1,
+  ordinal_position int NOT NULL,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+CREATE TABLE IF NOT EXISTS "flydata_ctl_tables"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  attribute varchar(128) NOT NULL,
+  value varchar(max),
+  created_at timestamp DEFAULT SYSDATE,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+BEGIN;
+DROP TABLE IF EXISTS "test_table_flydata20160125232857";
+CREATE TABLE IF NOT EXISTS "test_table" (
+  "id" int4,
+  "tag" varchar(144) DEFAULT NULL,
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+ALTER TABLE "test_table" RENAME TO "test_table_flydata20160125232857";
+CREATE TABLE "test_table" (
+  "id" int4,
+  "tag" varchar(144) DEFAULT NULL,
+  PRIMARY KEY ("id")
+) DISTKEY("id") SORTKEY("id");
+DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('test_table', 'id', 'int4(11)', 1),
+('test_table', 'tag', 'varchar(144)', 2);
+DELETE FROM "flydata_ctl_tables" WHERE table_name = 'test_table';
+INSERT INTO "flydata_ctl_tables" (table_name, attribute, value) VALUES
+('test_table', 'cs', 'ISO_8859_1'),
+('test_table', 'revision', 1),
+('test_table', 'src_ddl', 'CREATE TABLE `test_table` (  `id` int(11) NOT NULL AUTO_INCREMENT,  `tag` varchar(48) DEFAULT NULL,  PRIMARY KEY (`id`)) ENGINE=InnoDB DEFAULT CHARSET=latin1;');
+COMMIT;
+DROP TABLE IF EXISTS "test_table_flydata20160125232857";
 EOT
         end
       end

data/flydata-core/spec/table_def/mysqldump_test_blackhole_engine.dump

@@ -0,0 +1,41 @@
+-- MySQL dump 10.13  Distrib 5.5.46, for debian-linux-gnu (x86_64)
+--
+-- Host: rdss-chie.c6dj7ech9e6t.us-east-1.rds.amazonaws.com    Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `blackhole_table`
+--
+
+DROP TABLE IF EXISTS `blackhole_table`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `blackhole_table` (
+  `id` int(11) NOT NULL,
+  `tag` char(10) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=BLACKHOLE DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01 18:30:41

data/flydata-core/spec/table_def/mysqldump_test_memory_engine.dump

@@ -0,0 +1,41 @@
+-- MySQL dump 10.13  Distrib 5.5.46, for debian-linux-gnu (x86_64)
+--
+-- Host: rdss-chie.c6dj7ech9e6t.us-east-1.rds.amazonaws.com    Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `memory_table`
+--
+
+DROP TABLE IF EXISTS `memory_table`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `memory_table` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tag` varchar(24) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=MEMORY DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01 18:31:14

data/flydata-core/spec/table_def/mysqldump_test_null_uk.dump

@@ -0,0 +1,56 @@
+-- MySQL dump 10.13  Distrib 5.6.25, for Linux (x86_64)
+--
+-- Host: localhost    Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `null_uk`
+--
+
+DROP TABLE IF EXISTS `null_uk`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `null_uk` (
+  `id` int(11) DEFAULT '0',
+  `field` int(11) DEFAULT '0',
+  `uid` int(11) NOT NULL DEFAULT '0',
+  `null_id` int(11) DEFAULT NULL,
+  UNIQUE KEY `null_id` (`null_id`),
+  UNIQUE KEY `id` (`id`),
+  UNIQUE KEY `multi_id` (`id`,`uid`),
+  KEY `id_2` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `null_uk`
+--
+
+LOCK TABLES `null_uk` WRITE;
+/*!40000 ALTER TABLE `null_uk` DISABLE KEYS */;
+INSERT INTO `null_uk` VALUES (NULL,0,0,NULL),(NULL,0,0,NULL),(NULL,0,0,NULL),(NULL,0,0,NULL);
+/*!40000 ALTER TABLE `null_uk` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01 23:56:08

data/flydata-core/spec/table_def/mysqldump_test_table_only_uk.dump

@@ -0,0 +1,52 @@
+-- MySQL dump 10.13  Distrib 5.6.25, for Linux (x86_64)
+--
+-- Host: localhost    Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `only_uk`
+--
+
+DROP TABLE IF EXISTS `only_uk`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `only_uk` (
+  `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `tag` varchar(43) DEFAULT NULL,
+  `comment` varchar(24) NOT NULL,
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `only_uk`
+--
+
+LOCK TABLES `only_uk` WRITE;
+/*!40000 ALTER TABLE `only_uk` DISABLE KEYS */;
+INSERT INTO `only_uk` VALUES (1,'test DDE',1),(2,'test DDE',2);
+/*!40000 ALTER TABLE `only_uk` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01 21:03:42

data/flydata-core/spec/table_def/mysqldump_test_view1.dump

@@ -0,0 +1,57 @@
+-- MySQL dump 10.13  Distrib 5.6.25, for Linux (x86_64)
+--
+-- Host: localhost   Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Temporary view structure for view `sample_view`
+--
+
+DROP TABLE IF EXISTS `sample_view`;
+/*!50001 DROP VIEW IF EXISTS `sample_view`*/;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+/*!50001 CREATE VIEW `sample_view` AS SELECT 
+ 1 AS `name`*/;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Final view structure for view `sample_view`
+--
+
+/*!50001 DROP VIEW IF EXISTS `sample_view`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`admin`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `sample_view` AS select distinct `sample`.`name` AS `name` from `sample` */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01  6:50:24

data/flydata-core/spec/table_def/mysqldump_test_view_and_table.dump

@@ -0,0 +1,80 @@
+-- MySQL dump 10.13  Distrib 5.6.25, for Linux (x86_64)
+--
+-- Host: localhost    Database: synctest
+-- ------------------------------------------------------
+-- Server version	5.6.23-log
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `test_table`
+--
+
+DROP TABLE IF EXISTS `test_table`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `test_table` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `tag` varchar(48) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `test_table`
+--
+
+LOCK TABLES `test_table` WRITE;
+/*!40000 ALTER TABLE `test_table` DISABLE KEYS */;
+/*!40000 ALTER TABLE `test_table` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Temporary view structure for view `sample_view`
+--
+
+DROP TABLE IF EXISTS `sample_view`;
+/*!50001 DROP VIEW IF EXISTS `sample_view`*/;
+SET @saved_cs_client     = @@character_set_client;
+SET character_set_client = utf8;
+/*!50001 CREATE VIEW `sample_view` AS SELECT 
+ 1 AS `name`*/;
+SET character_set_client = @saved_cs_client;
+
+--
+-- Final view structure for view `sample_view`
+--
+
+/*!50001 DROP VIEW IF EXISTS `sample_view`*/;
+/*!50001 SET @saved_cs_client          = @@character_set_client */;
+/*!50001 SET @saved_cs_results         = @@character_set_results */;
+/*!50001 SET @saved_col_connection     = @@collation_connection */;
+/*!50001 SET character_set_client      = utf8 */;
+/*!50001 SET character_set_results     = utf8 */;
+/*!50001 SET collation_connection      = utf8_general_ci */;
+/*!50001 CREATE ALGORITHM=UNDEFINED */
+/*!50013 DEFINER=`admin`@`%` SQL SECURITY DEFINER */
+/*!50001 VIEW `sample_view` AS select distinct `sample`.`name` AS `name` from `sample` */;
+/*!50001 SET character_set_client      = @saved_cs_client */;
+/*!50001 SET character_set_results     = @saved_cs_results */;
+/*!50001 SET collation_connection      = @saved_col_connection */;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2016-07-01 22:47:38

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -48,7 +48,7 @@ class SyncGenerateTableDdl < Component
     end
 
     if missing_tables
-      missing_tables.each {|missing_table| error_list << { error: "table does not exist in the #{data_source_type_display_name}", table: missing_table } }
+      missing_tables.each {|missing_table| error_list << { error: "Table does not exist in the #{data_source_type_display_name}", table: missing_table } }
     end
 
     [flydata_tabledefs, error_list, uk_as_pk_override]

data/lib/flydata/source_mysql/mysql_compatibility_check.rb

@@ -78,13 +78,6 @@ module SourceMysql
       end
     end
 
-    # If table_type='VIEW' or engine='MEMORY', raise error.
-    def check_mysql_table_types
-      return if @tables.empty?
-      option = @db_opts.dup.merge(tables: @tables)
-      FlydataCore::Mysql::TableTypeChecker.new(option).do_check
-    end
-
     def run_mysql_retention_check
       FlydataCore::Mysql::NonRdsRetentionChecker.new(@db_opts).do_check
     end

data/spec/flydata/command/sync_spec.rb

@@ -202,9 +202,9 @@ module Flydata
         let(:invalid_tables_str) { 'error_fullsync_1,error_append_2' }
         let(:tbl_attrs_for_invalid_tables) {[
           {"table_name"=>"error_fullsync_1", "status"=>"init_sync_pending",
-           "invalid_table_reason"=>"no primary key defined"},
+           "invalid_table_reason"=>"No primary key defined"},
           {"table_name"=>"error_append_2", "omit_events"=>["delete"], "status"=>"init_sync_pending",
-           "invalid_table_reason"=>"table does not exist in the MySQL database"},
+           "invalid_table_reason"=>"Table does not exist in the MySQL database"},
         ]}
 
         let(:pk_override_hash) { {"Users"=>["id"]} }

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -17,7 +17,7 @@ module Fluent
   tables test_table,test_table_1,test_table_2
   tables_append_only test_table_3,error_table_4
   pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
-  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
+  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"Table does not exist in the MySQL database"}]
   position_file #{TEST_POSITION_FILE}
   host localhost
   port 3306
@@ -244,7 +244,7 @@ EOT
         {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
         {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
         {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
-        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
+        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"Table does not exist in the MySQL database"}]
       }
       before do
         Test.configure_plugin(plugin, config)

data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb

@@ -16,7 +16,7 @@ module Fluent
   tables test_table,test_table_1,test_table_2
   tables_append_only test_table_3,error_table_4
   pk_override {"test_table_1":["id1","id2"],"test_table_3":["uid"]}
-  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"table does not exist in the MySQL database"}]
+  table_attributes [{"table_name":"test_table","status":"init_sync_completed"},{"table_name":"test_table_1","status":"init_sync_completed","pk_override":["id1","id2"]},{"table_name":"test_table_2","status":"init_sync_pending"},{"table_name":"test_table_3","omit_events":["delete"],"status":"init_sync_pending","uk_as_pk":["uid"]},{"table_name":"error_table_4","omit_events":["delete"],"status":"init_sync_pending","invalid_table_reason":"Table does not exist in the MySQL database"}]
   position_file #{TEST_POSITION_FILE}
   host localhost
   port 5433
@@ -39,7 +39,7 @@ EOT
         {"table_name"=>"test_table_1", "status"=>"init_sync_completed", "pk_override"=>["id1", "id2"]},
         {"table_name"=>"test_table_2", "status"=>"init_sync_pending"},
         {"table_name"=>"test_table_3", "omit_events"=>["delete"], "status"=>"init_sync_pending", "uk_as_pk"=>["uid"]},
-        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"table does not exist in the MySQL database"}]
+        {"table_name"=>"error_table_4", "omit_events"=>["delete"], "status"=>"init_sync_pending", "invalid_table_reason"=>"Table does not exist in the MySQL database"}]
       }
       before do
         create_file(TEST_POSITION_FILE, "1001:1001:\t\t")

data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb

@@ -41,7 +41,7 @@ module SourceMysql
       it "calls all check methods" do
         %w(compatibility56_variable mysql_user_compat mysql_protocol_tcp_compat
            mysql_parameters_compat rds_master_status mysql_binlog_retention
-           mysql_table_types writing_permissions).each do |method_name|
+           writing_permissions).each do |method_name|
           expect(subject_object).to receive("check_#{method_name}".to_sym)
         end
 
@@ -194,53 +194,6 @@ module SourceMysql
       end
     end
 
-    describe "#check_mysql_table_types" do
-      subject { subject_object.check_mysql_table_types }
-
-      let(:de_hash) do
-        { "host" => "test",
-          "port" => 1234,
-          "username" => "test",
-          "password" => "password",
-          "database" => "test_db",
-          "tables"=>["normal_table", "engine_table", "view_table"] }
-      end
-      let(:normal_table) { {"table_name"=>"normal_table", "table_type"=>"BASE TABLE", "engine"=>"InnoDB"} }
-      let(:engine_table) { {"table_name"=>"engine_table", "table_type"=>"BASE TABLE", "engine"=>"MEMORY"} }
-      let(:blackhole_table) { {"table_name"=>"blackhole_table", "table_type"=>"BASE TABLE", "engine"=>"BLACKHOLE"} }
-      let(:view) { {"table_name"=>"view_table", "table_type"=>"VIEW", "engine"=>nil} }
-      let(:error) { FlydataCore::MysqlCompatibilityError }
-      let(:base_error_msg) { "FlyData does not support VIEW and MEMORY,BLACKHOLE STORAGE ENGINE table. Remove following tables from data entry: %s" }
-      before do
-        allow(client).to receive(:query).and_return(table_list)
-        allow(client).to receive(:escape).and_return("aaa")
-      end
-      context "where data entry has VIEW and MEMORY engine table" do
-        let(:error_msg) { base_error_msg %  engine_table['table_name'] + ', ' + view['table_name'] }
-        let(:table_list) { [ engine_table, view ] }
-        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
-      end
-      context "where data entry has MEMORY engine table" do
-        let(:error_msg) { base_error_msg %  engine_table['table_name'] }
-        let(:table_list) { [ engine_table ] }
-        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
-      end
-      context "where data entry has BLACKHOLE engine table" do
-        let(:error_msg) { base_error_msg %  blackhole_table['table_name'] }
-        let(:table_list) { [ blackhole_table ] }
-        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
-      end
-      context "where data entry has the VIEW" do
-        let(:error_msg) { base_error_msg % view['table_name'] }
-        let(:table_list) { [ view ] }
-        it { expect{subject}.to raise_error(error, /#{error_msg}/) }
-      end
-      context "where data entry does not have either VIEW and ENGINE table" do
-        let(:table_list) { [ normal_table ] }
-        it { expect{subject}.to_not raise_error }
-      end
-    end
-
     describe "#check_rds_master_status" do
       subject { subject_object.check_rds_master_status }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.7.7
+  version: 0.7.8
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-30 00:00:00.000000000 Z
+date: 2016-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -590,6 +590,7 @@ files:
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
+- flydata-core/spec/table_def/mysqldump_test_blackhole_engine.dump
 - flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump
 - flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump
 - flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump
@@ -597,16 +598,21 @@ files:
 - flydata-core/spec/table_def/mysqldump_test_col_name_with_space.dump
 - flydata-core/spec/table_def/mysqldump_test_column_charset.dump
 - flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
+- flydata-core/spec/table_def/mysqldump_test_memory_engine.dump
+- flydata-core/spec/table_def/mysqldump_test_null_uk.dump
 - flydata-core/spec/table_def/mysqldump_test_table_all.dump
 - flydata-core/spec/table_def/mysqldump_test_table_column_comment.dump
 - flydata-core/spec/table_def/mysqldump_test_table_column_comment_including_invalid_char.dump
 - flydata-core/spec/table_def/mysqldump_test_table_enum.dump
 - flydata-core/spec/table_def/mysqldump_test_table_multi_pk.dump
 - flydata-core/spec/table_def/mysqldump_test_table_no_pk.dump
+- flydata-core/spec/table_def/mysqldump_test_table_only_uk.dump
 - flydata-core/spec/table_def/mysqldump_test_unique_key.dump
 - flydata-core/spec/table_def/mysqldump_test_unique_key2.dump
 - flydata-core/spec/table_def/mysqldump_test_unique_key3.dump
 - flydata-core/spec/table_def/mysqldump_test_unsigned.dump
+- flydata-core/spec/table_def/mysqldump_test_view1.dump
+- flydata-core/spec/table_def/mysqldump_test_view_and_table.dump
 - flydata-core/spec/table_def/postgresql_table_def_spec.rb
 - flydata-core/spec/table_def/redshift_table_def_spec.rb
 - flydata-core/spec/table_def/sync_redshift_table_def_spec.rb