flydata 0.1.12 → 0.1.13

data/lib/flydata/table_def/redshift_table_def.rb

@@ -31,17 +31,19 @@ class RedshiftTableDef
   }
   def self.from_flydata_tabledef(flydata_tabledef, options = {})
     options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
+    schema_name = options[:schema_name]
 
     tabledef = ""
-    tabledef += create_flydata_ctl_table_sql if options[:flydata_ctl_table]
-    tabledef += create_table_sql(flydata_tabledef)
-    tabledef += comment_sql(flydata_tabledef)
-    tabledef += flydata_ctl_sql(flydata_tabledef)
+    tabledef += create_flydata_ctl_table_sql(schema_name) if options[:flydata_ctl_table]
+    tabledef += create_table_sql(flydata_tabledef, schema_name)
+    tabledef += comment_sql(flydata_tabledef, schema_name)
+    tabledef += flydata_ctl_sql(flydata_tabledef, schema_name)
   end
 
+  FLYDATA_CTL_COLUMNS_TABLE = "flydata_ctl_columns"
   CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
-DROP TABLE flydata_ctl_columns;
-CREATE TABLE flydata_ctl_columns (
+DROP TABLE %s;
+CREATE TABLE %s(
   id integer NOT NULL IDENTITY(1,1),
   table_name varchar(128) NOT NULL,
   column_name varchar(128) NOT NULL,
@@ -51,9 +53,19 @@ CREATE TABLE flydata_ctl_columns (
   PRIMARY KEY(id)
 ) DISTKEY(table_name) SORTKEY(table_name);
 EOS
-  def self.create_flydata_ctl_table_sql
+
+  def self.table_name_for_ddl(table_name, schema_name)
+    schema_name.to_s.empty? ? "\"#{table_name}\"" : "\"#{schema_name}\".\"#{table_name}\""
+  end
+
+  def self.flydata_ctl_table_for_ddl(schema_name)
+    table_name_for_ddl(FLYDATA_CTL_COLUMNS_TABLE, schema_name)
+  end
+
+  def self.create_flydata_ctl_table_sql(schema_name)
     # No drop table here intentionally because losing the data is fatal.
-    CREATE_FLYDATA_CTL_TABLE_SQL
+    tbl = flydata_ctl_table_for_ddl(schema_name)
+    CREATE_FLYDATA_CTL_TABLE_SQL % [tbl, tbl]
   end
 
   CREATE_TABLE_SQL = <<EOS
@@ -63,7 +75,7 @@ CREATE TABLE %s (
 );
 EOS
 
-  def self.create_table_sql(flydata_tabledef)
+  def self.create_table_sql(flydata_tabledef, schema_name)
     lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
     pk_def = primary_key_sql(flydata_tabledef)
     lines << pk_def if pk_def
@@ -71,7 +83,8 @@ EOS
     contents = lines.join(",\n")
 
     table_name = flydata_tabledef[:table_name]
-    CREATE_TABLE_SQL % [table_name, table_name, contents]
+    redshift_tbl = table_name_for_ddl(table_name, schema_name)
+    CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents]
   end
 
   def self.column_def_sql(column)
@@ -118,13 +131,13 @@ EOS
     pks.empty? ? nil : "  PRIMARY KEY (#{pks.join(',')})"
   end
 
-  def self.comment_sql(flydata_tabledef)
+  def self.comment_sql(flydata_tabledef, schema_name)
     sql = ""
     flydata_tabledef[:columns].each do |col|
       next unless col[:comment]
 
       sql += <<EOS
-COMMENT ON COLUMN #{flydata_tabledef[:table_name]}."#{col[:name]}"
+COMMENT ON COLUMN #{table_name_for_ddl(flydata_tabledef[:table_name], schema_name)}."#{col[:name]}"
   IS '#{col[:comment]}';
 EOS
     end
@@ -132,11 +145,12 @@ EOS
   end
 
   FLYDATA_CTL_COLUMNS_SQL = <<EOS
-DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
-INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type, ordinal_position) VALUES
+DELETE FROM %s WHERE table_name = '%s';
+INSERT INTO %s (table_name, column_name, src_data_type, ordinal_position) VALUES
 EOS
-  def self.flydata_ctl_sql(flydata_tabledef)
-    sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
+  def self.flydata_ctl_sql(flydata_tabledef, schema_name)
+    flydata_ctl_tbl = flydata_ctl_table_for_ddl(schema_name)
+    sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_ctl_tbl, flydata_tabledef[:table_name], flydata_ctl_tbl ]
     values = []
     flydata_tabledef[:columns].each.with_index(1) do |col, i|
       values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{escape(col[:type])}', #{i})"

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -280,16 +280,11 @@ EOT
           end
         end
 
-        context 'when received alter table event' do
-          it do
-            expect_no_emitted_record(alter_table_add_column_event)
-          end
-        end
-
-        #TODO: Uncomment the following test and delete the above one after supporting alter table
         context 'when received alter table add column event' do
           it do
-            skip "Now alter table add column is not supported"
+# TODO Replace the expectation when enabling alter table
+            expect_no_emitted_record(alter_table_add_column_event)
+=begin
             expect_emitted_records(alter_table_add_column_event, {
               type: :alter_table,
               table_name: "test_table",
@@ -300,12 +295,15 @@ EOT
               actions: [{
                 action: :add_column, column: "sum", :type=>'int4'}],
             })
+=end
           end
         end
 
         context 'when received alter table drop column event' do
           it do
-            skip "Now alter table drop column is not supported"
+# TODO Replace the expectation when enabling alter table
+            expect_no_emitted_record(alter_table_drop_column_event)
+=begin
             expect_emitted_records(alter_table_drop_column_event, {
               type: :alter_table,
               table_name: "test_table",
@@ -316,6 +314,7 @@ EOT
               actions: [{
                 action: :drop_column, column: "sum"}],
             })
+=end
           end
         end
 

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -234,6 +234,60 @@ describe 'MysqlAlterTableParser' do
       end
     end
 
+    context 'when table_name is wrapped with backquote' do
+      let(:query) { "alter table `test_table` drop value" }
+      it do
+        expect(subject).to eq(
+          type: :alter_table,
+          table_name: "test_table",
+          actions: [{
+            action: :drop_column,
+            column: "value"
+          }])
+      end
+    end
+
+    context 'when keywords are capitalized' do
+      let(:query) { "ALTER TABLE `test_table` DROP value" }
+      it do
+        expect(subject).to eq(
+          type: :alter_table,
+          table_name: "test_table",
+          actions: [{
+            action: :drop_column,
+            column: "value"
+          }])
+      end
+    end
+
+    context 'when table_name includes schema' do
+      let(:query) { "alter table test_schema.test_table drop value" }
+      it do
+        expect(subject).to eq(
+          type: :alter_table,
+          schema_name: "test_schema",
+          table_name: "test_table",
+          actions: [{
+            action: :drop_column,
+            column: "value"
+          }])
+      end
+    end
+
+    context 'when table_name includes schema with quotes' do
+      let(:query) { "alter table `test_schema`.`test_table` drop `value`" }
+      it do
+        expect(subject).to eq(
+          type: :alter_table,
+          schema_name: "test_schema",
+          table_name: "test_table",
+          actions: [{
+            action: :drop_column,
+            column: "value"
+          }])
+      end
+    end
+
     context 'when query has unsupported drop dextension' do
       subject { parser.parse(query) }
 

data/spec/flydata/table_def/redshift_table_def_spec.rb

@@ -6,6 +6,136 @@ module TableDef
 
 describe RedshiftTableDef do
 
+  describe '.from_flydata_tabledef' do
+    let(:flydata_tabledef) { {
+        table_name: "test_table",
+        columns: [
+          { name: "id", type: "int4(11)", not_null: true, primary_key: true },
+          { name: "value", type: "text" },
+        ],
+        default_charset: "utf8;"} }
+    let(:option) { { flydata_ctl_table: false } }
+
+    subject { described_class.from_flydata_tabledef(flydata_tabledef, option) }
+
+    context 'with simple flydatadef' do
+      it 'should return ddl' do
+        expect(subject).to eq( <<EOT.strip )
+DROP TABLE "test_table";
+CREATE TABLE "test_table" (
+  "id" int4 NOT NULL,
+  "value" varchar(max),
+  PRIMARY KEY (id)
+);
+DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('test_table', 'id', 'int4(11)', 1),
+('test_table', 'value', 'text', 2);
+EOT
+      end
+    end
+
+    context 'with flydata_ctl_columns option true' do
+      let(:option) { { flydata_ctl_columns: true } }
+
+      it 'should return ddl including flydata_ctl_columns creation' do
+        expect(subject).to eq( <<EOT.strip )
+DROP TABLE "flydata_ctl_columns";
+CREATE TABLE "flydata_ctl_columns"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  column_name varchar(128) NOT NULL,
+  src_data_type varchar(1024) NOT NULL,
+  revision int NOT NULL DEFAULT 1,
+  ordinal_position int NOT NULL,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+DROP TABLE "test_table";
+CREATE TABLE "test_table" (
+  "id" int4 NOT NULL,
+  "value" varchar(max),
+  PRIMARY KEY (id)
+);
+DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('test_table', 'id', 'int4(11)', 1),
+('test_table', 'value', 'text', 2);
+EOT
+      end
+    end
+
+    context 'with flydatadef with comment' do
+      let(:flydata_tabledef) { {
+        table_name: "test_table",
+        columns: [
+          { name: "id", type: "int4(11)", not_null: true, primary_key: true },
+          { name: "value", type: "text" },
+          { name: "cmnt", type: "text", comment: "helloworld" },
+        ],
+        default_charset: "utf8;"} }
+
+      it 'should add comment creation ddl' do
+        expect(subject).to eq( <<EOT.strip )
+DROP TABLE "test_table";
+CREATE TABLE "test_table" (
+  "id" int4 NOT NULL,
+  "value" varchar(max),
+  "cmnt" varchar(max),
+  PRIMARY KEY (id)
+);
+COMMENT ON COLUMN "test_table"."cmnt"
+  IS 'helloworld';
+DELETE FROM "flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO "flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('test_table', 'id', 'int4(11)', 1),
+('test_table', 'value', 'text', 2),
+('test_table', 'cmnt', 'text', 3);
+EOT
+      end
+    end
+
+    context 'with schema_name' do
+      let(:flydata_tabledef) { {
+        table_name: "test_table",
+        columns: [
+          { name: "id", type: "int4(11)", not_null: true, primary_key: true },
+          { name: "value", type: "text" },
+          { name: "cmnt", type: "text", comment: "helloworld" },
+        ],
+        default_charset: "utf8;"} }
+      let(:option) { { flydata_ctl_table: true, schema_name: 'test_schema' } }
+
+      it 'should preappend schema name to table name' do
+        expect(subject).to eq( <<EOT.strip )
+DROP TABLE "test_schema"."flydata_ctl_columns";
+CREATE TABLE "test_schema"."flydata_ctl_columns"(
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  column_name varchar(128) NOT NULL,
+  src_data_type varchar(1024) NOT NULL,
+  revision int NOT NULL DEFAULT 1,
+  ordinal_position int NOT NULL,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+DROP TABLE "test_schema"."test_table";
+CREATE TABLE "test_schema"."test_table" (
+  "id" int4 NOT NULL,
+  "value" varchar(max),
+  "cmnt" varchar(max),
+  PRIMARY KEY (id)
+);
+COMMENT ON COLUMN "test_schema"."test_table"."cmnt"
+  IS 'helloworld';
+DELETE FROM "test_schema"."flydata_ctl_columns" WHERE table_name = 'test_table';
+INSERT INTO "test_schema"."flydata_ctl_columns" (table_name, column_name, src_data_type, ordinal_position) VALUES
+('test_table', 'id', 'int4(11)', 1),
+('test_table', 'value', 'text', 2),
+('test_table', 'cmnt', 'text', 3);
+EOT
+      end
+    end
+  end
+
   describe '.column_def_sql' do
     let(:column) { {} }
     subject { described_class.column_def_sql(column) }