flydata 0.2.8 → 0.2.9

data/spec/flydata/sync_file_manager_spec.rb

@@ -0,0 +1,159 @@
+# coding: utf-8
+require 'spec_helper'
+
+module Flydata
+  module FileUtil
+    describe SyncFileManager do
+      let(:default_mysqldump_dir) do
+        File.join('/tmp', "sync_dump_#{Time.now.to_i}")
+      end
+      let(:default_data_entry) do
+        {"id"=>93,
+        "name"=>"flydata_sync_mysql",
+        "data_port_id"=>52,
+        "display_name"=>"flydata_sync_mysql",
+        "enabled"=>true,
+        "heroku_log_type"=>nil,
+        "heroku_resource_id"=>nil,
+        "log_deletion"=>nil,
+        "log_file_delimiter"=>nil,
+        "log_file_type"=>nil,
+        "log_path"=>nil,
+        "redshift_schema_name"=>"",
+        "redshift_table_name"=>nil,
+        "created_at"=>"2014-01-22T18:58:43Z",
+        "updated_at"=>"2014-01-30T02:42:26Z",
+        "type"=>"RedshiftMysqlDataEntry",
+        "tag_name"=>"flydata.a458c641_dp52.flydata_mysql",
+        "tag_name_dev"=>"flydata.a458c641_dp52.flydata_mysql.dev",
+        "data_port_key"=>"a458c641",
+        "mysql_data_entry_preference" =>
+          { "host"=>"localhost", "port"=>3306, "username"=>"masashi",
+            "password"=>"welcome", "database"=>"sync_test", "tables"=>nil,
+            "mysqldump_dir"=>default_mysqldump_dir, "forwarder" => "tcpforwarder",
+            "data_servers"=>"localhost:9905" }
+         }
+      end
+      let(:default_sfm) { SyncFileManager.new(default_data_entry) }
+
+      let (:status) { 'PARSING' }
+      let (:table_name) { 'test_table' }
+      let (:last_pos) { 9999 }
+      let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111}  }
+      let (:state) { 'CREATE_TABLE' }
+      let (:substate) { nil }
+
+      after :each do
+        if Dir.exists?(default_mysqldump_dir)
+          Dir.delete(default_mysqldump_dir) rescue nil
+        end
+        if File.exists?(default_mysqldump_dir)
+          File.delete(default_mysqldump_dir) rescue nil
+        end
+      end
+
+      describe '#dump_file_path' do
+        context 'when mysqldump_dir param is nil' do
+          before do
+            default_data_entry['mysql_data_entry_preference'].delete('mysqldump_dir')
+          end
+          it do
+            expect(default_sfm.dump_file_path).to eq(
+              File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
+          end
+        end
+        context 'when file exists' do
+          before { `touch #{default_mysqldump_dir}`}
+          it do
+            expect{default_sfm.dump_file_path}.to raise_error
+          end
+        end
+        context 'when directory exists' do
+          before { `mkdir -p #{default_mysqldump_dir}`}
+          it do
+            expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
+            expect(default_sfm.dump_file_path).to eq(
+              File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
+          end
+        end
+        context 'when directory or file does not exist' do
+          it do
+            expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
+            expect(default_sfm.dump_file_path).to eq(
+              File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
+          end
+        end
+        context 'when file name includes "~"' do
+          let(:default_mysqldump_dir) { "~/tmp/dump/sync_spec_#{Time.now.to_i}" }
+          it do
+            expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
+            expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
+            expect(default_sfm.dump_file_path).to eq(
+              File.join(expected_dir, 'flydata_sync_mysql.dump'))
+          end
+        end
+      end
+
+      describe '#dump_pos_path' do
+        it { expect(default_sfm.dump_pos_path).to eq(
+          File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
+      end
+
+      describe '#save_dump_pos' do
+        context 'without mysql marshal data' do
+          it do
+            expect{default_sfm.save_dump_pos(
+              status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
+              expect(default_sfm.load_dump_pos).to eq({
+                status: status, table_name: table_name, last_pos: last_pos,
+                binlog_pos: binlog_pos, state: state, substate: substate,
+                mysql_table: nil
+              })
+          end
+        end
+      end
+
+      describe '#load_dump_pos' do
+        let (:mysql_table) do
+          Flydata::Parser::Mysql::MysqlTable.new(
+            table_name, { 'id' => { format_type: 'int' }, 'value' => { format_type: 'text' } }
+          )
+        end
+
+        context 'with mysql marshal data' do
+          before do
+            default_sfm.save_mysql_table_marshal_dump(mysql_table)
+            default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
+          end
+          it do
+            ret = default_sfm.load_dump_pos
+            mt = ret.delete(:mysql_table)
+            expect(ret).to eq({
+              status: status, table_name: table_name, last_pos: last_pos,
+              binlog_pos: binlog_pos, state: state, substate: substate,
+            })
+            expect(mt.table_name).to eq(table_name)
+            expect(mt.columns).to eq({
+              'id' => { format_type: 'int' },
+              'value' => { format_type: 'text' },
+            })
+          end
+        end
+      end
+
+      describe '#save_binlog' do
+        let(:binfile) { 'mysqlbinlog.000001' }
+        let(:pos) { 107 }
+        let(:binlog_pos) { {binfile: binfile, pos: pos} }
+        it do
+          default_sfm.save_binlog(binlog_pos)
+          expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
+        end
+      end
+
+      describe '#binlog_path' do
+        it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
+      end
+    end
+  end
+end

data/spec/flydata/table_def/mysql_table_def_spec.rb

@@ -31,7 +31,7 @@ describe MysqlTableDef do
         expect(subject[:columns]).to eq(
           [
             {:column=>"id", :type=>"int8(20)", :auto_increment=>true, :not_null=>true, :primary_key=>true},
-            {:column=>"col_binary", :type=>"binary(100)", :default=>nil},
+            {:column=>"col_binary", :type=>"binary(202)", :default=>nil},
             {:column=>"col_blob", :type=>"varbinary(65535)"},
             {:column=>"col_bool", :type=>"int1(1)", :default=>"0"},
             {:column=>"col_char", :type=>"varchar(18)", :default=>nil},
@@ -55,7 +55,7 @@ describe MysqlTableDef do
             {:column=>"col_tinyblob", :type=>"varbinary(255)"},
             {:column=>"col_tinyint", :type=>"int1(4)", :default=>nil},
             {:column=>"col_tinytext", :type=>"text"},
-            {:column=>"col_varbinary", :type=>"varbinary(255)", :default=>nil},
+            {:column=>"col_varbinary", :type=>"varbinary(512)", :default=>nil},
             {:column=>"col_varchar", :type=>"varchar(372)", :default=>nil}
           ]
         )

data/spec/flydata/table_def/redshift_table_def_spec.rb

@@ -140,74 +140,229 @@ EOT
   end
 
   describe '.column_def_sql' do
+    subject { described_class.column_def_sql(column, opt) }
     let(:column) { {} }
-    subject { described_class.column_def_sql(column) }
 
-    context 'with varchar column def' do
-      context 'when size is smaller than max size' do
-        let(:column) { {:column=>"col_char", :type=>"varchar(18)", :default=>nil} }
-        it do
-          expect(subject).to eq('  "col_char" varchar(18) DEFAULT NULL')
+    let(:no_default_sql) { "" }
+    shared_examples "default values" do |*examples|
+      context "default nil" do
+        before do
+          column[:default] = nil
         end
+        let(:default_sql) { " DEFAULT NULL" }
+        it_behaves_like *examples
       end
-
-      context 'when size exceeds varchar max size' do
-        let(:column) { {:column=>"col_char", :type=>"varchar(1000000000)", :default=>nil} }
-        it do
-          expect(subject).to eq('  "col_char" varchar(65535) DEFAULT NULL')
+      context "no default" do
+        let(:default_sql) { no_default_sql }
+        it_behaves_like *examples
+      end
+      context "defalut value" do
+        before do
+          column[:default] = default_value
         end
+        let(:default_sql) { " DEFAULT #{default_value_sql}" }
+        it_behaves_like *examples
       end
+    end
 
-      context 'when size is max size' do
-        let(:column) { {:column=>"col_char", :type=>"varchar(max)", :default=>nil} }
-        it do
-          expect(subject).to eq('  "col_char" varchar(max) DEFAULT NULL')
+    shared_examples "not null values" do |*examples|
+      context "without not null" do
+        let(:not_null_sql) { "" }
+        it_behaves_like *examples
+      end
+      context "with not null false" do
+        before do
+          column[:not_null] = false
         end
+        let(:not_null_sql) { "" }
+        it_behaves_like *examples
       end
-    end
-    
-    context 'with medium blob column def' do
-      context 'when size exceeds max size' do
-        let(:column) { {:column=>"col_medium_blob", :type=>"varbinary(16777215)", :default=>nil} }
-        it do
-          expect(subject).to eq('  "col_medium_blob" varchar(65535) DEFAULT NULL')
+      context "with not null true" do
+        before do
+          column[:not_null] = true
         end
-      end    
+        let(:not_null_sql) { " NOT NULL" }
+        it_behaves_like *examples
+      end
     end
-    
-    context 'with unsigned column def' do
-      let(:column) { {:column=>"value_small_int", :type=>"int2(5) unsigned", :default=>nil} }
+
+    shared_examples "test generated Redshift SQL for column" do
       it do
-        expect(subject).to eq('  "value_small_int" int4 DEFAULT NULL')
+         expect(subject).to eq(expected_query)
       end
     end
 
-    context 'with decimal column def' do
-      context 'when precision exceeds max allowed' do
-        let(:column) { {:column=>"value", :type=>"numeric(65,30)", :default=>nil} }
-        it 'should replace precision with max value' do
-          expect(subject).to eq('  "value" numeric(38,30) DEFAULT NULL')
+    shared_examples "test column types" do |*examples|
+      context 'with varchar column def' do
+        context 'when size is smaller than max size' do
+          let(:default_value) { "something" }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT ''" }
+          before do
+            column[:column] = "col_char"
+            column[:type] = "varchar(18)"
+          end
+          let(:type_sql) { %Q|"col_char" varchar(18)| }
+
+          it_behaves_like *examples
+        end
+
+        context 'when size exceeds varchar max size' do
+          let(:default_value) { "something" }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT ''" }
+          before do
+            column[:column] = "col_char"
+            column[:type] = "varchar(1000000000)"
+          end
+          let(:type_sql) { %Q|"col_char" varchar(65535)| }
+
+          it_behaves_like *examples
+        end
+
+        context 'when size is max size' do
+          let(:default_value) { "something" }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT ''" }
+          before do
+            column[:column] = "col_char"
+            column[:type] = "varchar(max)"
+          end
+          let(:type_sql) { %Q|"col_char" varchar(max)| }
+
+          it_behaves_like *examples
         end
       end
-      context 'when scale exceeds max allowed' do #not possible
-        let(:column) { {:column=>"value", :type=>"numeric(45,44)", :default=>nil} }
-        it 'should replace scale with max value' do
-          expect(subject).to eq('  "value" numeric(38,37) DEFAULT NULL')
+      
+      context 'with medium blob column def' do
+        context 'when size exceeds max size' do
+          let(:default_value) { "something" }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT ''" }
+          before do
+            column[:column] = "col_medium_blob"
+            column[:type] = "varbinary(16777215)"
+          end
+          let(:type_sql) { %Q|"col_medium_blob" varchar(65535)| }
+
+          it_behaves_like *examples
+        end    
+      end
+      
+      context 'with unsigned column def' do
+        let(:default_value) { 4 }
+        let(:default_value_sql) { "'#{default_value}'" }
+        let(:not_null_default_sql) { " DEFAULT '0'" }
+        before do
+          column[:column] = "value_small_int"
+          column[:type] = "int2(5) unsigned"
         end
+        let(:type_sql) { %Q|"value_small_int" int4| }
+
+        it_behaves_like *examples
       end
 
-      context 'when precision and scale exceeds max allowed (for unsigned)' do
-        let(:column) { {:column=>"value", :type=>"numeric(65,44) unsigned", :default=>nil} }
-        it 'should replace precision with max value' do
-          expect(subject).to eq('  "value" numeric(38,37) DEFAULT NULL')
+      context 'with datetime column def' do
+        let(:default_value) { '2014-10-17 22:22:22' }
+        let(:default_value_sql) { "'#{default_value}'" }
+        let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
+        before do
+          column[:column] = "value_datetime"
+          column[:type] = "datetime"
         end
+        let(:type_sql) { %Q|"value_datetime" timestamp| }
+
+        it_behaves_like *examples
       end
 
-      context 'when precision and scale are within limit (for unsigned)' do
-        let(:column) { {:column=>"value", :type=>"numeric(37,29) unsigned", :default=>nil} }
-        it 'should do nothing' do
-          expect(subject).to eq('  "value" numeric(37,29) DEFAULT NULL')
+      context 'with decimal column def' do
+        context 'when precision exceeds max allowed' do
+          let(:default_value) { 4 }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT '0'" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric(65,30)"
+          end
+          let(:type_sql) { %Q|"value" numeric(38,30)| }
+
+          it_behaves_like *examples
         end
+        context 'when scale exceeds max allowed' do #not possible
+          let(:default_value) { 4 }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT '0'" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric(45,44)"
+          end
+          let(:type_sql) { %Q|"value" numeric(38,37)| }
+
+          it_behaves_like *examples
+        end
+
+        context 'when precision and scale exceeds max allowed (for unsigned)' do
+          let(:default_value) { 4 }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT '0'" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric(65,44) unsigned"
+          end
+          let(:type_sql) {%Q|"value" numeric(38,37)| }
+
+          it_behaves_like *examples
+        end
+
+        context 'when precision and scale are within limit (for unsigned)' do
+          let(:default_value) { 4 }
+          let(:default_value_sql) { "'#{default_value}'" }
+          let(:not_null_default_sql) { " DEFAULT '0'" }
+          before do
+            column[:column] = "value"
+            column[:type] = "numeric(37,29) unsigned"
+          end
+          let(:type_sql) { %Q|"value" numeric(37,29)| }
+
+          it_behaves_like *examples
+        end
+      end
+    end
+
+    context 'for create table' do
+      let(:opt) { {} }
+      let(:expected_query) { %Q|  #{type_sql}#{not_null_sql}#{default_sql}| }
+      let(:no_default_sql) { "" }
+
+      it_behaves_like "test column types", "not null values", "default values", "test generated Redshift SQL for column"
+    end
+
+    context 'for alter table' do
+      let(:opt) { { for: :alter_table } }
+
+      context "without not null" do
+        let(:expected_query) { %Q|  #{type_sql}#{default_sql}| }
+        let(:no_default_sql) { "" }
+    
+        it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
+      end
+      context "with not null false" do
+        let(:expected_query) { %Q|  #{type_sql}#{default_sql}| }
+        before do
+          column[:not_null] = false
+        end
+        let(:no_default_sql) { "" }
+    
+        it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
+      end
+      context "with not null true" do
+        let(:expected_query) { %Q|  #{type_sql} NOT NULL#{default_sql}| }
+        before do
+          column[:not_null] = true
+        end
+        let(:no_default_sql) { not_null_default_sql }
+    
+        it_behaves_like "test column types", "default values", "test generated Redshift SQL for column"
       end
     end
   end