flydata 0.3.16 → 0.3.17

data/spec/flydata/parser/mysql/alter_table_parser_spec.rb

@@ -421,6 +421,24 @@ describe 'MysqlAlterTableParser' do
           })
         end
       end
+      context 'with charset option' do
+        let(:length) { 45 }
+        let(:query) { "alter table `test_schema`.`test_table` add column `extra` varchar(#{length}) character set 'sjis' collate 'sjis_japanese_ci'" }
+        it do
+          expect(subject).to eq(
+            type: :alter_table,
+            schema_name: "test_schema",
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "extra",
+              type: "varchar(#{length * 3})",
+              cs: "SHIFT_JIS",
+              query: "add column `extra` varchar(#{length}) character set 'sjis' collate 'sjis_japanese_ci'"
+            }])
+        end
+      end
+
     end
 
     context 'with alter table drop column syntax' do
@@ -830,10 +848,11 @@ describe 'MysqlAlterTableParser' do
       end
     end
     shared_examples "test charsetname" do |*examples|
-      ["default","binary","utf8","latin1"].each do |charsetname|
+      {"default"=>"DEFAULT","binary"=>"ASCII_8BIT","utf8"=>"UTF_8","latin1"=>"ISO_8859_1"}.each do |charsetname, flydata_charset|
         context "charsetname is #{charsetname}" do
           it_behaves_like *examples do
             let(:charsetname){"#{charsetname}"}
+            let(:flydata_charset){flydata_charset}
           end
         end
       end
@@ -1327,35 +1346,44 @@ describe 'MysqlAlterTableParser' do
         it_behaves_like "test simple identifier", "test order", "a parser parsing a breaking query"
       end
     end
+    let(:charset_action_hash) {
+      {
+        action: action,
+        query: alter_table_action,
+        cs: flydata_charset,
+      }
+    }
     context "convert to charset" do
       let(:action) { :convert_charset }
+      let(:action_hash) { [charset_action_hash] }
       context "test with collate" do
         let(:alter_table_action) { "convert to #{charset} #{charsetname} #{collate}" }
-        it_behaves_like "test charset", "test charsetname", "test collate", "a parser parsing a breaking query"
+        it_behaves_like "test charset", "test charsetname", "test collate", "a parser parsing a single action alter table query"
       end
       context "test without collate" do
         let(:alter_table_action) { "convert to #{charset} #{charsetname}" }
-        it_behaves_like "test charset", "test charsetname", "a parser parsing a breaking query"
+        it_behaves_like "test charset", "test charsetname", "a parser parsing a single action alter table query"
       end
     end
     context "table options" do
       context "default charset" do
+        let(:action) { :default_charset }
         context "with collate" do
-          let(:action) { [:default_charset, :default_collate] }
           let(:alter_table_action) { "#{default}#{charset}=#{charsetname} #{collate}" }
-          it_behaves_like "test optional default", "test charset",  "test charsetname", "test collate", "a parser parsing a breaking query"
-          it_behaves_like "test optional default",  "test charset", "test charsetname", "test collate with equals", "a parser parsing a breaking query"
+          let(:action_hash) { [charset_action_hash, {action: :default_collate, query: alter_table_action}] }
+          it_behaves_like "test optional default", "test charset", "test charsetname", "test collate", "a parser parsing a single action alter table query"
+          it_behaves_like "test optional default", "test charset", "test charsetname", "test collate with equals", "a parser parsing a single action alter table query"
         end
         context "without collate" do
-          let(:action) { :default_charset }
           let(:alter_table_action) { "#{default}#{charset} #{charsetname}" }
-          it_behaves_like "test optional default", "test charset",  "test charsetname", "a parser parsing a breaking query"
+          let(:action_hash) { [charset_action_hash] }
+          it_behaves_like "test optional default", "test charset",  "test charsetname", "a parser parsing a single action alter table query"
         end
         context "without charset" do
         #The doc at http://dev.mysql.com/doc/refman/5.6/en/alter-table.html does not indicate that
         #charset is optional. But this works (tested on mysql 5.5.38)
-          let(:alter_table_action) { "#{default}#{collate}" }
           let(:action) { :default_collate }
+          let(:alter_table_action) { "#{default}#{collate}" }
           it_behaves_like  "test optional default","test collate", "a parser parsing a breaking query"
           it_behaves_like  "test optional default","test collate with equals", "a parser parsing a breaking query"
         end

data/spec/flydata/sync_file_manager_spec.rb

@@ -5,6 +5,8 @@ require 'flydata/parser/mysql/dump_parser'
 
 module Flydata
   describe SyncFileManager do
+    let(:subject_object) { described_class.new(default_data_entry) }
+
     let(:default_mysqldump_dir) do
       File.join('/tmp', "sync_dump_#{Time.now.to_i}")
     end
@@ -35,10 +37,10 @@ module Flydata
           "data_servers"=>"localhost:9905" }
        }
     end
-    let(:default_sfm) { SyncFileManager.new(default_data_entry) }
 
     let (:status) { 'PARSING' }
     let (:table_name) { 'test_table' }
+    let (:table_name2) { 'test_table2' }
     let (:last_pos) { 9999 }
     let (:binlog_pos) { {binfile: 'mysqlbin.00001', pos: 111}  }
     let (:state) { 'CREATE_TABLE' }
@@ -60,28 +62,28 @@ module Flydata
         end
         it do
           stub_const('Flydata::FileUtil::SyncFileManager::DUMP_DIR', File.join(Flydata::FLYDATA_HOME, 'dump'))
-          expect(default_sfm.dump_file_path).to eq(
+          expect(subject_object.dump_file_path).to eq(
             File.join(Flydata::FLYDATA_HOME, 'dump', 'flydata_sync_mysql.dump'))
         end
       end
       context 'when file exists' do
         before { `touch #{default_mysqldump_dir}`}
         it do
-          expect{default_sfm.dump_file_path}.to raise_error
+          expect{subject_object.dump_file_path}.to raise_error
         end
       end
       context 'when directory exists' do
         before { `mkdir -p #{default_mysqldump_dir}`}
         it do
           expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).never
-          expect(default_sfm.dump_file_path).to eq(
+          expect(subject_object.dump_file_path).to eq(
             File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
         end
       end
       context 'when directory or file does not exist' do
         it do
           expect(FileUtils).to receive(:mkdir_p).with(default_mysqldump_dir).once
-          expect(default_sfm.dump_file_path).to eq(
+          expect(subject_object.dump_file_path).to eq(
             File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump'))
         end
       end
@@ -90,23 +92,23 @@ module Flydata
         it do
           expected_dir = File.join(ENV['HOME'], default_mysqldump_dir[1..-1])
           expect(FileUtils).to receive(:mkdir_p).with(expected_dir).once
-          expect(default_sfm.dump_file_path).to eq(
+          expect(subject_object.dump_file_path).to eq(
             File.join(expected_dir, 'flydata_sync_mysql.dump'))
         end
       end
     end
 
     describe '#dump_pos_path' do
-      it { expect(default_sfm.dump_pos_path).to eq(
+      it { expect(subject_object.dump_pos_path).to eq(
         File.join(default_mysqldump_dir, 'flydata_sync_mysql.dump.pos')) }
     end
 
     describe '#save_dump_pos' do
       context 'without mysql marshal data' do
         it do
-          expect{default_sfm.save_dump_pos(
+          expect{subject_object.save_dump_pos(
             status, table_name, last_pos, binlog_pos, state, substate)}.not_to raise_error
-            expect(default_sfm.load_dump_pos).to eq({
+            expect(subject_object.load_dump_pos).to eq({
               status: status, table_name: table_name, last_pos: last_pos,
               binlog_pos: binlog_pos, state: state, substate: substate,
               mysql_table: nil
@@ -124,11 +126,11 @@ module Flydata
 
       context 'with mysql marshal data' do
         before do
-          default_sfm.save_mysql_table_marshal_dump(mysql_table)
-          default_sfm.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
+          subject_object.save_mysql_table_marshal_dump(mysql_table)
+          subject_object.save_dump_pos(status, table_name, last_pos, binlog_pos, state, substate)
         end
         it do
-          ret = default_sfm.load_dump_pos
+          ret = subject_object.load_dump_pos
           mt = ret.delete(:mysql_table)
           expect(ret).to eq({
             status: status, table_name: table_name, last_pos: last_pos,
@@ -148,8 +150,8 @@ module Flydata
       let(:pos) { 107 }
       let(:binlog_pos) { {binfile: binfile, pos: pos} }
       it do
-        default_sfm.save_binlog(binlog_pos)
-        expect(`cat #{default_sfm.binlog_path}`).to eq("#{binfile}\t#{pos}")
+        subject_object.save_binlog(binlog_pos)
+        expect(`cat #{subject_object.binlog_path}`).to eq("#{binfile}\t#{pos}")
       end
     end
 
@@ -158,61 +160,61 @@ module Flydata
       let(:pos) { 107 }
       let(:binlog_pos) { {binfile: binfile, pos: pos} }
 
-      subject { default_sfm.load_binlog }
+      subject { subject_object.load_binlog }
 
       context 'when binlog pos does not exist' do
         before do
-          File.delete(default_sfm.binlog_path) if File.exist?(default_sfm.binlog_path)
+          File.delete(subject_object.binlog_path) if File.exist?(subject_object.binlog_path)
         end
         it { is_expected.to be_nil }
       end
 
       context 'when binlog pos exists' do
         before do
-          default_sfm.save_binlog(binlog_pos)
+          subject_object.save_binlog(binlog_pos)
         end
         it { is_expected.to eq(binlog_pos) }
       end
     end
 
     describe '#binlog_path' do
-      it { expect(default_sfm.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
+      it { expect(subject_object.binlog_path).to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.pos") }
     end
 
     describe '#sent_binlog_path' do
       context 'with no args' do
-        subject { default_sfm.sent_binlog_path }
+        subject { subject_object.sent_binlog_path }
         it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.binlog.sent.pos") }
       end
       context 'with invalid args' do
-        subject { default_sfm.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.pos') }
+        subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.pos') }
         it { expect{subject}.to raise_error(ArgumentError) }
       end
       context 'with valid args' do
-        subject { default_sfm.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
+        subject { subject_object.sent_binlog_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
         it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.binlog.sent.pos') }
       end
     end
 
     describe '#ssl_ca_path' do
       context 'with no args' do
-        subject { default_sfm.ssl_ca_path }
+        subject { subject_object.ssl_ca_path }
         it { is_expected.to eq("#{FLYDATA_HOME}/flydata_sync_mysql.ssl_ca.pem") }
       end
       context 'with invalid args' do
-        subject { default_sfm.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.pos') }
+        subject { subject_object.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.pos') }
         it { expect{subject}.to raise_error(ArgumentError) }
       end
       context 'with valid args' do
-        subject { default_sfm.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
+        subject { subject_object.ssl_ca_path('/home/ec2-user/.flydata/flydata_sync.binlog.pos') }
         it { is_expected.to eq('/home/ec2-user/.flydata/flydata_sync.ssl_ca.pem') }
       end
     end
 
     describe '#save_ssl_ca_path' do
-      let(:ssl_ca_path) { default_sfm.ssl_ca_path }
+      let(:ssl_ca_path) { subject_object.ssl_ca_path }
       let(:ssl_ca_content) { 'aaaaabbbbbccccc' }
-      subject { default_sfm.save_ssl_ca(ssl_ca_content) }
+      subject { subject_object.save_ssl_ca(ssl_ca_content) }
 
       after do
         File.delete(ssl_ca_path)
@@ -232,7 +234,7 @@ module Flydata
         table_pos_file = "#{table_pos_dir}/#{test_table}.pos"
         FileUtils.rm(table_pos_file) if File.exist?(table_pos_file)
       end
-      subject { default_sfm }
+      subject { subject_object }
       context 'when an exception happens in the block' do
         let(:first_pos) { 1 }
         let(:last_pos) { first_pos + 1 }
@@ -254,5 +256,78 @@ module Flydata
         end
       end
     end
+    describe '#save_generated_ddl' do
+      subject { subject_object.save_generated_ddl(table_names, contents) }
+
+      let(:contents) { '2' }
+      let(:io) { double('io') }
+
+      shared_examples 'writing to per-table .generated_ddl files in positions directory' do
+        it do
+          table_name_array.each do |tn|
+            expect(File).to receive(:open).with("#{FLYDATA_HOME}/positions/#{tn}.generated_ddl", 'w').and_yield(io)
+            expect(io).to receive(:write).with(contents)
+          end
+
+          subject
+        end
+      end
+
+      context 'single table name' do
+        let(:table_names) { table_name }
+        let(:table_name_array) { [ table_name ] }
+
+        it_behaves_like 'writing to per-table .generated_ddl files in positions directory'
+      end
+      context 'multiple table names' do
+        let(:table_names) { [ table_name, table_name2 ] }
+        let(:table_name_array) { table_names }
+
+        it_behaves_like 'writing to per-table .generated_ddl files in positions directory'
+      end
+    end
+    describe "#load_generated_ddl" do
+      subject { subject_object.load_generated_ddl(table_names) }
+
+      let(:contents) { { table_name => '2', table_name2 => '3' } }
+      let(:io) { double('io') }
+      let(:expected_result) { table_name_array.collect{|tn| contents[tn] } }
+
+      shared_examples 'reading from per-table .generated_ddl files in positions directory' do
+        it do
+          table_name_array.each do |tn|
+            expect(File).to receive(:open).with("#{FLYDATA_HOME}/positions/#{tn}.generated_ddl").and_yield(io)
+            expect(io).to receive(:read).and_return(contents[tn])
+          end
+
+          is_expected.to eq expected_result
+        end
+      end
+
+      context 'single table name' do
+        let(:table_names) { table_name }
+        let(:table_name_array) { [ table_name ] }
+
+        it_behaves_like 'reading from per-table .generated_ddl files in positions directory'
+      end
+      context 'multiple table names' do
+        let(:table_names) { [ table_name, table_name2 ] }
+        let(:table_name_array) { table_names }
+
+        context 'normal case' do
+          it_behaves_like 'reading from per-table .generated_ddl files in positions directory'
+        end
+        context 'first file is missing' do
+          let(:table_name_array) { [ table_name2 ] }
+          let(:expected_result) { [nil, contents[table_name2]] }
+
+          let(:file_path) { "#{FLYDATA_HOME}/positions/#{table_name}.generated_ddl" }
+          before do
+            expect(File).to receive(:open).with(file_path).and_raise(Errno::ENOENT.new(file_path))
+          end
+          it_behaves_like 'reading from per-table .generated_ddl files in positions directory'
+        end
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.3.16
+  version: 0.3.17
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -470,6 +470,7 @@ files:
 - flydata-core/lib/flydata-core/fluent-plugins/multi_buffer.rb
 - flydata-core/lib/flydata-core/fluent/config_helper.rb
 - flydata-core/lib/flydata-core/logger.rb
+- flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/table_def.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
@@ -481,6 +482,7 @@ files:
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
+- flydata-core/spec/table_def/mysqldump_test_column_charset.dump
 - flydata-core/spec/table_def/mysqldump_test_foreign_key.dump
 - flydata-core/spec/table_def/mysqldump_test_table_all.dump
 - flydata-core/spec/table_def/mysqldump_test_table_column_comment.dump
@@ -525,7 +527,6 @@ files:
 - lib/flydata/fluent-plugins/idle_event_detector.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
 - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
-- lib/flydata/fluent-plugins/mysql/binlog_position.rb
 - lib/flydata/fluent-plugins/mysql/binlog_position_file.rb
 - lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb
 - lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb
@@ -544,6 +545,9 @@ files:
 - lib/flydata/heroku/configuration_methods.rb
 - lib/flydata/heroku/instance_methods.rb
 - lib/flydata/log_monitor.rb
+- lib/flydata/mysql/binlog_position.rb
+- lib/flydata/mysql/mysql_util.rb
+- lib/flydata/mysql/table_ddl.rb
 - lib/flydata/output/forwarder.rb
 - lib/flydata/parser/mysql/dump_parser.rb
 - lib/flydata/parser/mysql/mysql_alter_table.treetop
@@ -552,7 +556,6 @@ files:
 - lib/flydata/proxy.rb
 - lib/flydata/sync_file_manager.rb
 - lib/flydata/util/encryptor.rb
-- lib/flydata/util/mysql_util.rb
 - spec/fluent_plugins_spec_helper.rb
 - spec/fly_data_model_spec.rb
 - spec/flydata/api/data_entry_spec.rb
@@ -578,19 +581,21 @@ files:
 - spec/flydata/fluent-plugins/idle_event_detector_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
 - spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
-- spec/flydata/fluent-plugins/mysql/binlog_position_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
 - spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb
+- spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
 - spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
 - spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
 - spec/flydata/heroku_spec.rb
+- spec/flydata/mysql/binlog_position_spec.rb
+- spec/flydata/mysql/mysql_util_spec.rb
+- spec/flydata/mysql/table_ddl_spec.rb
 - spec/flydata/output/forwarder_spec.rb
 - spec/flydata/parser/mysql/alter_table_parser_spec.rb
 - spec/flydata/parser/mysql/dump_parser_spec.rb
 - spec/flydata/sync_file_manager_spec.rb
 - spec/flydata/util/encryptor_spec.rb
-- spec/flydata/util/mysql_util_spec.rb
 - spec/flydata_spec.rb
 - spec/spec_helper.rb
 - tmpl/redshift_mysql_data_entry.conf.tmpl
@@ -614,7 +619,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: FlyData Agent