flydata 0.6.11 → 0.6.12

data/spec/flydata/source_postgresql/generate_source_dump_spec.rb

@@ -0,0 +1,144 @@
+require 'flydata/source_postgresql/generate_source_dump'
+
+module Flydata
+module SourcePostgresql
+
+describe GenerateSourceDump do
+  let(:subject_object) { described_class.new(source, dp, options) }
+
+  let(:source) { Source.create(de) }
+  let(:de) { { "type" => "RedshiftPostgresqlDataEntry",
+               "postgresql_data_entry_preference" => de_prefs } }
+  let(:de_prefs) {
+    { "host" => "localhost", "port" => 5432,
+      "username" => "testuser", "password" => "testpass",
+      "database" => "testdb", "schema" => "testschema",
+    }
+  }
+  let(:dp) { double('dp') }
+  let(:options) { double('options') }
+
+  describe '#dump' do
+    subject { subject_object.dump(tables, file_path, &src_pos_callback) }
+
+    let(:tables) { %w( test1 test2 ) }
+    let(:src_pos_callback) { Proc.new{} }
+
+    context 'when file_path is given' do
+      let(:file_path) { double('file_path') }
+      before do
+        allow(File).to receive(:open).with(file_path, "w").
+          and_return(io)
+        allow(PGClient).to receive(:new).with(de_prefs).and_return(cli)
+        allow(subject_object).to receive(:get_source_pos).
+          with(cli, &src_pos_callback).and_return(source_pos)
+        allow(source).to receive(:sync_generate_table_ddl).with(dp, nil).
+          and_return context
+        allow(subject_object).to receive(:dump_table).
+          with(tabledef1, source_pos, io, cli)
+        allow(subject_object).to receive(:dump_table).
+          with(tabledef2, source_pos, io, cli)
+      end
+      let(:io) { double('io') }
+      before do
+        allow(io).to receive(:close)
+      end
+
+      let(:cli) { double('cli') }
+      let(:source_pos) { double('source_pos') }
+      let(:context) { double('context') }
+      let(:tabledef1) { double('tabledef1') }
+      let(:tabledef2) { double('tabledef2') }
+      let(:missing_tables) { [] }
+      let(:error) { double('error') }
+
+      it 'does expected things' do
+        expect(File).to receive(:open).with(file_path, "w").
+          and_return(io)
+        expect(PGClient).to receive(:new).with(de_prefs).and_return(cli)
+        expect(subject_object).to receive(:get_source_pos).
+          with(cli, &src_pos_callback).and_return(source_pos)
+        expect(source).to receive(:sync_generate_table_ddl).with(dp, nil).
+          and_return context
+        expect(context).to receive(:each_source_tabledef).
+          with(tables, de_prefs).
+          and_yield(tabledef1, nil).and_yield(tabledef2, nil).
+          and_return missing_tables
+        expect(subject_object).to receive(:dump_table).
+          with(tabledef1, source_pos, io, cli)
+        expect(subject_object).to receive(:dump_table).
+          with(tabledef2, source_pos, io, cli)
+
+        subject
+      end
+
+      context 'when a table is missing' do
+        let(:missing_tables) { [ "test2" ] }
+        before do
+          allow(context).to receive(:each_source_tabledef).
+            with(tables, de_prefs).
+            and_yield(tabledef1, nil).
+            and_return missing_tables
+        end
+        it "does nothing for the missing table.  we might need to add an error
+            handling" do
+          expect(subject_object).to receive(:dump_table).
+            with(tabledef1, source_pos, io, cli)
+          expect(subject_object).not_to receive(:dump_table).
+            with(tabledef2, source_pos, io, cli)
+
+          subject
+        end
+      end
+
+      context 'when a table has an error' do
+        before do
+          allow(context).to receive(:each_source_tabledef).
+            with(tables, de_prefs).
+            and_yield(nil, error).
+            and_yield(tabledef2, nil).
+            and_return missing_tables
+        end
+        it "does nothing for the error table.  we might need to add an error
+            handling" do
+          expect(subject_object).not_to receive(:dump_table).
+            with(tabledef1, source_pos, io, cli)
+          expect(subject_object).to receive(:dump_table).
+            with(tabledef2, source_pos, io, cli)
+
+          subject
+        end
+      end
+    end
+    context 'when file_path is nil' do
+      let(:file_path) { nil }
+      it 'raises error because the feature is not supported yet' do
+        expect{subject}.to raise_error /not been implemented yet/
+      end
+    end
+  end
+
+  describe '#pk_conditions' do
+    subject { subject_object.send(:pk_conditions, pk_columns) }
+
+    context 'with a single primary key' do
+      let(:pk_columns) { [ 'id' ] }
+
+      it { is_expected.to eq %Q| AND ("id" > $1)| }
+    end
+
+    context 'with two primary keys' do
+      let(:pk_columns) { [ 'pk1', 'pk2' ] }
+
+      it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2)))| }
+    end
+    context 'with three primary keys' do
+      let(:pk_columns) { [ 'pk1', 'pk2', 'pk3' ] }
+
+      it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2 OR ("pk2" = $2 AND ("pk3" > $3)))))| }
+    end
+  end
+end
+
+end
+end

data/spec/flydata/sync_file_manager_spec.rb

@@ -165,7 +165,7 @@ module Flydata
       end
       it do
         expect(subject.table_name).to eq(table_name)
-        expect(subject.columns).to eq(columns_hash)
+        expect(subject.instance_variable_get(:@columns)).to eq(columns_hash)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.11
+  version: 0.6.12
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2016-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -262,7 +262,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.11
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
@@ -272,7 +272,7 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.1.10
+        version: 0.1.11
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
@@ -544,6 +544,7 @@ files:
 - flydata-core/lib/flydata-core/mysql/config.rb
 - flydata-core/lib/flydata-core/mysql/ssl.rb
 - flydata-core/lib/flydata-core/option_validator.rb
+- flydata-core/lib/flydata-core/postgresql/source_pos.rb
 - flydata-core/lib/flydata-core/query_job.rb
 - flydata-core/lib/flydata-core/record/record.rb
 - flydata-core/lib/flydata-core/redshift/string.rb
@@ -571,10 +572,12 @@ files:
 - flydata-core/spec/mysql/compatibility_checker_spec.rb
 - flydata-core/spec/mysql/config_spec.rb
 - flydata-core/spec/option_validator_spec.rb
+- flydata-core/spec/postgresql/source_pos_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
 - flydata-core/spec/string_utils_spec.rb
 - flydata-core/spec/table_def/autoload_redshift_table_def_spec.rb
+- flydata-core/spec/table_def/base_spec.rb
 - flydata-core/spec/table_def/mysql_table_def_spec.rb
 - flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb
 - flydata-core/spec/table_def/mysqldump_test_bit_table.dump
@@ -640,18 +643,6 @@ files:
 - lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
-- lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
-- lib/flydata/fluent-plugins/mysql/binlog_position_file.rb
-- lib/flydata/fluent-plugins/mysql/binlog_query_dispatcher.rb
-- lib/flydata/fluent-plugins/mysql/binlog_query_handler.rb
-- lib/flydata/fluent-plugins/mysql/binlog_record_dispatcher.rb
-- lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb
-- lib/flydata/fluent-plugins/mysql/context.rb
-- lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb
-- lib/flydata/fluent-plugins/mysql/dml_record_handler.rb
-- lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb
-- lib/flydata/fluent-plugins/mysql/table_meta.rb
-- lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
 - lib/flydata/flydata_crontab.sh
 - lib/flydata/helper/action/agent_action.rb
@@ -679,6 +670,8 @@ files:
 - lib/flydata/parser.rb
 - lib/flydata/parser/parser_provider.rb
 - lib/flydata/parser/source_table.rb
+- lib/flydata/plugin_support/context.rb
+- lib/flydata/plugin_support/sync_record_emittable.rb
 - lib/flydata/preference/data_entry_preference.rb
 - lib/flydata/proxy.rb
 - lib/flydata/queueable_thread.rb
@@ -706,12 +699,30 @@ files:
 - lib/flydata/source_mysql/parser/.gitignore
 - lib/flydata/source_mysql/parser/dump_parser.rb
 - lib/flydata/source_mysql/parser/mysql_alter_table.treetop
+- lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/binlog_position_file.rb
+- lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb
+- lib/flydata/source_mysql/plugin_support/binlog_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb
+- lib/flydata/source_mysql/plugin_support/binlog_record_handler.rb
+- lib/flydata/source_mysql/plugin_support/context.rb
+- lib/flydata/source_mysql/plugin_support/ddl_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/dml_record_handler.rb
+- lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
+- lib/flydata/source_mysql/plugin_support/table_meta.rb
+- lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
 - lib/flydata/source_mysql/setup.rb
 - lib/flydata/source_mysql/source_pos.rb
 - lib/flydata/source_mysql/sync.rb
 - lib/flydata/source_mysql/sync_generate_table_ddl.rb
 - lib/flydata/source_mysql/table_ddl.rb
 - lib/flydata/source_postgresql/data_entry.rb
+- lib/flydata/source_postgresql/generate_source_dump.rb
+- lib/flydata/source_postgresql/parse_dump_and_send.rb
+- lib/flydata/source_postgresql/pg_client.rb
+- lib/flydata/source_postgresql/postgresql_component.rb
+- lib/flydata/source_postgresql/setup.rb
+- lib/flydata/source_postgresql/source_pos.rb
 - lib/flydata/source_postgresql/sync.rb
 - lib/flydata/source_postgresql/sync_generate_table_ddl.rb
 - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
@@ -747,14 +758,6 @@ files:
 - spec/flydata/compatibility_check_spec.rb
 - spec/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
-- spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
-- spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb
-- spec/flydata/fluent-plugins/mysql/ddl_query_handler_spec.rb
-- spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb
-- spec/flydata/fluent-plugins/mysql/drop_database_query_handler_spec.rb
-- spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb
-- spec/flydata/fluent-plugins/mysql/table_meta_spec.rb
-- spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb
 - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
 - spec/flydata/helper/action/check_remote_actions_spec.rb
 - spec/flydata/helper/action/restart_agent_spec.rb
@@ -769,12 +772,23 @@ files:
 - spec/flydata/heroku_spec.rb
 - spec/flydata/json/json_ext_spec.rb
 - spec/flydata/output/forwarder_spec.rb
+- spec/flydata/plugin_support/context_spec.rb
 - spec/flydata/source_mysql/generate_source_dump_spec.rb
 - spec/flydata/source_mysql/mysql_compatibility_check_spec.rb
 - spec/flydata/source_mysql/parser/alter_table_parser_spec.rb
 - spec/flydata/source_mysql/parser/dump_parser_spec.rb
+- spec/flydata/source_mysql/plugin_support/alter_table_query_handler_spec.rb
+- spec/flydata/source_mysql/plugin_support/binlog_query_dispatcher_spec.rb
+- spec/flydata/source_mysql/plugin_support/context_spec.rb
+- spec/flydata/source_mysql/plugin_support/ddl_query_handler_spec.rb
+- spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb
+- spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb
+- spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
+- spec/flydata/source_mysql/plugin_support/table_meta_spec.rb
+- spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
 - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
 - spec/flydata/source_mysql/table_ddl_spec.rb
+- spec/flydata/source_postgresql/generate_source_dump_spec.rb
 - spec/flydata/source_spec.rb
 - spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb
 - spec/flydata/sync_file_manager_spec.rb

data/lib/flydata/fluent-plugins/mysql/context.rb

@@ -1,25 +0,0 @@
-module Mysql
-  class Context
-    MANDATORY_OPTS = [
-      :database, :tables, :tag, :sync_fm, :omit_events, :table_meta, :table_revs,
-    ]
-    OPTIONAL_OPTS = [
-      :current_binlog_file
-    ]
-
-    (MANDATORY_OPTS + OPTIONAL_OPTS).each do |opt|
-      attr_accessor opt
-    end
-
-    def initialize(opts)
-      missing_opts = MANDATORY_OPTS - opts.keys
-      unless (missing_opts.empty?)
-        raise "Mandatory option(s) are missing: #{missing_opts.join(', ')}"
-      end
-
-      opts.each do |k, v|
-        self.instance_variable_set(:"@#{k}", v)
-      end
-    end
-  end
-end