flydata 0.6.14 → 0.7.0

data/spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb

@@ -0,0 +1,213 @@
+require 'spec_helper'
+require 'flydata/source_postgresql/query_based_sync/diff_query_generator'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+describe DiffQueryGenerator do
+  let(:table) { 'test_table' }
+  let(:schema) { 'public' }
+  let(:from_sid) { '0001:0001:' }
+  let(:to_sid) { '0002:0002:' }
+  let(:columns) { [
+    { column: 'id', type: 'int4' },
+    { column: 'value', type: value_data_type},
+  ] }
+  let(:value_data_type) { 'varchar(48)' }
+  let(:last_pks) { ['1000'] }
+  let(:pk_columns) { ['id'] }
+  let(:limit) { 100 }
+  let(:query_cond) { {
+    from_sid: from_sid,
+    to_sid: to_sid,
+    columns: columns,
+    pk_columns: pk_columns,
+    last_pks: last_pks,
+    limit: limit,
+  } }
+  let(:subject_object) { described_class.new(table, schema, query_cond) }
+
+  describe '#initialize' do
+    subject { subject_object }
+    context 'when no required option is missing' do
+      it { expect{subject}.not_to raise_error }
+    end
+
+    context 'when to_sid is missing' do
+      before { query_cond.delete(:to_sid) }
+      it { expect{subject}.to raise_error(ArgumentError, /to_sid is required/) }
+    end
+
+    context 'when pk_columns is missing' do
+      before { query_cond.delete(:pk_columns) }
+      it { expect{subject}.to raise_error(ArgumentError, /pk_columns is required/) }
+    end
+
+    context 'when columns is missing' do
+      before { query_cond.delete(:columns) }
+      it { expect{subject}.to raise_error(ArgumentError, /columns is required/) }
+    end
+  end
+
+  describe '#build_query' do
+    let(:binding_param_enabled) { true }
+    subject { subject_object.build_query(binding_param_enabled) }
+
+    context 'when all options are given' do
+      it { is_expected.to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+      }
+    end
+
+    context 'when schema is not given' do
+      let(:schema) { nil }
+      it { is_expected.to eq(<<EOS)
+SELECT "id", "value" FROM "test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+      }
+    end
+
+    context 'when columns includes a money type' do
+      let(:value_data_type) { 'money(19,2)' }
+      it 'returns money values as numeric' do
+        is_expected.to eq(<<EOS)
+SELECT "id", "value"::numeric FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+      end
+    end
+
+    context 'when columns includes a bit type' do
+      let(:value_data_type) { 'bit(4)' }
+      it 'returns a query with an overrider proc for bit' do
+        result = subject
+        expect(result).to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+       expect(result.value_overriders).to eq({"value" => DiffQueryGenerator::VALUE_OVERRIDERS['bit']})
+      end
+    end
+
+    context 'when columns includes a varbit type' do
+      let(:value_data_type) { 'varbit' }
+      it 'returns a query with an overrider proc for varbit' do
+        result = subject
+        expect(result).to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+       expect(result.value_overriders).to eq({"value" => DiffQueryGenerator::VALUE_OVERRIDERS['varbit']})
+      end
+    end
+
+    context 'when from_sid is not given' do
+      let(:from_sid) { nil }
+      it { is_expected.to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+      }
+    end
+
+    context 'when binding_param_enabled is false' do
+      let(:binding_param_enabled) { false }
+      it { is_expected.to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > '1000') ORDER BY "id" LIMIT 100;
+EOS
+      }
+      it { expect(subject.binding_params).to be_nil }
+    end
+
+    context 'when binding_param_enabled is true ' do
+      let(:binding_param_enabled) { true }
+      it { is_expected.to eq(<<EOS)
+SELECT "id", "value" FROM "public"."test_table" WHERE txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0002:0002:') AND NOT txid_visible_in_snapshot(xmin::TEXT::BIGINT, '0001:0001:')  AND ("id" > $1) ORDER BY "id" LIMIT 100;
+EOS
+      }
+      it { expect(subject.binding_params).to eq(last_pks)}
+    end
+  end
+
+  describe 'VALUE_OVERRIDERS' do
+    subject { DiffQueryGenerator::VALUE_OVERRIDERS[type].call(value) }
+
+    context 'when type is bit' do
+      let(:type) { 'bit' }
+      let(:value) { '0110000010' }
+      it { is_expected.to eq '386' }
+    end
+
+    context 'when type is varbit' do
+      let(:type) { 'varbit' }
+      let(:value) { '1111111111111111111111111111111111111111111111111111111111111101' }
+
+      it { is_expected.to eq "18446744073709551613" }
+    end
+
+    context 'when value is nil' do
+      let(:type) { 'bit' }
+      let(:value) { nil }
+      it { is_expected.to eq nil }
+    end
+  end
+
+  describe '#pk_conditions' do
+    let(:binding_param_enabled) { true }
+    subject { subject_object.send(:pk_conditions, pk_columns, last_pks, binding_param_enabled) }
+
+    context 'with a single primary key' do
+      let(:pk_columns) { [ 'id' ] }
+
+      it { is_expected.to eq %Q| AND ("id" > $1)| }
+    end
+
+    context 'with two primary keys' do
+      let(:pk_columns) { [ 'pk1', 'pk2' ] }
+
+      it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2)))| }
+    end
+    context 'with three primary keys' do
+      let(:pk_columns) { [ 'pk1', 'pk2', 'pk3' ] }
+
+      it { is_expected.to eq %Q| AND ("pk1" > $1 OR ("pk1" = $1 AND ("pk2" > $2 OR ("pk2" = $2 AND ("pk3" > $3)))))| }
+    end
+
+    context 'with binding_param_enabled option' do
+      let(:pk_columns) { [ 'pk1', 'pk2', 'pk3' ] }
+      let(:last_pks) { [ 1000, 2000, 'apple' ] }
+      let(:binding_param_enabled) { false }
+      it { is_expected.to eq %Q| AND ("pk1" > '1000' OR ("pk1" = '1000' AND ("pk2" > '2000' OR ("pk2" = '2000' AND ("pk3" > 'apple')))))| }
+    end
+  end
+
+  describe '#column_list' do
+    subject { subject_object.send(:column_list, columns, types) }
+
+    let(:columns) { [ 'id', 'value' ] }
+    let(:types) { [ 'int4', value_data_type ] }
+    let(:value_data_type) { 'varchar' }
+
+    context 'an ordinary case' do
+      it 'builds a column list string off of the given columns' do
+        is_expected.to eq %Q["id", "value"]
+      end
+    end
+
+    context 'with a binary column' do
+      let(:value_data_type) { 'bytea' } # FlyData data type not PostgreSQL
+      it 'generates a list which outputs binary values in the FlyData binary format' do
+        is_expected.to eq %Q["id", '0x' || encode("value", 'hex') AS "value"]
+      end
+    end
+
+    context 'with a money column' do
+      let(:value_data_type) { 'money' }
+      it 'generates a list which outputs money values as numeric' do
+        is_expected.to eq %Q["id", "value"::numeric]
+      end
+    end
+  end
+end
+
+end
+end
+end

data/spec/flydata/source_postgresql/query_based_sync/query_based_sync_postgresql_context.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+require 'flydata-core/postgresql/snapshot'
+require 'flydata/source_postgresql/plugin_support/context'
+require 'flydata/source_postgresql/plugin_support/source_position_file'
+require 'flydata/query_based_sync/query_based_sync_context'
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+shared_context 'query based sync postgresql context' do
+  include_context 'query based sync context'
+
+  def create_src_pos_file(fp, snapshot, to_snapshot=nil, pk_values=nil)
+    spf = Flydata::SourcePostgresql::PluginSupport::SourcePositionFile.new(fp)
+    spf.save(snapshot, to_snapshot, pk_values)
+    spf
+  end
+
+  def create_ss(snapshot_txt)
+    FlydataCore::Postgresql::Snapshot.new(snapshot_txt)
+  end
+
+  let(:previous_snapshot) { create_ss('1010:1010:') }
+  let(:current_snapshot) { create_ss('1020:1020:') }
+  let(:cur_src_pos_file_path) { "test_position.binlog.pos" }
+  let(:cur_src_pos_file) { create_src_pos_file(cur_src_pos_file_path, previous_snapshot) }
+
+  let(:table_1_src_pos_file_path) { File.join(FLYDATA_TABLE_POSITIONS_DIR, 'test_1.binlog.pos') }
+  let(:table_1_snapshot) { create_ss('1010:1010:') }
+  let(:table_1_to_snapshot) { nil }
+  let(:table_1_pk_values) { nil }
+  let(:table_1_src_pos_file) {
+    create_src_pos_file(table_1_src_pos_file_path,
+      table_1_snapshot, table_1_to_snapshot, table_1_pk_values) }
+
+  let(:context) {
+    Flydata::SourcePostgresql::PluginSupport::Context.new(
+      cur_src_pos_file: cur_src_pos_file,
+      tables: %w(table_1 table_2 table_3),
+      tag: 'test_tag',
+      sync_fm: sync_fm,
+
+      omit_events: { 'table_3' => [:delete, :truncate] },
+      table_revs: { 'table_1' => 1, 'table_2' => 1, 'table_3' => 1 },
+      table_meta: table_meta,
+      table_src_pos_files: {
+        table_1: table_1_src_pos_file,
+        table_2: table_2_src_pos_file,
+        table_3: table_3_src_pos_file,
+      },
+      params: {
+        fetch_interval: 10,
+        retry_interval: 5,
+      },
+      dbconf: {
+        host: 'test-host',
+        port: 5555,
+        username: 'test-user',
+        password: 'test-password',
+        database: 'test-db',
+        schema: 'test-schema',
+      },
+      database: 'test-db',
+    )
+  }
+
+  # Override
+  def setup_dummy_objects
+  end
+
+end
+
+end
+end
+end

data/spec/flydata/source_postgresql/query_based_sync/resource_requester_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+require 'flydata/source_postgresql/query_based_sync/query_based_sync_postgresql_context'
+require 'flydata/source_postgresql/query_based_sync/resource_requester'
+
+
+module Flydata
+module SourcePostgresql
+module QueryBasedSync
+
+describe ResourceRequester do
+  include_context 'query based sync postgresql context'
+
+  let(:subject_object) { described_class.new(context) }
+
+  describe '#create_resource_client' do
+    subject { subject_object.create_resource_client(context) }
+    it { is_expected.to be_kind_of(FlydataCore::Postgresql::PGClient) }
+  end
+
+  describe '#fetch_responses_once' do
+    let(:table_name) { 'table_1' }
+    let(:result) { double('result') }
+    let(:response) { double('response') }
+    let(:query_cond) { {} }
+    let(:latest_src_pos) { FlydataCore::Postgresql::SourcePos.new(current_snapshot) }
+
+    subject { subject_object.fetch_responses_once(table_name, latest_src_pos) }
+
+    shared_examples 'returning expected responses' do
+      it 'runs a query and return responses' do
+        expect_any_instance_of(FlydataCore::Postgresql::PGClient).
+          to receive(:query).and_return(result)
+        expect(Response).to receive(:create_responses).with(context, table_name, result, query_cond).and_return([response])
+        expect(subject).to eq([response])
+      end
+    end
+
+    context 'when table_src_pos file exists but does not have pk_values' do
+      let(:table_1_snapshot) { create_ss('1015:1015:') }
+      let(:query_cond) { {from_sid: create_ss('1015:1015:'), to_sid: create_ss('1020:1020:'), pk_values: nil} }
+      it_behaves_like 'returning expected responses'
+    end
+
+    context 'when table_src_pos file does not exist' do
+      before do
+        FileUtils.rm(table_1_src_pos_file.path)
+      end
+      let(:query_cond) { {from_sid: create_ss('1010:1010:'), to_sid: create_ss('1020:1020:'), pk_values: nil} }
+      it_behaves_like 'returning expected responses'
+    end
+
+    context 'when table_src_pos has pk_values' do
+      let(:table_1_snapshot) { create_ss('1012:1012:') }
+      let(:table_1_to_snapshot) { create_ss('1020:1020:') }
+      let(:table_1_pk_values) { [{"id"=>2}] }
+
+      let(:query_cond) { {from_sid: create_ss('1012:1012:'), to_sid: create_ss('1020:1020:'), pk_values: [{"id"=>2}]} }
+      it_behaves_like 'returning expected responses'
+    end
+
+    context 'when table_src_pos snapshot is up to date' do
+      let(:table_1_snapshot) { current_snapshot }
+      it { is_expected.to be_nil }
+    end
+  end
+end
+
+end
+end
+end

data/spec/flydata/source_postgresql/table_meta_spec.rb

@@ -0,0 +1,77 @@
+require 'fluent_plugins_spec_helper'
+require 'flydata/source_postgresql/table_meta'
+
+module Flydata::SourcePostgresql
+  describe TableMeta do
+    let(:dbconf) { { host: 'test-host', port: 3306, username: 'test-user', password: 'test-pswd', database: 'test-db', schema: 'test-schema' } }
+    let(:database) { 'test-db' }
+    let(:schema) { 'test-schema' }
+    let(:tables) { %w(a_table b_table c_table) }
+
+    let(:conn) { double('conn') }
+    let(:subject_object) { described_class.new(dbconf, tables, schema) }
+
+
+    def build_column(table_name,column_name,column_default,is_nullable,is_primary,data_type,character_octet_length,numeric_precision,numeric_scale)
+      {'table_name'=>table_name,
+       'column_name'=>column_name,
+       'column_default'=>column_default,
+       'is_nullable'=>is_nullable,
+       'is_primary'=>is_primary,
+       'data_type'=>data_type,
+       'character_octet_length'=>character_octet_length,
+       'numeric_precision'=>numeric_precision,
+       'numeric_scale'=>numeric_scale,
+      }
+    end
+
+    let(:raw_columns) { [
+      build_column('table_a', 'id', nil, 'NO', true, 'int', nil, 4, 0),
+      build_column('table_a', 'val', nil, 'YES', false, 'character varying', 24, nil, nil),
+      build_column('table_b', 'name', nil, 'NO', true, 'character varying', 36, nil, nil),
+      build_column('table_b', 'count', nil, 'YES', false, 'int', nil, 4, 0),
+    ] }
+    let(:current_snapshots) { [{'txid_current_snapshot'=>'1010:1010:'}] }
+
+    before do
+      allow(conn).to receive(:close)
+      allow(FlydataCore::Postgresql::PGClient).to receive(:new).and_return(conn)
+    end
+
+    describe '.reload' do
+      subject { subject_object.reload(input_cli) }
+
+      context 'when params is nil' do
+        let(:input_cli) { nil }
+        before do
+          expect(conn).to receive(:query).and_return(raw_columns, current_snapshots)
+          expect(conn).to receive(:close)
+        end
+        it do
+          expect(subject[:table_a][:table_name]).to eq('table_a')
+          expect(subject[:table_a][:primary_keys]).to eq(%w(id))
+          expect(subject[:table_a][:pk_positions]).to eq(%w(1))
+          expect(subject[:table_a][:max_num_rows_per_query]).to be_kind_of(Integer)
+          expect(subject[:table_a][:columns]).to be_kind_of(Array)
+          expect(subject[:table_a][:table_def]).to be_kind_of(FlydataCore::TableDef::Base)
+        end
+      end
+
+      context 'when params is not nil' do
+        let(:input_cli) { conn }
+        before do
+          expect(conn).to receive(:query).and_return(raw_columns, current_snapshots)
+          expect(conn).to receive(:close).never
+        end
+        it do
+          expect(subject[:table_a][:table_name]).to eq('table_a')
+          expect(subject[:table_a][:primary_keys]).to eq(%w(id))
+          expect(subject[:table_a][:pk_positions]).to eq(%w(1))
+          expect(subject[:table_a][:max_num_rows_per_query]).to be_kind_of(Integer)
+          expect(subject[:table_a][:columns]).to be_kind_of(Array)
+          expect(subject[:table_a][:table_def]).to be_kind_of(FlydataCore::TableDef::Base)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.14
+  version: 0.7.0
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -544,6 +544,10 @@ files:
 - flydata-core/lib/flydata-core/mysql/config.rb
 - flydata-core/lib/flydata-core/mysql/ssl.rb
 - flydata-core/lib/flydata-core/option_validator.rb
+- flydata-core/lib/flydata-core/postgresql/compatibility_checker.rb
+- flydata-core/lib/flydata-core/postgresql/config.rb
+- flydata-core/lib/flydata-core/postgresql/pg_client.rb
+- flydata-core/lib/flydata-core/postgresql/snapshot.rb
 - flydata-core/lib/flydata-core/postgresql/source_pos.rb
 - flydata-core/lib/flydata-core/query_job.rb
 - flydata-core/lib/flydata-core/record/record.rb
@@ -572,6 +576,10 @@ files:
 - flydata-core/spec/mysql/compatibility_checker_spec.rb
 - flydata-core/spec/mysql/config_spec.rb
 - flydata-core/spec/option_validator_spec.rb
+- flydata-core/spec/postgresql/compatibility_checker_spec.rb
+- flydata-core/spec/postgresql/config_spec.rb
+- flydata-core/spec/postgresql/pg_client_spec.rb
+- flydata-core/spec/postgresql/snapshot_spec.rb
 - flydata-core/spec/postgresql/source_pos_spec.rb
 - flydata-core/spec/redshift/string_spec.rb
 - flydata-core/spec/spec_helper.rb
@@ -638,12 +646,16 @@ files:
 - lib/flydata/errors.rb
 - lib/flydata/event/api_event_sender.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/fluent_log_ext.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/flush_support.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb
 - lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
+- lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
 - lib/flydata/flydata_crontab.sh
 - lib/flydata/helper/action/agent_action.rb
@@ -672,9 +684,15 @@ files:
 - lib/flydata/parser/parser_provider.rb
 - lib/flydata/parser/source_table.rb
 - lib/flydata/plugin_support/context.rb
+- lib/flydata/plugin_support/source_position_file.rb
 - lib/flydata/plugin_support/sync_record_emittable.rb
 - lib/flydata/preference/data_entry_preference.rb
 - lib/flydata/proxy.rb
+- lib/flydata/query_based_sync/client.rb
+- lib/flydata/query_based_sync/record_size_estimator.rb
+- lib/flydata/query_based_sync/resource_requester.rb
+- lib/flydata/query_based_sync/response.rb
+- lib/flydata/query_based_sync/response_handler.rb
 - lib/flydata/queueable_thread.rb
 - lib/flydata/source.rb
 - lib/flydata/source/component.rb
@@ -701,7 +719,6 @@ files:
 - lib/flydata/source_mysql/parser/dump_parser.rb
 - lib/flydata/source_mysql/parser/mysql_alter_table.treetop
 - lib/flydata/source_mysql/plugin_support/alter_table_query_handler.rb
-- lib/flydata/source_mysql/plugin_support/binlog_position_file.rb
 - lib/flydata/source_mysql/plugin_support/binlog_query_dispatcher.rb
 - lib/flydata/source_mysql/plugin_support/binlog_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/binlog_record_dispatcher.rb
@@ -710,26 +727,35 @@ files:
 - lib/flydata/source_mysql/plugin_support/ddl_query_handler.rb
 - lib/flydata/source_mysql/plugin_support/dml_record_handler.rb
 - lib/flydata/source_mysql/plugin_support/drop_database_query_handler.rb
-- lib/flydata/source_mysql/plugin_support/table_meta.rb
+- lib/flydata/source_mysql/plugin_support/source_position_file.rb
 - lib/flydata/source_mysql/plugin_support/truncate_table_query_handler.rb
 - lib/flydata/source_mysql/setup.rb
 - lib/flydata/source_mysql/source_pos.rb
 - lib/flydata/source_mysql/sync.rb
 - lib/flydata/source_mysql/sync_generate_table_ddl.rb
 - lib/flydata/source_mysql/table_ddl.rb
+- lib/flydata/source_mysql/table_meta.rb
 - lib/flydata/source_postgresql/data_entry.rb
 - lib/flydata/source_postgresql/generate_source_dump.rb
 - lib/flydata/source_postgresql/parse_dump_and_send.rb
-- lib/flydata/source_postgresql/pg_client.rb
+- lib/flydata/source_postgresql/plugin_support/context.rb
+- lib/flydata/source_postgresql/plugin_support/source_position_file.rb
 - lib/flydata/source_postgresql/postgresql_component.rb
+- lib/flydata/source_postgresql/query_based_sync/client.rb
+- lib/flydata/source_postgresql/query_based_sync/diff_query_generator.rb
+- lib/flydata/source_postgresql/query_based_sync/resource_requester.rb
+- lib/flydata/source_postgresql/query_based_sync/response.rb
+- lib/flydata/source_postgresql/query_based_sync/response_handler.rb
 - lib/flydata/source_postgresql/setup.rb
 - lib/flydata/source_postgresql/source_pos.rb
 - lib/flydata/source_postgresql/sync.rb
 - lib/flydata/source_postgresql/sync_generate_table_ddl.rb
+- lib/flydata/source_postgresql/table_meta.rb
 - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
 - lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb
 - lib/flydata/sync_file_manager.rb
 - lib/flydata/table_ddl.rb
+- lib/flydata/table_meta.rb
 - lib/flydata/util/encryptor.rb
 - lib/flydata/util/file_util.rb
 - lib/flydata/util/shell.rb
@@ -757,8 +783,14 @@ files:
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/command/version_spec.rb
 - spec/flydata/compatibility_check_spec.rb
+- spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context.rb
+- spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based_shared_examples.rb
+- spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based_shared_examples.rb
+- spec/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_shared_examples.rb
 - spec/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
+- spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb
+- spec/flydata/fluent-plugins/sync_source_plugin_context.rb
 - spec/flydata/helper/action/check_abnormal_shutdown_spec.rb
 - spec/flydata/helper/action/check_remote_actions_spec.rb
 - spec/flydata/helper/action/restart_agent_spec.rb
@@ -774,6 +806,12 @@ files:
 - spec/flydata/json/json_ext_spec.rb
 - spec/flydata/output/forwarder_spec.rb
 - spec/flydata/plugin_support/context_spec.rb
+- spec/flydata/query_based_sync/client_spec.rb
+- spec/flydata/query_based_sync/query_based_sync_context.rb
+- spec/flydata/query_based_sync/record_size_estimator_spec.rb
+- spec/flydata/query_based_sync/resource_requester_spec.rb
+- spec/flydata/query_based_sync/response_handler_spec.rb
+- spec/flydata/query_based_sync/response_spec.rb
 - spec/flydata/source_mysql/generate_source_dump_spec.rb
 - spec/flydata/source_mysql/mysql_compatibility_check_spec.rb
 - spec/flydata/source_mysql/parser/alter_table_parser_spec.rb
@@ -785,11 +823,16 @@ files:
 - spec/flydata/source_mysql/plugin_support/dml_record_handler_spec.rb
 - spec/flydata/source_mysql/plugin_support/drop_database_query_handler_spec.rb
 - spec/flydata/source_mysql/plugin_support/shared_query_handler_context.rb
-- spec/flydata/source_mysql/plugin_support/table_meta_spec.rb
+- spec/flydata/source_mysql/plugin_support/source_position_file_spec.rb
 - spec/flydata/source_mysql/plugin_support/truncate_query_handler_spec.rb
 - spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb
 - spec/flydata/source_mysql/table_ddl_spec.rb
+- spec/flydata/source_mysql/table_meta_spec.rb
 - spec/flydata/source_postgresql/generate_source_dump_spec.rb
+- spec/flydata/source_postgresql/query_based_sync/diff_query_generator_spec.rb
+- spec/flydata/source_postgresql/query_based_sync/query_based_sync_postgresql_context.rb
+- spec/flydata/source_postgresql/query_based_sync/resource_requester_spec.rb
+- spec/flydata/source_postgresql/table_meta_spec.rb
 - spec/flydata/source_spec.rb
 - spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb
 - spec/flydata/sync_file_manager_spec.rb