flydata 0.6.14 → 0.7.0

data/spec/flydata/fluent-plugins/in_postgresql_query_based_flydata_spec.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+require 'fluent_plugins_spec_helper'
+require 'in_postgresql_query_based_flydata'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based_shared_examples'
+
+module Fluent
+
+  describe PostgresqlQueryBasedFlydataInput do
+    let(:plugin) { PostgresqlQueryBasedFlydataInput.new }
+    let(:default_config) { <<EOT
+  data_entry_name test_name
+  data_entry_type RedshiftMysqlDataEntry
+  tag #{TEST_TAG}
+  database #{TEST_DB}
+  tables test_table,test_table_1,test_table_2
+  tables_append_only test_table_3
+  position_file #{TEST_POSITION_FILE}
+  host localhost
+  port 3306
+  username test_admin
+  password test_password
+  database test_db
+  schema test_schema
+  emit_chunk_limit 64m
+  fetch_interval 2m
+EOT
+    }
+    include_examples 'flydata sync query based shared examples'
+
+    describe '#configure' do
+      let(:config) { default_config }
+      subject { Test.configure_plugin(plugin, config) }
+
+      before do
+        create_file(TEST_POSITION_FILE, "1001:1001:\t\t")
+        create_table_pos_file('test_table', '0')
+        subject
+      end
+      after do
+        delete_file(TEST_POSITION_FILE)
+      end
+
+      it 'sets data entry object' do
+        expect(plugin.instance_variable_get(:@data_entry)).to eq(
+          'name' => 'test_name',
+          'type' => 'RedshiftMysqlDataEntry',
+          'postgresql_data_entry_preference' => {
+            'database' => 'test_db',
+            'tables' => 'test_table,test_table_1,test_table_2',
+            'tables_append_only' => 'test_table_3',
+            'host' => 'localhost',
+            'username' => 'test_admin',
+            'password' => 'test_password',
+            'schema' => 'test_schema',
+          }
+        )
+      end
+
+      it 'sets data tables as array' do
+        expect(plugin.instance_variable_get(:@tables)).to eq(%w(test_table))
+      end
+
+      it 'sets sync_fm' do
+        expect(plugin.instance_variable_get(:@sync_fm)).to be_kind_of(Flydata::SyncFileManager)
+      end
+
+      it 'sets omit_events' do
+        expect(plugin.instance_variable_get(:@omit_events)).to eq({'test_table_3' => [:delete, :truncate_table]})
+      end
+
+      it 'sets context' do
+        expect(plugin.instance_variable_get(:@context)).to be_kind_of(Flydata::SourcePostgresql::PluginSupport::Context)
+      end
+
+      it 'sets client' do
+        expect(plugin.instance_variable_get(:@client)).to be_kind_of(Flydata::SourcePostgresql::QueryBasedSync::Client)
+      end
+    end
+  end
+
+end

data/spec/flydata/fluent-plugins/sync_source_plugin_context.rb

@@ -0,0 +1,29 @@
+require 'timecop'
+require 'fileutils'
+
+module Fluent
+  FLYDATA_HOME = Flydata::FLYDATA_HOME
+  TEST_TAG = "test_tag"
+  TEST_DB = "test_db"
+  TEST_TABLE = "test_table"
+  TEST_SEQUENCE_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.pos")
+  TEST_SEQUENCE_NUM = 1
+  TEST_TABLE_BINLOG_POS = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.binlog.pos")
+  TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
+  TEST_POSITION_FILE = "test_position.binlog.pos"
+  TEST_SENT_POSITION_FILE = "test_position.binlog.sent.pos"
+  TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
+  TEST_TIMESTAMP = 1389214083
+  TEST_TABLE_APPEND_ONLY = "test_table_4"
+  TEST_SEQUENCE_FILE_1 = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE_APPEND_ONLY}.pos")
+
+  shared_context 'sync source plugin context' do
+    def create_file(file_path, content)
+      File.open(file_path, 'w') {|f| f.write(content)}
+    end
+
+    def delete_file(file_path)
+      FileUtils.rm(file_path) if File.exists?(file_path)
+    end
+  end
+end

data/spec/flydata/plugin_support/context_spec.rb

@@ -2,16 +2,38 @@ require 'fluent_plugins_spec_helper'
 require 'flydata/plugin_support/context'
 
 describe ::Flydata::PluginSupport::Context do
-  let(:subject_object) { described_class.new(params) }
+
+  class DummySourcePos
+    def initialize(str)
+      @str = str
+    end
+    def self.load(str)
+      self.new(str)
+    end
+    attr_reader :str
+  end
+
+  class TestContext < described_class
+    SOURCE_POS_CLASS = DummySourcePos
+  end
+
+  let(:subject_object) { TestContext.new(params) }
+  let(:sync_fm) { double('sync_fm') }
 
   describe '#initialize' do
     subject { subject_object }
 
+    before do
+      allow(sync_fm).to receive(:get_table_source_raw_pos) do |tbl|
+        "#{tbl}_binlog_pos"
+      end
+    end
+
     context 'when no missing params' do
       let(:params) { {
         tables: %w(table_a table_b table_c),
         tag: 'test_tag',
-        sync_fm: 'test_sync_fm',
+        sync_fm: sync_fm,
         omit_events: {'table_c' => %i(delete truncate_table)},
         table_revs: {'table_a' => 1, 'table_b' => 2, 'table_c' => 3},
       } }
@@ -19,9 +41,21 @@ describe ::Flydata::PluginSupport::Context do
       it { expect{subject}.not_to raise_error }
       it { expect(subject.tables).to eq(%w(table_a table_b table_c)) }
       it { expect(subject.tag).to eq('test_tag') }
-      it { expect(subject.sync_fm).to eq('test_sync_fm') }
+      it { expect(subject.sync_fm).to eq(sync_fm) }
       it { expect(subject.omit_events).to eq({'table_c' => %i(delete truncate_table)}) }
       it { expect(subject.table_revs).to eq({'table_a' => 1, 'table_b' => 2, 'table_c' => 3}) }
+=begin
+      it do
+        binlog_pos = subject.table_binlog_pos
+        expect(binlog_pos.keys.size).to eq(3)
+        expect(binlog_pos['table_a']).to be_kind_of(DummySourcePos)
+        expect(binlog_pos['table_a'].str).to eq('table_a_binlog_pos')
+        expect(binlog_pos['table_b']).to be_kind_of(DummySourcePos)
+        expect(binlog_pos['table_b'].str).to eq('table_b_binlog_pos')
+        expect(binlog_pos['table_c']).to be_kind_of(DummySourcePos)
+        expect(binlog_pos['table_c'].str).to eq('table_c_binlog_pos')
+      end
+=end
     end
   end
 end

data/spec/flydata/query_based_sync/client_spec.rb

@@ -0,0 +1,79 @@
+require 'spec_helper'
+require 'flydata/query_based_sync/query_based_sync_context'
+
+
+module Flydata
+module QueryBasedSync
+
+describe Client do
+  include_context 'query based sync context'
+
+  let(:subject_object) { DummyClient.new(context) }
+
+  describe '#initialize' do
+    subject { subject_object }
+
+    before do
+      subject
+    end
+
+    it { expect(subject.context).to eq(context) }
+    it { expect(subject.resource_requester).to be_kind_of(DummyResourceRequester) }
+    it { expect(subject.response_handler).to be_kind_of(DummyResponseHandler) }
+    it { expect(subject.instance_variable_get(:@fetch_interval)).to eq(10) }
+    it { expect(subject.instance_variable_get(:@retry_interval)).to eq(5) }
+  end
+
+  describe '#start' do
+    def start_client(wait = 0.0)
+      @thread = Thread.new(&method(:run_client))
+      sleep wait
+      expect(@thread.alive?).to be(true)
+    end
+
+    def run_client
+      subject_object.start
+    rescue Exception => e
+      $stderr.puts "Caught an exception during running client - error:#{e}\n  #{e.backtrace.join("\n  ")}"
+      raise e
+    end
+
+    def stop_client
+      if @thread and @thread.alive?
+        subject_object.stop_request
+        @thread.join
+      end
+    end
+
+    before do
+      allow_any_instance_of(DummyResourceRequester).to receive(:start) do |my_instance, &block|
+        block.call(my_instance)
+      end
+      allow_any_instance_of(DummyResourceRequester).to receive(:each_response).and_yield(response)
+      allow_any_instance_of(DummyResponseHandler).to receive(:handle)
+    end
+
+    after do
+      stop_client
+      expect(subject_object.stop_requested?).to eq(true)
+      expect(@thread.alive?).to eq(false)
+    end
+
+    it 'starts and stops client' do
+      expect_any_instance_of(DummyResponseHandler).to receive(:handle).with(response)
+      start_client(0.2)
+      stop_client
+    end
+
+    context 'when calls start two times' do
+      it 'gets an error for the second try' do
+        start_client(0.2)
+        expect{subject_object.start}.to raise_error(/Already started/)
+      end
+    end
+  end
+end
+
+end
+end
+

data/spec/flydata/query_based_sync/query_based_sync_context.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+require 'flydata/plugin_support/context'
+require 'flydata/query_based_sync/client'
+require 'flydata/query_based_sync/resource_requester'
+require 'flydata/query_based_sync/response_handler'
+require 'flydata/query_based_sync/response'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_context'
+
+
+module Flydata
+module QueryBasedSync
+
+class DummyResourceRequester < ResourceRequester
+end
+
+class DummyResponseHandler < ResponseHandler
+end
+
+class DummyResponse < Response
+end
+
+class DummyClient < Client
+  RESOURCE_REQUESTER_CLASS = DummyResourceRequester
+  RESPONSE_HANDLER_CLASS = DummyResponseHandler
+end
+
+class DummySourcePos
+  include Comparable
+
+  def initialize(*args)
+    @snapshot = args.first
+    @args = args
+  end
+
+  attr_reader :snapshot
+  attr_reader :args
+
+  def <=>(other)
+    snapshot <=> other.snapshot
+  end
+end
+
+shared_context 'query based sync context' do
+  include_context 'flydata sync context'
+
+  def create_response(context, table_name, records, query_cond = {})
+    DummyResponse.new(context, table_name, records, query_cond)
+  end
+
+  let(:resource_client) { double('client') }
+  let(:records) { [
+    {'1'=>1, '2'=>'a'},
+    {'1'=>2, '2'=>'b'},
+    {'1'=>3, '2'=>'c'},
+  ] }
+  let(:query_cond) { {from_sid: previous_snapshot, to_sid: current_snapshot} }
+  let(:response) { create_response(context, 'table_1', records, query_cond) }
+
+  let(:sync_fm) { double('sync_fm') }
+  let(:table_meta) { double('table_meta') }
+  let(:table_meta_1) { {
+      table_name: 'table_1',
+      table_schema: nil,
+      pk_positions: [1],
+      primary_keys: ['id'],
+      columns: [
+        { column: 'id', type: 'int4' },
+        { column: 'value', type: 'varchar(24)' },
+      ]
+  } }
+  let(:cur_src_pos_file) { double('cur_src_pos_file') }
+  let(:table_1_src_pos_file) { double('table_1_src_pos_file') }
+  let(:table_2_src_pos_file) { double('table_2_src_pos_file') }
+  let(:table_3_src_pos_file) { double('table_3_src_pos_file') }
+  let(:context) {
+    Flydata::PluginSupport::Context.new(
+      cur_src_pos_file: cur_src_pos_file,
+      tables: %w(table_1 table_2 table_3),
+      tag: 'test_tag',
+      sync_fm: sync_fm,
+
+      omit_events: { 'table_3' => [:delete, :truncate] },
+      table_revs: { 'table_1' => 1, 'table_2' => 1, 'table_3' => 1 },
+      table_meta: table_meta,
+      table_src_pos_files: {
+        table_1: table_1_src_pos_file,
+        table_2: table_2_src_pos_file,
+        table_3: table_3_src_pos_file,
+      },
+      params: {
+        fetch_interval: 10,
+        retry_interval: 5,
+      }
+    )
+  }
+  let(:previous_snapshot) { double('previous_snapshot') }
+  let(:current_snapshot) { double('current_snapshot') }
+
+  def setup_dummy_objects
+    allow_any_instance_of(DummyResourceRequester).to receive(:create_resource_client).and_return(resource_client)
+    allow_any_instance_of(DummyResourceRequester).to receive(:close)
+    allow(context).to receive(:source_pos_class).and_return(DummySourcePos)
+    response.instance_variable_set(:@new_source_pos, DummySourcePos.new(current_snapshot))
+  end
+
+  before do
+    allow(table_meta).to receive(:[]).with(:table_1).and_return(table_meta_1)
+    allow(table_meta).to receive(:current_snapshot).and_return(current_snapshot)
+    allow(cur_src_pos_file).to receive(:save).with(current_snapshot)
+    setup_dummy_objects
+  end
+end
+
+
+end
+end

data/spec/flydata/query_based_sync/record_size_estimator_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'flydata/query_based_sync/record_size_estimator'
+
+module Flydata
+module QueryBasedSync
+
+describe RecordSizeEstimator do
+  def actual_record_size(table_name, record)
+    j = {
+      row: record,
+      type: :update,
+      respect_order: true,
+      src_pos: "10051:10051:\t10520:10052:\t{\"id\":\"1000000000\"}",
+      v: 2,
+      seq: 1000000000,
+      table_name: table_name,
+      table_rev: 1,
+    }.to_json.size
+  end
+
+  let(:table_name) { 'table_1' }
+  let(:num_columns) { 2 }
+  let(:subject_object) { described_class.new(table_name, num_columns) }
+
+  describe '#calc_record_size' do
+
+    def create_record(num_records = 10)
+      1.upto(num_records).inject({}){|h, i| h[i.to_s]=i.to_s ;h}
+    end
+
+    let(:num_columns) { record.values.count }
+    subject { subject_object.calc_record_size(record) }
+    let(:actual_size) { actual_record_size(table_name, record) }
+
+    context 'when number of columns is small' do
+      let(:record) { create_record(2) }
+      it { is_expected.to eq(actual_size) }
+    end
+
+    context 'when number of columns is medium' do
+      let(:record) { create_record(12) }
+      it { is_expected.to eq(actual_size) }
+    end
+
+    context 'when number of columns is big' do
+      let(:record) { create_record(102) }
+      it { is_expected.to eq(actual_size) }
+    end
+  end
+end
+
+end
+end
+

data/spec/flydata/query_based_sync/resource_requester_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+require 'flydata/query_based_sync/query_based_sync_context'
+
+
+module Flydata
+module QueryBasedSync
+
+describe ResourceRequester do
+  include_context 'query based sync context'
+
+  let(:subject_object) { DummyResourceRequester.new(context) }
+
+  describe '#initialize' do
+    subject { subject_object }
+
+    before do
+      subject
+    end
+
+    it { expect(subject.context).to eq(context) }
+    it { expect(subject.instance_variable_get(:@resource_client)).to eq(resource_client) }
+  end
+
+  describe '#start' do
+    it do
+      expect(table_meta).to receive(:reload).with(resource_client)
+      expect(resource_client).to receive(:close).once
+      expect{|b| subject_object.start(&b)}.to yield_with_args(subject_object)
+    end
+  end
+
+  describe '#each_response' do
+    let(:table_name) { 'table_1' }
+    let(:returned_responses) { nil }
+
+    before do
+      allow(subject_object).to receive(:fetch_responses_once).and_return(returned_responses)
+    end
+
+    context 'when no response' do
+      let(:returned_responses) { [] }
+      it do
+        expect(subject_object.each_response(table_name)).to be_nil
+      end
+    end
+
+    context 'when returning a response' do
+      let(:returned_responses) { [response] }
+      it do
+        expect{|b| subject_object.each_response(table_name, 0.1, &b)}.
+          to yield_with_args(response)
+      end
+    end
+  end
+end
+
+end
+end