flydata 0.6.4 → 0.6.5

data/spec/flydata/fluent-plugins/{idle_event_detector_spec.rb → flydata_plugin_ext/idle_event_detector_spec.rb}

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 require 'fluent_plugins_spec_helper'
-require 'idle_event_detector'
+require 'flydata_plugin_ext/idle_event_detector'
 require 'timecop'
 
 module Fluent

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -16,6 +16,7 @@ module Fluent
   TEST_TABLE_BINLOG_POS = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.binlog.pos")
   TEST_TABLES = "#{TEST_TABLE},test_table_1,test_table_2,test_table_3"
   TEST_POSITION_FILE = "test_position.binlog.pos"
+  TEST_SENT_POSITION_FILE = "test_position.binlog.sent.pos"
   TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
   TEST_TIMESTAMP = 1389214083
   TEST_TABLE_APPEND_ONLY = "test_table_4"
@@ -203,6 +204,13 @@ EOT
       f
     end
 
+    let(:sent_position_file) do
+      f = double('sent_position_file')
+      allow(f).to receive(:exists?).and_return(true)
+      allow(f).to receive(:read).and_return('test_sent_position')
+      f
+    end
+
     def create_file(file_path, content)
       File.open(file_path, 'w') {|f| f.write(content)}
     end
@@ -228,6 +236,8 @@ EOT
       setup_initial_flydata_files
       allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
       allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
+      allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_SENT_POSITION_FILE).
+        and_return(sent_position_file)
       allow(Flydata::Mysql::TableMeta).to receive(:update)
       allow(plugin).to receive(:`).with("mysql -V").and_return("mysql  Ver 14.14 Distrib 5.5.40")
       allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")

data/spec/flydata/source_mysql/parser/alter_table_parser_spec.rb

@@ -33,6 +33,86 @@ describe 'MysqlAlterTableParser' do
         end
       end
 
+      context 'with space between type and meta' do
+        let(:query) { "alter table test_table add column value varchar (26)" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar (26)"
+            }]
+          })
+        end
+      end
+
+      context 'with space before meta value' do
+        let(:query) { "alter table test_table add column value varchar( 26)" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar( 26)"
+            }]
+          })
+        end
+      end
+
+      context 'with space after meta value' do
+        let(:query) { "alter table test_table add column value varchar(26 )" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar(26 )"
+            }]
+          })
+        end
+      end
+
+      context 'with space before and after meta value' do
+        let(:query) { "alter table test_table add column value varchar( 26 )" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar( 26 )"
+            }]
+          })
+        end
+      end
+
+      context 'with spaces everywhere' do
+        let(:query) { "alter table test_table add column value varchar  (  26    )" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "varchar(78)",
+              query: "add column value varchar  (  26    )"
+            }]
+          })
+        end
+      end
+
       context 'with a non-ascii table name' do
         let(:query) { "alter table `テスト` add column value varchar(26)" }
         it do
@@ -83,6 +163,23 @@ describe 'MysqlAlterTableParser' do
         end
       end
 
+      context 'data_type without meta' do
+        let(:query) { "alter table test_table add value int not null" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value",
+              type: "int4(11)",
+              not_null: true,
+              query: "add value int not null"
+            }]
+          })
+        end
+      end
+
       context 'with multiple columns' do
         shared_examples 'test result hash' do
           it do
@@ -103,6 +200,7 @@ describe 'MysqlAlterTableParser' do
             })
           end
         end
+
         context 'with spaces before opening bracket' do
           let(:query) { "alter table test_table add column (value1 varchar(26), value2 varchar(26))" }
           let(:expected_query) {"add column (value1 varchar(26), value2 varchar(26))"}
@@ -115,6 +213,27 @@ describe 'MysqlAlterTableParser' do
         end
       end
 
+      context 'with multiple columns without bracket without the keyword "COLUMN"' do
+        let(:query) { "alter table test_table add value1 varchar(26), add value2 varchar(26)" }
+        it do
+          expect(subject).to eq({
+            type: :alter_table,
+            table_name: "test_table",
+            actions: [{
+              action: :add_column,
+              column: "value1",
+              type: "varchar(78)",
+              query: "add value1 varchar(26)"
+            },{
+              action: :add_column,
+              column: "value2",
+              type: "varchar(78)",
+              query: "add value2 varchar(26)"
+            }]
+          })
+        end
+      end
+
       context 'with multiple columns and without the keyword "COLUMN"' do
         let(:query) { "alter table test_table add ( value1 varchar(26), value2 varchar(26) )" }
         it do

data/spec/flydata/source_mysql/parser/dump_parser_spec.rb

@@ -322,16 +322,29 @@ EOT
                   expect(source_table.table_name).to eq('users_login')
                   expect(values_set).to eq(expected_values)
                   nil
+=begin
+# Use this when _parse2 is used
                 }.once
+=end
+# =begin
+# Use this when _parse is used
+                }.never
+# =end
               end
               expect(insert_record_block).to receive(:call).never
 
-              # insert_record_block
+              # check_point_block
               [
                 {state: Flydata::Parser::State::CREATE_TABLE},
                 {state: Flydata::Parser::State::INSERT_RECORD},
+=begin
+# Use this when _parse2 is used
                 {state: Flydata::Parser::State::CREATE_TABLE,
                  last_pos: index_after(dump_content, 'UNLOCK TABLES;')}
+=end
+# =begin
+# Use this when _parse is used
+# =end
               ].each do |expected_params|
                 expect(check_point_block).to receive(:call) { |source_table, last_pos, bytesize, binlog_pos, state, substate|
                   expect(source_table.table_name).to eq('users_login') if source_table
@@ -346,10 +359,21 @@ EOT
               end
               expect(check_point_block).to receive(:call).never
 
+=begin
+# Use this when _parse2 is used
               binlog_pos = default_parser.parse(
                 dump_io, create_table_block, insert_record_block, check_point_block
               )
               expect(binlog_pos).to eq(default_binlog_pos)
+=end
+# =begin
+# Use this when _parse is used
+              expect {
+                binlog_pos = default_parser.parse(
+                  dump_io, create_table_block, insert_record_block,
+                  check_point_block)
+              }.to raise_error(DumpParseError)
+# =end
             end
           end
 
@@ -851,7 +875,14 @@ EOT
             end
             context 'with various escape chars 1' do
               let(:value) { %Q|\\r\t\\Z| }
+=begin
+# Use this when _parse2 is used
               let(:expected_value) { %Q|\r\t\x1a| }
+=end
+# =begin
+# Use this when _parse is used
+              let(:expected_value) { %Q|\r\t\\Z| }
+# =end
               it 'returns correct chars' do
                 expect(subject).to eq([[expected_value]])
               end

data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb

@@ -12,7 +12,7 @@ describe SyncGenerateTableDdl do
       'host' => 'localhost',
       'port' => 3306,
       'username' => 'masashi',
-      'password' => 'welcome',
+      'password' => 'welcomes',
       'database' => 'sync_test',
     }
   } }
@@ -42,7 +42,7 @@ describe SyncGenerateTableDdl do
     context 'with full options' do
       it 'issues mysqldump command with expected parameters' do
         expect(Open3).to receive(:popen3).with(
-         'mysqldump -h localhost -P 3306 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+         'mysqldump -h localhost -P 3306 -umasashi -pwelcomes --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
 
         subject
       end
@@ -65,7 +65,7 @@ describe SyncGenerateTableDdl do
       end
       it "uses the default port" do
         expect(Open3).to receive(:popen3).with(
-         'mysqldump -h localhost -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+         'mysqldump -h localhost -umasashi -pwelcomes --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
         subject
       end
     end
@@ -75,7 +75,7 @@ describe SyncGenerateTableDdl do
       end
       it "uses the specified port" do
         expect(Open3).to receive(:popen3).with(
-         'mysqldump -h localhost -P 1234 -umasashi -pwelcome --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
+         'mysqldump -h localhost -P 1234 -umasashi -pwelcomes --default-character-set=utf8 --protocol=tcp -d sync_test table1 table2 table4 table3')
         subject
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.6.5
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -162,6 +162,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.4
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
@@ -536,7 +550,9 @@ files:
 - flydata-core/lib/flydata-core/string_utils.rb
 - flydata-core/lib/flydata-core/table_def.rb
 - flydata-core/lib/flydata-core/table_def/autoload_redshift_table_def.rb
+- flydata-core/lib/flydata-core/table_def/base.rb
 - flydata-core/lib/flydata-core/table_def/mysql_table_def.rb
+- flydata-core/lib/flydata-core/table_def/postgresql_table_def.rb
 - flydata-core/lib/flydata-core/table_def/redshift_table_def.rb
 - flydata-core/lib/flydata-core/table_def/sync_redshift_table_def.rb
 - flydata-core/lib/flydata-core/thread_context.rb
@@ -578,6 +594,7 @@ files:
 - flydata-core/spec/table_def/mysqldump_test_unique_key2.dump
 - flydata-core/spec/table_def/mysqldump_test_unique_key3.dump
 - flydata-core/spec/table_def/mysqldump_test_unsigned.dump
+- flydata-core/spec/table_def/postgresql_table_def_spec.rb
 - flydata-core/spec/table_def/redshift_table_def_spec.rb
 - flydata-core/spec/table_def/sync_redshift_table_def_spec.rb
 - flydata-core/spec/wrapper_forwardable_spec.rb
@@ -616,7 +633,12 @@ files:
 - lib/flydata/cron.rb
 - lib/flydata/errors.rb
 - lib/flydata/event/api_event_sender.rb
-- lib/flydata/fluent-plugins/idle_event_detector.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/base.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/flush_support.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb
+- lib/flydata/fluent-plugins/flydata_plugin_ext/transaction_support.rb
 - lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb
 - lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb
 - lib/flydata/fluent-plugins/mysql/binlog_position_file.rb
@@ -631,7 +653,6 @@ files:
 - lib/flydata/fluent-plugins/mysql/table_meta.rb
 - lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb
 - lib/flydata/fluent-plugins/out_forward_ssl.rb
-- lib/flydata/fluent-plugins/preference.rb
 - lib/flydata/flydata_crontab.sh
 - lib/flydata/helper/action/agent_action.rb
 - lib/flydata/helper/action/check_abnormal_shutdown.rb
@@ -663,6 +684,7 @@ files:
 - lib/flydata/queueable_thread.rb
 - lib/flydata/source.rb
 - lib/flydata/source/component.rb
+- lib/flydata/source/data_entry.rb
 - lib/flydata/source/errors.rb
 - lib/flydata/source/generate_source_dump.rb
 - lib/flydata/source/parse_dump_and_send.rb
@@ -676,6 +698,7 @@ files:
 - lib/flydata/source_mysql/command/mysql_command_base.rb
 - lib/flydata/source_mysql/command/mysqlbinlog.rb
 - lib/flydata/source_mysql/command/mysqldump.rb
+- lib/flydata/source_mysql/data_entry.rb
 - lib/flydata/source_mysql/generate_source_dump.rb
 - lib/flydata/source_mysql/mysql_compatibility_check.rb
 - lib/flydata/source_mysql/parse_dump_and_send.rb
@@ -687,6 +710,9 @@ files:
 - lib/flydata/source_mysql/sync.rb
 - lib/flydata/source_mysql/sync_generate_table_ddl.rb
 - lib/flydata/source_mysql/table_ddl.rb
+- lib/flydata/source_postgresql/data_entry.rb
+- lib/flydata/source_postgresql/sync.rb
+- lib/flydata/source_postgresql/sync_generate_table_ddl.rb
 - lib/flydata/source_zendesk/sync_generate_table_ddl.rb
 - lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb
 - lib/flydata/sync_file_manager.rb
@@ -718,7 +744,7 @@ files:
 - spec/flydata/command/sync_spec.rb
 - spec/flydata/command/version_spec.rb
 - spec/flydata/compatibility_check_spec.rb
-- spec/flydata/fluent-plugins/idle_event_detector_spec.rb
+- spec/flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector_spec.rb
 - spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb
 - spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb
 - spec/flydata/fluent-plugins/mysql/binlog_query_dispatcher_spec.rb