flydata 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9c27e5aaf80b5e422b45d86afd09ce77e50e89a
-  data.tar.gz: cf978a752a26f2700aefc8882c9240278a75f051
+  metadata.gz: 0a3fbdc5a0e42708b1dd6f173c366810e4f58892
+  data.tar.gz: 85a647490961e9b85baea5524b9c0f4f395a9d64
 SHA512:
-  metadata.gz: bfc17189f1057dc79b2e1e969d005bcc587f28bff69340e444d2e6a11176688a7b26ef3074045b8df212a7b652a471b626d1a36a0e470064a9fdc5f2515e2170
-  data.tar.gz: 4578998bd6fb444acc86acb82ae141396f279661d4c5aacb4592c67540951eb5651212566ba3fd61f6259c4b0201ca9e16f4c9c64d98ab981557bd46e25389d6
+  metadata.gz: 200c29e2a64610cd4561ae127365f934b9cef0ef9dad264fed54d191bc3ec779eb22fe507d4d55c6373928a560c807dafa0cc0883243b0865723a62649349d40
+  data.tar.gz: 2a5b4a1e246b448e6fffe76c6fff8416298d22c4e46719109d6d9c2fca99c3f2ddb1cc3e70cb5b5bf9afbac388c3048b1b840c05952bcc0ab9e14a53c89b901f

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

data/bin/serverinfo

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+detect_system() {
+  # refer to https://github.com/wayneeseguin/rvm/blob/master/scripts/functions/utility_system
+  unset  _system_info _system_lib_type
+  export _system_info _system_lib_type
+  _system_info="$(command uname -a)"
+  _system_lib_type="unknown"
+  case "$(command uname)" in
+    (Linux|GNU*)
+      if [[ -f /etc/lsb-release ]]; then
+        _system_lib_type="debian"
+      elif [[ -f /etc/debian_version ]]; then
+        _system_lib_type="debian"
+      elif [[ -f /etc/os-release ]]; then
+        _system_lib_type="debian"
+      elif [[ -f /etc/system-release ]]; then
+        _system_lib_type="centos"
+      elif [[ -f /etc/centos-release ]]; then
+        _system_lib_type="centos"
+      elif [[ -f /etc/redhat-release ]]; then
+        _system_lib_type="centos"
+      fi
+      ;;
+    #(Darwin)
+    #  _system_lib_type="osx"
+    #  ;;
+    (*)
+      ;;
+  esac
+}
+
+log_server_details()
+{
+  echo "Logging ip address"
+  ip addr show
+
+  echo "Logging ulimit and uname"
+  ulimit -a
+  uname -a
+
+  echo "Logging repo details"
+  detect_system
+  if [ "${_system_lib_type}" = "debian" ]; then
+    grep -RoPish --include="*.list" "(?<=^deb\s).*?(?=#|$)" /etc/apt
+  else
+    yum repolist all
+  fi
+
+  echo "Logging release details"
+  cat /etc/*-release
+}
+
+log_server_details

data/flydata.gemspec

@@ -2,19 +2,19 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.2.1 ruby lib
+# stub: flydata 0.2.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.2.1"
+  s.version = "0.2.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa"]
-  s.date = "2014-08-20"
+  s.date = "2014-09-12"
   s.description = "FlyData Command Line Interface"
   s.email = "sysadmin@flydata.co"
-  s.executables = ["fdmysqldump", "flydata"]
+  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
   s.files = [
     ".gitignore",
     ".rspec",
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/fdmysqldump",
     "bin/flydata",
+    "bin/serverinfo",
     "flydata.gemspec",
     "lib/fly_data_model.rb",
     "lib/flydata.rb",

data/lib/flydata/cli.rb

@@ -33,6 +33,7 @@ module Flydata
         puts
         print_usage
         raise e if FLYDATA_DEBUG
+        exit 1
       end
     end
 

data/lib/flydata/command/sender.rb

@@ -6,7 +6,7 @@ module Flydata
           on 'n', 'no-daemon', 'Start FlyData agent as a regular program'
         end
       end
-      def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward 
+      def start(options_or_show_final_message = {show_final_message: true}) # For backward compatibility. Use only as options going forward
         if options_or_show_final_message.kind_of? Hash
           options = options_or_show_final_message
         else
@@ -23,6 +23,7 @@ module Flydata
         # Start sender(fluentd) process
         say('Starting sender process.') unless options[:quiet]
         Dir.chdir(FLYDATA_HOME){
+          Kernel.system("bash #{File.dirname(__FILE__)}/../../../bin/serverinfo", :out => ["#{FLYDATA_HOME}/flydata.log",'a'], :err => ["#{FLYDATA_HOME}/flydata.log",'a'])
           daemon_option = opts.no_daemon? ? "" : "-d #{FLYDATA_HOME}/flydata.pid"
           Kernel.system("fluentd #{daemon_option} -l #{FLYDATA_HOME}/flydata.log -c #{FLYDATA_HOME}/flydata.conf -p #{File.dirname(__FILE__)}/../fluent-plugins")
         }

data/lib/flydata/command/sync.rb

@@ -35,16 +35,10 @@ module Flydata
           end
           exit 1
         end
-        de = retrieve_data_entries.first
-        raise "There are no data entry." unless de
-        case de['type']
-        when 'RedshiftMysqlDataEntry'
-          de = load_sync_info(override_tables(de, tables))
-          flush_buffer_and_stop  unless de['mysql_data_entry_preference']['initial_sync']
-          sync_mysql_to_redshift(de)
-        else
-          raise "No supported data entry. Only mysql-redshift sync is supported."
-        end
+        de = retrieve_data_entry
+        de = load_sync_info(override_tables(de, tables))
+        flush_buffer_and_stop  unless de['mysql_data_entry_preference']['initial_sync']
+        sync_mysql_to_redshift(de)
       end
 
       def flush
@@ -65,7 +59,7 @@ module Flydata
         sender.flush_client_buffer # TODO We should rather delete buffer files
         sender.stop
 
-        de = retrieve_data_entries.first
+        de = retrieve_data_entry
         wait_for_server_buffer
         cleanup_sync_server(de, tables) unless opts.client?
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
@@ -114,7 +108,7 @@ module Flydata
       end
 
       def check
-        de = retrieve_data_entries.first
+        de = retrieve_data_entry
         retry_on(RestClient::Exception) do
           status = do_check(de)
           if status['complete']
@@ -127,7 +121,7 @@ module Flydata
 
       # skip initial sync
       def skip
-        de = retrieve_data_entries.first
+        de = retrieve_data_entry
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         binlog_path = sync_fm.binlog_path
         `touch #{binlog_path}`
@@ -137,20 +131,28 @@ module Flydata
       end
 
       def generate_table_ddl(*tables)
-        de = retrieve_data_entries.first
+        de = retrieve_data_entry
         Flydata::Mysql::CompatibilityCheck.new(de['mysql_data_entry_preference']).check
+        do_generate_table_ddl(override_tables(de, tables))
+      end
 
+      private
+
+      def retrieve_data_entry
+        de = retrieve_data_entries.first
         raise "There are no data entry." unless de
         case de['type']
         when 'RedshiftMysqlDataEntry'
-          do_generate_table_ddl(override_tables(de, tables))
+          mp = de['mysql_data_entry_preference']
+          if mp['tables_append_only']
+            mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only'].split(",")).uniq.join(",")
+          end
         else
           raise "No supported data entry. Only mysql-redshift sync is supported."
         end
+        de
       end
 
-      private
-
       def cleanup_sync_server(de, tables = [])
         puts "Cleaning the server"
         flydata.data_entry.cleanup_sync(de['id'], tables)
@@ -412,7 +414,7 @@ What's next?
 Thank you for using FlyData!
       EOM
       def complete
-        de = load_sync_info(retrieve_data_entries.first)
+        de = load_sync_info(retrieve_data_entry)
         sync_fm = Flydata::FileUtil::SyncFileManager.new(de)
         info = sync_fm.load_dump_pos
         if info[:status] == STATUS_COMPLETE

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -47,6 +47,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   config_param :database, :string
   config_param :tables, :string
+  config_param :tables_append_only, :string
 
   def configure(conf)
     super
@@ -55,13 +56,18 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       raise "No position file(#{@position_file}). Initial synchronization is required before starting."
     end
     load_custom_conf
-    $log.info "mysql host:\"#{@host}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\""
+    $log.info "mysql host:\"#{@host}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
     @tables = @tables.split(/,\s*/)
+    @omit_events = Hash.new
+    @tables_append_only.split(/,\s*/).each do |table|
+      @tables << table unless @tables.include?(table)
+      @omit_events[table] = [:delete]
+    end
     sync_fm = Flydata::FileUtil::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
 
     @context = Mysql::Context.new(
       database: @database, tables: @tables,
-      tag: @tag, sync_fm: sync_fm
+      tag: @tag, sync_fm: sync_fm, omit_events: @omit_events
     )
     @record_dispatcher = Mysql::FlydataBinlogRecordDispatcher.new(@context)
   end

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -50,6 +50,10 @@ module Mysql
       acceptable
     end
 
+    def acceptable_event?(type, table)
+      @context.omit_events[table].nil? || !@context.omit_events[table].include?(type)
+    end
+
     def emit_record(type, record, opt = {})
       return unless acceptable_db?(record)
       return unless record["table_name"].nil? or acceptable_table?(record, record["table_name"])
@@ -62,7 +66,7 @@ module Mysql
 
       table = records.first[TABLE_NAME] || record['table_name']
       raise "Missing table name. #{record}" if table.to_s.empty?
-      return unless acceptable_table?(record, table)
+      return unless acceptable_table?(record, table) && acceptable_event?(type, table)
 
       table_rev = @context.sync_fm.table_rev(table)
       position = record['next_position'] - record['event_length']

data/lib/flydata/fluent-plugins/mysql/context.rb

@@ -1,10 +1,10 @@
 module Mysql
   class Context
     MANDATORY_OPTS = [
-      :database, :tables, :tag, :sync_fm,
+      :database, :tables, :tag, :sync_fm, :omit_events
     ]
     OPTIONAL_OPTS = [
-      :current_binlog_file,
+      :current_binlog_file
     ]
 
     (MANDATORY_OPTS + OPTIONAL_OPTS).each do |opt|

data/lib/flydata/fluent-plugins/preference.rb

@@ -72,6 +72,7 @@ module Fluent
       mysql_data_entry_preference: {
         database: {},
         tables: {},
+        tables_append_only: {},
         host: {},
         username: {},
         password: {encrypted: true},

data/spec/flydata/cli_spec.rb

@@ -20,7 +20,7 @@ module Flydata
                 expect(a.to_hash).to be_empty
               }.and_return(command_obj)
               expect(command_obj).to receive(:run)
-  
+
               subject.run
             end
           end
@@ -30,7 +30,7 @@ module Flydata
               expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
               allow(subject).to receive(:puts)
 
-              subject.run
+              expect{ subject.run }.to terminate.with_code(1)
             end
           end
           context 'with arguments' do
@@ -65,7 +65,7 @@ module Flydata
               expect(subject).to receive(:puts).with('! Unknown options -a, --host-name').once
               allow(subject).to receive(:puts)
 
-              subject.run
+              expect{ subject.run }.to terminate.with_code(1)
             end
           end
         end

data/spec/flydata/command/sender_spec.rb

@@ -21,6 +21,8 @@ module Flydata
           let(:args) { [] }
           it "starts fluend with daemon option" do
             expect(Kernel).to receive(:system).with( Regexp.new(
+"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
+            expect(Kernel).to receive(:system).with( Regexp.new(
 "fluentd -d .+/flydata.pid -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
             subject.start(false)
           end
@@ -28,6 +30,8 @@ module Flydata
         context "as regular process" do
           let(:args) { ["-n"] }
           it "starts fluentd with no daemon option" do
+            expect(Kernel).to receive(:system).with( Regexp.new(
+"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with(Regexp.new(
 "fluentd  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
             subject.start(false)
@@ -36,6 +40,8 @@ module Flydata
         context "as regular process with long option" do
           let(:args) { ["--no-daemon"] }
           it "starts fluentd with no daemon option" do
+            expect(Kernel).to receive(:system).with( Regexp.new(
+"bash .+/../../../bin/serverinfo"), :out => [Regexp.new(".+/flydata.log"),'a'], :err => [Regexp.new(".+/flydata.log"),'a'])
             expect(Kernel).to receive(:system).with(Regexp.new(
 "fluentd  -l .+/flydata.log -c .+/flydata.conf -p .+/\.\./fluent-plugins"))
             subject.start(false)

data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb

@@ -18,10 +18,27 @@ module Fluent
   TEST_POSITION_FILE = "test_position.log"
   TEST_REVISION_FILE = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE}.rev")
   TEST_TIMESTAMP = 1389214083
+  TEST_TABLE_APPEND_ONLY = "test_table_4"
+  TEST_SEQUENCE_FILE_1 = File.join(FLYDATA_HOME, "positions/#{TEST_TABLE_APPEND_ONLY}.pos")
   TEST_CONFIG = <<EOT
   tag #{TEST_TAG}
   database #{TEST_DB}
   tables #{TEST_TABLES}
+  tables_append_only
+  position_file #{TEST_POSITION_FILE}
+EOT
+  TEST_TABLES_APPEND_ONLY_CONFIG = <<EOT
+  tag #{TEST_TAG}
+  database #{TEST_DB}
+  tables #{TEST_TABLES}
+  tables_append_only #{TEST_TABLE_APPEND_ONLY}
+  position_file #{TEST_POSITION_FILE}
+EOT
+  TEST_TABLES_DUPLICATE_CONFIG = <<EOT
+  tag #{TEST_TAG}
+  database #{TEST_DB}
+  tables #{TEST_TABLES},#{TEST_TABLE_APPEND_ONLY}
+  tables_append_only #{TEST_TABLE_APPEND_ONLY}
   position_file #{TEST_POSITION_FILE}
 EOT
 
@@ -189,11 +206,13 @@ EOT
     def setup_initial_flydata_files
       %w(positions dump conf).each{|f| FileUtils.mkdir_p(File.join(FLYDATA_HOME, f))}
       create_file(TEST_SEQUENCE_FILE, TEST_SEQUENCE_NUM.to_s)
+      create_file(TEST_SEQUENCE_FILE_1, TEST_SEQUENCE_NUM.to_s)
     end
 
     def cleanup_flydata_files
       %w(positions dump conf).each{|f| FileUtils.rm_rf(File.join(FLYDATA_HOME, f))}
       delete_file(TEST_SEQUENCE_FILE)
+      delete_file(TEST_SEQUENCE_FILE_1)
     end
 
     before do
@@ -409,6 +428,56 @@ EOT
           expect_no_emitted_record(event)
         end
       end
+
+      context 'for append only' do
+        shared_examples 'emits records correctly' do
+          it 'emits records when it receives an insert event for append only table' do
+            event = insert_event
+            event['table_name'] = TEST_TABLE_APPEND_ONLY
+            expect_emitted_records_with_rows(event, :insert, TEST_TABLE_APPEND_ONLY, 628, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          end
+
+          it 'does not emit a record when it receives a delete event for append only table' do
+            event = delete_event
+            event['table_name'] = TEST_TABLE_APPEND_ONLY
+            expect_no_emitted_record(event)
+          end
+
+          it 'emits records when it receives an update event for append only table' do
+            event = update_event
+            event['table_name'] = TEST_TABLE_APPEND_ONLY
+            expect_emitted_records_with_rows(event, :update, TEST_TABLE_APPEND_ONLY, 2528, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"wow"}, {"1"=>"0SL00000003", "2"=>"fuga"}])
+          end
+
+          it 'emits a record when it receives a delete event for non-append only table' do
+            expect_emitted_records_with_rows(delete_event, :delete, TEST_TABLE, 5324, "mysql-bin.000048",
+              [{"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          end
+
+          it 'emits records when it receives an insert event for non-append only table' do
+            expect_emitted_records_with_rows(insert_event, :insert, TEST_TABLE, 628, "mysql-bin.000048",
+              [{"1"=>"0SL00000001", "2"=>"foo"}, {"1"=>"0SL00000002", "2"=>"var"}, {"1"=>"0SL00000003", "2"=>"hoge"}])
+          end
+        end
+
+        context 'no duplicate entries in tables and tables_append_only' do
+          before do
+            Test.configure_plugin(plugin, TEST_TABLES_APPEND_ONLY_CONFIG)
+            plugin.event_listener(rotate_event)
+          end
+          include_examples 'emits records correctly'
+        end
+
+        context 'duplicate entries in tables and tables_append_only' do
+          before do
+            Test.configure_plugin(plugin, TEST_TABLES_DUPLICATE_CONFIG)
+            plugin.event_listener(rotate_event)
+          end
+          include_examples 'emits records correctly'
+        end
+      end
     end
   end
 

data/spec/spec_helper.rb

@@ -33,5 +33,43 @@ RSpec.configure do |config|
       end
     end
   end
+end
+
+# https://gist.github.com/stevenharman/2355172
+RSpec::Matchers.define :terminate do |code|
+  actual = nil
+
+  def supports_block_expectations?
+    true
+  end
+
+  match do |block|
+    begin
+      block.call
+    rescue SystemExit => e
+      actual = e.status
+    end
+    actual and actual == status_code
+  end
+
+  chain :with_code do |status_code|
+    @status_code = status_code
+  end
+
+  failure_message_for_should do |block|
+    "expected block to call exit(#{status_code}) but exit" +
+      (actual.nil? ? " not called" : "(#{actual}) was called")
+  end
+
+  failure_message_for_should_not do |block|
+    "expected block not to call exit(#{status_code})"
+  end
+
+  description do
+    "expect block to call exit(#{status_code})"
+  end
 
+  def status_code
+    @status_code ||= 0
+  end
 end

data/tmpl/redshift_mysql_data_entry.conf.tmpl

@@ -8,4 +8,5 @@ mysql_data_entry_preference:
   #password: abcd
   #database: dev
   #tables: users,country,rows
+  #tables_append_only: employees
   #mysqldump_dir: /mnt/dump

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Koichi Fujikawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-20 00:00:00.000000000 Z
+date: 2014-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -377,6 +377,7 @@ email: sysadmin@flydata.co
 executables:
 - fdmysqldump
 - flydata
+- serverinfo
 extensions: []
 extra_rdoc_files: []
 files:
@@ -388,6 +389,7 @@ files:
 - VERSION
 - bin/fdmysqldump
 - bin/flydata
+- bin/serverinfo
 - flydata.gemspec
 - lib/fly_data_model.rb
 - lib/flydata.rb