flydata 0.5.11 → 0.5.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 48cf9129bf088b6ab52c5b1933c344f772cc208d
-  data.tar.gz: 577ead8b2fd3962be51b610542ae52030ecd88d0
+  metadata.gz: 82270f47e1280076e3dc0b40a86cc548e0da6636
+  data.tar.gz: bed0f9307f53602e2956369b5419e3173410d8bd
 SHA512:
-  metadata.gz: e6ed2e5faf4aab3e8347b5d56ce7c5fc08e9213e16629fbdd5ab46084ec6a213f1a3917ae308b46d58701c10b2be926d32268416a4a38341198f014700f61349
-  data.tar.gz: 38e9f620d6b617cb3461918de1159adfb7a2084f6a3667bb2d7f8f68533d159d13be0ff85b2d08016fe8ef4bbcdb925b212c9ae122b3efd400c1bd6350ea52f8
+  metadata.gz: e765cd79b44de76a883867a81718e9c7b4ab4834c0ae97d59faa57728bea4e1888bc97d8c993edd6103ea6c5da1786ca33b58f18f532c5eb04c3b6e518f95a2f
+  data.tar.gz: e62f777a303bae1fee4810bc1a1133ef2ca3d12b212ab4932666605e822d57f3ebcaf0ced16433c1b9894e43a286bd824edf30a210f24ab0a764a706420429d4

data/VERSION

@@ -1 +1 @@
-0.5.11
+0.5.12

data/flydata.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: flydata 0.5.11 ruby lib
+# stub: flydata 0.5.12 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "flydata"
-  s.version = "0.5.11"
+  s.version = "0.5.12"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
-  s.date = "2015-10-06"
+  s.date = "2015-10-08"
   s.description = "FlyData Agent"
   s.email = "sysadmin@flydata.com"
   s.executables = ["fdmysqldump", "flydata", "serverinfo"]

data/lib/flydata/command/sync.rb

@@ -598,15 +598,11 @@ EOS
         complete(de)
       end
 
+      # Don't call the method unless init sync needs to be run for a table
       def sync_mysql_to_redshift(de, options = {})
         dp = flydata.data_port.get
         sync_fm = create_sync_file_manager(de)
 
-        # Check client condition
-        if @full_initial_sync && !options[:sync_resumed] && File.exists?(sync_fm.binlog_path)
-          raise "Already synchronized. If you want to do initial sync, run 'flydata sync:reset'"
-        end
-
         # Copy template if not exists
         unless Flydata::Preference::DataEntryPreference.conf_exists?(de)
           Flydata::Command::Conf.new.copy_templates

data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb

@@ -2,7 +2,7 @@ require 'flydata/parser_provider'
 require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 
 module Mysql
-  class AlterTableQueryHandler < DdlQueryHandler
+  class AlterTableQueryHandler < TableDdlQueryHandler
     PATTERN = /^ALTER TABLE/i
 
     def initialize(context)

data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb

@@ -4,8 +4,14 @@ require 'flydata/mysql/binlog_position'
 require 'flydata-core/record/record'
 
 module Mysql
-
   class BinlogRecordHandler
+    TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
+    TYPE = :type
+    SEQ = :seq
+    RESPECT_ORDER = :respect_order
+    SRC_POS = :src_pos
+    TABLE_REV = :table_rev
+    V = :v  # FlyData record format version
 
     def initialize(context)
       @context = context
@@ -53,7 +59,7 @@ module Mysql
 
     def emit_record(type, record)
       return unless acceptable_db?(record)
-      return if record["table_name"] and not acceptable_table?(record, record["table_name"])
+      return unless record["table_name"].nil? or acceptable_table?(record, record["table_name"])
 
       check_empty_binlog
 
@@ -62,135 +68,47 @@ module Mysql
       return if records.nil? # skip
       records = [records] unless records.kind_of?(Array)
 
-      FlydataJsonHandlerFactory.create(records, record, @context, type, opt[:increment_table_rev])
-    end
+      table = records.first[TABLE_NAME] || record['table_name']
+      raise "Missing table name. #{record}" if table.to_s.empty?
+      return unless acceptable_table?(record, table) && acceptable_event?(type, table)
 
-    def check_empty_binlog
-      #Log one warning per consecutive records that have empty binlog filename
-      if @context.current_binlog_file.to_s.empty?
-        if @first_empty_binlog
-          $log.warn "Binlog file name is empty. Rotate event not received!"
-          @first_empty_binlog = false
-        end
-      else
-        @first_empty_binlog = true
-      end
-    end
-  end
-  class FlydataJsonHandlerFactory
-
-    def self.create(records, record, context, type, increment_table_rev)
-      table = records.first[:table_name] || record['table_name']
-      database = records.first[:db_name] || record['db_name']
-
-      case table
-      when nil
-        DatabaseFlydataJsonHandler.new(records, record, context, type)
-      else
-        TableFlydataJsonHandler.new(records, record, context, type, increment_table_rev)
-      end
-    end
+      table_rev = @context.table_revs[table]
+      position = record['next_position'] - record['event_length']
 
-    class FlydataJsonHandler
-      TABLE_NAME = :table_name # A Flydata JSON tag to specify a table name
-      TYPE = :type
-      SEQ = :seq
-      RESPECT_ORDER = :respect_order
-      SRC_POS = :src_pos
-      TABLE_REV = :table_rev
-      V = :v  # FlyData record format version
-      DB_NAME = :db_name
-
-      def initialize(records, record, context, type, increment_table_rev=nil)
-        @context = context
-      end
-
-      def emit(timestamp, row)
-      end
-    end
-
-    class TableFlydataJsonHandler < FlydataJsonHandler
-      def initialize(records, record, context, type, increment_table_rev=nil)
-        super
-        table = records.first[TABLE_NAME] || record['table_name']
-        raise "Missing table name. #{record}" if table.to_s.empty?
-
-        return unless acceptable_event?(type, table)
-
-        table_rev = @context.table_revs[table]
-        position = record['next_position'] - record['event_length']
-
-        # Add common information to each record
-        records.each do |r|
-          if increment_table_rev
-            table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
-            @context.table_revs[table] = table_rev
-          end
-          r[TYPE] = type
-          r[RESPECT_ORDER] = true
-          r[TABLE_NAME] = table
-          r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
-          r[TABLE_REV] = table_rev
-          r[V] = FlydataCore::Record::V2
-        end
-
-        # Use binlog's timestamp
-        timestamp = record["timestamp"].to_i
-        records.each do |row|
-          emit(timestamp, row)
+      # Add common information to each record
+      records.each do |r|
+        if opt[:increment_table_rev]
+          table_rev = @context.sync_fm.increment_table_rev(table, table_rev)
+          @context.table_revs[table] = table_rev
         end
+        r[TYPE] = type
+        r[RESPECT_ORDER] = true
+        r[TABLE_NAME] = table
+        r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+        r[TABLE_REV] = table_rev
+        r[V] = FlydataCore::Record::V2
       end
 
-      def emit(timestamp, row)
+      # Use binlog's timestamp
+      timestamp = record["timestamp"].to_i
+      records.each do |row|
         @context.sync_fm.increment_and_save_table_position(row[TABLE_NAME]) do |seq|
           row[SEQ] = seq
           Fluent::Engine.emit(@context.tag, timestamp, row)
         end
       end
-
-      private
-      def acceptable_event?(type, table)
-        @context.omit_events[table].nil? || !@context.omit_events[table].include?(type)
-      end
-
     end
 
-    class DatabaseFlydataJsonHandler < FlydataJsonHandler
-      def initialize(records, record, context, type, increment_table_rev=nil)
-        super
-        database = records.first[DB_NAME] || record['db_name']
-
-        return unless acceptable_event?(type)
-
-        position = record['next_position'] - record['event_length']
-
-        # Add common information to each record
-        records.each do |r|
-          r[TYPE] = type
-          r[RESPECT_ORDER] = true
-          r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
-          r[V] = FlydataCore::Record::V2
-        end
-
-        # Use binlog's timestamp
-        timestamp = record["timestamp"].to_i
-        records.each do |row|
-          emit(timestamp, row)
+    def check_empty_binlog
+      #Log one warning per consecutive records that have empty binlog filename
+      if @context.current_binlog_file.to_s.empty?
+        if @first_empty_binlog
+          $log.warn "Binlog file name is empty. Rotate event not received!"
+          @first_empty_binlog = false
         end
+      else
+        @first_empty_binlog = true
       end
-
-      def emit(timestamp, row)
-        Fluent::Engine.emit(@context.tag, timestamp, row)
-      end
-
-      private
-      #TODO: No support on DS for database records currently so no record should be emitted
-      def acceptable_event?(type)
-        false
-      end
-
     end
-
   end
-
 end

data/lib/flydata/fluent-plugins/mysql/ddl_query_handler.rb

@@ -25,4 +25,40 @@ class DdlQueryHandler < BinlogQueryHandler
   end
 end
 
+class TableDdlQueryHandler < DdlQueryHandler
+end
+
+class DatabaseDdlQueryHandler < DdlQueryHandler
+  def emit_record(type, record)
+    return unless acceptable_db?(record)
+
+    check_empty_binlog
+
+    opt = {}
+    records = yield(opt) # The block may set options as necessary
+    return if records.nil? # skip
+    records = [records] unless records.kind_of?(Array)
+
+    database = records.first[DB_NAME] || record['db_name']
+
+    return unless acceptable_event?(type)
+
+    position = record['next_position'] - record['event_length']
+
+    # Add common information to each record
+    records.each do |r|
+      r[TYPE] = type
+      r[RESPECT_ORDER] = true
+      r[SRC_POS] = "#{@context.current_binlog_file}\t#{position}"
+      r[V] = FlydataCore::Record::V2
+    end
+
+    # Use binlog's timestamp
+    timestamp = record["timestamp"].to_i
+    records.each do |row|
+      Fluent::Engine.emit(@context.tag, timestamp, row)
+    end
+  end
+end
+
 end

data/lib/flydata/fluent-plugins/mysql/drop_database_query_handler.rb

@@ -1,8 +1,7 @@
-require 'flydata/parser_provider'
 require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 
 module Mysql
-  class DropDatabaseQueryHandler < DdlQueryHandler
+  class DropDatabaseQueryHandler < DatabaseDdlQueryHandler
     # For MySQL, database and schema are exchangable
     PATTERN = /^DROP DATABASE|SCHEMA/i
 

data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb

@@ -2,7 +2,7 @@ require 'flydata/parser_provider'
 require 'flydata/fluent-plugins/mysql/ddl_query_handler'
 
 module Mysql
-  class TruncateTableQueryHandler < DdlQueryHandler
+  class TruncateTableQueryHandler < TableDdlQueryHandler
     PATTERN = /^TRUNCATE/i
 
     def initialize(context)
@@ -22,4 +22,4 @@ module Mysql
       end
     end
   end
-end    
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flydata
 version: !ruby/object:Gem::Version
-  version: 0.5.11
+  version: 0.5.12
 platform: ruby
 authors:
 - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client