flydata 0.6.14 → 0.7.0

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync.rb

@@ -1,4 +1,5 @@
 require 'flydata/sync_file_manager'
+require 'flydata/source'
 require 'flydata-core/fluent/config_helper'
 require 'flydata/fluent-plugins/flydata_plugin_ext/flush_support'
 require 'flydata/fluent-plugins/flydata_plugin_ext/transaction_support'
@@ -11,33 +12,35 @@ module Fluent
         include FlushSupport
         prepend TransactionSupport
 
+        config_param :data_entry_name, :string, default: nil  # data entry name
+        config_param :data_entry_type, :string, default: nil  # data entry type
         config_param :tables, :string
         config_param :tables_append_only, :string
-
-        # binlog plugin
         config_param :tag, :string
-        config_param :position_file, :string, default: 'position.log'
+        config_param :position_file, :string, default: 'position.binlog.pos'
       end
     end
 
     def configure(conf)
       super
 
-      @binlog_position_file = self.class::BINLOG_POSITION_FILE_CLASS.new(@position_file)
-      unless @binlog_position_file.exists?
-        raise "No position file(#{@binlog_position_file.path}). Initial synchronization is required before starting."
+      @source_position_file = self.class::SOURCE_POSITION_FILE_CLASS.new(@position_file)
+      unless @source_position_file.exists?
+        raise "No position file(#{@source_position_file.path}). Initial synchronization is required before starting."
       end
 
-      @sync_fm = Flydata::SyncFileManager.new(nil) # Passing nil for data_entry as this class does not use methods which require data_entry
+      load_custom_conf # preference module needs to be included
+      @data_entry = build_data_entry
+      @source = Flydata::Source.create(@data_entry)
+
+      @sync_fm = Flydata::SyncFileManager.new(@data_entry, @source)
       sent_position_file_path = @sync_fm.sent_source_pos_path(@position_file)
-      @sent_position_file = self.class::BINLOG_POSITION_FILE_CLASS.new(sent_position_file_path)
+      @sent_position_file = self.class::SOURCE_POSITION_FILE_CLASS.new(sent_position_file_path)
 
       # Create positions dir
       positions_path = @sync_fm.table_positions_dir_path
       Dir.mkdir positions_path unless File.exists? positions_path
 
-      load_custom_conf
-
       @tables = @tables.split(/(?:\s*,\s*|\s+)/)
       @omit_events = Hash.new
       @tables_append_only.split(/(?:\s*,\s*|\s+)/).each do |table|
@@ -49,6 +52,28 @@ module Fluent
       new_tables = @sync_fm.get_new_table_list(@tables, "pos")
       @tables -= new_tables
       $log.info "Not watching these tables: #{new_tables.join(", ")}"
+
+      # Set table revisions
+      @table_revs = @tables.inject({}) do |h, table_name|
+        h[table_name] = @sync_fm.table_rev(table_name)
+        h
+      end
+
+      $log.info("Source position - resume_pos:'#{@source_position_file.read rescue IOError}' " +
+                "sent_pos:'#{@sent_position_file.read rescue nil}'")
+    end
+
+    def build_data_entry(base_object = {})
+      de = base_object || {}
+      de['name'] = @data_entry_name
+      de['type'] = @data_entry_type
+      de.merge!(@data_entry_preferences || {})
+      de
     end
   end
+
+  def shutdown
+    super
+    @sync_fm.close if @sync_fm
+  end
 end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based.rb

@@ -0,0 +1,29 @@
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync'
+
+module Fluent
+
+  module FlydataSyncDiffBased
+    def self.include_modules(base)
+      base.send(:include, FlydataSync)
+      base.send(:include, self)
+    end
+
+    def self.included(base)
+      base.class_eval do
+        config_param :fetch_interval, :time, default: 60   # 1 minute
+        config_param :retry_interval, :time, default: 30   # 30 seconds
+        config_param :emit_chunk_limit, :size, default: 64 * (1024 ** 2)   # 64MB, should be equal or less than buffer_chunk_limit
+      end
+    end
+
+    def configure(conf)
+      super
+      @table_src_pos_files = @tables.inject({}) do |ret, table_name|
+        ret[table_name.to_sym] = self.class::SOURCE_POSITION_FILE_CLASS.new(
+          @sync_fm.table_source_pos_file_path(table_name)
+        )
+        ret
+      end
+    end
+  end
+end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based.rb

@@ -0,0 +1,26 @@
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_diff_based'
+
+module Fluent
+
+  module FlydataSyncQueryBased
+    def self.include_modules(base)
+      FlydataSyncDiffBased.include_modules(base)
+      base.send(:include, self)
+    end
+
+    def self.included(base)
+      base.class_eval do
+        config_param :host, :string
+        config_param :port, :integer
+        config_param :username, :string
+        config_param :password, :string
+        config_param :database, :string
+        config_param :schema, :string, default: nil
+      end
+    end
+
+    def configure(conf)
+      super
+    end
+  end
+end

data/lib/flydata/fluent-plugins/flydata_plugin_ext/preference.rb

@@ -1,14 +1,15 @@
 require 'yaml'
 require 'flydata/util/encryptor'
 require 'flydata/source_mysql/data_entry'
+require 'flydata/source_postgresql/data_entry'
 
 module Fluent
   module DataEntryPreferenceConfigurable
-    @@supported_custom_confs = Hash.new{|h,k| h[k] = {}}
 
     def self.included(base)
-      base.extend ClassMethods
-      base.class_eval do
+      base.instance_eval do
+        extend ClassMethods
+        @supported_custom_confs = Hash.new{|h,k| h[k] = {}}
         config_param :custom_conf_path, :string, default: nil
         config_param :key, :string, default: nil
       end
@@ -20,10 +21,10 @@ module Fluent
                     else
                       nil
                     end
-      @@supported_custom_confs.each do |type, settings|
+      self.class.instance_variable_get(:@supported_custom_confs).each do |type, settings|
         settings.each do |key, option|
           apply_custom_conf(custom_conf, key, type, option) if custom_conf
-          apply_custom_option(key, option)
+          apply_custom_option(key, option, type)
         end
       end
     end
@@ -35,7 +36,7 @@ module Fluent
       end
     end
 
-    def apply_custom_option(key, option)
+    def apply_custom_option(key, option, type)
       var_name = option[:var_name] || key
       original_value = instance_variable_get(:"@#{var_name}")
       value = original_value
@@ -46,11 +47,21 @@ module Fluent
       if original_value != value
         instance_variable_set(:"@#{var_name}", value)
       end
+      @data_entry_preferences ||= Hash.new{|h,k| h[k] = {}}
+      @data_entry_preferences[type.to_s][var_name.to_s] = value
     end
 
     module ClassMethods
+      def register_config_params(config_params)
+        config_params.each do |type, custom_conf|
+          custom_conf.each do |key, option|
+            custom_config_param key, type, option
+          end
+        end
+      end
+
       def custom_config_param(key, type, option = {})
-        conf = class_variable_get(:@@supported_custom_confs)
+        conf = @supported_custom_confs
         conf[type.to_sym][key.to_sym] = option
       end
     end
@@ -70,13 +81,18 @@ module Fluent
 
   module MysqlBinlogFlydataInputPreference
     def self.included(base)
-      base.class_eval do
+      base.instance_eval do
         include DataEntryPreferenceConfigurable
-        Flydata::SourceMysql::DataEntry::CONFIG_PARAMS.each do |type, custom_conf|
-          custom_conf.each do |key, option|
-            custom_config_param key, type, option
-          end
-        end
+        register_config_params Flydata::SourceMysql::DataEntry::CONFIG_PARAMS
+      end
+    end
+  end
+
+  module PostgresqlQueryBasedSyncPreference
+    def self.included(base)
+      base.instance_eval do
+        include DataEntryPreferenceConfigurable
+        register_config_params Flydata::SourcePostgresql::DataEntry::CONFIG_PARAMS
       end
     end
   end

data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb

@@ -11,10 +11,10 @@ require 'flydata-core/mysql/command_generator'
 require 'flydata/fluent-plugins/flydata_plugin_ext/preference'
 require 'flydata/fluent-plugins/flydata_plugin_ext/idle_event_detector'
 require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync'
-require 'flydata/source_mysql/plugin_support/binlog_position_file'
+require 'flydata/source_mysql/plugin_support/source_position_file'
 require 'flydata/source_mysql/plugin_support/binlog_record_dispatcher'
 require 'flydata/source_mysql/plugin_support/context'
-require 'flydata/source_mysql/plugin_support/table_meta'
+require 'flydata/source_mysql/table_meta'
 require 'flydata/source_mysql/table_ddl'
 require 'flydata-core/fluent/config_helper'
 require 'flydata-core/mysql/ssl'
@@ -26,7 +26,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
   Plugin.register_input('mysql_binlog_flydata', self)
 
-  BINLOG_POSITION_FILE_CLASS = Flydata::SourceMysql::PluginSupport::BinLogPositionFile
+  SOURCE_POSITION_FILE_CLASS = Flydata::SourceMysql::PluginSupport::SourcePositionFile
 
   config_param :database, :string
   config_param :initial_idle_interval, :integer, :default => 30     # 30 sec
@@ -56,7 +56,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
 
     # SSL configuration
     unless @ssl_ca_content.to_s.strip.empty?
-      @ssl_ca_path = @sync_fm.ssl_ca_path(@binlog_position_file.path)
+      @ssl_ca_path = @sync_fm.ssl_ca_path(@source_position_file.path)
       @sync_fm.save_ssl_ca(FlydataCore::Fluent::ConfigHelper.unescape_conf(@ssl_ca_content), @ssl_ca_path)
     end
 
@@ -75,18 +75,13 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
       #exit 1   # causes retry loop
     end
 
-    table_meta = Flydata::SourceMysql::PluginSupport::TableMeta.new(@db_opts.merge(tables: @tables))
-
-    table_revs = tables.inject({}) do |h, table_name|
-      h[table_name] = @sync_fm.table_rev(table_name)
-      h
-    end
+    table_meta = Flydata::SourceMysql::TableMeta.new(@db_opts.merge(tables: @tables))
 
     # Set context
     @context = Flydata::SourceMysql::PluginSupport::Context.new(
       database: @database, tables: @tables,
       tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
-      table_meta: table_meta, table_revs: table_revs,
+      table_meta: table_meta, table_revs: @table_revs, dbconf: @db_opts
     )
     @record_dispatcher = Flydata::SourceMysql::PluginSupport::FlydataBinlogRecordDispatcher.new(@context)
     @idle_event_detector = IdleEventDetector.new(@initial_idle_interval, @continuous_idle_interval, @check_interval, @idle_timeout)
@@ -128,9 +123,9 @@ EOS
   def run
     return if process_aborted?
 
-    @context.table_meta.update
+    @context.table_meta.reload
     Flydata::SourceMysql::TableDdl.migrate_tables(@context.tables, @db_opts,
-                                            @context.sync_fm, @binlog_position_file.path,
+                                            @context.sync_fm, @source_position_file.path,
                                             @context) do |event|
       @record_dispatcher.dispatch(event)
     end
@@ -142,7 +137,7 @@ EOS
       begin
         begin
           start_kodama(mysql_url) do |c|
-            c.binlog_position_file = @binlog_position_file.path
+            c.binlog_position_file = @source_position_file.path
             if @ssl_ca_path.to_s != '' && c.respond_to?(:ssl_ca=)
               $log.info "SSL is enabled. (ssl_ca: #{@ssl_ca_path})"
               c.ssl_ca = @ssl_ca_path
@@ -157,9 +152,6 @@ EOS
               c.sent_binlog_position_file = @sent_position_file.path
             end
 
-            $log.info("Binlog position - resume_pos:'#{@binlog_position_file.read rescue IOError}' " +
-                      "sent_pos:'#{@sent_position_file.read rescue IOError}'")
-
             c.connection_retry_limit = @retry_limit
             c.connection_retry_wait = @retry_wait
             c.log_level = @log_level.to_sym
@@ -215,7 +207,7 @@ EOS
     @idle_event_detector.notify
     @record_dispatcher.dispatch(event)
   rescue Exception => e
-    position = @binlog_position_file.read
+    position = @source_position_file.read
     $log.error "error occurred while processing #{event.event_type} event at #{position}, event_class:#{event.class.to_s}\n#{e.message}\n#{$!.backtrace.join("\n")}"
     raise
   end

data/lib/flydata/fluent-plugins/in_postgresql_query_based_flydata.rb

@@ -0,0 +1,64 @@
+require_relative 'flydata_plugin_ext/base'
+
+require 'flydata/fluent-plugins/flydata_plugin_ext/preference'
+require 'flydata/fluent-plugins/flydata_plugin_ext/flydata_sync_query_based'
+require 'flydata/source_postgresql/plugin_support/source_position_file'
+require 'flydata/source_postgresql/plugin_support/context'
+require 'flydata/source_postgresql/table_meta'
+require 'flydata/source_postgresql/query_based_sync/client'
+require 'flydata-core/postgresql/config'
+
+module Fluent
+
+class PostgresqlQueryBasedFlydataInput < Input
+  include PostgresqlQueryBasedSyncPreference
+  FlydataSyncQueryBased.include_modules(self)
+
+  Plugin.register_input('postgresql_query_based_sync_flydata', self)
+
+  SOURCE_POSITION_FILE_CLASS = Flydata::SourcePostgresql::PluginSupport::SourcePositionFile
+
+  def configure(conf)
+    super
+    @dbconf = FlydataCore::Postgresql::Config.opts_for_pg(@data_entry['postgresql_data_entry_preference'])
+    $log.info "postgresql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{@tables_append_only}\""
+
+    @table_meta = Flydata::SourcePostgresql::TableMeta.new(
+      @dbconf, @tables, @schema)
+
+    @context = Flydata::SourcePostgresql::PluginSupport::Context.new(
+      database: @database, tables: @tables,
+      tag: @tag, sync_fm: @sync_fm, omit_events: @omit_events,
+      table_revs: @table_revs, dbconf: @dbconf,
+      cur_src_pos_file: @source_position_file,
+      table_src_pos_files: @table_src_pos_files,
+      table_meta: @table_meta,
+      params: {
+        fetch_interval: @fetch_interval,
+        retry_interval: @retry_interval,
+        emit_chunk_limit: @emit_chunk_limit,
+      },
+    )
+
+    @client = Flydata::SourcePostgresql::QueryBasedSync::Client.new(@context)
+  end
+
+  def start
+   super
+   @thread = Thread.new(&method(:run))
+  end
+
+  def run
+    @client.start
+  end
+
+  def shutdown
+    if @thread and @thread.alive?
+      @client.stop_request
+      @thread.join
+    end
+    super
+  end
+end
+
+end

data/lib/flydata/helpers.rb

@@ -14,9 +14,7 @@ Usage: flydata COMMAND
   restart                  # restart flydata process
   status                   # check flydata process
   version                  # show flydata version
-  conf                     # show configuration
-  sync [tables]            # initial sync (only for mysql database)
-  sync:generate_table_ddl [tables | -all-tables] # Generate CREATE TABLE script for Redshift
+  sync:generate_table_ddl  # Generate CREATE TABLE script for Redshift
   sync:reset [tables]      # reset sync and stop flydata process
   sync:flush               # flush your current buffer and stop flydata process
       EOM

data/lib/flydata/plugin_support/context.rb

@@ -28,8 +28,16 @@ module PluginSupport
       nil
     end
 
-    register_mandatory_opts :tables, :tag, :sync_fm, :omit_events, :table_revs
-    register_optional_opts :current_binlog_file
+    register_mandatory_opts :tables,
+                            :tag,
+                            :sync_fm,
+                            :omit_events,
+                            :table_revs,
+                            :table_meta
+
+    register_optional_opts  :cur_src_pos_file,
+                            :table_src_pos_files,
+                            :params
 
     def initialize(opts)
       missing_opts = self.class.mandatory_opts - opts.keys
@@ -41,6 +49,10 @@ module PluginSupport
         self.instance_variable_set(:"@#{k}", v)
       end
     end
+
+    def source_pos_class
+      self.class::SOURCE_POS_CLASS
+    end
   end
 end
 end

data/lib/flydata/plugin_support/source_position_file.rb

@@ -0,0 +1,35 @@
+module Flydata
+
+module PluginSupport
+  class SourcePositionFile
+    attr_accessor :path
+
+    def initialize(path)
+      @path = path
+    end
+
+    def exists?
+      File.exists?(@path)
+    end
+
+    def read
+      File.open(@path) {|f| f.read }
+    end
+
+    def pos
+      if exists?
+        self.class::SOURCE_POS_CLASS.load(read)
+      else
+        nil
+      end
+    end
+
+    def save(*args)
+      c = self.class::SOURCE_POS_CLASS.new(*args)
+      File.open(@path, 'w') {|f| f.write(c.to_s) }
+      c
+    end
+  end
+end
+
+end