flydata 0.6.4 → 0.6.5

data/lib/flydata/fluent-plugins/mysql/dml_record_handler.rb

@@ -91,10 +91,15 @@ module Mysql
             # It's a signless integer.
             intval = record[row_type][position]
             next unless (intval.kind_of?(Numeric) || intval =~ /^-?[\d]+$/)
-            width = INTEGER_TYPES[column_type] * 2 # * 2 because a single byte requires two characters (e.g. ff)
-            signless_val = SIGNLESS_INTEGER_PREFIX
-            signless_val += sprintf("%0#{width}x", intval).gsub(/\.\.f/, 'f' * width).slice(-width..-1)
-            record[row_type][position] = signless_val
+            begin
+              width = INTEGER_TYPES[column_type] * 2 # * 2 because a single byte requires two characters (e.g. ff)
+              signless_val = SIGNLESS_INTEGER_PREFIX
+              signless_val += sprintf("%0#{width}x", intval).gsub(/\.\.f/, 'f' * width).slice(-width..-1)
+              record[row_type][position] = signless_val
+            rescue => e
+              $log.debug "failed to encode signless integer. - exception:`#{e.class.to_s}` record:`#{record[row_type][position]}` column_type:`#{column_type}` width:`#{width}` intval:`#{intval}`"
+              raise
+            end
           end
         end
       end

data/lib/flydata/helper/server.rb

@@ -1,5 +1,12 @@
 require 'serverengine'
 require 'flydata-core/logger'
+require 'flydata/helper/worker'
+
+# Require all helper files -
+lib_dir = File.absolute_path(File.join(__FILE__, '../../..'))
+FileUtils.cd(lib_dir) do
+  Dir["flydata/helper/**/*.rb"].each { |file| require file }
+end
 
 module Flydata
   module Helper

data/lib/flydata/preference/data_entry_preference.rb

@@ -1,17 +1,14 @@
 require 'fileutils'
-require 'flydata/fluent-plugins/preference'
+require 'flydata/fluent-plugins/flydata_plugin_ext/preference'
 
 module Flydata
   module Preference
     class DataEntryPreference
       CONFS_HOME = File.join(FLYDATA_HOME, 'confs')
-      CUSTOM_CONFIG_PARAMS = {
-        RedshiftMysqlDataEntry: ::Fluent::MysqlBinlogFlydataInputPreference::CUSTOM_CONFIG_PARAMS
-      }
 
       class << self
         # data_entry must be hash
-        def load_conf(data_entry)
+        def load_conf(data_entry, source)
           path = conf_path(data_entry)
           raise "Conf file does not exist. path:#{path}" unless File.exists?(path)
           custom_conf = YAML::load(File.open(path, 'r'))
@@ -25,12 +22,12 @@ module Flydata
               data_entry[k] = v
             end
           end
-          filter_data_entry(data_entry)
+          filter_data_entry(data_entry, source)
           data_entry
         end
 
-        def filter_data_entry(de)
-          configurable_params = CUSTOM_CONFIG_PARAMS[de['type'].to_sym]
+        def filter_data_entry(de, source)
+          configurable_params = source.data_entry.config_params
           return de if configurable_params.nil? or configurable_params.empty?
 
           configurable_params.each do |pref_name, param_info|
@@ -87,11 +84,6 @@ module Flydata
           type = ActiveSupport::Inflector.underscore(data_entry['type'])
           "#{type}.conf.tmpl"
         end
-
-        def create(type)
-          custom_config_params = CUSTOM_CONFIG_PARAMS[type]
-          return nil unless custom_config_params
-        end
       end
     end
   end

data/lib/flydata/source.rb

@@ -22,7 +22,7 @@ module Source
     "RedshiftMysqlDataEntry" => :source_mysql,
     "RedshiftFileDataEntry" => :source_file,
     "FileDataEntry" => :source_file,
-    "RedshiftPostgresDataEntry" => :source_postgres,
+    "RedshiftPostgresqlDataEntry" => :source_postgresql,
     "RedshiftZendeskDataEntry" => :source_zendesk,
   }
   def self.component_class_for(component_sym, de)

data/lib/flydata/source/data_entry.rb

@@ -0,0 +1,29 @@
+require 'flydata/source'
+require 'flydata/source/component'
+require 'flydata/source/errors'
+
+module Flydata
+module Source
+
+class DataEntry < Component
+  def self.inherited(child_class)
+    Source.register(child_class, self)
+  end
+
+  # Public Interface CONFIG_PARAMS
+  #
+  # Has the definition of data entry parameters specific to the data source.
+  # Override
+  CONFIG_PARAMS = {
+    sourcename_data_entry_preference: {
+      source_specific_param1: { key: "subclass must define" },
+    }
+  }
+
+  def config_params
+    self.class::CONFIG_PARAMS
+  end
+end
+
+end
+end

data/lib/flydata/source/sync.rb

@@ -50,6 +50,25 @@ class Sync < Component
   def forwarder
     raise UnsupportedSourceError, "subclass must implement"
   end
+
+  private
+
+  def setup_table_prefs(prefs)
+    if prefs['tables_append_only']
+      prefs['tables_append_only'] =
+        prefs['tables_append_only'].split(/(?:\s*,\s*|\s+)/).uniq
+      prefs['tables'] = (prefs['tables'].split(/(?:\s*,\s*|\s+)/) +
+                         prefs['tables_append_only']).uniq
+    else
+      prefs['tables'] = prefs['tables'].split(/(?:\s*,\s*|\s+)/).uniq
+    end
+    prefs['invalid_tables'] =
+      prefs['invalid_tables'].kind_of?(String) ?
+        prefs['invalid_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
+    prefs['new_tables'] =
+      prefs['new_tables'].kind_of?(String) ?
+        prefs['new_tables'].split(/(?:\s*,\s*|\s+)/).uniq : []
+  end
 end
 
 end

data/lib/flydata/source/sync_generate_table_ddl.rb

@@ -26,15 +26,55 @@ class SyncGenerateTableDdl < Component
     raise UnsupportedSourceError,  "subclass must implement"
   end
 
-  # Public Interface: Generate FlyData table definitions for given tables
+  def generate_flydata_tabledef(tables, options)
+    prefs = data_entry_prefs
+    options = options.merge(prefs)
+    flydata_tabledefs = []
+    error_list = []
+    missing_tables = each_source_tabledef(tables, options) do |source_tabledef, error|
+      if error
+        error_list << error.err_hash
+        next
+      end
+      flydata_tabledefs << source_tabledef.to_flydata_tabledef
+    end
+    if missing_tables
+      missing_tables.each {|missing_table| error_list << { error: "table does not exist in the #{data_source_type_display_name}", table: missing_table } }
+    end
+ 
+    [flydata_tabledefs, error_list]
+  end
+
+  private
+
+  # Returns the namne of the data source type.  The name will be used in an error message.
+  def data_source_type_display_name
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Returns a data entry preference hash
+  def data_entry_prefs
+    raise UnsupportedSourceError,  "subclass must implement"
+  end
+
+  # Calls `block` with the source tabledef or error for each table.
   #
-  # tables - An array of table names
-  # options - A hash of options
+  #   tables:  An array of table names
   #
-  # Returns flydata_tabledefs, errors
-  #  flydata_tablesdefs - An array of FlyData tabledefs
-  #  errors - An array of error hashes for tables whose tabledef generation failed
-  def generate_flydata_tabledef(tables, options)
+  #   options: A hash including options.  It includes the contents of
+  #   `data_entry_prefs` and :skip_primary_key_check.  When the option is true,
+  #   the method must call the `block` for a table which is missing the primary
+  #   key.  Otherwise, it should call the `block` with an error.
+  #
+  #   block: A callback block called for each table with the following
+  #   arguments:
+  #     source_tabledef: A TableDef object of the source (e.g. MysqlTableDef)
+  #     The value will be nil if it failed to create the object.
+  #     error: A FlydataCore::TableDefError object.  If no error ocurred,
+  #     this will be nil.
+  #
+  # Returns an array of tables which do not exist in the source
+  def each_source_tabledef(tables, options, &block)
     raise UnsupportedSourceError,  "subclass must implement"
   end
 end

data/lib/flydata/source_mysql/data_entry.rb

@@ -0,0 +1,22 @@
+require 'flydata/source/data_entry'
+
+module Flydata
+module SourceMysql
+
+class DataEntry < Source::DataEntry
+  CONFIG_PARAMS = {
+    mysql_data_entry_preference: {
+      database: {},
+      tables: {},
+      tables_append_only: {},
+      host: {},
+      username: {},
+      password: {encrypted: true},
+      ssl_ca_content: {},
+      ssl_cipher: {},
+    },
+  }
+end
+
+end
+end

data/lib/flydata/source_mysql/parser/dump_parser.rb

@@ -478,7 +478,7 @@ EOS
           def parse(line)
             start_ruby_prof
             bench_start_time = Time.now
-            _parse2(line)
+            _parse(line)
           ensure
             stop_ruby_prof
             if ENV['FLYDATA_BENCHMARK']

data/lib/flydata/source_mysql/parser/mysql_alter_table.treetop

@@ -745,7 +745,7 @@ grammar MysqlAlterTable
   rule data_type
     data_type_name meta_text unsigned zerofill {
       def data_type
-        meta = (meta_text.text_value.size > 1) ? meta_text.text_value : ''
+        meta = (meta_text.text_value.size > 1) ? meta_text.text_value.strip : ''
         type = data_type_name.text_value.downcase + meta
         type = FlydataCore::TableDef::MysqlTableDef.convert_to_flydata_type(type)
         type << " unsigned" if !unsigned.terminal?
@@ -769,7 +769,7 @@ grammar MysqlAlterTable
   end
 
   rule meta_text
-    '(' meta_value ')' / ''
+    ( nsp '(' nsp meta_value nsp ')' )?
   end
 
   rule meta_value
@@ -1154,12 +1154,17 @@ grammar MysqlAlterTable
   end
 
   rule value
-    quoted_value { def raw_value; text_raw_value; end }
+    single_quoted_value { def raw_value; text_raw_value; end }
     / double_quoted_value { def raw_value; text_raw_value; end }
     / ident_sym { def raw_value; text_value; end }
   end
 
   rule quoted_value
+    single_quoted_value { def raw_value; text_raw_value; end }
+    / double_quoted_value { def raw_value; text_raw_value; end }
+  end
+
+  rule single_quoted_value
     "'" text "'" { def text_raw_value; text.text_value; end }
   end
 

data/lib/flydata/source_mysql/sync.rb

@@ -7,14 +7,7 @@ class Sync < Source::Sync
   def setup
     mp = de['mysql_data_entry_preference']
 
-    if mp['tables_append_only']
-      mp['tables_append_only'] = mp['tables_append_only'].split(",").uniq
-      mp['tables'] = (mp['tables'].split(",") + mp['tables_append_only']).uniq
-    else
-      mp['tables'] = mp['tables'].split(",").uniq
-    end
-    mp['invalid_tables'] = mp['invalid_tables'].kind_of?(String) ? mp['invalid_tables'].split(",").uniq : []
-    mp['new_tables'] = mp['new_tables'].kind_of?(String) ? mp['new_tables'].split(",").uniq : []
+    setup_table_prefs(mp)
 
     unless mp['ssl_ca_content'].to_s.strip.empty?
       sync_fm = SyncFileManager.new(de)

data/lib/flydata/source_mysql/sync_generate_table_ddl.rb

@@ -16,23 +16,18 @@ class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
     end
   end
 
-  def generate_flydata_tabledef(tables, options)
-    mp = de['mysql_data_entry_preference']
-    options = options.merge(mp)
-    flydata_tabledefs = []
-    error_list = []
-    missing_tables = FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, options) do |mysql_tabledef, error|
-      if error
-        error_list << error.err_hash
-        next
-      end
-      flydata_tabledefs << mysql_tabledef.to_flydata_tabledef
-    end
-    if missing_tables
-      missing_tables.each {|missing_table| error_list << { error: 'table does not exist in the MySQL database', table: missing_table } }
-    end
+  private
+
+  def data_source_type_display_name
+    "MySQL database"
+  end
+
+  def data_entry_prefs
+    de['mysql_data_entry_preference']
+  end
 
-    [flydata_tabledefs, error_list]
+  def each_source_tabledef(tables, options, &block)
+    FlydataCore::Mysql::CommandGenerator.each_mysql_tabledef(tables, options, &block) 
   end
 end
 

data/lib/flydata/source_postgresql/data_entry.rb

@@ -0,0 +1,21 @@
+require 'flydata/source/data_entry'
+
+module Flydata
+module SourcePostgresql
+
+class DataEntry < Source::DataEntry
+  CONFIG_PARAMS = {
+    postgresql_data_entry_preference: {
+      database: {},
+      tables: {},
+      tables_append_only: {},
+      host: {},
+      username: {},
+      password: {encrypted: true},
+      schema: {},
+    }
+  }
+end
+
+end
+end

data/lib/flydata/source_postgresql/sync.rb

@@ -0,0 +1,29 @@
+require 'flydata/source/sync'
+
+module Flydata
+module SourcePostgresql
+
+class Sync < Source::Sync
+  def setup
+    setup_table_prefs(de['postgresql_data_entry_preference'])
+  end
+
+  def supported?
+    true
+  end
+
+  def table_lists
+    de['postgresql_data_entry_preference'].select {|key, value| %w(tables new_tables invalid_tables tables_append_only).include?(key)}
+  end
+
+  def data_servers
+    de['postgresql_data_entry_preference']['data_servers']
+  end
+
+  def forwarder
+    de['postgresql_data_entry_preference']['forwarder']
+  end
+end
+
+end
+end

data/lib/flydata/source_postgresql/sync_generate_table_ddl.rb

@@ -0,0 +1,126 @@
+require 'flydata/source/sync_generate_table_ddl'
+require 'flydata-core/table_def/postgresql_table_def'
+require 'pg'
+
+module Flydata
+module SourcePostgresql
+
+class SyncGenerateTableDdl < Source::SyncGenerateTableDdl
+  def run_compatibility_check
+    # do nothing for now
+  end
+
+  def data_source_type_display_name
+    "PostgreSQL database"
+  end
+
+  def data_entry_prefs
+    de['postgresql_data_entry_preference']
+  end
+
+  def each_source_tabledef(tables, options, &block)
+    pg_opts = {
+      host: options['host'],
+      port: options['port'],
+      dbname: options['database'],
+      user: options['username'],
+      password: options['password'],
+      sslmode: :prefer,
+    }
+    # PostgreSQL options.
+    tables = tables.clone
+    missing_tables = []
+    begin
+      if tables.to_s == '' || tables.to_s == '[]'
+        raise ArgumentError, "tables is nil or empty"
+      end
+      _each_tabledef(tables, options, pg_opts, &block)
+    rescue TableMissingError => e
+      tables.delete e.table
+      missing_tables << e.table
+      return missing_tables if tables.empty?
+      retry
+    end
+    missing_tables
+  end
+
+  private
+
+  COLUMNS_QUERY = <<EOS
+SELECT c.table_name, c.column_name, c.data_type, c.character_octet_length,
+  c.numeric_precision, c.numeric_scale, c.is_nullable, c.column_default, i.indisprimary AS is_primary
+FROM pg_index i
+JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
+RIGHT JOIN
+(SELECT (table_catalog ||'.'|| table_schema ||'.'|| table_name)::regclass AS regid, *
+FROM information_schema.columns) c
+ ON i.indrelid = c.regid AND a.attname = c.column_name
+WHERE c.table_schema = $1 AND c.table_name IN (%s)
+ORDER BY c.table_name, c.ordinal_position;
+EOS
+  TABLE_PLACEHOLDER_START_NUM = 2 # because $1 is used by table_schema
+
+  def _each_tabledef(tables, options, pg_opts, &block)
+    cli = PG::Connection.new(pg_opts)
+
+    # TODO call the query for every 50 tables
+    placeholders = placeholder_string(tables.size, TABLE_PLACEHOLDER_START_NUM)
+    query = COLUMNS_QUERY % [placeholders]
+    res = cli.query(query, [options['schema']] + tables)
+
+    create_opt = {}
+    if options.has_key?(:skip_primary_key_check)
+      create_opt[:skip_primary_key_check] = options[:skip_primary_key_check]
+    end
+    current_table = nil
+    columns = []
+    completed_tables = []
+    res.each do |row|
+      table_name = row["table_name"]
+      unless table_name == current_table
+        unless columns.empty?
+          tabledef = create_tabledef_and_yield(columns, create_opt, &block)
+          completed_tables << current_table
+          columns = []
+          break unless tabledef
+        end
+        current_table = table_name
+      end
+      columns << row
+    end
+    unless columns.empty?
+      create_tabledef_and_yield(columns, create_opt, &block)
+      completed_tables << current_table
+    end
+    missing_tables = tables - completed_tables
+    unless missing_tables.empty?
+      raise TableMissingError.new("Table is missing", missing_tables.first)
+    end
+  end
+
+  def placeholder_string(num_items, start_num)
+    num_items.times.collect{|i| "$#{i + start_num}"}.join(",")
+  end
+
+  def create_tabledef_and_yield(columns, create_opt, &block)
+    pg_tabledef = nil
+    begin
+      pg_tabledef = FlydataCore::TableDef::PostgresqlTableDef.create(columns, create_opt)
+      yield(pg_tabledef, nil) if pg_tabledef
+    rescue FlydataCore::TableDefError => e
+      yield(nil, e)
+    end
+    pg_tabledef
+  end
+
+  class TableMissingError < RuntimeError
+    def initialize(message, table)
+      super(message)
+      @table = table
+    end
+     attr_reader :table
+  end
+end
+
+end
+end