flydata 0.0.5.6 → 0.1.0

data/lib/flydata/helpers.rb

@@ -1,5 +1,6 @@
 module Flydata
   module Helpers
+    @@development_mode = nil
     module_function
     def parse_command(cmd)
       klass = Flydata::Command::Base
@@ -19,6 +20,8 @@ Usage: flydata COMMAND
   start                    # start flydata process
   stop                     # stop flydata process
   restart                  # restart flydata process
+  conf                     # show configuration
+  sync                     # initial sync(only for mysql database)
 
 If you encountered login or any other errors during setup,
 please setup flydata again by following commands.
@@ -36,7 +39,16 @@ You can check the logs of sender(flydata) process.
     end
 
     def development?
-      File.open(flydata_conf_file).first.strip.end_with?('dev')
+      return @@development_mode unless @@development_mode.nil?
+      @@development_mode = !!(File.open(flydata_conf_file).first.strip.end_with?('dev'))
+    end
+
+    def env_mode
+      development? ? 'dev' : ''
+    end
+
+    def env_suffix
+      development? ? '_dev' : ''
     end
 
     # format text

data/lib/flydata/preference/data_entry_preference.rb

@@ -0,0 +1,98 @@
+require 'fileutils'
+require_relative '../fluent-plugins/preference'
+
+module Flydata
+  module Preference
+    class DataEntryPreference
+      CONFS_HOME = File.join(FLYDATA_HOME, 'confs')
+      CUSTOM_CONFIG_PARAMS = {
+        RedshiftMysqlDataEntry: ::Fluent::MysqlBinlogFlydataInputPreference::CUSTOM_CONFIG_PARAMS
+      }
+
+      class << self
+        # data_entry must be hash
+        def load_conf(data_entry)
+          path = conf_path(data_entry)
+          raise "Conf file does not exist. path:#{path}" unless File.exists?(path)
+          custom_conf = YAML::load(File.open(path, 'r'))
+          data_entry.each_key do |k|
+            v = custom_conf[k]
+            case v
+            when Hash
+              data_entry[k].merge!(v)
+            when NilClass
+            else
+              data_entry[k] = v
+            end
+          end
+          filter_data_entry(data_entry)
+          data_entry
+        end
+
+        def filter_data_entry(de)
+          configurable_params = CUSTOM_CONFIG_PARAMS[de['type'].to_sym]
+          return de if configurable_params.nil? or configurable_params.empty?
+
+          configurable_params.each do |pref_name, param_info|
+            # skip if the data entry doesn't have the information
+            de_pref = de[pref_name.to_s]
+            next unless de_pref
+
+            param_info.each do |param_name, options|
+              next if options.nil? or options.empty?
+              param_name_str = param_name.to_s
+              next if de_pref[param_name_str].nil? or de_pref[param_name_str].empty?
+
+              options.each do |option_name, option_value|
+                de_pref[param_name_str] = Fluent::DataEntryPreferenceConfigurable.replace_value_with_option(
+                  param_name_str, de_pref[param_name_str], option_name, option_value, key: de['data_port_key'])
+              end
+            end
+          end
+        end
+
+
+        def copy_template(data_entry)
+          return unless File.exists?(conf_template_path(data_entry))
+          FileUtils.mkdir_p(CONFS_HOME)
+          FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry)+".tmpl")
+
+          # Do not overwrite a current conf file
+          unless File.exists?(conf_path(data_entry))
+            FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry))
+          end
+        end
+
+        def conf_path(data_entry)
+          "#{File.join(CONFS_HOME, data_entry['name'])}.conf"
+        end
+
+        def conf_exists?(data_entry)
+          File.exists?(conf_path(data_entry))
+        end
+
+        def conf_name(data_entry)
+          "#{data_entry['name']}.conf"
+        end
+
+        def configurable?(data_entry)
+          File.exists?(conf_template_path(data_entry))
+        end
+
+        def conf_template_path(data_entry)
+          File.join(FLYDATA_TMPL_DIR, conf_template_name(data_entry))
+        end
+
+        def conf_template_name(data_entry)
+          type = ActiveSupport::Inflector.underscore(data_entry['type'])
+          "#{type}.conf.tmpl"
+        end
+
+        def create(type)
+          custom_config_params = CUSTOM_CONFIG_PARAMS[type]
+          return nil unless custom_config_params
+        end
+      end
+    end
+  end
+end

data/lib/flydata/sync_file_manager.rb

@@ -0,0 +1,120 @@
+module Flydata
+  module FileUtil
+    class SyncFileManager
+      DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
+      TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
+      def initialize(data_entry)
+        @data_entry = data_entry
+      end
+
+      def dump_file_path
+        dump_dir = @data_entry['mysql_data_entry_preference']['mysqldump_dir']
+        if dump_dir
+          dump_dir = dump_dir.dup
+          dump_dir[0] = ENV['HOME'] if dump_dir.match(/^~$|^~\//)
+        else
+          dump_dir = DUMP_DIR.dup
+        end
+        if File.exists?(dump_dir) and not Dir.exists?(dump_dir)
+          raise "'mysqldump_dir'(#{dump_dir}) must be a directory."
+        end
+        FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
+        File.join(dump_dir, @data_entry['name']) + ".dump"
+      end
+
+      # dump pos file for resume
+      def dump_pos_path
+        dump_file_path + ".pos"
+      end
+
+      def save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
+        File.open(dump_pos_path, 'w') do |f|
+          f.write(dump_pos_content(status, table_name, last_pos, binlog_pos, state, substate))
+        end
+      end
+
+      def load_dump_pos
+        path = dump_pos_path
+        return nil unless File.exists?(path)
+        items = File.open(path, 'r').readline.split("\t")
+        raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
+        mysql_table = load_mysql_table_marshal_dump
+        { status: items[0], table_name: items[1], last_pos: items[2].to_i,
+          binlog_pos: {binfile: items[3], pos: items[4].to_i},
+          state: items[5], substate: items[6], mysql_table: mysql_table}
+      end
+
+      # MysqlTable marshal file
+      def mysql_table_marshal_dump_path
+        dump_file_path + ".mysql_table"
+      end
+
+      def save_mysql_table_marshal_dump(mysql_table)
+        File.open(mysql_table_marshal_dump_path, 'w') do |f|
+          f.write Marshal.dump(mysql_table)
+        end
+      end
+
+      # binlog.pos file
+      def save_binlog(binlog_pos)
+        path = binlog_path
+        File.open(path, 'w') do |f|
+          f.write(binlog_content(binlog_pos))
+        end
+      end
+
+      def binlog_path
+        File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos")
+      end
+
+      def table_positions_dir_path
+        TABLE_POSITIONS_DIR
+      end
+
+      def table_position_file_paths
+        Dir.glob(File.join(table_positions_dir_path, '*.pos'))
+      end
+
+      # Read a sequence number from the table's position file, 
+      # increment the number and pass the number to a block.
+      # After executing the block, saves the value to the position
+      # file.
+      def increment_and_save_table_position(table_name)
+        file = File.join(table_positions_dir_path, table_name + ".pos")
+        retry_count = 0
+        begin
+          File.open(file, "r+") do |f|
+            seq = f.read
+            seq = seq.to_i + 1
+            yield(seq)
+            f.rewind
+            f.truncate(0)
+            f.write(seq)
+          end
+        rescue Errno::ENOENT
+          raise if retry_count > 0 # Already retried.  Must be a differentfile causing the error
+          # File not exist.  Create one with initial value of '0'
+          File.open(file, "w") {|f| f.write('0') }
+          retry_count += 1
+          retry
+        end
+      end
+
+      private
+
+      def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
+        [status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
+      end
+
+      def binlog_content(binlog_pos)
+        [binlog_pos[:binfile], binlog_pos[:pos]].join("\t")
+      end
+
+      def load_mysql_table_marshal_dump
+        path = mysql_table_marshal_dump_path
+        return nil unless File.exists?(path)
+        Marshal.load(File.open(path, 'r'))
+      end
+    end
+  end
+end

data/lib/flydata/table_def.rb

@@ -0,0 +1,2 @@
+require_relative 'table_def/redshift_table_def'
+require_relative 'table_def/mysql_table_def'

data/lib/flydata/table_def/mysql_table_def.rb

@@ -0,0 +1,128 @@
+module Flydata
+module Sync
+
+class MysqlTableDef
+  TYPE_MAP_M2F = {
+    'bigint' => 'int8',
+    'binary' => 'binary',
+    'blob' => 'varbinary',
+    'bool' => 'int1',
+    'boolean' => 'int1',
+    'char' => 'varchar',
+    'date' => 'date',
+    'datetime' => 'datetime',
+    'dec' => 'numeric',
+    'decimal' => 'numeric',
+    'double' => 'float8',
+    'double precision' => 'float8',
+    'fixed' => 'numeric',
+    'float' => 'float4',
+    'int' => 'int4',
+    'longblob' => 'varbinary',
+    'longtext' => 'text',
+    'mediumblob' => 'varbinary',
+    'mediumint' => 'int3',
+    'mediumtext' => 'text',
+    'numeric' => 'numeric',
+    'smallint' => 'int2',
+    'text' => 'text',
+    'time' => 'time',
+    'timestamp' => 'datetime',
+    'tinyblob' => 'varbinary',
+    'tinyint' => 'int1',
+    'tinytext' => 'text',
+    'varbinary' => 'varbinary',
+    'varchar' => 'varchar',
+  }
+
+  def self.create(io)
+    params = _create(io)
+    params ? self.new(*params) : nil
+  end
+
+  def initialize(table_def, table_name, columns, default_charset, comment)
+    @table_def = table_def
+    @table_name = table_name
+    @columns = columns
+    @default_charset = default_charset
+    @comment = comment
+  end
+
+  def self._create(io)
+    
+    table_def = ''
+    table_name = nil
+    columns = []
+    default_charset = nil
+    comment = nil
+
+    position = :before_create_table
+    io.each_line do |line|
+      if line =~ /CREATE TABLE `(.*?)`/
+        position = :in_create_table
+        table_name = $1
+      end
+
+      next unless position == :in_create_table
+
+      table_def += line.chomp
+
+      if line =~ /^\s*`(.*?)`\s+([a-z()0-9,\s]+)/ # column def
+        name = $1
+        type = $2.strip
+        type = type[0..-2] if type.end_with?(',')
+        TYPE_MAP_M2F.each do |mysql_type, flydata_type|
+          type.gsub!(/\b#{mysql_type}\b/, flydata_type)
+        end
+        column = {name: name, type: type}
+        column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
+        column[:not_null] = true if line =~ /NOT NULL/
+        if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/.match(line)
+          val = $1
+          val = val.slice(1..-1) if val.start_with?("'")
+          val = val.slice(0..-2) if val.end_with?("'")
+          column[:default] = val == "NULL" ? nil : val
+        end
+        if /COMMENT\s+'(((?:\\'|[^'])*))'/.match(line)
+          column[:comment] = $1
+        end
+        columns << column
+      end
+
+      if line =~ /PRIMARY KEY/
+        primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
+        primary_keys.each do |primary_key|
+          column = columns.detect {|column|
+            column[:name] === primary_key
+          }
+          raise "PK #{primary_key} must exist in the definition " if column.nil?
+          column[:primary_key] = true
+        end
+      end
+
+      if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
+        default_charset = $1
+      end
+
+      if line =~ /ENGINE=.*/  # TODO Need better parsing.
+        position = :after_create_table
+        break
+      end
+    end
+    position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
+  end
+  attr_reader :columns, :table_name
+
+  def to_flydata_tabledef
+    tabledef = { table_name: @table_name,
+                 columns: @columns,
+               }
+    tabledef[:default_charset] = @default_charset if @default_charset
+    tabledef[:comment] = @comment if @comment
+
+    tabledef
+  end
+end
+
+end
+end

data/lib/flydata/table_def/redshift_table_def.rb

@@ -0,0 +1,144 @@
+module Flydata
+module Sync
+
+class RedshiftTableDef
+  TYPE_MAP_F2R = {
+    'binary' => {type: 'varchar', use_params: true},
+    'char' => {type: 'char', use_params: true},
+    'date' => {type: 'date'},
+    'datetime' => {type: 'timestamp'},
+    'float4' => {type: 'float4'},
+    'float4 unsigned' => {type: 'float4'},
+    'float8' => {type: 'float8'},
+    'float8 unsigned' => {type: 'float8'},
+    'int1' => {type: 'int2'},
+    'int1 unsigned' => {type: 'int2', unsigned: true},
+    'int2' => {type: 'int2'},
+    'int2 unsigned' => {type: 'int4', unsigned: true},
+    'int3' => {type: 'int4'},
+    'int3 unsigned' => {type: 'int4', unsigned: true},
+    'int4' => {type: 'int4'},
+    'int4 unsigned' => {type: 'int8', unsigned: true},
+    'int8' => {type: 'int8'},
+    'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
+    'numeric' => {type: 'numeric', use_params: true},
+    'numeric unsigned' => {type: 'numeric', use_params: true},
+    'text' => {type: 'varchar(max)'},
+    'time' => {type: 'timestamp'},
+    'varbinary' => {type: 'varchar', use_params: true},
+    'varchar' => {type: 'varchar', use_params: true},
+  }
+  def self.from_flydata_tabledef(flydata_tabledef, options = {})
+    options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
+
+    tabledef = ""
+    tabledef += create_flydata_ctl_table_sql if options[:flydata_ctl_table]
+    tabledef += create_table_sql(flydata_tabledef)
+    tabledef += comment_sql(flydata_tabledef)
+    tabledef += flydata_ctl_sql(flydata_tabledef)
+  end
+
+  CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
+CREATE TABLE flydata_ctl_columns (
+  id integer NOT NULL IDENTITY(1,1),
+  table_name varchar(128) NOT NULL,
+  column_name varchar(128) NOT NULL,
+  src_data_type varchar(1024) NOT NULL,
+  PRIMARY KEY(id)
+) DISTKEY(table_name) SORTKEY(table_name);
+EOS
+  def self.create_flydata_ctl_table_sql
+    # No drop table here intentionally because losing the data is fatal.
+    CREATE_FLYDATA_CTL_TABLE_SQL
+  end
+
+  CREATE_TABLE_SQL = <<EOS
+DROP TABLE %s;
+CREATE TABLE %s (
+%s
+);
+EOS
+
+  def self.create_table_sql(flydata_tabledef)
+    lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
+    pk_def = primary_key_sql(flydata_tabledef)
+    lines << pk_def if pk_def
+
+    contents = lines.join(",\n")
+
+    table_name = flydata_tabledef[:table_name]
+    CREATE_TABLE_SQL % [table_name, table_name, contents]
+  end
+
+  def self.column_def_sql(column)
+    type = column[:type]
+    if type =~ /\(.*?\)/
+      type = $` + $'
+      params = $&
+    end
+
+    type_info = TYPE_MAP_F2R[type]
+    raise "Unsupported type '#{column[:type]}'" if type_info.nil?
+
+    rs_type = (type_info[:use_params] && params && !params.nil?) ?
+                 type_info[:type] + "#{params}"
+               : type_info[:type]
+    line = %Q|  "#{column[:name]}" #{rs_type}|
+    line += " NOT NULL" if column[:not_null]
+    if (column.has_key?(:default))
+      val = replace_default_value(type_info[:type], column[:default])
+      line += " DEFAULT #{val}"
+    end
+# Commented out because no IDENTITY column must be used for a replicated table.
+# Values come from the master.
+#    line += " IDENTITY(1, 1)" if (column[:auto_increment])
+
+    line
+  end
+
+  def self.replace_default_value(type, default_value)
+    val = default_value ? "'#{default_value}'" : "NULL"
+    case type
+    when 'timestamp'
+      (val.upcase == "'CURRENT_TIMESTAMP'") ? 'SYSDATE' : val
+    else
+      val
+    end
+  end
+
+  def self.primary_key_sql(flydata_tabledef)
+    pks = flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:name]}
+    pks.empty? ? nil : "  PRIMARY KEY (#{pks.join(',')})"
+  end
+
+  def self.comment_sql(flydata_tabledef)
+    sql = ""
+    flydata_tabledef[:columns].each do |col|
+      next unless col[:comment]
+
+      sql += <<EOS
+COMMENT ON COLUMN #{flydata_tabledef[:table_name]}."#{col[:name]}"
+  IS '#{col[:comment]}';
+EOS
+    end
+    sql
+  end
+
+  FLYDATA_CTL_COLUMNS_SQL = <<EOS
+DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
+INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type) VALUES
+EOS
+  def self.flydata_ctl_sql(flydata_tabledef)
+    sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
+    values = []
+    flydata_tabledef[:columns].each do |col|
+      values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{col[:type]}')"
+    end
+    sql += values.join(",\n") + ';'
+    sql
+  end
+end
+
+end
+end
+