flydata 0.0.5.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  module Flydata
2
2
  module Helpers
3
+ @@development_mode = nil
3
4
  module_function
4
5
  def parse_command(cmd)
5
6
  klass = Flydata::Command::Base
@@ -19,6 +20,8 @@ Usage: flydata COMMAND
19
20
  start # start flydata process
20
21
  stop # stop flydata process
21
22
  restart # restart flydata process
23
+ conf # show configuration
24
+ sync # initial sync(only for mysql database)
22
25
 
23
26
  If you encountered login or any other errors during setup,
24
27
  please setup flydata again by following commands.
@@ -36,7 +39,16 @@ You can check the logs of sender(flydata) process.
36
39
  end
37
40
 
38
41
  def development?
39
- File.open(flydata_conf_file).first.strip.end_with?('dev')
42
+ return @@development_mode unless @@development_mode.nil?
43
+ @@development_mode = !!(File.open(flydata_conf_file).first.strip.end_with?('dev'))
44
+ end
45
+
46
+ def env_mode
47
+ development? ? 'dev' : ''
48
+ end
49
+
50
+ def env_suffix
51
+ development? ? '_dev' : ''
40
52
  end
41
53
 
42
54
  # format text
@@ -0,0 +1,98 @@
1
+ require 'fileutils'
2
+ require_relative '../fluent-plugins/preference'
3
+
4
+ module Flydata
5
+ module Preference
6
+ class DataEntryPreference
7
+ CONFS_HOME = File.join(FLYDATA_HOME, 'confs')
8
+ CUSTOM_CONFIG_PARAMS = {
9
+ RedshiftMysqlDataEntry: ::Fluent::MysqlBinlogFlydataInputPreference::CUSTOM_CONFIG_PARAMS
10
+ }
11
+
12
+ class << self
13
+ # data_entry must be hash
14
+ def load_conf(data_entry)
15
+ path = conf_path(data_entry)
16
+ raise "Conf file does not exist. path:#{path}" unless File.exists?(path)
17
+ custom_conf = YAML::load(File.open(path, 'r'))
18
+ data_entry.each_key do |k|
19
+ v = custom_conf[k]
20
+ case v
21
+ when Hash
22
+ data_entry[k].merge!(v)
23
+ when NilClass
24
+ else
25
+ data_entry[k] = v
26
+ end
27
+ end
28
+ filter_data_entry(data_entry)
29
+ data_entry
30
+ end
31
+
32
+ def filter_data_entry(de)
33
+ configurable_params = CUSTOM_CONFIG_PARAMS[de['type'].to_sym]
34
+ return de if configurable_params.nil? or configurable_params.empty?
35
+
36
+ configurable_params.each do |pref_name, param_info|
37
+ # skip if the data entry doesn't have the information
38
+ de_pref = de[pref_name.to_s]
39
+ next unless de_pref
40
+
41
+ param_info.each do |param_name, options|
42
+ next if options.nil? or options.empty?
43
+ param_name_str = param_name.to_s
44
+ next if de_pref[param_name_str].nil? or de_pref[param_name_str].empty?
45
+
46
+ options.each do |option_name, option_value|
47
+ de_pref[param_name_str] = Fluent::DataEntryPreferenceConfigurable.replace_value_with_option(
48
+ param_name_str, de_pref[param_name_str], option_name, option_value, key: de['data_port_key'])
49
+ end
50
+ end
51
+ end
52
+ end
53
+
54
+
55
+ def copy_template(data_entry)
56
+ return unless File.exists?(conf_template_path(data_entry))
57
+ FileUtils.mkdir_p(CONFS_HOME)
58
+ FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry)+".tmpl")
59
+
60
+ # Do not overwrite a current conf file
61
+ unless File.exists?(conf_path(data_entry))
62
+ FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry))
63
+ end
64
+ end
65
+
66
+ def conf_path(data_entry)
67
+ "#{File.join(CONFS_HOME, data_entry['name'])}.conf"
68
+ end
69
+
70
+ def conf_exists?(data_entry)
71
+ File.exists?(conf_path(data_entry))
72
+ end
73
+
74
+ def conf_name(data_entry)
75
+ "#{data_entry['name']}.conf"
76
+ end
77
+
78
+ def configurable?(data_entry)
79
+ File.exists?(conf_template_path(data_entry))
80
+ end
81
+
82
+ def conf_template_path(data_entry)
83
+ File.join(FLYDATA_TMPL_DIR, conf_template_name(data_entry))
84
+ end
85
+
86
+ def conf_template_name(data_entry)
87
+ type = ActiveSupport::Inflector.underscore(data_entry['type'])
88
+ "#{type}.conf.tmpl"
89
+ end
90
+
91
+ def create(type)
92
+ custom_config_params = CUSTOM_CONFIG_PARAMS[type]
93
+ return nil unless custom_config_params
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,120 @@
1
+ module Flydata
2
+ module FileUtil
3
+ class SyncFileManager
4
+ DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
5
+ TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
6
+ def initialize(data_entry)
7
+ @data_entry = data_entry
8
+ end
9
+
10
+ def dump_file_path
11
+ dump_dir = @data_entry['mysql_data_entry_preference']['mysqldump_dir']
12
+ if dump_dir
13
+ dump_dir = dump_dir.dup
14
+ dump_dir[0] = ENV['HOME'] if dump_dir.match(/^~$|^~\//)
15
+ else
16
+ dump_dir = DUMP_DIR.dup
17
+ end
18
+ if File.exists?(dump_dir) and not Dir.exists?(dump_dir)
19
+ raise "'mysqldump_dir'(#{dump_dir}) must be a directory."
20
+ end
21
+ FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
22
+ File.join(dump_dir, @data_entry['name']) + ".dump"
23
+ end
24
+
25
+ # dump pos file for resume
26
+ def dump_pos_path
27
+ dump_file_path + ".pos"
28
+ end
29
+
30
+ def save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
31
+ File.open(dump_pos_path, 'w') do |f|
32
+ f.write(dump_pos_content(status, table_name, last_pos, binlog_pos, state, substate))
33
+ end
34
+ end
35
+
36
+ def load_dump_pos
37
+ path = dump_pos_path
38
+ return nil unless File.exists?(path)
39
+ items = File.open(path, 'r').readline.split("\t")
40
+ raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
41
+ mysql_table = load_mysql_table_marshal_dump
42
+ { status: items[0], table_name: items[1], last_pos: items[2].to_i,
43
+ binlog_pos: {binfile: items[3], pos: items[4].to_i},
44
+ state: items[5], substate: items[6], mysql_table: mysql_table}
45
+ end
46
+
47
+ # MysqlTable marshal file
48
+ def mysql_table_marshal_dump_path
49
+ dump_file_path + ".mysql_table"
50
+ end
51
+
52
+ def save_mysql_table_marshal_dump(mysql_table)
53
+ File.open(mysql_table_marshal_dump_path, 'w') do |f|
54
+ f.write Marshal.dump(mysql_table)
55
+ end
56
+ end
57
+
58
+ # binlog.pos file
59
+ def save_binlog(binlog_pos)
60
+ path = binlog_path
61
+ File.open(path, 'w') do |f|
62
+ f.write(binlog_content(binlog_pos))
63
+ end
64
+ end
65
+
66
+ def binlog_path
67
+ File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos")
68
+ end
69
+
70
+ def table_positions_dir_path
71
+ TABLE_POSITIONS_DIR
72
+ end
73
+
74
+ def table_position_file_paths
75
+ Dir.glob(File.join(table_positions_dir_path, '*.pos'))
76
+ end
77
+
78
+ # Read a sequence number from the table's position file,
79
+ # increment the number and pass the number to a block.
80
+ # After executing the block, saves the value to the position
81
+ # file.
82
+ def increment_and_save_table_position(table_name)
83
+ file = File.join(table_positions_dir_path, table_name + ".pos")
84
+ retry_count = 0
85
+ begin
86
+ File.open(file, "r+") do |f|
87
+ seq = f.read
88
+ seq = seq.to_i + 1
89
+ yield(seq)
90
+ f.rewind
91
+ f.truncate(0)
92
+ f.write(seq)
93
+ end
94
+ rescue Errno::ENOENT
95
+ raise if retry_count > 0 # Already retried. Must be a differentfile causing the error
96
+ # File not exist. Create one with initial value of '0'
97
+ File.open(file, "w") {|f| f.write('0') }
98
+ retry_count += 1
99
+ retry
100
+ end
101
+ end
102
+
103
+ private
104
+
105
+ def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
106
+ [status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
107
+ end
108
+
109
+ def binlog_content(binlog_pos)
110
+ [binlog_pos[:binfile], binlog_pos[:pos]].join("\t")
111
+ end
112
+
113
+ def load_mysql_table_marshal_dump
114
+ path = mysql_table_marshal_dump_path
115
+ return nil unless File.exists?(path)
116
+ Marshal.load(File.open(path, 'r'))
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,2 @@
1
+ require_relative 'table_def/redshift_table_def'
2
+ require_relative 'table_def/mysql_table_def'
@@ -0,0 +1,128 @@
1
+ module Flydata
2
+ module Sync
3
+
4
+ class MysqlTableDef
5
+ TYPE_MAP_M2F = {
6
+ 'bigint' => 'int8',
7
+ 'binary' => 'binary',
8
+ 'blob' => 'varbinary',
9
+ 'bool' => 'int1',
10
+ 'boolean' => 'int1',
11
+ 'char' => 'varchar',
12
+ 'date' => 'date',
13
+ 'datetime' => 'datetime',
14
+ 'dec' => 'numeric',
15
+ 'decimal' => 'numeric',
16
+ 'double' => 'float8',
17
+ 'double precision' => 'float8',
18
+ 'fixed' => 'numeric',
19
+ 'float' => 'float4',
20
+ 'int' => 'int4',
21
+ 'longblob' => 'varbinary',
22
+ 'longtext' => 'text',
23
+ 'mediumblob' => 'varbinary',
24
+ 'mediumint' => 'int3',
25
+ 'mediumtext' => 'text',
26
+ 'numeric' => 'numeric',
27
+ 'smallint' => 'int2',
28
+ 'text' => 'text',
29
+ 'time' => 'time',
30
+ 'timestamp' => 'datetime',
31
+ 'tinyblob' => 'varbinary',
32
+ 'tinyint' => 'int1',
33
+ 'tinytext' => 'text',
34
+ 'varbinary' => 'varbinary',
35
+ 'varchar' => 'varchar',
36
+ }
37
+
38
+ def self.create(io)
39
+ params = _create(io)
40
+ params ? self.new(*params) : nil
41
+ end
42
+
43
+ def initialize(table_def, table_name, columns, default_charset, comment)
44
+ @table_def = table_def
45
+ @table_name = table_name
46
+ @columns = columns
47
+ @default_charset = default_charset
48
+ @comment = comment
49
+ end
50
+
51
+ def self._create(io)
52
+
53
+ table_def = ''
54
+ table_name = nil
55
+ columns = []
56
+ default_charset = nil
57
+ comment = nil
58
+
59
+ position = :before_create_table
60
+ io.each_line do |line|
61
+ if line =~ /CREATE TABLE `(.*?)`/
62
+ position = :in_create_table
63
+ table_name = $1
64
+ end
65
+
66
+ next unless position == :in_create_table
67
+
68
+ table_def += line.chomp
69
+
70
+ if line =~ /^\s*`(.*?)`\s+([a-z()0-9,\s]+)/ # column def
71
+ name = $1
72
+ type = $2.strip
73
+ type = type[0..-2] if type.end_with?(',')
74
+ TYPE_MAP_M2F.each do |mysql_type, flydata_type|
75
+ type.gsub!(/\b#{mysql_type}\b/, flydata_type)
76
+ end
77
+ column = {name: name, type: type}
78
+ column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
79
+ column[:not_null] = true if line =~ /NOT NULL/
80
+ if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/.match(line)
81
+ val = $1
82
+ val = val.slice(1..-1) if val.start_with?("'")
83
+ val = val.slice(0..-2) if val.end_with?("'")
84
+ column[:default] = val == "NULL" ? nil : val
85
+ end
86
+ if /COMMENT\s+'(((?:\\'|[^'])*))'/.match(line)
87
+ column[:comment] = $1
88
+ end
89
+ columns << column
90
+ end
91
+
92
+ if line =~ /PRIMARY KEY/
93
+ primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
94
+ primary_keys.each do |primary_key|
95
+ column = columns.detect {|column|
96
+ column[:name] === primary_key
97
+ }
98
+ raise "PK #{primary_key} must exist in the definition " if column.nil?
99
+ column[:primary_key] = true
100
+ end
101
+ end
102
+
103
+ if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
104
+ default_charset = $1
105
+ end
106
+
107
+ if line =~ /ENGINE=.*/ # TODO Need better parsing.
108
+ position = :after_create_table
109
+ break
110
+ end
111
+ end
112
+ position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
113
+ end
114
+ attr_reader :columns, :table_name
115
+
116
+ def to_flydata_tabledef
117
+ tabledef = { table_name: @table_name,
118
+ columns: @columns,
119
+ }
120
+ tabledef[:default_charset] = @default_charset if @default_charset
121
+ tabledef[:comment] = @comment if @comment
122
+
123
+ tabledef
124
+ end
125
+ end
126
+
127
+ end
128
+ end
@@ -0,0 +1,144 @@
1
+ module Flydata
2
+ module Sync
3
+
4
+ class RedshiftTableDef
5
+ TYPE_MAP_F2R = {
6
+ 'binary' => {type: 'varchar', use_params: true},
7
+ 'char' => {type: 'char', use_params: true},
8
+ 'date' => {type: 'date'},
9
+ 'datetime' => {type: 'timestamp'},
10
+ 'float4' => {type: 'float4'},
11
+ 'float4 unsigned' => {type: 'float4'},
12
+ 'float8' => {type: 'float8'},
13
+ 'float8 unsigned' => {type: 'float8'},
14
+ 'int1' => {type: 'int2'},
15
+ 'int1 unsigned' => {type: 'int2', unsigned: true},
16
+ 'int2' => {type: 'int2'},
17
+ 'int2 unsigned' => {type: 'int4', unsigned: true},
18
+ 'int3' => {type: 'int4'},
19
+ 'int3 unsigned' => {type: 'int4', unsigned: true},
20
+ 'int4' => {type: 'int4'},
21
+ 'int4 unsigned' => {type: 'int8', unsigned: true},
22
+ 'int8' => {type: 'int8'},
23
+ 'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
24
+ 'numeric' => {type: 'numeric', use_params: true},
25
+ 'numeric unsigned' => {type: 'numeric', use_params: true},
26
+ 'text' => {type: 'varchar(max)'},
27
+ 'time' => {type: 'timestamp'},
28
+ 'varbinary' => {type: 'varchar', use_params: true},
29
+ 'varchar' => {type: 'varchar', use_params: true},
30
+ }
31
+ def self.from_flydata_tabledef(flydata_tabledef, options = {})
32
+ options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
33
+
34
+ tabledef = ""
35
+ tabledef += create_flydata_ctl_table_sql if options[:flydata_ctl_table]
36
+ tabledef += create_table_sql(flydata_tabledef)
37
+ tabledef += comment_sql(flydata_tabledef)
38
+ tabledef += flydata_ctl_sql(flydata_tabledef)
39
+ end
40
+
41
+ CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
42
+ CREATE TABLE flydata_ctl_columns (
43
+ id integer NOT NULL IDENTITY(1,1),
44
+ table_name varchar(128) NOT NULL,
45
+ column_name varchar(128) NOT NULL,
46
+ src_data_type varchar(1024) NOT NULL,
47
+ PRIMARY KEY(id)
48
+ ) DISTKEY(table_name) SORTKEY(table_name);
49
+ EOS
50
+ def self.create_flydata_ctl_table_sql
51
+ # No drop table here intentionally because losing the data is fatal.
52
+ CREATE_FLYDATA_CTL_TABLE_SQL
53
+ end
54
+
55
+ CREATE_TABLE_SQL = <<EOS
56
+ DROP TABLE %s;
57
+ CREATE TABLE %s (
58
+ %s
59
+ );
60
+ EOS
61
+
62
+ def self.create_table_sql(flydata_tabledef)
63
+ lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
64
+ pk_def = primary_key_sql(flydata_tabledef)
65
+ lines << pk_def if pk_def
66
+
67
+ contents = lines.join(",\n")
68
+
69
+ table_name = flydata_tabledef[:table_name]
70
+ CREATE_TABLE_SQL % [table_name, table_name, contents]
71
+ end
72
+
73
+ def self.column_def_sql(column)
74
+ type = column[:type]
75
+ if type =~ /\(.*?\)/
76
+ type = $` + $'
77
+ params = $&
78
+ end
79
+
80
+ type_info = TYPE_MAP_F2R[type]
81
+ raise "Unsupported type '#{column[:type]}'" if type_info.nil?
82
+
83
+ rs_type = (type_info[:use_params] && params && !params.nil?) ?
84
+ type_info[:type] + "#{params}"
85
+ : type_info[:type]
86
+ line = %Q| "#{column[:name]}" #{rs_type}|
87
+ line += " NOT NULL" if column[:not_null]
88
+ if (column.has_key?(:default))
89
+ val = replace_default_value(type_info[:type], column[:default])
90
+ line += " DEFAULT #{val}"
91
+ end
92
+ # Commented out because no IDENTITY column must be used for a replicated table.
93
+ # Values come from the master.
94
+ # line += " IDENTITY(1, 1)" if (column[:auto_increment])
95
+
96
+ line
97
+ end
98
+
99
+ def self.replace_default_value(type, default_value)
100
+ val = default_value ? "'#{default_value}'" : "NULL"
101
+ case type
102
+ when 'timestamp'
103
+ (val.upcase == "'CURRENT_TIMESTAMP'") ? 'SYSDATE' : val
104
+ else
105
+ val
106
+ end
107
+ end
108
+
109
+ def self.primary_key_sql(flydata_tabledef)
110
+ pks = flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:name]}
111
+ pks.empty? ? nil : " PRIMARY KEY (#{pks.join(',')})"
112
+ end
113
+
114
+ def self.comment_sql(flydata_tabledef)
115
+ sql = ""
116
+ flydata_tabledef[:columns].each do |col|
117
+ next unless col[:comment]
118
+
119
+ sql += <<EOS
120
+ COMMENT ON COLUMN #{flydata_tabledef[:table_name]}."#{col[:name]}"
121
+ IS '#{col[:comment]}';
122
+ EOS
123
+ end
124
+ sql
125
+ end
126
+
127
+ FLYDATA_CTL_COLUMNS_SQL = <<EOS
128
+ DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
129
+ INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type) VALUES
130
+ EOS
131
+ def self.flydata_ctl_sql(flydata_tabledef)
132
+ sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
133
+ values = []
134
+ flydata_tabledef[:columns].each do |col|
135
+ values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{col[:type]}')"
136
+ end
137
+ sql += values.join(",\n") + ';'
138
+ sql
139
+ end
140
+ end
141
+
142
+ end
143
+ end
144
+