flydata 0.0.5.6 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +8 -0
- data/Gemfile.lock +36 -1
- data/VERSION +1 -1
- data/bin/fdmysqldump +59 -0
- data/flydata.gemspec +49 -5
- data/lib/flydata.rb +3 -1
- data/lib/flydata/api/data_entry.rb +4 -0
- data/lib/flydata/api/redshift_cluster.rb +15 -0
- data/lib/flydata/cli.rb +1 -0
- data/lib/flydata/command/base.rb +8 -2
- data/lib/flydata/command/conf.rb +48 -0
- data/lib/flydata/command/encrypt.rb +18 -0
- data/lib/flydata/command/sender.rb +10 -3
- data/lib/flydata/command/setlogdel.rb +1 -1
- data/lib/flydata/command/setup.rb +26 -3
- data/lib/flydata/command/sync.rb +962 -0
- data/lib/flydata/command/version.rb +10 -0
- data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +305 -0
- data/lib/flydata/fluent-plugins/out_forward_ssl.rb +91 -0
- data/lib/flydata/fluent-plugins/preference.rb +92 -0
- data/lib/flydata/helpers.rb +13 -1
- data/lib/flydata/preference/data_entry_preference.rb +98 -0
- data/lib/flydata/sync_file_manager.rb +120 -0
- data/lib/flydata/table_def.rb +2 -0
- data/lib/flydata/table_def/mysql_table_def.rb +128 -0
- data/lib/flydata/table_def/redshift_table_def.rb +144 -0
- data/lib/flydata/util/encryptor.rb +53 -0
- data/spec/fluent_plugins_spec_helper.rb +19 -0
- data/spec/flydata/command/sender_spec.rb +3 -29
- data/spec/flydata/command/sync_spec.rb +1049 -0
- data/spec/flydata/fluent-plugins/in_mysql_binlog_flydata_spec.rb +204 -0
- data/spec/flydata/util/encryptor_spec.rb +96 -0
- data/spec/spec_helper.rb +1 -0
- data/tmpl/redshift_mysql_data_entry.conf.tmpl +11 -0
- metadata +153 -4
data/lib/flydata/helpers.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
module Flydata
|
2
2
|
module Helpers
|
3
|
+
@@development_mode = nil
|
3
4
|
module_function
|
4
5
|
def parse_command(cmd)
|
5
6
|
klass = Flydata::Command::Base
|
@@ -19,6 +20,8 @@ Usage: flydata COMMAND
|
|
19
20
|
start # start flydata process
|
20
21
|
stop # stop flydata process
|
21
22
|
restart # restart flydata process
|
23
|
+
conf # show configuration
|
24
|
+
sync # initial sync(only for mysql database)
|
22
25
|
|
23
26
|
If you encountered login or any other errors during setup,
|
24
27
|
please setup flydata again by following commands.
|
@@ -36,7 +39,16 @@ You can check the logs of sender(flydata) process.
|
|
36
39
|
end
|
37
40
|
|
38
41
|
def development?
|
39
|
-
|
42
|
+
return @@development_mode unless @@development_mode.nil?
|
43
|
+
@@development_mode = !!(File.open(flydata_conf_file).first.strip.end_with?('dev'))
|
44
|
+
end
|
45
|
+
|
46
|
+
def env_mode
|
47
|
+
development? ? 'dev' : ''
|
48
|
+
end
|
49
|
+
|
50
|
+
def env_suffix
|
51
|
+
development? ? '_dev' : ''
|
40
52
|
end
|
41
53
|
|
42
54
|
# format text
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require_relative '../fluent-plugins/preference'
|
3
|
+
|
4
|
+
module Flydata
|
5
|
+
module Preference
|
6
|
+
class DataEntryPreference
|
7
|
+
CONFS_HOME = File.join(FLYDATA_HOME, 'confs')
|
8
|
+
CUSTOM_CONFIG_PARAMS = {
|
9
|
+
RedshiftMysqlDataEntry: ::Fluent::MysqlBinlogFlydataInputPreference::CUSTOM_CONFIG_PARAMS
|
10
|
+
}
|
11
|
+
|
12
|
+
class << self
|
13
|
+
# data_entry must be hash
|
14
|
+
def load_conf(data_entry)
|
15
|
+
path = conf_path(data_entry)
|
16
|
+
raise "Conf file does not exist. path:#{path}" unless File.exists?(path)
|
17
|
+
custom_conf = YAML::load(File.open(path, 'r'))
|
18
|
+
data_entry.each_key do |k|
|
19
|
+
v = custom_conf[k]
|
20
|
+
case v
|
21
|
+
when Hash
|
22
|
+
data_entry[k].merge!(v)
|
23
|
+
when NilClass
|
24
|
+
else
|
25
|
+
data_entry[k] = v
|
26
|
+
end
|
27
|
+
end
|
28
|
+
filter_data_entry(data_entry)
|
29
|
+
data_entry
|
30
|
+
end
|
31
|
+
|
32
|
+
def filter_data_entry(de)
|
33
|
+
configurable_params = CUSTOM_CONFIG_PARAMS[de['type'].to_sym]
|
34
|
+
return de if configurable_params.nil? or configurable_params.empty?
|
35
|
+
|
36
|
+
configurable_params.each do |pref_name, param_info|
|
37
|
+
# skip if the data entry doesn't have the information
|
38
|
+
de_pref = de[pref_name.to_s]
|
39
|
+
next unless de_pref
|
40
|
+
|
41
|
+
param_info.each do |param_name, options|
|
42
|
+
next if options.nil? or options.empty?
|
43
|
+
param_name_str = param_name.to_s
|
44
|
+
next if de_pref[param_name_str].nil? or de_pref[param_name_str].empty?
|
45
|
+
|
46
|
+
options.each do |option_name, option_value|
|
47
|
+
de_pref[param_name_str] = Fluent::DataEntryPreferenceConfigurable.replace_value_with_option(
|
48
|
+
param_name_str, de_pref[param_name_str], option_name, option_value, key: de['data_port_key'])
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
def copy_template(data_entry)
|
56
|
+
return unless File.exists?(conf_template_path(data_entry))
|
57
|
+
FileUtils.mkdir_p(CONFS_HOME)
|
58
|
+
FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry)+".tmpl")
|
59
|
+
|
60
|
+
# Do not overwrite a current conf file
|
61
|
+
unless File.exists?(conf_path(data_entry))
|
62
|
+
FileUtils.cp(conf_template_path(data_entry), conf_path(data_entry))
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def conf_path(data_entry)
|
67
|
+
"#{File.join(CONFS_HOME, data_entry['name'])}.conf"
|
68
|
+
end
|
69
|
+
|
70
|
+
def conf_exists?(data_entry)
|
71
|
+
File.exists?(conf_path(data_entry))
|
72
|
+
end
|
73
|
+
|
74
|
+
def conf_name(data_entry)
|
75
|
+
"#{data_entry['name']}.conf"
|
76
|
+
end
|
77
|
+
|
78
|
+
def configurable?(data_entry)
|
79
|
+
File.exists?(conf_template_path(data_entry))
|
80
|
+
end
|
81
|
+
|
82
|
+
def conf_template_path(data_entry)
|
83
|
+
File.join(FLYDATA_TMPL_DIR, conf_template_name(data_entry))
|
84
|
+
end
|
85
|
+
|
86
|
+
def conf_template_name(data_entry)
|
87
|
+
type = ActiveSupport::Inflector.underscore(data_entry['type'])
|
88
|
+
"#{type}.conf.tmpl"
|
89
|
+
end
|
90
|
+
|
91
|
+
def create(type)
|
92
|
+
custom_config_params = CUSTOM_CONFIG_PARAMS[type]
|
93
|
+
return nil unless custom_config_params
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
module Flydata
|
2
|
+
module FileUtil
|
3
|
+
class SyncFileManager
|
4
|
+
DUMP_DIR = ENV['FLYDATA_DUMP'] || File.join(FLYDATA_HOME, 'dump')
|
5
|
+
TABLE_POSITIONS_DIR = ENV['FLYDATA_TABLE_POSITIONS'] || File.join(FLYDATA_HOME, 'positions')
|
6
|
+
def initialize(data_entry)
|
7
|
+
@data_entry = data_entry
|
8
|
+
end
|
9
|
+
|
10
|
+
def dump_file_path
|
11
|
+
dump_dir = @data_entry['mysql_data_entry_preference']['mysqldump_dir']
|
12
|
+
if dump_dir
|
13
|
+
dump_dir = dump_dir.dup
|
14
|
+
dump_dir[0] = ENV['HOME'] if dump_dir.match(/^~$|^~\//)
|
15
|
+
else
|
16
|
+
dump_dir = DUMP_DIR.dup
|
17
|
+
end
|
18
|
+
if File.exists?(dump_dir) and not Dir.exists?(dump_dir)
|
19
|
+
raise "'mysqldump_dir'(#{dump_dir}) must be a directory."
|
20
|
+
end
|
21
|
+
FileUtils.mkdir_p(dump_dir) unless Dir.exists?(dump_dir)
|
22
|
+
File.join(dump_dir, @data_entry['name']) + ".dump"
|
23
|
+
end
|
24
|
+
|
25
|
+
# dump pos file for resume
|
26
|
+
def dump_pos_path
|
27
|
+
dump_file_path + ".pos"
|
28
|
+
end
|
29
|
+
|
30
|
+
def save_dump_pos(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
|
31
|
+
File.open(dump_pos_path, 'w') do |f|
|
32
|
+
f.write(dump_pos_content(status, table_name, last_pos, binlog_pos, state, substate))
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def load_dump_pos
|
37
|
+
path = dump_pos_path
|
38
|
+
return nil unless File.exists?(path)
|
39
|
+
items = File.open(path, 'r').readline.split("\t")
|
40
|
+
raise "Invalid dump.pos file: #{path}" unless items.length >= 5 && items.length <= 7
|
41
|
+
mysql_table = load_mysql_table_marshal_dump
|
42
|
+
{ status: items[0], table_name: items[1], last_pos: items[2].to_i,
|
43
|
+
binlog_pos: {binfile: items[3], pos: items[4].to_i},
|
44
|
+
state: items[5], substate: items[6], mysql_table: mysql_table}
|
45
|
+
end
|
46
|
+
|
47
|
+
# MysqlTable marshal file
|
48
|
+
def mysql_table_marshal_dump_path
|
49
|
+
dump_file_path + ".mysql_table"
|
50
|
+
end
|
51
|
+
|
52
|
+
def save_mysql_table_marshal_dump(mysql_table)
|
53
|
+
File.open(mysql_table_marshal_dump_path, 'w') do |f|
|
54
|
+
f.write Marshal.dump(mysql_table)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# binlog.pos file
|
59
|
+
def save_binlog(binlog_pos)
|
60
|
+
path = binlog_path
|
61
|
+
File.open(path, 'w') do |f|
|
62
|
+
f.write(binlog_content(binlog_pos))
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def binlog_path
|
67
|
+
File.join(FLYDATA_HOME, @data_entry['name'] + ".binlog.pos")
|
68
|
+
end
|
69
|
+
|
70
|
+
def table_positions_dir_path
|
71
|
+
TABLE_POSITIONS_DIR
|
72
|
+
end
|
73
|
+
|
74
|
+
def table_position_file_paths
|
75
|
+
Dir.glob(File.join(table_positions_dir_path, '*.pos'))
|
76
|
+
end
|
77
|
+
|
78
|
+
# Read a sequence number from the table's position file,
|
79
|
+
# increment the number and pass the number to a block.
|
80
|
+
# After executing the block, saves the value to the position
|
81
|
+
# file.
|
82
|
+
def increment_and_save_table_position(table_name)
|
83
|
+
file = File.join(table_positions_dir_path, table_name + ".pos")
|
84
|
+
retry_count = 0
|
85
|
+
begin
|
86
|
+
File.open(file, "r+") do |f|
|
87
|
+
seq = f.read
|
88
|
+
seq = seq.to_i + 1
|
89
|
+
yield(seq)
|
90
|
+
f.rewind
|
91
|
+
f.truncate(0)
|
92
|
+
f.write(seq)
|
93
|
+
end
|
94
|
+
rescue Errno::ENOENT
|
95
|
+
raise if retry_count > 0 # Already retried. Must be a differentfile causing the error
|
96
|
+
# File not exist. Create one with initial value of '0'
|
97
|
+
File.open(file, "w") {|f| f.write('0') }
|
98
|
+
retry_count += 1
|
99
|
+
retry
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
private
|
104
|
+
|
105
|
+
def dump_pos_content(status, table_name, last_pos, binlog_pos, state = nil, substate = nil)
|
106
|
+
[status, table_name, last_pos, binlog_content(binlog_pos), state, substate].join("\t")
|
107
|
+
end
|
108
|
+
|
109
|
+
def binlog_content(binlog_pos)
|
110
|
+
[binlog_pos[:binfile], binlog_pos[:pos]].join("\t")
|
111
|
+
end
|
112
|
+
|
113
|
+
def load_mysql_table_marshal_dump
|
114
|
+
path = mysql_table_marshal_dump_path
|
115
|
+
return nil unless File.exists?(path)
|
116
|
+
Marshal.load(File.open(path, 'r'))
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module Flydata
|
2
|
+
module Sync
|
3
|
+
|
4
|
+
class MysqlTableDef
|
5
|
+
TYPE_MAP_M2F = {
|
6
|
+
'bigint' => 'int8',
|
7
|
+
'binary' => 'binary',
|
8
|
+
'blob' => 'varbinary',
|
9
|
+
'bool' => 'int1',
|
10
|
+
'boolean' => 'int1',
|
11
|
+
'char' => 'varchar',
|
12
|
+
'date' => 'date',
|
13
|
+
'datetime' => 'datetime',
|
14
|
+
'dec' => 'numeric',
|
15
|
+
'decimal' => 'numeric',
|
16
|
+
'double' => 'float8',
|
17
|
+
'double precision' => 'float8',
|
18
|
+
'fixed' => 'numeric',
|
19
|
+
'float' => 'float4',
|
20
|
+
'int' => 'int4',
|
21
|
+
'longblob' => 'varbinary',
|
22
|
+
'longtext' => 'text',
|
23
|
+
'mediumblob' => 'varbinary',
|
24
|
+
'mediumint' => 'int3',
|
25
|
+
'mediumtext' => 'text',
|
26
|
+
'numeric' => 'numeric',
|
27
|
+
'smallint' => 'int2',
|
28
|
+
'text' => 'text',
|
29
|
+
'time' => 'time',
|
30
|
+
'timestamp' => 'datetime',
|
31
|
+
'tinyblob' => 'varbinary',
|
32
|
+
'tinyint' => 'int1',
|
33
|
+
'tinytext' => 'text',
|
34
|
+
'varbinary' => 'varbinary',
|
35
|
+
'varchar' => 'varchar',
|
36
|
+
}
|
37
|
+
|
38
|
+
def self.create(io)
|
39
|
+
params = _create(io)
|
40
|
+
params ? self.new(*params) : nil
|
41
|
+
end
|
42
|
+
|
43
|
+
def initialize(table_def, table_name, columns, default_charset, comment)
|
44
|
+
@table_def = table_def
|
45
|
+
@table_name = table_name
|
46
|
+
@columns = columns
|
47
|
+
@default_charset = default_charset
|
48
|
+
@comment = comment
|
49
|
+
end
|
50
|
+
|
51
|
+
def self._create(io)
|
52
|
+
|
53
|
+
table_def = ''
|
54
|
+
table_name = nil
|
55
|
+
columns = []
|
56
|
+
default_charset = nil
|
57
|
+
comment = nil
|
58
|
+
|
59
|
+
position = :before_create_table
|
60
|
+
io.each_line do |line|
|
61
|
+
if line =~ /CREATE TABLE `(.*?)`/
|
62
|
+
position = :in_create_table
|
63
|
+
table_name = $1
|
64
|
+
end
|
65
|
+
|
66
|
+
next unless position == :in_create_table
|
67
|
+
|
68
|
+
table_def += line.chomp
|
69
|
+
|
70
|
+
if line =~ /^\s*`(.*?)`\s+([a-z()0-9,\s]+)/ # column def
|
71
|
+
name = $1
|
72
|
+
type = $2.strip
|
73
|
+
type = type[0..-2] if type.end_with?(',')
|
74
|
+
TYPE_MAP_M2F.each do |mysql_type, flydata_type|
|
75
|
+
type.gsub!(/\b#{mysql_type}\b/, flydata_type)
|
76
|
+
end
|
77
|
+
column = {name: name, type: type}
|
78
|
+
column[:auto_increment] = true if line =~ /AUTO_INCREMENT/
|
79
|
+
column[:not_null] = true if line =~ /NOT NULL/
|
80
|
+
if /DEFAULT\s+((?:[^'\s]+\b)|(?:'(?:\\'|[^'])*'))/.match(line)
|
81
|
+
val = $1
|
82
|
+
val = val.slice(1..-1) if val.start_with?("'")
|
83
|
+
val = val.slice(0..-2) if val.end_with?("'")
|
84
|
+
column[:default] = val == "NULL" ? nil : val
|
85
|
+
end
|
86
|
+
if /COMMENT\s+'(((?:\\'|[^'])*))'/.match(line)
|
87
|
+
column[:comment] = $1
|
88
|
+
end
|
89
|
+
columns << column
|
90
|
+
end
|
91
|
+
|
92
|
+
if line =~ /PRIMARY KEY/
|
93
|
+
primary_keys = line.scan(/`(.*?)`/).collect{|item| item[0]}
|
94
|
+
primary_keys.each do |primary_key|
|
95
|
+
column = columns.detect {|column|
|
96
|
+
column[:name] === primary_key
|
97
|
+
}
|
98
|
+
raise "PK #{primary_key} must exist in the definition " if column.nil?
|
99
|
+
column[:primary_key] = true
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
if line =~ /DEFAULT CHARSET\s*=\s*([^\s]+)/
|
104
|
+
default_charset = $1
|
105
|
+
end
|
106
|
+
|
107
|
+
if line =~ /ENGINE=.*/ # TODO Need better parsing.
|
108
|
+
position = :after_create_table
|
109
|
+
break
|
110
|
+
end
|
111
|
+
end
|
112
|
+
position == :after_create_table ? [table_def, table_name, columns, default_charset, comment] : nil
|
113
|
+
end
|
114
|
+
attr_reader :columns, :table_name
|
115
|
+
|
116
|
+
def to_flydata_tabledef
|
117
|
+
tabledef = { table_name: @table_name,
|
118
|
+
columns: @columns,
|
119
|
+
}
|
120
|
+
tabledef[:default_charset] = @default_charset if @default_charset
|
121
|
+
tabledef[:comment] = @comment if @comment
|
122
|
+
|
123
|
+
tabledef
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
module Flydata
|
2
|
+
module Sync
|
3
|
+
|
4
|
+
class RedshiftTableDef
|
5
|
+
TYPE_MAP_F2R = {
|
6
|
+
'binary' => {type: 'varchar', use_params: true},
|
7
|
+
'char' => {type: 'char', use_params: true},
|
8
|
+
'date' => {type: 'date'},
|
9
|
+
'datetime' => {type: 'timestamp'},
|
10
|
+
'float4' => {type: 'float4'},
|
11
|
+
'float4 unsigned' => {type: 'float4'},
|
12
|
+
'float8' => {type: 'float8'},
|
13
|
+
'float8 unsigned' => {type: 'float8'},
|
14
|
+
'int1' => {type: 'int2'},
|
15
|
+
'int1 unsigned' => {type: 'int2', unsigned: true},
|
16
|
+
'int2' => {type: 'int2'},
|
17
|
+
'int2 unsigned' => {type: 'int4', unsigned: true},
|
18
|
+
'int3' => {type: 'int4'},
|
19
|
+
'int3 unsigned' => {type: 'int4', unsigned: true},
|
20
|
+
'int4' => {type: 'int4'},
|
21
|
+
'int4 unsigned' => {type: 'int8', unsigned: true},
|
22
|
+
'int8' => {type: 'int8'},
|
23
|
+
'int8 unsigned' => {type: 'numeric(20,0)', unsigned: true},
|
24
|
+
'numeric' => {type: 'numeric', use_params: true},
|
25
|
+
'numeric unsigned' => {type: 'numeric', use_params: true},
|
26
|
+
'text' => {type: 'varchar(max)'},
|
27
|
+
'time' => {type: 'timestamp'},
|
28
|
+
'varbinary' => {type: 'varchar', use_params: true},
|
29
|
+
'varchar' => {type: 'varchar', use_params: true},
|
30
|
+
}
|
31
|
+
def self.from_flydata_tabledef(flydata_tabledef, options = {})
|
32
|
+
options[:flydata_ctl_table] = true unless options.has_key?(:flydata_ctl_table)
|
33
|
+
|
34
|
+
tabledef = ""
|
35
|
+
tabledef += create_flydata_ctl_table_sql if options[:flydata_ctl_table]
|
36
|
+
tabledef += create_table_sql(flydata_tabledef)
|
37
|
+
tabledef += comment_sql(flydata_tabledef)
|
38
|
+
tabledef += flydata_ctl_sql(flydata_tabledef)
|
39
|
+
end
|
40
|
+
|
41
|
+
CREATE_FLYDATA_CTL_TABLE_SQL = <<EOS
|
42
|
+
CREATE TABLE flydata_ctl_columns (
|
43
|
+
id integer NOT NULL IDENTITY(1,1),
|
44
|
+
table_name varchar(128) NOT NULL,
|
45
|
+
column_name varchar(128) NOT NULL,
|
46
|
+
src_data_type varchar(1024) NOT NULL,
|
47
|
+
PRIMARY KEY(id)
|
48
|
+
) DISTKEY(table_name) SORTKEY(table_name);
|
49
|
+
EOS
|
50
|
+
def self.create_flydata_ctl_table_sql
|
51
|
+
# No drop table here intentionally because losing the data is fatal.
|
52
|
+
CREATE_FLYDATA_CTL_TABLE_SQL
|
53
|
+
end
|
54
|
+
|
55
|
+
CREATE_TABLE_SQL = <<EOS
|
56
|
+
DROP TABLE %s;
|
57
|
+
CREATE TABLE %s (
|
58
|
+
%s
|
59
|
+
);
|
60
|
+
EOS
|
61
|
+
|
62
|
+
def self.create_table_sql(flydata_tabledef)
|
63
|
+
lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
|
64
|
+
pk_def = primary_key_sql(flydata_tabledef)
|
65
|
+
lines << pk_def if pk_def
|
66
|
+
|
67
|
+
contents = lines.join(",\n")
|
68
|
+
|
69
|
+
table_name = flydata_tabledef[:table_name]
|
70
|
+
CREATE_TABLE_SQL % [table_name, table_name, contents]
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.column_def_sql(column)
|
74
|
+
type = column[:type]
|
75
|
+
if type =~ /\(.*?\)/
|
76
|
+
type = $` + $'
|
77
|
+
params = $&
|
78
|
+
end
|
79
|
+
|
80
|
+
type_info = TYPE_MAP_F2R[type]
|
81
|
+
raise "Unsupported type '#{column[:type]}'" if type_info.nil?
|
82
|
+
|
83
|
+
rs_type = (type_info[:use_params] && params && !params.nil?) ?
|
84
|
+
type_info[:type] + "#{params}"
|
85
|
+
: type_info[:type]
|
86
|
+
line = %Q| "#{column[:name]}" #{rs_type}|
|
87
|
+
line += " NOT NULL" if column[:not_null]
|
88
|
+
if (column.has_key?(:default))
|
89
|
+
val = replace_default_value(type_info[:type], column[:default])
|
90
|
+
line += " DEFAULT #{val}"
|
91
|
+
end
|
92
|
+
# Commented out because no IDENTITY column must be used for a replicated table.
|
93
|
+
# Values come from the master.
|
94
|
+
# line += " IDENTITY(1, 1)" if (column[:auto_increment])
|
95
|
+
|
96
|
+
line
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.replace_default_value(type, default_value)
|
100
|
+
val = default_value ? "'#{default_value}'" : "NULL"
|
101
|
+
case type
|
102
|
+
when 'timestamp'
|
103
|
+
(val.upcase == "'CURRENT_TIMESTAMP'") ? 'SYSDATE' : val
|
104
|
+
else
|
105
|
+
val
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.primary_key_sql(flydata_tabledef)
|
110
|
+
pks = flydata_tabledef[:columns].select{|col| col[:primary_key]}.collect{|col| col[:name]}
|
111
|
+
pks.empty? ? nil : " PRIMARY KEY (#{pks.join(',')})"
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.comment_sql(flydata_tabledef)
|
115
|
+
sql = ""
|
116
|
+
flydata_tabledef[:columns].each do |col|
|
117
|
+
next unless col[:comment]
|
118
|
+
|
119
|
+
sql += <<EOS
|
120
|
+
COMMENT ON COLUMN #{flydata_tabledef[:table_name]}."#{col[:name]}"
|
121
|
+
IS '#{col[:comment]}';
|
122
|
+
EOS
|
123
|
+
end
|
124
|
+
sql
|
125
|
+
end
|
126
|
+
|
127
|
+
FLYDATA_CTL_COLUMNS_SQL = <<EOS
|
128
|
+
DELETE FROM flydata_ctl_columns WHERE table_name = '%s';
|
129
|
+
INSERT INTO flydata_ctl_columns (table_name, column_name, src_data_type) VALUES
|
130
|
+
EOS
|
131
|
+
def self.flydata_ctl_sql(flydata_tabledef)
|
132
|
+
sql = FLYDATA_CTL_COLUMNS_SQL % [ flydata_tabledef[:table_name] ]
|
133
|
+
values = []
|
134
|
+
flydata_tabledef[:columns].each do |col|
|
135
|
+
values << "('#{flydata_tabledef[:table_name]}', '#{col[:name]}', '#{col[:type]}')"
|
136
|
+
end
|
137
|
+
sql += values.join(",\n") + ';'
|
138
|
+
sql
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|