flydata 0.6.3 → 0.6.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +2 -2
  3. data/VERSION +1 -1
  4. data/bin/fdredshift +78 -0
  5. data/circle.yml +1 -1
  6. data/ext/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  7. data/ext/flydata/{parser/mysql → source_mysql/parser}/dump_parser_ext.cpp +3 -3
  8. data/ext/flydata/source_mysql/parser/extconf.rb +3 -0
  9. data/ext/flydata/{parser/mysql → source_mysql/parser}/parser.txt +0 -0
  10. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.cpp +0 -0
  11. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.h +0 -0
  12. data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +34 -32
  13. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +20 -0
  14. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +12 -4
  15. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +60 -6
  16. data/flydata-core/spec/mysql/binlog_pos_spec.rb +474 -0
  17. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +57 -0
  18. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +174 -20
  19. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump +43 -0
  20. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump +43 -0
  21. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump +43 -0
  22. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump +43 -0
  23. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +41 -8
  24. data/flydata.gemspec +0 -0
  25. data/lib/flydata/cli.rb +11 -5
  26. data/lib/flydata/command/base.rb +14 -1
  27. data/lib/flydata/command/exclusive_runnable.rb +42 -12
  28. data/lib/flydata/command/helper.rb +6 -6
  29. data/lib/flydata/command/sender.rb +4 -3
  30. data/lib/flydata/command/setup.rb +30 -381
  31. data/lib/flydata/command/stop.rb +1 -0
  32. data/lib/flydata/command/sync.rb +273 -301
  33. data/lib/flydata/compatibility_check.rb +24 -117
  34. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +3 -3
  35. data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +2 -2
  36. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -6
  37. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +0 -1
  38. data/lib/flydata/parser.rb +14 -0
  39. data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb} +6 -4
  40. data/lib/flydata/parser/source_table.rb +33 -0
  41. data/lib/flydata/source.rb +105 -0
  42. data/lib/flydata/source/component.rb +21 -0
  43. data/lib/flydata/source/errors.rb +7 -0
  44. data/lib/flydata/source/generate_source_dump.rb +72 -0
  45. data/lib/flydata/source/parse_dump_and_send.rb +52 -0
  46. data/lib/flydata/source/setup.rb +31 -0
  47. data/lib/flydata/source/source_pos.rb +45 -0
  48. data/lib/flydata/source/sync.rb +56 -0
  49. data/lib/flydata/source/sync_generate_table_ddl.rb +43 -0
  50. data/lib/flydata/source_file/setup.rb +17 -0
  51. data/lib/flydata/source_file/sync.rb +14 -0
  52. data/lib/flydata/{command → source_mysql/command}/mysql.rb +2 -1
  53. data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb +2 -4
  54. data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb +2 -1
  55. data/lib/flydata/{command → source_mysql/command}/mysqldump.rb +2 -1
  56. data/lib/flydata/source_mysql/generate_source_dump.rb +53 -0
  57. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +114 -0
  58. data/lib/flydata/source_mysql/parse_dump_and_send.rb +28 -0
  59. data/lib/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  60. data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb +32 -67
  61. data/lib/flydata/{parser/mysql → source_mysql/parser}/mysql_alter_table.treetop +0 -0
  62. data/lib/flydata/source_mysql/setup.rb +24 -0
  63. data/lib/flydata/source_mysql/source_pos.rb +21 -0
  64. data/lib/flydata/source_mysql/sync.rb +45 -0
  65. data/lib/flydata/source_mysql/sync_generate_table_ddl.rb +40 -0
  66. data/lib/flydata/{mysql → source_mysql}/table_ddl.rb +6 -17
  67. data/lib/flydata/source_zendesk/sync_generate_table_ddl.rb +30 -0
  68. data/lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb +133 -0
  69. data/lib/flydata/sync_file_manager.rb +132 -73
  70. data/lib/flydata/table_ddl.rb +18 -0
  71. data/spec/flydata/cli_spec.rb +1 -0
  72. data/spec/flydata/command/exclusive_runnable_spec.rb +19 -8
  73. data/spec/flydata/command/sender_spec.rb +1 -1
  74. data/spec/flydata/command/setup_spec.rb +4 -4
  75. data/spec/flydata/command/sync_spec.rb +97 -134
  76. data/spec/flydata/compatibility_check_spec.rb +16 -289
  77. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -3
  78. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +1 -1
  79. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +4 -2
  80. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +1 -1
  81. data/spec/flydata/source_mysql/generate_source_dump_spec.rb +69 -0
  82. data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +280 -0
  83. data/spec/flydata/{parser/mysql → source_mysql/parser}/alter_table_parser_spec.rb +2 -2
  84. data/spec/flydata/{parser/mysql → source_mysql/parser}/dump_parser_spec.rb +75 -70
  85. data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb +137 -0
  86. data/spec/flydata/{mysql → source_mysql}/table_ddl_spec.rb +2 -2
  87. data/spec/flydata/source_spec.rb +140 -0
  88. data/spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb +33 -0
  89. data/spec/flydata/sync_file_manager_spec.rb +157 -77
  90. data/tmpl/redshift_mysql_data_entry.conf.tmpl +1 -1
  91. metadata +56 -23
  92. data/ext/flydata/parser/mysql/extconf.rb +0 -3
  93. data/lib/flydata/mysql/binlog_position.rb +0 -22
  94. data/spec/flydata/mysql/binlog_position_spec.rb +0 -35
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 45df6073e7d4b59b307a695058cd1ee70366eb89
4
- data.tar.gz: cdac2f4feb8dfdd4531c14f97a38f1dd2df91f48
3
+ metadata.gz: 12c8fd94c861658f352698075555d8a868d3ca6d
4
+ data.tar.gz: ff5061d23bb1cee12430d471834131034bc925b2
5
5
  SHA512:
6
- metadata.gz: df38e5c265ee0b0e4f9cfcde1a6f219db1fa0b4df013d4f79d1a9dc648d4d5684210560153d91ace7a5267cde76c4d26d00d5ee95e757c0acee2867b4ea6be9c
7
- data.tar.gz: 7ec85d6e13314f445649d64c1769597f2fd192d35c5a6b76b42043accfca4be1d2fc5d95b7c40de7f00e462876bd67076e0102efa0634f755af4d88ebba2f2f0
6
+ metadata.gz: 0c956f29080dbc9a517a08b2b865aceef102f26a509061cb2152ee519d0f2d99794348546001d09c48f9b7ae2ed8015b12a593520780ffe1517614a9fc1dedde
7
+ data.tar.gz: 731ecc427a94b7aab2afb638c8dbbb532b6055f080ed938574d26a19b187478d1fecbb75dc1c09a066b816983dfb0611711a27b8123b14370efda8128f535da0
data/Rakefile CHANGED
@@ -36,8 +36,8 @@ task :default => :spec
36
36
  require 'rake/extensiontask'
37
37
 
38
38
  Rake::ExtensionTask.new "dump_parser_ext" do |ext|
39
- ext.ext_dir = "ext/flydata/parser/mysql"
40
- ext.lib_dir = "lib/flydata/parser/mysql"
39
+ ext.ext_dir = "ext/flydata/source_mysql/parser"
40
+ ext.lib_dir = "lib/flydata/source_mysql/parser"
41
41
  end
42
42
 
43
43
  Rake::ExtensionTask.new "json_ext" do |ext|
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.3
1
+ 0.6.4
data/bin/fdredshift ADDED
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+
6
+ require 'rubygems'
7
+ require 'flydata'
8
+ require 'shellwords'
9
+
10
+ # Write your code from here
11
+ class FdRedshift < Flydata::Command::Base
12
+ def run(*args)
13
+ validate_required_values
14
+ rs_info = retrieve_redshift_info
15
+ cmd = generate_psql_command(rs_info, args)
16
+ $stderr.puts "cmd:#{cmd}" if ENV['FLYDATA_DEBUG']
17
+ if $stdin.tty?
18
+ # interactive shell
19
+ system cmd
20
+ else
21
+ # execute queries given to $stdin
22
+ Open3.popen2e(cmd) do |i, o, wt|
23
+ $stdin.each_line do |line|
24
+ i.print line
25
+ end
26
+ i.close
27
+ while line = o.gets
28
+ print line
29
+ end
30
+ end
31
+ end
32
+ end
33
+
34
+ def generate_psql_command(rs_info,args)
35
+ password = "PGPASSWORD=#{rs_info['password'].shellescape}"
36
+ cmd = "psql"
37
+ dbname = "-d #{rs_info['dbname'].shellescape}"
38
+ host = "-h #{rs_info['host'].shellescape}"
39
+ port = "-p #{rs_info['port'].to_s.shellescape}"
40
+ username = "-U #{rs_info['username'].shellescape}"
41
+ options = args.join(' ')
42
+ #"PGPASSWORD='#{password}' psql -d '#{dbname}' -h '#{host}' -p '#{port}' -U '#{username}'"
43
+ [password,cmd,dbname,host,port,username,options].join(' ')
44
+ end
45
+
46
+ def retrieve_redshift_info(token = Flydata::Credentials.new.token)
47
+ admin_agent_key = read_admin_agent_api_key
48
+ headers = {'AUTHORIZATION' => %Q|Token token="#{admin_agent_key}"|}
49
+
50
+ host = read_admin_agent_api_host
51
+ path = "/api/v1/redshift_clusters/show_default?user_auth_token=#{token}"
52
+ url = host + path
53
+ response = RestClient::Request.execute(method: :get,
54
+ url: url,
55
+ headers: headers)
56
+ JSON.parse(response)
57
+ end
58
+
59
+ def read_admin_agent_api_host
60
+ ENV['FLYDATA_AGENT_FLYDATA_ADMIN_API_HOST']
61
+ end
62
+
63
+ def read_admin_agent_api_key
64
+ ENV['FLYDATA_AGENT_FLYDATA_ADMIN_API_KEY']
65
+ end
66
+
67
+ def validate_required_values
68
+ unless read_admin_agent_api_host && read_admin_agent_api_key
69
+ $stderr.puts "ERROR! This command is not supported in your environment."
70
+ exit 1
71
+ end
72
+ end
73
+ end
74
+
75
+
76
+ args = ARGV.dup
77
+ ARGV.clear # for 'gets'
78
+ FdRedshift.new().run(args)
data/circle.yml CHANGED
@@ -12,7 +12,7 @@ dependencies:
12
12
  - sudo apt-get install -y libboost-thread1.54.0
13
13
  - sudo apt-get install -y libboost-thread1.54-dev
14
14
  - sudo apt-get install -y libboost1.54-dev
15
- - wget --no-check-certificate -O mysql-replication-listener.zip https://github.com/hapyrus/mysql-replication-listener/archive/master.zip
15
+ - wget --no-check-certificate -O mysql-replication-listener.zip https://github.com/flydata/mysql-replication-listener/archive/master.zip
16
16
  - unzip mysql-replication-listener.zip
17
17
  - cd mysql-replication-listener-master; cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DBoost_NO_BOOST_CMAKE=ON .; make -j4; sudo make install
18
18
 
@@ -97,9 +97,9 @@ static VALUE rb_insert_parser_parse2(VALUE self, VALUE sql)
97
97
  void Init_dump_parser_ext()
98
98
  {
99
99
  VALUE mFlydata = rb_define_module("Flydata");
100
- VALUE mParser = rb_define_module_under(mFlydata, "Parser");
101
- VALUE mMysql = rb_define_module_under(mParser, "Mysql");
102
- VALUE cMysqlDumpParser = rb_define_class_under(mMysql, "MysqlDumpParser", rb_cObject);
100
+ VALUE mSourceMysql = rb_define_module_under(mFlydata, "SourceMysql");
101
+ VALUE mParser = rb_define_module_under(mSourceMysql, "Parser");
102
+ VALUE cMysqlDumpParser = rb_define_class_under(mParser, "MysqlDumpParser", rb_cObject);
103
103
  VALUE cInsertParser = rb_define_class_under(cMysqlDumpParser, "InsertParser", rb_cObject);
104
104
 
105
105
  rb_define_private_method(cInsertParser, "_parse2", __F(&rb_insert_parser_parse2), 1);
@@ -0,0 +1,3 @@
1
+ require "mkmf"
2
+
3
+ create_makefile "flydata/source_mysql/parser/dump_parser_ext"
@@ -2,17 +2,26 @@ module FlydataCore
2
2
  module Mysql
3
3
 
4
4
  class BinlogPos
5
+ include Comparable
5
6
  def initialize(binlog_str_or_filename_or_hash, binlog_pos = nil)
6
7
  arg = binlog_str_or_filename_or_hash
7
8
  if binlog_pos
8
9
  @pos = binlog_pos
9
10
  @filename = arg
10
11
  elsif arg.kind_of?(String)
11
- @filename, @pos = arg.split("\t")
12
+ if arg == "-"
13
+ set_empty
14
+ else
15
+ @filename, @pos = arg.split("\t")
16
+ end
12
17
  elsif arg.kind_of?(Hash)
13
- @pos = arg[:pos]
14
- @filename = arg[:filename]
15
- @filename ||= arg[:binfile]
18
+ if arg[:filename] == "-" || arg[:binfile] == "-"
19
+ set_empty
20
+ else
21
+ @pos = arg[:pos]
22
+ @filename = arg[:filename]
23
+ @filename ||= arg[:binfile]
24
+ end
16
25
  end
17
26
  if @filename.nil? || @pos.nil?
18
27
  raise "Invalid initialize argument (#{arg}, #{binlog_pos})"
@@ -22,50 +31,43 @@ class BinlogPos
22
31
 
23
32
  attr_reader :filename, :pos
24
33
 
34
+ def empty?
35
+ @filename && @filename == "-"
36
+ end
37
+
25
38
  def <=>(other_pos)
26
39
  if other_pos.nil?
27
40
  raise ArgumentError.new("comparison of BinlosPos with nil failed")
28
41
  end
29
42
 
30
43
  other_pos = BinlogPos.new(other_pos) unless other_pos.kind_of?(BinlogPos)
31
-
32
- if @filename < other_pos.filename
33
- -1
34
- elsif @filename == other_pos.filename
35
- if @pos < other_pos.pos
36
- -1
37
- elsif @pos == other_pos.pos
38
- 0
39
- else
40
- 1
41
- end
44
+ if empty? && other_pos.empty?
45
+ return 0
46
+ elsif empty? || other_pos.empty?
47
+ raise ArgumentError.new("comparison with empty BinlogPos failed")
42
48
  else
43
- 1
49
+ (@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
44
50
  end
45
51
  end
46
- def <(other_pos)
47
- self.<=>(other_pos) < 0
48
- end
49
- def <=(other_pos)
50
- self.<=>(other_pos) <= 0
51
- end
52
+
52
53
  def ==(other_pos)
53
54
  return false if other_pos.nil?
55
+ return @filename == other_pos.filename if empty? || other_pos.empty?
54
56
  self.<=>(other_pos) == 0
55
57
  end
56
- def !=(other_pos)
57
- !self.==(other_pos)
58
- end
59
- def >(other_pos)
60
- self.<=>(other_pos) > 0
61
- end
62
- def >=(other_pos)
63
- self.<=>(other_pos) >= 0
64
- end
65
58
 
66
59
  def to_s
67
- "#{@filename}\t#{@pos}"
60
+ empty? ? "-" : "#{@filename}\t#{@pos}"
68
61
  end
62
+
63
+ private
64
+ def set_empty
65
+ # @filename == "-" is the definition of `empty`. @pos could be anything but nul
66
+ @filename = "-"
67
+ @pos = -1
68
+ end
69
+
70
+
69
71
  end
70
72
 
71
73
  end
@@ -226,6 +226,26 @@ EOT
226
226
  end
227
227
  end
228
228
 
229
+ class MySqlVersion57CompabilityChecker < MysqlCompatibilityChecker
230
+
231
+ def create_query(option = @option)
232
+ "show variables where Variable_name in ('version','show_compatibility_56');"
233
+ end
234
+
235
+ def check_result(result, option = @option)
236
+ mysqlversion = nil
237
+ compability56 = nil
238
+ result.each { |value|
239
+ mysqlversion = value['Value'] if value['Variable_name'] == 'version'
240
+ compability56 = value['Value'] if value['Variable_name'] == 'show_compatibility_56'
241
+ }
242
+ if mysqlversion.start_with?('5.7') && compability56 == "OFF"
243
+ raise FlydataCore::MysqlCompatibilityError,
244
+ "Make sure you set 'show_compatibility_56 = ON' in your MySQL configuration. Once you update your MySQL configuration, restart the MySQL server for the change to take effect."
245
+ end
246
+ end
247
+ end
248
+
229
249
  class TableTypeChecker < MysqlCompatibilityChecker
230
250
  SELECT_TABLE_INFO_TMPLT =
231
251
  "SELECT table_name, table_type, engine " +
@@ -171,6 +171,11 @@ class MysqlTableDef
171
171
 
172
172
  tabledef
173
173
  end
174
+
175
+ # Replaces a MySQL string with an empty string ('')
176
+ def self.strip_string(line)
177
+ line.gsub(/'(\\'|''|[^'])*'/, "''")
178
+ end
174
179
 
175
180
  def self.parse_one_column_def(query)
176
181
  line = query.strip
@@ -207,11 +212,14 @@ class MysqlTableDef
207
212
  column[:type] = convert_to_flydata_type(type)
208
213
 
209
214
  cond = :options
215
+
210
216
  when :options
211
- column[:type] += ' unsigned' if line =~ /unsigned/i
212
- column[:auto_increment] = true if line =~ /AUTO_INCREMENT/i
213
- column[:not_null] = true if line =~ /NOT NULL/i
214
- column[:unique] = true if line =~ /UNIQUE/i
217
+ strip_line = strip_string(line)
218
+ column[:type] += ' unsigned' if strip_line =~ /unsigned/i
219
+ column[:auto_increment] = true if strip_line =~ /AUTO_INCREMENT/i
220
+ column[:not_null] = true if strip_line =~ /NOT NULL/i
221
+ column[:unique] = true if strip_line =~ /UNIQUE/i
222
+
215
223
  if /DEFAULT\s+((?:[bx]?'(?:\\'|[^'])*')|(?:[^'\s]+\b))/i.match(line)
216
224
  val = $1
217
225
  column[:default] = val == "NULL" ? nil : val
@@ -45,9 +45,14 @@ class RedshiftTableDef
45
45
  tabledef = ""
46
46
  tabledef += create_schema_sql(schema_name) if options[:flydata_ctl_table] && !schema_name.to_s.empty?
47
47
  tabledef += create_flydata_ctl_table_sql(schema_name) if options[:flydata_ctl_table]
48
+ options[:backup_postfix] = "_flydata#{Time.now.strftime('%Y%m%d%H%M%S')}"
49
+ tabledef += "BEGIN;\n"
50
+ tabledef += remove_table_sql(flydata_tabledef, schema_name, options) unless options[:ctl_only]
48
51
  tabledef += create_table_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
49
52
  tabledef += comment_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
50
53
  tabledef += flydata_ctl_sql(flydata_tabledef, schema_name)
54
+ tabledef += "COMMIT;\n"
55
+ tabledef += drop_backup_table_sql(flydata_tabledef, schema_name, options) unless options[:ctl_only]
51
56
  rescue => e
52
57
  # Catch errors from generating schema. Generally an unsupported data type
53
58
  raise TableDefError, {error: "errors generating schema. Please contact us for further instructions", table: flydata_tabledef[:table_name]}
@@ -124,14 +129,61 @@ EOS
124
129
  CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
125
130
  end
126
131
 
127
- CREATE_TABLE_SQL = <<EOS
132
+ DROP_TABLE_SQL = <<EOS
128
133
  DROP TABLE IF EXISTS %s;
134
+ EOS
135
+
136
+ RENAME_TABLE_SQL = <<EOS
137
+ ALTER TABLE %s RENAME TO %s;
138
+ EOS
139
+
140
+ CREATE_TABLE_SQL = <<EOS
129
141
  CREATE TABLE %s (
130
142
  %s
131
143
  )%s;
132
144
  EOS
133
145
 
134
- def self.create_table_sql(flydata_tabledef, schema_name)
146
+ CREATE_TABLE_IF_NOT_EXISTS_SQL = <<EOS
147
+ CREATE TABLE IF NOT EXISTS %s (
148
+ %s
149
+ )%s;
150
+ EOS
151
+
152
+ def self.remove_table_sql(flydata_tabledef, schema_name, options)
153
+ return "" if options[:skip_drop_table]
154
+ table_name = flydata_tabledef[:table_name]
155
+ redshift_tbl = table_name_for_ddl(table_name, schema_name)
156
+ sql = ""
157
+ if options[:backup_postfix]
158
+ # drop backup table if exists
159
+ sql += drop_backup_table_sql(flydata_tabledef, schema_name, options)
160
+ # create an empty table to prevent RENAME TABLE query from failing
161
+ sql += create_table_sql(flydata_tabledef, schema_name,
162
+ create_table_sql: CREATE_TABLE_IF_NOT_EXISTS_SQL)
163
+ backup_tbl = table_name_for_ddl(
164
+ "#{table_name}#{options[:backup_postfix]}", nil)
165
+ sql += RENAME_TABLE_SQL % [redshift_tbl, backup_tbl]
166
+ else
167
+ sql += DROP_TABLE_SQL % redshift_tbl
168
+ end
169
+ sql
170
+ end
171
+
172
+ def self.drop_backup_table_sql(flydata_tabledef, schema_name, options)
173
+ return "" if options[:skip_drop_table]
174
+ sql = ""
175
+ if options[:backup_postfix]
176
+ table_name = flydata_tabledef[:table_name]
177
+ backup_tbl = table_name_for_ddl(
178
+ "#{table_name}#{options[:backup_postfix]}", schema_name)
179
+ sql += DROP_TABLE_SQL % backup_tbl
180
+ end
181
+
182
+ sql
183
+ end
184
+
185
+ def self.create_table_sql(flydata_tabledef, schema_name, options = {})
186
+ ct_sql = options[:create_table_sql] || CREATE_TABLE_SQL
135
187
  lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
136
188
  pk_def = primary_key_sql(flydata_tabledef)
137
189
  lines << pk_def if pk_def
@@ -142,7 +194,9 @@ EOS
142
194
 
143
195
  table_name = flydata_tabledef[:table_name]
144
196
  redshift_tbl = table_name_for_ddl(table_name, schema_name)
145
- CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents, dk_sk_def]
197
+ sql = ""
198
+ sql += ct_sql % [redshift_tbl, contents, dk_sk_def]
199
+ sql
146
200
  end
147
201
 
148
202
  def self.column_def_sql(column, opt = {})
@@ -261,7 +315,7 @@ EOS
261
315
  end
262
316
 
263
317
  def self.flydata_ctl_sql(flydata_tabledef, schema_name)
264
- flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
318
+ flydata_ctl_columns_sql(flydata_tabledef, schema_name) +
265
319
  flydata_ctl_tables_sql(flydata_tabledef, schema_name)
266
320
  end
267
321
  FLYDATA_CTL_COLUMNS_SQL = <<EOS
@@ -276,7 +330,7 @@ EOS
276
330
  charset = col[:charset] ? " cs:#{col[:charset]}" : ""
277
331
  values << "('#{escape(flydata_tabledef[:table_name])}', '#{escape(col[:column])}', '#{escape(col[:type])}#{charset}', #{i})"
278
332
  end
279
- sql += values.join(",\n") + ';'
333
+ sql += values.join(",\n") + ";\n"
280
334
  sql
281
335
  end
282
336
 
@@ -292,7 +346,7 @@ EOS
292
346
  values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
293
347
  values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
294
348
  values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
295
- sql += values.join(",\n") + ';'
349
+ sql += values.join(",\n") + ";\n"
296
350
  sql
297
351
  end
298
352
 
@@ -0,0 +1,474 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/mysql/binlog_pos.rb'
3
+
4
+ module FlydataCore
5
+ module Mysql
6
+ describe BinlogPos do
7
+ let(:subject_object) { described_class.new(binlog_str) }
8
+ let(:binlog_str) { "#{filename}\t#{pos}" }
9
+ let(:filename) { 'mysql-bin.000064' }
10
+ let(:pos) { 104 }
11
+
12
+ describe '#pos' do
13
+ subject { subject_object.pos }
14
+ it { is_expected.to eq(pos) }
15
+ end
16
+ describe '#filename' do
17
+ subject { subject_object.filename }
18
+ it { is_expected.to eq(filename) }
19
+ end
20
+ describe '#to_s' do
21
+ subject { subject_object.to_s }
22
+ it { is_expected.to eq(binlog_str) }
23
+ end
24
+
25
+ describe '.new' do
26
+ subject { described_class.new(*args) }
27
+ shared_examples "properly constructed" do
28
+ it do
29
+ expect(subject.filename).to eq(filename)
30
+ expect(subject.pos).to eq(pos)
31
+ end
32
+ end
33
+
34
+ context 'when given two arguments' do
35
+ let(:args) { [filename, pos] }
36
+ it_behaves_like "properly constructed"
37
+ end
38
+
39
+ let(:args) { [ arg1 ] }
40
+ context 'when given a String' do
41
+ let(:arg1) { binlog_str }
42
+ it_behaves_like "properly constructed"
43
+ end
44
+
45
+ context 'when given a Hash' do
46
+ let(:arg1) { { filename: filename, pos: pos } }
47
+ it_behaves_like "properly constructed"
48
+ end
49
+ end
50
+
51
+ let(:arg) { described_class.new (arg_str) }
52
+ let(:arg_str) { arg_binlog_str }
53
+ let(:arg_binlog_str) { "#{arg_filename}\t#{arg_pos}" }
54
+ let(:older_filename) { 'mysql-bin.000061' }
55
+ let(:newer_filename) { 'mysql-bin.000065' }
56
+
57
+ shared_examples 'returning the same result no matter what pos value is' do
58
+ context 'when the target pos is smaller' do
59
+ let(:arg_pos) { pos - 10 }
60
+ it { is_expected.to eq expected_result }
61
+ end
62
+ context 'when the target pos is the same' do
63
+ let(:arg_pos) { pos }
64
+ it { is_expected.to eq expected_result }
65
+ end
66
+ context 'when the target pos is larger' do
67
+ let(:arg_pos) { pos + 10 }
68
+ it { is_expected.to eq expected_result }
69
+ end
70
+ end
71
+
72
+ describe '#<=>' do
73
+ subject { subject_object <=> arg }
74
+
75
+ context 'when the target is nil' do
76
+ let(:arg) { nil }
77
+ it { expect { subject }.to raise_error(ArgumentError) }
78
+ end
79
+ context 'when the target has the same value' do
80
+ let(:arg_filename) { filename }
81
+ let(:arg_pos) { pos }
82
+ it { is_expected.to eq 0 }
83
+ end
84
+
85
+ context 'when the target binlog filename is the same' do
86
+ let(:arg_filename) { filename }
87
+ context 'when the target pos is smaller' do
88
+ let(:arg_pos) { pos - 10 }
89
+ it { is_expected.to eq 1 }
90
+ end
91
+ context 'when the target pos is the same' do
92
+ let(:arg_pos) { pos }
93
+ it { is_expected.to eq 0 }
94
+ end
95
+ context 'when the target pos is larger' do
96
+ let(:arg_pos) { pos + 10 }
97
+ it { is_expected.to eq -1 }
98
+ end
99
+ end
100
+
101
+ context 'when the target binlog filename is older' do
102
+ let(:arg_filename) { older_filename }
103
+ let(:expected_result) { 1 }
104
+ it_behaves_like 'returning the same result no matter what pos value is'
105
+ end
106
+ context 'when the target binlog filename is newer' do
107
+ let(:arg_filename) { newer_filename }
108
+ let(:expected_result) { -1 }
109
+ it_behaves_like 'returning the same result no matter what pos value is'
110
+ end
111
+
112
+ context 'when the target is empty' do
113
+ let(:arg_str) { "-" }
114
+ it { expect { subject }.to raise_error(ArgumentError) }
115
+ end
116
+ context 'when self is empty' do
117
+ let(:binlog_str) { "-" }
118
+ context 'when target is empty' do
119
+ let(:arg_str) { "-" }
120
+ it { is_expected.to eq 0 }
121
+ end
122
+ context 'when target is not empty' do
123
+ let(:arg) { double('arg') }
124
+ before do
125
+ allow(arg).to receive(:kind_of?).with(BinlogPos).and_return(true)
126
+ allow(arg).to receive(:empty?).and_return(false)
127
+ end
128
+ it { expect { subject }.to raise_error(ArgumentError) }
129
+ end
130
+ end
131
+ end
132
+
133
+ describe '#<' do
134
+ subject { subject_object < arg }
135
+
136
+ context 'when the target has the same value' do
137
+ let(:arg_filename) { filename }
138
+ let(:arg_pos) { pos }
139
+ it { is_expected.to be_falsey }
140
+ end
141
+
142
+ context 'when the target binlog filename is the same' do
143
+ let(:arg_filename) { filename }
144
+ context 'when the target pos is smaller' do
145
+ let(:arg_pos) { pos - 10 }
146
+ it { is_expected.to be_falsey }
147
+ end
148
+ context 'when the target pos is the same' do
149
+ let(:arg_pos) { pos }
150
+ it { is_expected.to be_falsey }
151
+ end
152
+ context 'when the target pos is larger' do
153
+ let(:arg_pos) { pos + 10 }
154
+ it { is_expected.to be_truthy }
155
+ end
156
+ end
157
+
158
+ context 'when the target binlog filename is older' do
159
+ let(:arg_filename) { older_filename }
160
+ let(:expected_result) { false }
161
+ it_behaves_like 'returning the same result no matter what pos value is'
162
+ end
163
+ context 'when the target binlog filename is newer' do
164
+ let(:arg_filename) { newer_filename }
165
+ let(:expected_result) { true }
166
+ it_behaves_like 'returning the same result no matter what pos value is'
167
+ end
168
+
169
+ context 'when the target is empty' do
170
+ let(:arg_str) { "-" }
171
+ it { expect { subject }.to raise_error(ArgumentError) }
172
+ end
173
+ context 'when self is empty' do
174
+ let(:binlog_str) { "-" }
175
+ context 'when target is empty' do
176
+ let(:arg_str) { "-" }
177
+ it { is_expected.to be_falsey }
178
+ end
179
+ context 'when target is not empty' do
180
+ let(:arg_str) { arg_binlog_str }
181
+ let(:arg_filename) { filename }
182
+ let(:arg_pos) { pos }
183
+ it { expect { subject }.to raise_error(ArgumentError) }
184
+ end
185
+ end
186
+ end
187
+
188
+ describe '#<=' do
189
+ subject { subject_object <= arg }
190
+
191
+ context 'when the target has the same value' do
192
+ let(:arg_filename) { filename }
193
+ let(:arg_pos) { pos }
194
+ it { is_expected.to be_truthy }
195
+ end
196
+
197
+ context 'when the target binlog filename is the same' do
198
+ let(:arg_filename) { filename }
199
+ context 'when the target pos is smaller' do
200
+ let(:arg_pos) { pos - 10 }
201
+ it { is_expected.to be_falsey }
202
+ end
203
+ context 'when the target pos is the same' do
204
+ let(:arg_pos) { pos }
205
+ it { is_expected.to be_truthy }
206
+ end
207
+ context 'when the target pos is larger' do
208
+ let(:arg_pos) { pos + 10 }
209
+ it { is_expected.to be_truthy }
210
+ end
211
+ end
212
+
213
+ context 'when the target binlog filename is older' do
214
+ let(:arg_filename) { older_filename }
215
+ let(:expected_result) { false }
216
+ it_behaves_like 'returning the same result no matter what pos value is'
217
+ end
218
+ context 'when the target binlog filename is newer' do
219
+ let(:arg_filename) { newer_filename }
220
+ let(:expected_result) { true }
221
+ it_behaves_like 'returning the same result no matter what pos value is'
222
+ end
223
+
224
+ context 'when the target is empty' do
225
+ let(:arg_str) { "-" }
226
+ it { expect { subject }.to raise_error(ArgumentError) }
227
+ end
228
+ context 'when self is empty' do
229
+ let(:binlog_str) { "-" }
230
+ context 'when target is empty' do
231
+ let(:arg_str) { "-" }
232
+ it { is_expected.to be_truthy }
233
+ end
234
+ context 'when target is not empty' do
235
+ let(:arg_str) { arg_binlog_str }
236
+ let(:arg_filename) { filename }
237
+ let(:arg_pos) { pos }
238
+ it { expect { subject }.to raise_error(ArgumentError) }
239
+ end
240
+ end
241
+ end
242
+
243
+ describe '#>=' do
244
+ subject { subject_object >= arg }
245
+
246
+ context 'when the target has the same value' do
247
+ let(:arg_filename) { filename }
248
+ let(:arg_pos) { pos }
249
+ it { is_expected.to be_truthy }
250
+ end
251
+
252
+ context 'when the target binlog filename is the same' do
253
+ let(:arg_filename) { filename }
254
+ context 'when the target pos is smaller' do
255
+ let(:arg_pos) { pos - 10 }
256
+ it { is_expected.to be_truthy }
257
+ end
258
+ context 'when the target pos is the same' do
259
+ let(:arg_pos) { pos }
260
+ it { is_expected.to be_truthy }
261
+ end
262
+ context 'when the target pos is larger' do
263
+ let(:arg_pos) { pos + 10 }
264
+ it { is_expected.to be_falsey }
265
+ end
266
+ end
267
+
268
+ context 'when the target binlog filename is older' do
269
+ let(:arg_filename) { older_filename }
270
+ let(:expected_result) { true }
271
+ it_behaves_like 'returning the same result no matter what pos value is'
272
+ end
273
+ context 'when the target binlog filename is newer' do
274
+ let(:arg_filename) { newer_filename }
275
+ let(:expected_result) { false }
276
+ it_behaves_like 'returning the same result no matter what pos value is'
277
+ end
278
+
279
+ context 'when the target is empty' do
280
+ let(:arg_str) { "-" }
281
+ it { expect { subject }.to raise_error(ArgumentError) }
282
+ end
283
+ context 'when self is empty' do
284
+ let(:binlog_str) { "-" }
285
+ context 'when target is empty' do
286
+ let(:arg_str) { "-" }
287
+ it { is_expected.to be_truthy }
288
+ end
289
+ context 'when target is not empty' do
290
+ let(:arg_str) { arg_binlog_str }
291
+ let(:arg_filename) { filename }
292
+ let(:arg_pos) { pos }
293
+ it { expect { subject }.to raise_error(ArgumentError) }
294
+ end
295
+ end
296
+ end
297
+
298
+ describe '#>' do
299
+ subject { subject_object > arg }
300
+
301
+ context 'when the target has the same value' do
302
+ let(:arg_filename) { filename }
303
+ let(:arg_pos) { pos }
304
+ it { is_expected.to be_falsey }
305
+ end
306
+
307
+ context 'when the target binlog filename is the same' do
308
+ let(:arg_filename) { filename }
309
+ context 'when the target pos is smaller' do
310
+ let(:arg_pos) { pos - 10 }
311
+ it { is_expected.to be_truthy }
312
+ end
313
+ context 'when the target pos is the same' do
314
+ let(:arg_pos) { pos }
315
+ it { is_expected.to be_falsey }
316
+ end
317
+ context 'when the target pos is larger' do
318
+ let(:arg_pos) { pos + 10 }
319
+ it { is_expected.to be_falsey }
320
+ end
321
+ end
322
+
323
+ context 'when the target binlog filename is older' do
324
+ let(:arg_filename) { older_filename }
325
+ let(:expected_result) { true }
326
+ it_behaves_like 'returning the same result no matter what pos value is'
327
+ end
328
+ context 'when the target binlog filename is newer' do
329
+ let(:arg_filename) { newer_filename }
330
+ let(:expected_result) { false }
331
+ it_behaves_like 'returning the same result no matter what pos value is'
332
+ end
333
+
334
+ context 'when the target is empty' do
335
+ let(:arg_str) { "-" }
336
+ it { expect { subject }.to raise_error(ArgumentError) }
337
+ end
338
+ context 'when self is empty' do
339
+ let(:binlog_str) { "-" }
340
+ context 'when target is empty' do
341
+ let(:arg_str) { "-" }
342
+ it { is_expected.to be_falsey }
343
+ end
344
+ context 'when target is not empty' do
345
+ let(:arg_str) { arg_binlog_str }
346
+ let(:arg_filename) { filename }
347
+ let(:arg_pos) { pos }
348
+ it { expect { subject }.to raise_error(ArgumentError) }
349
+ end
350
+ end
351
+ end
352
+
353
+ describe '#==' do
354
+ subject { subject_object == arg }
355
+
356
+ context 'when comparing with nil' do
357
+ let(:arg) { nil }
358
+ it { is_expected.to be_falsey }
359
+ end
360
+
361
+ context 'when the target has the same value' do
362
+ let(:arg_filename) { filename }
363
+ let(:arg_pos) { pos }
364
+ it { is_expected.to be_truthy }
365
+ end
366
+
367
+ context 'when the target binlog filename is the same' do
368
+ let(:arg_filename) { filename }
369
+ context 'when the target pos is smaller' do
370
+ let(:arg_pos) { pos - 10 }
371
+ it { is_expected.to be_falsey }
372
+ end
373
+ context 'when the target pos is the same' do
374
+ let(:arg_pos) { pos }
375
+ it { is_expected.to be_truthy }
376
+ end
377
+ context 'when the target pos is larger' do
378
+ let(:arg_pos) { pos + 10 }
379
+ it { is_expected.to be_falsey }
380
+ end
381
+ end
382
+
383
+ context 'when the target binlog filename is older' do
384
+ let(:arg_filename) { older_filename }
385
+ let(:expected_result) { false }
386
+ it_behaves_like 'returning the same result no matter what pos value is'
387
+ end
388
+ context 'when the target binlog filename is newer' do
389
+ let(:arg_filename) { newer_filename }
390
+ let(:expected_result) { false }
391
+ it_behaves_like 'returning the same result no matter what pos value is'
392
+ end
393
+
394
+ context 'when the target is empty' do
395
+ let(:arg_str) { "-" }
396
+ it { is_expected.to be_falsey }
397
+ end
398
+ context 'when self is empty' do
399
+ let(:binlog_str) { "-" }
400
+ context 'when target is empty' do
401
+ let(:arg_str) { "-" }
402
+ it { is_expected.to be_truthy }
403
+ end
404
+ context 'when target is not empty' do
405
+ let(:arg_str) { arg_binlog_str }
406
+ let(:arg_filename) { filename }
407
+ let(:arg_pos) { pos }
408
+ it { is_expected.to be_falsey }
409
+ end
410
+ end
411
+ end
412
+
413
+ describe '#!=' do
414
+ subject { subject_object != arg }
415
+
416
+ context 'when comparing with nil' do
417
+ let(:arg) { nil }
418
+ it { is_expected.to be_truthy }
419
+ end
420
+
421
+ context 'when the target has the same value' do
422
+ let(:arg_filename) { filename }
423
+ let(:arg_pos) { pos }
424
+ it { is_expected.to be_falsey }
425
+ end
426
+
427
+ context 'when the target binlog filename is the same' do
428
+ let(:arg_filename) { filename }
429
+ context 'when the target pos is smaller' do
430
+ let(:arg_pos) { pos - 10 }
431
+ it { is_expected.to be_truthy }
432
+ end
433
+ context 'when the target pos is the same' do
434
+ let(:arg_pos) { pos }
435
+ it { is_expected.to be_falsey }
436
+ end
437
+ context 'when the target pos is larger' do
438
+ let(:arg_pos) { pos + 10 }
439
+ it { is_expected.to be_truthy }
440
+ end
441
+ end
442
+
443
+ context 'when the target binlog filename is older' do
444
+ let(:arg_filename) { older_filename }
445
+ let(:expected_result) { true }
446
+ it_behaves_like 'returning the same result no matter what pos value is'
447
+ end
448
+ context 'when the target binlog filename is newer' do
449
+ let(:arg_filename) { newer_filename }
450
+ let(:expected_result) { true }
451
+ it_behaves_like 'returning the same result no matter what pos value is'
452
+ end
453
+
454
+ context 'when the target is empty' do
455
+ let(:arg_str) { "-" }
456
+ it { is_expected.to be_truthy }
457
+ end
458
+ context 'when self is empty' do
459
+ let(:binlog_str) { "-" }
460
+ context 'when target is empty' do
461
+ let(:arg_str) { "-" }
462
+ it { is_expected.to be_falsey }
463
+ end
464
+ context 'when target is not empty' do
465
+ let(:arg_str) { arg_binlog_str }
466
+ let(:arg_filename) { filename }
467
+ let(:arg_pos) { pos }
468
+ it { is_expected.to be_truthy }
469
+ end
470
+ end
471
+ end
472
+ end
473
+ end
474
+ end