flydata 0.6.3 → 0.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +2 -2
  3. data/VERSION +1 -1
  4. data/bin/fdredshift +78 -0
  5. data/circle.yml +1 -1
  6. data/ext/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  7. data/ext/flydata/{parser/mysql → source_mysql/parser}/dump_parser_ext.cpp +3 -3
  8. data/ext/flydata/source_mysql/parser/extconf.rb +3 -0
  9. data/ext/flydata/{parser/mysql → source_mysql/parser}/parser.txt +0 -0
  10. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.cpp +0 -0
  11. data/ext/flydata/{parser/mysql → source_mysql/parser}/sql_parser.h +0 -0
  12. data/flydata-core/lib/flydata-core/mysql/binlog_pos.rb +34 -32
  13. data/flydata-core/lib/flydata-core/mysql/compatibility_checker.rb +20 -0
  14. data/flydata-core/lib/flydata-core/table_def/mysql_table_def.rb +12 -4
  15. data/flydata-core/lib/flydata-core/table_def/redshift_table_def.rb +60 -6
  16. data/flydata-core/spec/mysql/binlog_pos_spec.rb +474 -0
  17. data/flydata-core/spec/table_def/mysql_table_def_spec.rb +57 -0
  18. data/flydata-core/spec/table_def/mysql_to_redshift_table_def_spec.rb +174 -20
  19. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_AUTO_INCREMENT_keyword.dump +43 -0
  20. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_not_null_keyword.dump +43 -0
  21. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unique_keyword.dump +43 -0
  22. data/flydata-core/spec/table_def/mysqldump_test_col_comment_with_unsigned_keyword.dump +43 -0
  23. data/flydata-core/spec/table_def/redshift_table_def_spec.rb +41 -8
  24. data/flydata.gemspec +0 -0
  25. data/lib/flydata/cli.rb +11 -5
  26. data/lib/flydata/command/base.rb +14 -1
  27. data/lib/flydata/command/exclusive_runnable.rb +42 -12
  28. data/lib/flydata/command/helper.rb +6 -6
  29. data/lib/flydata/command/sender.rb +4 -3
  30. data/lib/flydata/command/setup.rb +30 -381
  31. data/lib/flydata/command/stop.rb +1 -0
  32. data/lib/flydata/command/sync.rb +273 -301
  33. data/lib/flydata/compatibility_check.rb +24 -117
  34. data/lib/flydata/fluent-plugins/in_mysql_binlog_flydata.rb +3 -3
  35. data/lib/flydata/fluent-plugins/mysql/alter_table_query_handler.rb +2 -2
  36. data/lib/flydata/fluent-plugins/mysql/binlog_record_handler.rb +6 -6
  37. data/lib/flydata/fluent-plugins/mysql/truncate_table_query_handler.rb +0 -1
  38. data/lib/flydata/parser.rb +14 -0
  39. data/lib/flydata/{parser_provider.rb → parser/parser_provider.rb} +6 -4
  40. data/lib/flydata/parser/source_table.rb +33 -0
  41. data/lib/flydata/source.rb +105 -0
  42. data/lib/flydata/source/component.rb +21 -0
  43. data/lib/flydata/source/errors.rb +7 -0
  44. data/lib/flydata/source/generate_source_dump.rb +72 -0
  45. data/lib/flydata/source/parse_dump_and_send.rb +52 -0
  46. data/lib/flydata/source/setup.rb +31 -0
  47. data/lib/flydata/source/source_pos.rb +45 -0
  48. data/lib/flydata/source/sync.rb +56 -0
  49. data/lib/flydata/source/sync_generate_table_ddl.rb +43 -0
  50. data/lib/flydata/source_file/setup.rb +17 -0
  51. data/lib/flydata/source_file/sync.rb +14 -0
  52. data/lib/flydata/{command → source_mysql/command}/mysql.rb +2 -1
  53. data/lib/flydata/{command → source_mysql/command}/mysql_command_base.rb +2 -4
  54. data/lib/flydata/{command → source_mysql/command}/mysqlbinlog.rb +2 -1
  55. data/lib/flydata/{command → source_mysql/command}/mysqldump.rb +2 -1
  56. data/lib/flydata/source_mysql/generate_source_dump.rb +53 -0
  57. data/lib/flydata/source_mysql/mysql_compatibility_check.rb +114 -0
  58. data/lib/flydata/source_mysql/parse_dump_and_send.rb +28 -0
  59. data/lib/flydata/{parser/mysql → source_mysql/parser}/.gitignore +0 -0
  60. data/lib/flydata/{parser/mysql → source_mysql/parser}/dump_parser.rb +32 -67
  61. data/lib/flydata/{parser/mysql → source_mysql/parser}/mysql_alter_table.treetop +0 -0
  62. data/lib/flydata/source_mysql/setup.rb +24 -0
  63. data/lib/flydata/source_mysql/source_pos.rb +21 -0
  64. data/lib/flydata/source_mysql/sync.rb +45 -0
  65. data/lib/flydata/source_mysql/sync_generate_table_ddl.rb +40 -0
  66. data/lib/flydata/{mysql → source_mysql}/table_ddl.rb +6 -17
  67. data/lib/flydata/source_zendesk/sync_generate_table_ddl.rb +30 -0
  68. data/lib/flydata/source_zendesk/zendesk_flydata_tabledefs.rb +133 -0
  69. data/lib/flydata/sync_file_manager.rb +132 -73
  70. data/lib/flydata/table_ddl.rb +18 -0
  71. data/spec/flydata/cli_spec.rb +1 -0
  72. data/spec/flydata/command/exclusive_runnable_spec.rb +19 -8
  73. data/spec/flydata/command/sender_spec.rb +1 -1
  74. data/spec/flydata/command/setup_spec.rb +4 -4
  75. data/spec/flydata/command/sync_spec.rb +97 -134
  76. data/spec/flydata/compatibility_check_spec.rb +16 -289
  77. data/spec/flydata/fluent-plugins/mysql/alter_table_query_handler_spec.rb +3 -3
  78. data/spec/flydata/fluent-plugins/mysql/dml_record_handler_spec.rb +1 -1
  79. data/spec/flydata/fluent-plugins/mysql/shared_query_handler_context.rb +4 -2
  80. data/spec/flydata/fluent-plugins/mysql/truncate_query_handler_spec.rb +1 -1
  81. data/spec/flydata/source_mysql/generate_source_dump_spec.rb +69 -0
  82. data/spec/flydata/source_mysql/mysql_compatibility_check_spec.rb +280 -0
  83. data/spec/flydata/{parser/mysql → source_mysql/parser}/alter_table_parser_spec.rb +2 -2
  84. data/spec/flydata/{parser/mysql → source_mysql/parser}/dump_parser_spec.rb +75 -70
  85. data/spec/flydata/source_mysql/sync_generate_table_ddl_spec.rb +137 -0
  86. data/spec/flydata/{mysql → source_mysql}/table_ddl_spec.rb +2 -2
  87. data/spec/flydata/source_spec.rb +140 -0
  88. data/spec/flydata/source_zendesk/sync_generate_table_ddl_spec.rb +33 -0
  89. data/spec/flydata/sync_file_manager_spec.rb +157 -77
  90. data/tmpl/redshift_mysql_data_entry.conf.tmpl +1 -1
  91. metadata +56 -23
  92. data/ext/flydata/parser/mysql/extconf.rb +0 -3
  93. data/lib/flydata/mysql/binlog_position.rb +0 -22
  94. data/spec/flydata/mysql/binlog_position_spec.rb +0 -35
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 45df6073e7d4b59b307a695058cd1ee70366eb89
4
- data.tar.gz: cdac2f4feb8dfdd4531c14f97a38f1dd2df91f48
3
+ metadata.gz: 12c8fd94c861658f352698075555d8a868d3ca6d
4
+ data.tar.gz: ff5061d23bb1cee12430d471834131034bc925b2
5
5
  SHA512:
6
- metadata.gz: df38e5c265ee0b0e4f9cfcde1a6f219db1fa0b4df013d4f79d1a9dc648d4d5684210560153d91ace7a5267cde76c4d26d00d5ee95e757c0acee2867b4ea6be9c
7
- data.tar.gz: 7ec85d6e13314f445649d64c1769597f2fd192d35c5a6b76b42043accfca4be1d2fc5d95b7c40de7f00e462876bd67076e0102efa0634f755af4d88ebba2f2f0
6
+ metadata.gz: 0c956f29080dbc9a517a08b2b865aceef102f26a509061cb2152ee519d0f2d99794348546001d09c48f9b7ae2ed8015b12a593520780ffe1517614a9fc1dedde
7
+ data.tar.gz: 731ecc427a94b7aab2afb638c8dbbb532b6055f080ed938574d26a19b187478d1fecbb75dc1c09a066b816983dfb0611711a27b8123b14370efda8128f535da0
data/Rakefile CHANGED
@@ -36,8 +36,8 @@ task :default => :spec
36
36
  require 'rake/extensiontask'
37
37
 
38
38
  Rake::ExtensionTask.new "dump_parser_ext" do |ext|
39
- ext.ext_dir = "ext/flydata/parser/mysql"
40
- ext.lib_dir = "lib/flydata/parser/mysql"
39
+ ext.ext_dir = "ext/flydata/source_mysql/parser"
40
+ ext.lib_dir = "lib/flydata/source_mysql/parser"
41
41
  end
42
42
 
43
43
  Rake::ExtensionTask.new "json_ext" do |ext|
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.3
1
+ 0.6.4
data/bin/fdredshift ADDED
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+
6
+ require 'rubygems'
7
+ require 'flydata'
8
+ require 'shellwords'
9
+
10
+ # Write your code from here
11
+ class FdRedshift < Flydata::Command::Base
12
+ def run(*args)
13
+ validate_required_values
14
+ rs_info = retrieve_redshift_info
15
+ cmd = generate_psql_command(rs_info, args)
16
+ $stderr.puts "cmd:#{cmd}" if ENV['FLYDATA_DEBUG']
17
+ if $stdin.tty?
18
+ # interactive shell
19
+ system cmd
20
+ else
21
+ # execute queries given to $stdin
22
+ Open3.popen2e(cmd) do |i, o, wt|
23
+ $stdin.each_line do |line|
24
+ i.print line
25
+ end
26
+ i.close
27
+ while line = o.gets
28
+ print line
29
+ end
30
+ end
31
+ end
32
+ end
33
+
34
+ def generate_psql_command(rs_info,args)
35
+ password = "PGPASSWORD=#{rs_info['password'].shellescape}"
36
+ cmd = "psql"
37
+ dbname = "-d #{rs_info['dbname'].shellescape}"
38
+ host = "-h #{rs_info['host'].shellescape}"
39
+ port = "-p #{rs_info['port'].to_s.shellescape}"
40
+ username = "-U #{rs_info['username'].shellescape}"
41
+ options = args.join(' ')
42
+ #"PGPASSWORD='#{password}' psql -d '#{dbname}' -h '#{host}' -p '#{port}' -U '#{username}'"
43
+ [password,cmd,dbname,host,port,username,options].join(' ')
44
+ end
45
+
46
+ def retrieve_redshift_info(token = Flydata::Credentials.new.token)
47
+ admin_agent_key = read_admin_agent_api_key
48
+ headers = {'AUTHORIZATION' => %Q|Token token="#{admin_agent_key}"|}
49
+
50
+ host = read_admin_agent_api_host
51
+ path = "/api/v1/redshift_clusters/show_default?user_auth_token=#{token}"
52
+ url = host + path
53
+ response = RestClient::Request.execute(method: :get,
54
+ url: url,
55
+ headers: headers)
56
+ JSON.parse(response)
57
+ end
58
+
59
+ def read_admin_agent_api_host
60
+ ENV['FLYDATA_AGENT_FLYDATA_ADMIN_API_HOST']
61
+ end
62
+
63
+ def read_admin_agent_api_key
64
+ ENV['FLYDATA_AGENT_FLYDATA_ADMIN_API_KEY']
65
+ end
66
+
67
+ def validate_required_values
68
+ unless read_admin_agent_api_host && read_admin_agent_api_key
69
+ $stderr.puts "ERROR! This command is not supported in your environment."
70
+ exit 1
71
+ end
72
+ end
73
+ end
74
+
75
+
76
+ args = ARGV.dup
77
+ ARGV.clear # for 'gets'
78
+ FdRedshift.new().run(args)
data/circle.yml CHANGED
@@ -12,7 +12,7 @@ dependencies:
12
12
  - sudo apt-get install -y libboost-thread1.54.0
13
13
  - sudo apt-get install -y libboost-thread1.54-dev
14
14
  - sudo apt-get install -y libboost1.54-dev
15
- - wget --no-check-certificate -O mysql-replication-listener.zip https://github.com/hapyrus/mysql-replication-listener/archive/master.zip
15
+ - wget --no-check-certificate -O mysql-replication-listener.zip https://github.com/flydata/mysql-replication-listener/archive/master.zip
16
16
  - unzip mysql-replication-listener.zip
17
17
  - cd mysql-replication-listener-master; cmake -DCMAKE_INSTALL_PREFIX:PATH=/usr -DBoost_NO_BOOST_CMAKE=ON .; make -j4; sudo make install
18
18
 
@@ -97,9 +97,9 @@ static VALUE rb_insert_parser_parse2(VALUE self, VALUE sql)
97
97
  void Init_dump_parser_ext()
98
98
  {
99
99
  VALUE mFlydata = rb_define_module("Flydata");
100
- VALUE mParser = rb_define_module_under(mFlydata, "Parser");
101
- VALUE mMysql = rb_define_module_under(mParser, "Mysql");
102
- VALUE cMysqlDumpParser = rb_define_class_under(mMysql, "MysqlDumpParser", rb_cObject);
100
+ VALUE mSourceMysql = rb_define_module_under(mFlydata, "SourceMysql");
101
+ VALUE mParser = rb_define_module_under(mSourceMysql, "Parser");
102
+ VALUE cMysqlDumpParser = rb_define_class_under(mParser, "MysqlDumpParser", rb_cObject);
103
103
  VALUE cInsertParser = rb_define_class_under(cMysqlDumpParser, "InsertParser", rb_cObject);
104
104
 
105
105
  rb_define_private_method(cInsertParser, "_parse2", __F(&rb_insert_parser_parse2), 1);
@@ -0,0 +1,3 @@
1
+ require "mkmf"
2
+
3
+ create_makefile "flydata/source_mysql/parser/dump_parser_ext"
@@ -2,17 +2,26 @@ module FlydataCore
2
2
  module Mysql
3
3
 
4
4
  class BinlogPos
5
+ include Comparable
5
6
  def initialize(binlog_str_or_filename_or_hash, binlog_pos = nil)
6
7
  arg = binlog_str_or_filename_or_hash
7
8
  if binlog_pos
8
9
  @pos = binlog_pos
9
10
  @filename = arg
10
11
  elsif arg.kind_of?(String)
11
- @filename, @pos = arg.split("\t")
12
+ if arg == "-"
13
+ set_empty
14
+ else
15
+ @filename, @pos = arg.split("\t")
16
+ end
12
17
  elsif arg.kind_of?(Hash)
13
- @pos = arg[:pos]
14
- @filename = arg[:filename]
15
- @filename ||= arg[:binfile]
18
+ if arg[:filename] == "-" || arg[:binfile] == "-"
19
+ set_empty
20
+ else
21
+ @pos = arg[:pos]
22
+ @filename = arg[:filename]
23
+ @filename ||= arg[:binfile]
24
+ end
16
25
  end
17
26
  if @filename.nil? || @pos.nil?
18
27
  raise "Invalid initialize argument (#{arg}, #{binlog_pos})"
@@ -22,50 +31,43 @@ class BinlogPos
22
31
 
23
32
  attr_reader :filename, :pos
24
33
 
34
+ def empty?
35
+ @filename && @filename == "-"
36
+ end
37
+
25
38
  def <=>(other_pos)
26
39
  if other_pos.nil?
27
40
  raise ArgumentError.new("comparison of BinlosPos with nil failed")
28
41
  end
29
42
 
30
43
  other_pos = BinlogPos.new(other_pos) unless other_pos.kind_of?(BinlogPos)
31
-
32
- if @filename < other_pos.filename
33
- -1
34
- elsif @filename == other_pos.filename
35
- if @pos < other_pos.pos
36
- -1
37
- elsif @pos == other_pos.pos
38
- 0
39
- else
40
- 1
41
- end
44
+ if empty? && other_pos.empty?
45
+ return 0
46
+ elsif empty? || other_pos.empty?
47
+ raise ArgumentError.new("comparison with empty BinlogPos failed")
42
48
  else
43
- 1
49
+ (@filename <=> other_pos.filename) == 0 ? (@pos <=> other_pos.pos) : (@filename <=> other_pos.filename)
44
50
  end
45
51
  end
46
- def <(other_pos)
47
- self.<=>(other_pos) < 0
48
- end
49
- def <=(other_pos)
50
- self.<=>(other_pos) <= 0
51
- end
52
+
52
53
  def ==(other_pos)
53
54
  return false if other_pos.nil?
55
+ return @filename == other_pos.filename if empty? || other_pos.empty?
54
56
  self.<=>(other_pos) == 0
55
57
  end
56
- def !=(other_pos)
57
- !self.==(other_pos)
58
- end
59
- def >(other_pos)
60
- self.<=>(other_pos) > 0
61
- end
62
- def >=(other_pos)
63
- self.<=>(other_pos) >= 0
64
- end
65
58
 
66
59
  def to_s
67
- "#{@filename}\t#{@pos}"
60
+ empty? ? "-" : "#{@filename}\t#{@pos}"
68
61
  end
62
+
63
+ private
64
+ def set_empty
65
+ # @filename == "-" is the definition of `empty`. @pos could be anything but nul
66
+ @filename = "-"
67
+ @pos = -1
68
+ end
69
+
70
+
69
71
  end
70
72
 
71
73
  end
@@ -226,6 +226,26 @@ EOT
226
226
  end
227
227
  end
228
228
 
229
+ class MySqlVersion57CompabilityChecker < MysqlCompatibilityChecker
230
+
231
+ def create_query(option = @option)
232
+ "show variables where Variable_name in ('version','show_compatibility_56');"
233
+ end
234
+
235
+ def check_result(result, option = @option)
236
+ mysqlversion = nil
237
+ compability56 = nil
238
+ result.each { |value|
239
+ mysqlversion = value['Value'] if value['Variable_name'] == 'version'
240
+ compability56 = value['Value'] if value['Variable_name'] == 'show_compatibility_56'
241
+ }
242
+ if mysqlversion.start_with?('5.7') && compability56 == "OFF"
243
+ raise FlydataCore::MysqlCompatibilityError,
244
+ "Make sure you set 'show_compatibility_56 = ON' in your MySQL configuration. Once you update your MySQL configuration, restart the MySQL server for the change to take effect."
245
+ end
246
+ end
247
+ end
248
+
229
249
  class TableTypeChecker < MysqlCompatibilityChecker
230
250
  SELECT_TABLE_INFO_TMPLT =
231
251
  "SELECT table_name, table_type, engine " +
@@ -171,6 +171,11 @@ class MysqlTableDef
171
171
 
172
172
  tabledef
173
173
  end
174
+
175
+ # Replaces a MySQL string with an empty string ('')
176
+ def self.strip_string(line)
177
+ line.gsub(/'(\\'|''|[^'])*'/, "''")
178
+ end
174
179
 
175
180
  def self.parse_one_column_def(query)
176
181
  line = query.strip
@@ -207,11 +212,14 @@ class MysqlTableDef
207
212
  column[:type] = convert_to_flydata_type(type)
208
213
 
209
214
  cond = :options
215
+
210
216
  when :options
211
- column[:type] += ' unsigned' if line =~ /unsigned/i
212
- column[:auto_increment] = true if line =~ /AUTO_INCREMENT/i
213
- column[:not_null] = true if line =~ /NOT NULL/i
214
- column[:unique] = true if line =~ /UNIQUE/i
217
+ strip_line = strip_string(line)
218
+ column[:type] += ' unsigned' if strip_line =~ /unsigned/i
219
+ column[:auto_increment] = true if strip_line =~ /AUTO_INCREMENT/i
220
+ column[:not_null] = true if strip_line =~ /NOT NULL/i
221
+ column[:unique] = true if strip_line =~ /UNIQUE/i
222
+
215
223
  if /DEFAULT\s+((?:[bx]?'(?:\\'|[^'])*')|(?:[^'\s]+\b))/i.match(line)
216
224
  val = $1
217
225
  column[:default] = val == "NULL" ? nil : val
@@ -45,9 +45,14 @@ class RedshiftTableDef
45
45
  tabledef = ""
46
46
  tabledef += create_schema_sql(schema_name) if options[:flydata_ctl_table] && !schema_name.to_s.empty?
47
47
  tabledef += create_flydata_ctl_table_sql(schema_name) if options[:flydata_ctl_table]
48
+ options[:backup_postfix] = "_flydata#{Time.now.strftime('%Y%m%d%H%M%S')}"
49
+ tabledef += "BEGIN;\n"
50
+ tabledef += remove_table_sql(flydata_tabledef, schema_name, options) unless options[:ctl_only]
48
51
  tabledef += create_table_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
49
52
  tabledef += comment_sql(flydata_tabledef, schema_name) unless options[:ctl_only]
50
53
  tabledef += flydata_ctl_sql(flydata_tabledef, schema_name)
54
+ tabledef += "COMMIT;\n"
55
+ tabledef += drop_backup_table_sql(flydata_tabledef, schema_name, options) unless options[:ctl_only]
51
56
  rescue => e
52
57
  # Catch errors from generating schema. Generally an unsupported data type
53
58
  raise TableDefError, {error: "errors generating schema. Please contact us for further instructions", table: flydata_tabledef[:table_name]}
@@ -124,14 +129,61 @@ EOS
124
129
  CREATE_FLYDATA_CTL_TABLES_SQL % [tables_tbl]
125
130
  end
126
131
 
127
- CREATE_TABLE_SQL = <<EOS
132
+ DROP_TABLE_SQL = <<EOS
128
133
  DROP TABLE IF EXISTS %s;
134
+ EOS
135
+
136
+ RENAME_TABLE_SQL = <<EOS
137
+ ALTER TABLE %s RENAME TO %s;
138
+ EOS
139
+
140
+ CREATE_TABLE_SQL = <<EOS
129
141
  CREATE TABLE %s (
130
142
  %s
131
143
  )%s;
132
144
  EOS
133
145
 
134
- def self.create_table_sql(flydata_tabledef, schema_name)
146
+ CREATE_TABLE_IF_NOT_EXISTS_SQL = <<EOS
147
+ CREATE TABLE IF NOT EXISTS %s (
148
+ %s
149
+ )%s;
150
+ EOS
151
+
152
+ def self.remove_table_sql(flydata_tabledef, schema_name, options)
153
+ return "" if options[:skip_drop_table]
154
+ table_name = flydata_tabledef[:table_name]
155
+ redshift_tbl = table_name_for_ddl(table_name, schema_name)
156
+ sql = ""
157
+ if options[:backup_postfix]
158
+ # drop backup table if exists
159
+ sql += drop_backup_table_sql(flydata_tabledef, schema_name, options)
160
+ # create an empty table to prevent RENAME TABLE query from failing
161
+ sql += create_table_sql(flydata_tabledef, schema_name,
162
+ create_table_sql: CREATE_TABLE_IF_NOT_EXISTS_SQL)
163
+ backup_tbl = table_name_for_ddl(
164
+ "#{table_name}#{options[:backup_postfix]}", nil)
165
+ sql += RENAME_TABLE_SQL % [redshift_tbl, backup_tbl]
166
+ else
167
+ sql += DROP_TABLE_SQL % redshift_tbl
168
+ end
169
+ sql
170
+ end
171
+
172
+ def self.drop_backup_table_sql(flydata_tabledef, schema_name, options)
173
+ return "" if options[:skip_drop_table]
174
+ sql = ""
175
+ if options[:backup_postfix]
176
+ table_name = flydata_tabledef[:table_name]
177
+ backup_tbl = table_name_for_ddl(
178
+ "#{table_name}#{options[:backup_postfix]}", schema_name)
179
+ sql += DROP_TABLE_SQL % backup_tbl
180
+ end
181
+
182
+ sql
183
+ end
184
+
185
+ def self.create_table_sql(flydata_tabledef, schema_name, options = {})
186
+ ct_sql = options[:create_table_sql] || CREATE_TABLE_SQL
135
187
  lines = flydata_tabledef[:columns].collect{|column| column_def_sql(column) }
136
188
  pk_def = primary_key_sql(flydata_tabledef)
137
189
  lines << pk_def if pk_def
@@ -142,7 +194,9 @@ EOS
142
194
 
143
195
  table_name = flydata_tabledef[:table_name]
144
196
  redshift_tbl = table_name_for_ddl(table_name, schema_name)
145
- CREATE_TABLE_SQL % [redshift_tbl, redshift_tbl, contents, dk_sk_def]
197
+ sql = ""
198
+ sql += ct_sql % [redshift_tbl, contents, dk_sk_def]
199
+ sql
146
200
  end
147
201
 
148
202
  def self.column_def_sql(column, opt = {})
@@ -261,7 +315,7 @@ EOS
261
315
  end
262
316
 
263
317
  def self.flydata_ctl_sql(flydata_tabledef, schema_name)
264
- flydata_ctl_columns_sql(flydata_tabledef, schema_name) + "\n" +
318
+ flydata_ctl_columns_sql(flydata_tabledef, schema_name) +
265
319
  flydata_ctl_tables_sql(flydata_tabledef, schema_name)
266
320
  end
267
321
  FLYDATA_CTL_COLUMNS_SQL = <<EOS
@@ -276,7 +330,7 @@ EOS
276
330
  charset = col[:charset] ? " cs:#{col[:charset]}" : ""
277
331
  values << "('#{escape(flydata_tabledef[:table_name])}', '#{escape(col[:column])}', '#{escape(col[:type])}#{charset}', #{i})"
278
332
  end
279
- sql += values.join(",\n") + ';'
333
+ sql += values.join(",\n") + ";\n"
280
334
  sql
281
335
  end
282
336
 
@@ -292,7 +346,7 @@ EOS
292
346
  values << "('#{flydata_tabledef[:table_name]}', 'cs', '#{escape(flydata_tabledef[:default_charset])}')"
293
347
  values << "('#{flydata_tabledef[:table_name]}', 'revision', 1)"
294
348
  values << "('#{flydata_tabledef[:table_name]}', 'src_ddl', '#{escape(flydata_tabledef[:src_ddl])}')"
295
- sql += values.join(",\n") + ';'
349
+ sql += values.join(",\n") + ";\n"
296
350
  sql
297
351
  end
298
352
 
@@ -0,0 +1,474 @@
1
+ require 'spec_helper'
2
+ require 'flydata-core/mysql/binlog_pos.rb'
3
+
4
+ module FlydataCore
5
+ module Mysql
6
+ describe BinlogPos do
7
+ let(:subject_object) { described_class.new(binlog_str) }
8
+ let(:binlog_str) { "#{filename}\t#{pos}" }
9
+ let(:filename) { 'mysql-bin.000064' }
10
+ let(:pos) { 104 }
11
+
12
+ describe '#pos' do
13
+ subject { subject_object.pos }
14
+ it { is_expected.to eq(pos) }
15
+ end
16
+ describe '#filename' do
17
+ subject { subject_object.filename }
18
+ it { is_expected.to eq(filename) }
19
+ end
20
+ describe '#to_s' do
21
+ subject { subject_object.to_s }
22
+ it { is_expected.to eq(binlog_str) }
23
+ end
24
+
25
+ describe '.new' do
26
+ subject { described_class.new(*args) }
27
+ shared_examples "properly constructed" do
28
+ it do
29
+ expect(subject.filename).to eq(filename)
30
+ expect(subject.pos).to eq(pos)
31
+ end
32
+ end
33
+
34
+ context 'when given two arguments' do
35
+ let(:args) { [filename, pos] }
36
+ it_behaves_like "properly constructed"
37
+ end
38
+
39
+ let(:args) { [ arg1 ] }
40
+ context 'when given a String' do
41
+ let(:arg1) { binlog_str }
42
+ it_behaves_like "properly constructed"
43
+ end
44
+
45
+ context 'when given a Hash' do
46
+ let(:arg1) { { filename: filename, pos: pos } }
47
+ it_behaves_like "properly constructed"
48
+ end
49
+ end
50
+
51
+ let(:arg) { described_class.new (arg_str) }
52
+ let(:arg_str) { arg_binlog_str }
53
+ let(:arg_binlog_str) { "#{arg_filename}\t#{arg_pos}" }
54
+ let(:older_filename) { 'mysql-bin.000061' }
55
+ let(:newer_filename) { 'mysql-bin.000065' }
56
+
57
+ shared_examples 'returning the same result no matter what pos value is' do
58
+ context 'when the target pos is smaller' do
59
+ let(:arg_pos) { pos - 10 }
60
+ it { is_expected.to eq expected_result }
61
+ end
62
+ context 'when the target pos is the same' do
63
+ let(:arg_pos) { pos }
64
+ it { is_expected.to eq expected_result }
65
+ end
66
+ context 'when the target pos is larger' do
67
+ let(:arg_pos) { pos + 10 }
68
+ it { is_expected.to eq expected_result }
69
+ end
70
+ end
71
+
72
+ describe '#<=>' do
73
+ subject { subject_object <=> arg }
74
+
75
+ context 'when the target is nil' do
76
+ let(:arg) { nil }
77
+ it { expect { subject }.to raise_error(ArgumentError) }
78
+ end
79
+ context 'when the target has the same value' do
80
+ let(:arg_filename) { filename }
81
+ let(:arg_pos) { pos }
82
+ it { is_expected.to eq 0 }
83
+ end
84
+
85
+ context 'when the target binlog filename is the same' do
86
+ let(:arg_filename) { filename }
87
+ context 'when the target pos is smaller' do
88
+ let(:arg_pos) { pos - 10 }
89
+ it { is_expected.to eq 1 }
90
+ end
91
+ context 'when the target pos is the same' do
92
+ let(:arg_pos) { pos }
93
+ it { is_expected.to eq 0 }
94
+ end
95
+ context 'when the target pos is larger' do
96
+ let(:arg_pos) { pos + 10 }
97
+ it { is_expected.to eq -1 }
98
+ end
99
+ end
100
+
101
+ context 'when the target binlog filename is older' do
102
+ let(:arg_filename) { older_filename }
103
+ let(:expected_result) { 1 }
104
+ it_behaves_like 'returning the same result no matter what pos value is'
105
+ end
106
+ context 'when the target binlog filename is newer' do
107
+ let(:arg_filename) { newer_filename }
108
+ let(:expected_result) { -1 }
109
+ it_behaves_like 'returning the same result no matter what pos value is'
110
+ end
111
+
112
+ context 'when the target is empty' do
113
+ let(:arg_str) { "-" }
114
+ it { expect { subject }.to raise_error(ArgumentError) }
115
+ end
116
+ context 'when self is empty' do
117
+ let(:binlog_str) { "-" }
118
+ context 'when target is empty' do
119
+ let(:arg_str) { "-" }
120
+ it { is_expected.to eq 0 }
121
+ end
122
+ context 'when target is not empty' do
123
+ let(:arg) { double('arg') }
124
+ before do
125
+ allow(arg).to receive(:kind_of?).with(BinlogPos).and_return(true)
126
+ allow(arg).to receive(:empty?).and_return(false)
127
+ end
128
+ it { expect { subject }.to raise_error(ArgumentError) }
129
+ end
130
+ end
131
+ end
132
+
133
+ describe '#<' do
134
+ subject { subject_object < arg }
135
+
136
+ context 'when the target has the same value' do
137
+ let(:arg_filename) { filename }
138
+ let(:arg_pos) { pos }
139
+ it { is_expected.to be_falsey }
140
+ end
141
+
142
+ context 'when the target binlog filename is the same' do
143
+ let(:arg_filename) { filename }
144
+ context 'when the target pos is smaller' do
145
+ let(:arg_pos) { pos - 10 }
146
+ it { is_expected.to be_falsey }
147
+ end
148
+ context 'when the target pos is the same' do
149
+ let(:arg_pos) { pos }
150
+ it { is_expected.to be_falsey }
151
+ end
152
+ context 'when the target pos is larger' do
153
+ let(:arg_pos) { pos + 10 }
154
+ it { is_expected.to be_truthy }
155
+ end
156
+ end
157
+
158
+ context 'when the target binlog filename is older' do
159
+ let(:arg_filename) { older_filename }
160
+ let(:expected_result) { false }
161
+ it_behaves_like 'returning the same result no matter what pos value is'
162
+ end
163
+ context 'when the target binlog filename is newer' do
164
+ let(:arg_filename) { newer_filename }
165
+ let(:expected_result) { true }
166
+ it_behaves_like 'returning the same result no matter what pos value is'
167
+ end
168
+
169
+ context 'when the target is empty' do
170
+ let(:arg_str) { "-" }
171
+ it { expect { subject }.to raise_error(ArgumentError) }
172
+ end
173
+ context 'when self is empty' do
174
+ let(:binlog_str) { "-" }
175
+ context 'when target is empty' do
176
+ let(:arg_str) { "-" }
177
+ it { is_expected.to be_falsey }
178
+ end
179
+ context 'when target is not empty' do
180
+ let(:arg_str) { arg_binlog_str }
181
+ let(:arg_filename) { filename }
182
+ let(:arg_pos) { pos }
183
+ it { expect { subject }.to raise_error(ArgumentError) }
184
+ end
185
+ end
186
+ end
187
+
188
+ describe '#<=' do
189
+ subject { subject_object <= arg }
190
+
191
+ context 'when the target has the same value' do
192
+ let(:arg_filename) { filename }
193
+ let(:arg_pos) { pos }
194
+ it { is_expected.to be_truthy }
195
+ end
196
+
197
+ context 'when the target binlog filename is the same' do
198
+ let(:arg_filename) { filename }
199
+ context 'when the target pos is smaller' do
200
+ let(:arg_pos) { pos - 10 }
201
+ it { is_expected.to be_falsey }
202
+ end
203
+ context 'when the target pos is the same' do
204
+ let(:arg_pos) { pos }
205
+ it { is_expected.to be_truthy }
206
+ end
207
+ context 'when the target pos is larger' do
208
+ let(:arg_pos) { pos + 10 }
209
+ it { is_expected.to be_truthy }
210
+ end
211
+ end
212
+
213
+ context 'when the target binlog filename is older' do
214
+ let(:arg_filename) { older_filename }
215
+ let(:expected_result) { false }
216
+ it_behaves_like 'returning the same result no matter what pos value is'
217
+ end
218
+ context 'when the target binlog filename is newer' do
219
+ let(:arg_filename) { newer_filename }
220
+ let(:expected_result) { true }
221
+ it_behaves_like 'returning the same result no matter what pos value is'
222
+ end
223
+
224
+ context 'when the target is empty' do
225
+ let(:arg_str) { "-" }
226
+ it { expect { subject }.to raise_error(ArgumentError) }
227
+ end
228
+ context 'when self is empty' do
229
+ let(:binlog_str) { "-" }
230
+ context 'when target is empty' do
231
+ let(:arg_str) { "-" }
232
+ it { is_expected.to be_truthy }
233
+ end
234
+ context 'when target is not empty' do
235
+ let(:arg_str) { arg_binlog_str }
236
+ let(:arg_filename) { filename }
237
+ let(:arg_pos) { pos }
238
+ it { expect { subject }.to raise_error(ArgumentError) }
239
+ end
240
+ end
241
+ end
242
+
243
+ describe '#>=' do
244
+ subject { subject_object >= arg }
245
+
246
+ context 'when the target has the same value' do
247
+ let(:arg_filename) { filename }
248
+ let(:arg_pos) { pos }
249
+ it { is_expected.to be_truthy }
250
+ end
251
+
252
+ context 'when the target binlog filename is the same' do
253
+ let(:arg_filename) { filename }
254
+ context 'when the target pos is smaller' do
255
+ let(:arg_pos) { pos - 10 }
256
+ it { is_expected.to be_truthy }
257
+ end
258
+ context 'when the target pos is the same' do
259
+ let(:arg_pos) { pos }
260
+ it { is_expected.to be_truthy }
261
+ end
262
+ context 'when the target pos is larger' do
263
+ let(:arg_pos) { pos + 10 }
264
+ it { is_expected.to be_falsey }
265
+ end
266
+ end
267
+
268
+ context 'when the target binlog filename is older' do
269
+ let(:arg_filename) { older_filename }
270
+ let(:expected_result) { true }
271
+ it_behaves_like 'returning the same result no matter what pos value is'
272
+ end
273
+ context 'when the target binlog filename is newer' do
274
+ let(:arg_filename) { newer_filename }
275
+ let(:expected_result) { false }
276
+ it_behaves_like 'returning the same result no matter what pos value is'
277
+ end
278
+
279
+ context 'when the target is empty' do
280
+ let(:arg_str) { "-" }
281
+ it { expect { subject }.to raise_error(ArgumentError) }
282
+ end
283
+ context 'when self is empty' do
284
+ let(:binlog_str) { "-" }
285
+ context 'when target is empty' do
286
+ let(:arg_str) { "-" }
287
+ it { is_expected.to be_truthy }
288
+ end
289
+ context 'when target is not empty' do
290
+ let(:arg_str) { arg_binlog_str }
291
+ let(:arg_filename) { filename }
292
+ let(:arg_pos) { pos }
293
+ it { expect { subject }.to raise_error(ArgumentError) }
294
+ end
295
+ end
296
+ end
297
+
298
+ describe '#>' do
299
+ subject { subject_object > arg }
300
+
301
+ context 'when the target has the same value' do
302
+ let(:arg_filename) { filename }
303
+ let(:arg_pos) { pos }
304
+ it { is_expected.to be_falsey }
305
+ end
306
+
307
+ context 'when the target binlog filename is the same' do
308
+ let(:arg_filename) { filename }
309
+ context 'when the target pos is smaller' do
310
+ let(:arg_pos) { pos - 10 }
311
+ it { is_expected.to be_truthy }
312
+ end
313
+ context 'when the target pos is the same' do
314
+ let(:arg_pos) { pos }
315
+ it { is_expected.to be_falsey }
316
+ end
317
+ context 'when the target pos is larger' do
318
+ let(:arg_pos) { pos + 10 }
319
+ it { is_expected.to be_falsey }
320
+ end
321
+ end
322
+
323
+ context 'when the target binlog filename is older' do
324
+ let(:arg_filename) { older_filename }
325
+ let(:expected_result) { true }
326
+ it_behaves_like 'returning the same result no matter what pos value is'
327
+ end
328
+ context 'when the target binlog filename is newer' do
329
+ let(:arg_filename) { newer_filename }
330
+ let(:expected_result) { false }
331
+ it_behaves_like 'returning the same result no matter what pos value is'
332
+ end
333
+
334
+ context 'when the target is empty' do
335
+ let(:arg_str) { "-" }
336
+ it { expect { subject }.to raise_error(ArgumentError) }
337
+ end
338
+ context 'when self is empty' do
339
+ let(:binlog_str) { "-" }
340
+ context 'when target is empty' do
341
+ let(:arg_str) { "-" }
342
+ it { is_expected.to be_falsey }
343
+ end
344
+ context 'when target is not empty' do
345
+ let(:arg_str) { arg_binlog_str }
346
+ let(:arg_filename) { filename }
347
+ let(:arg_pos) { pos }
348
+ it { expect { subject }.to raise_error(ArgumentError) }
349
+ end
350
+ end
351
+ end
352
+
353
+ describe '#==' do
354
+ subject { subject_object == arg }
355
+
356
+ context 'when comparing with nil' do
357
+ let(:arg) { nil }
358
+ it { is_expected.to be_falsey }
359
+ end
360
+
361
+ context 'when the target has the same value' do
362
+ let(:arg_filename) { filename }
363
+ let(:arg_pos) { pos }
364
+ it { is_expected.to be_truthy }
365
+ end
366
+
367
+ context 'when the target binlog filename is the same' do
368
+ let(:arg_filename) { filename }
369
+ context 'when the target pos is smaller' do
370
+ let(:arg_pos) { pos - 10 }
371
+ it { is_expected.to be_falsey }
372
+ end
373
+ context 'when the target pos is the same' do
374
+ let(:arg_pos) { pos }
375
+ it { is_expected.to be_truthy }
376
+ end
377
+ context 'when the target pos is larger' do
378
+ let(:arg_pos) { pos + 10 }
379
+ it { is_expected.to be_falsey }
380
+ end
381
+ end
382
+
383
+ context 'when the target binlog filename is older' do
384
+ let(:arg_filename) { older_filename }
385
+ let(:expected_result) { false }
386
+ it_behaves_like 'returning the same result no matter what pos value is'
387
+ end
388
+ context 'when the target binlog filename is newer' do
389
+ let(:arg_filename) { newer_filename }
390
+ let(:expected_result) { false }
391
+ it_behaves_like 'returning the same result no matter what pos value is'
392
+ end
393
+
394
+ context 'when the target is empty' do
395
+ let(:arg_str) { "-" }
396
+ it { is_expected.to be_falsey }
397
+ end
398
+ context 'when self is empty' do
399
+ let(:binlog_str) { "-" }
400
+ context 'when target is empty' do
401
+ let(:arg_str) { "-" }
402
+ it { is_expected.to be_truthy }
403
+ end
404
+ context 'when target is not empty' do
405
+ let(:arg_str) { arg_binlog_str }
406
+ let(:arg_filename) { filename }
407
+ let(:arg_pos) { pos }
408
+ it { is_expected.to be_falsey }
409
+ end
410
+ end
411
+ end
412
+
413
+ describe '#!=' do
414
+ subject { subject_object != arg }
415
+
416
+ context 'when comparing with nil' do
417
+ let(:arg) { nil }
418
+ it { is_expected.to be_truthy }
419
+ end
420
+
421
+ context 'when the target has the same value' do
422
+ let(:arg_filename) { filename }
423
+ let(:arg_pos) { pos }
424
+ it { is_expected.to be_falsey }
425
+ end
426
+
427
+ context 'when the target binlog filename is the same' do
428
+ let(:arg_filename) { filename }
429
+ context 'when the target pos is smaller' do
430
+ let(:arg_pos) { pos - 10 }
431
+ it { is_expected.to be_truthy }
432
+ end
433
+ context 'when the target pos is the same' do
434
+ let(:arg_pos) { pos }
435
+ it { is_expected.to be_falsey }
436
+ end
437
+ context 'when the target pos is larger' do
438
+ let(:arg_pos) { pos + 10 }
439
+ it { is_expected.to be_truthy }
440
+ end
441
+ end
442
+
443
+ context 'when the target binlog filename is older' do
444
+ let(:arg_filename) { older_filename }
445
+ let(:expected_result) { true }
446
+ it_behaves_like 'returning the same result no matter what pos value is'
447
+ end
448
+ context 'when the target binlog filename is newer' do
449
+ let(:arg_filename) { newer_filename }
450
+ let(:expected_result) { true }
451
+ it_behaves_like 'returning the same result no matter what pos value is'
452
+ end
453
+
454
+ context 'when the target is empty' do
455
+ let(:arg_str) { "-" }
456
+ it { is_expected.to be_truthy }
457
+ end
458
+ context 'when self is empty' do
459
+ let(:binlog_str) { "-" }
460
+ context 'when target is empty' do
461
+ let(:arg_str) { "-" }
462
+ it { is_expected.to be_falsey }
463
+ end
464
+ context 'when target is not empty' do
465
+ let(:arg_str) { arg_binlog_str }
466
+ let(:arg_filename) { filename }
467
+ let(:arg_pos) { pos }
468
+ it { is_expected.to be_truthy }
469
+ end
470
+ end
471
+ end
472
+ end
473
+ end
474
+ end