flydata 0.2.10 → 0.2.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: acf1cc6cb8257d42635142a15b10f63a7bf276b4
4
- data.tar.gz: 240b9f073f79c4a0fedc4dc1d454d8fbcba05077
3
+ metadata.gz: 1cf10dc736cbd2f2712294f21cf074728aab1be8
4
+ data.tar.gz: aba35db4a5431b33a673f4f1708601eeeaa46507
5
5
  SHA512:
6
- metadata.gz: 5d4c6dad5c51496f6b1748d112c9ac5851ae8ba1dc8db31045ec11d7bce44deff48f904c80ae04ec736330cb432b2da4a538879e4a87b1b49bc3313f9146758c
7
- data.tar.gz: 2b019366d7201f3d1a76367267d3f4a4189408bad14d650240f726eafdf1fd2e5eb8ff5b48b624790b0fb07f1fe705634fc95b4582a233e9699cc8c2eb15f156
6
+ metadata.gz: 90939617163c3050e124607859340cc0390242a4984451bfcfb383581a8acca374853263b8358c95f5297548984fa0c15fa4bb06a70f171aed0f99e2eca7fa37
7
+ data.tar.gz: f69a0c50a94ba780577fcc5877e10d4d2059f6de3ffaed1145b6509ca11b21500a7ea5a36cab7f47c559023e4b23ed25612c50c31c6ca7e3ed29c7262fd0c472
data/Gemfile CHANGED
@@ -25,4 +25,5 @@ group :development do
25
25
  gem 'activemodel', '~> 4.0', '>= 4.0.0'
26
26
  gem 'activerecord', '~> 4.0', '>= 4.0.0'
27
27
  gem 'protected_attributes', '~> 1.0', '>= 1.0.8'
28
+ gem 'pry'
28
29
  end
data/Gemfile.lock CHANGED
@@ -20,6 +20,7 @@ GEM
20
20
  addressable (2.3.6)
21
21
  arel (4.0.2)
22
22
  builder (3.1.4)
23
+ coderay (1.1.0)
23
24
  cool.io (1.2.4)
24
25
  diff-lcs (1.2.5)
25
26
  faraday (0.8.9)
@@ -60,6 +61,7 @@ GEM
60
61
  jwt (1.0.0)
61
62
  kodama (0.1.1)
62
63
  ruby-binlog (>= 0.1.9)
64
+ method_source (0.8.2)
63
65
  mime-types (2.3)
64
66
  minitest (4.7.5)
65
67
  msgpack (0.5.8)
@@ -78,6 +80,10 @@ GEM
78
80
  polyglot (0.3.5)
79
81
  protected_attributes (1.0.8)
80
82
  activemodel (>= 4.0.1, < 5.0)
83
+ pry (0.10.1)
84
+ coderay (~> 1.1.0)
85
+ method_source (~> 0.8.1)
86
+ slop (~> 3.4)
81
87
  rack (1.5.2)
82
88
  rake (10.3.2)
83
89
  rdoc (4.1.1)
@@ -124,6 +130,7 @@ DEPENDENCIES
124
130
  json (~> 1.8, >= 1.8.0)
125
131
  mysql2 (~> 0.3, >= 0.3.11)
126
132
  protected_attributes (~> 1.0, >= 1.0.8)
133
+ pry
127
134
  rest-client (~> 1.6, >= 1.6.7)
128
135
  rspec (~> 3.0)
129
136
  ruby-binlog (~> 1.0, >= 1.0.1)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.10
1
+ 0.2.11
@@ -0,0 +1,4 @@
1
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
2
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
3
+ require 'flydata'
4
+ require 'benchmark'
@@ -0,0 +1,28 @@
1
+ require_relative 'benchmark_helper'
2
+ require 'flydata/parser/mysql/dump_parser'
3
+ require 'zlib'
4
+
5
+ INSERT_TEST_FILE_NAME = case ARGV[0]
6
+ when 'num'
7
+ 'insert_parser_test_data_num_only.sql.gz'
8
+ else
9
+ 'insert_parser_test_data.sql.gz'
10
+ end
11
+ INSERT_TEST_FILE_PATH = File.realpath("data/#{INSERT_TEST_FILE_NAME}", File.dirname(__FILE__))
12
+ def readline_gz_file(gz_file_path = INSERT_TEST_FILE_PATH)
13
+ Zlib::GzipReader.open(gz_file_path) {|f| return f.readline}
14
+ end
15
+
16
+ @test_line = readline_gz_file
17
+
18
+ def subject
19
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
20
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
21
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
22
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
23
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
24
+ end
25
+
26
+ Benchmark.bm do |x|
27
+ x.report { 10.times{ subject } }
28
+ end
@@ -0,0 +1,24 @@
1
+ require_relative 'benchmark_helper'
2
+ require 'flydata/parser/mysql/dump_parser'
3
+ require 'zlib'
4
+ require 'ruby-prof'
5
+
6
+ INSERT_TEST_FILE_NAME = case ARGV[0]
7
+ when 'num'
8
+ 'insert_parser_test_data_num_only.sql.gz'
9
+ else
10
+ 'insert_parser_test_data.sql.gz'
11
+ end
12
+ INSERT_TEST_FILE_PATH = File.realpath("data/#{INSERT_TEST_FILE_NAME}", File.dirname(__FILE__))
13
+
14
+ def readline_gz_file(gz_file_path = INSERT_TEST_FILE_PATH)
15
+ Zlib::GzipReader.open(gz_file_path) {|f| return f.readline}
16
+ end
17
+
18
+ @test_line = readline_gz_file
19
+
20
+ def subject
21
+ Flydata::Parser::Mysql::MysqlDumpParser::InsertParser.new.parse(@test_line)
22
+ end
23
+
24
+ subject
data/flydata.gemspec CHANGED
@@ -2,16 +2,14 @@
2
2
  # DO NOT EDIT THIS FILE DIRECTLY
3
3
  # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
4
  # -*- encoding: utf-8 -*-
5
- # stub: flydata 0.2.10 ruby lib
6
5
 
7
6
  Gem::Specification.new do |s|
8
7
  s.name = "flydata"
9
- s.version = "0.2.10"
8
+ s.version = "0.2.11"
10
9
 
11
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
12
- s.require_paths = ["lib"]
13
11
  s.authors = ["Koichi Fujikawa", "Masashi Miyazaki", "Matthew Luu", "Mak Inada", "Sriram NS"]
14
- s.date = "2014-10-27"
12
+ s.date = "2014-10-31"
15
13
  s.description = "FlyData Agent"
16
14
  s.email = "sysadmin@flydata.com"
17
15
  s.executables = ["fdmysqldump", "flydata", "serverinfo"]
@@ -22,6 +20,11 @@ Gem::Specification.new do |s|
22
20
  "Gemfile.lock",
23
21
  "Rakefile",
24
22
  "VERSION",
23
+ "benchmark/benchmark_helper.rb",
24
+ "benchmark/data/insert_parser_test_data.sql.gz",
25
+ "benchmark/data/insert_parser_test_data_num_only.sql.gz",
26
+ "benchmark/insert_parser_bench.rb",
27
+ "benchmark/insert_parser_prof.rb",
25
28
  "bin/fdmysqldump",
26
29
  "bin/flydata",
27
30
  "bin/serverinfo",
@@ -115,7 +118,8 @@ Gem::Specification.new do |s|
115
118
  ]
116
119
  s.homepage = "http://flydata.com/"
117
120
  s.licenses = ["All right reserved."]
118
- s.rubygems_version = "2.2.2"
121
+ s.require_paths = ["lib"]
122
+ s.rubygems_version = "2.0.14"
119
123
  s.summary = "FlyData Agent"
120
124
 
121
125
  if s.respond_to? :specification_version then
@@ -141,6 +145,7 @@ Gem::Specification.new do |s|
141
145
  s.add_development_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
142
146
  s.add_development_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
143
147
  s.add_development_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
148
+ s.add_development_dependency(%q<pry>, [">= 0"])
144
149
  else
145
150
  s.add_dependency(%q<rest-client>, [">= 1.6.7", "~> 1.6"])
146
151
  s.add_dependency(%q<i18n>, [">= 0.6.5", "~> 0.6"])
@@ -161,6 +166,7 @@ Gem::Specification.new do |s|
161
166
  s.add_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
162
167
  s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
163
168
  s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
169
+ s.add_dependency(%q<pry>, [">= 0"])
164
170
  end
165
171
  else
166
172
  s.add_dependency(%q<rest-client>, [">= 1.6.7", "~> 1.6"])
@@ -182,6 +188,7 @@ Gem::Specification.new do |s|
182
188
  s.add_dependency(%q<activemodel>, [">= 4.0.0", "~> 4.0"])
183
189
  s.add_dependency(%q<activerecord>, [">= 4.0.0", "~> 4.0"])
184
190
  s.add_dependency(%q<protected_attributes>, [">= 1.0.8", "~> 1.0"])
191
+ s.add_dependency(%q<pry>, [">= 0"])
185
192
  end
186
193
  end
187
194
 
@@ -58,7 +58,7 @@ class MysqlBinlogFlydataInput < MysqlBinlogInput
58
58
  load_custom_conf
59
59
  $log.info "mysql host:\"#{@host}\" port:\"#{@port}\" username:\"#{@username}\" database:\"#{@database}\" tables:\"#{@tables}\" tables_append_only:\"#{tables_append_only}\""
60
60
  $log.info "mysql client version: #{`mysql -V`}"
61
- server_version = `echo 'select version();' | mysql -h #{@host} --port #{@port} -u #{@username} -p#{@password} 2>/dev/null`
61
+ server_version = `echo 'select version();' | MYSQL_PWD="#{@password}" mysql -h #{@host} --port #{@port} -u #{@username} 2>/dev/null`
62
62
  $log.info "mysql server version: #{server_version}"
63
63
 
64
64
  @tables = @tables.split(/,\s*/)
@@ -428,15 +428,6 @@ EOS
428
428
  # ex) INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
429
429
  class InsertParser
430
430
  #INSERT INTO `data_entries` VALUES (2,2,'access_log'), (2,3,'access_log2');
431
- module State
432
- IN_VALUE = 'IN_VALUE'
433
- NEXT_VALUES = 'NEXT_VALUES'
434
- end
435
-
436
- def initialize
437
- @values = []
438
- @values_set = []
439
- end
440
431
 
441
432
  def start_ruby_prof
442
433
  RubyProf.start if defined?(RubyProf) and not RubyProf.running?
@@ -466,6 +457,9 @@ EOS
466
457
  private
467
458
 
468
459
  def _parse(target_line)
460
+ values = []
461
+ values_set = []
462
+
469
463
  target_line = target_line.strip
470
464
  start_index = target_line.index('(')
471
465
  target_line = target_line[start_index..-2]
@@ -477,48 +471,49 @@ EOS
477
471
  # 'String#each_char' is twice as slow as the current storategy.
478
472
  items = target_line.split(',')
479
473
  index = 0
480
- cur_state = State::NEXT_VALUES
474
+ cur_state = :next_values
481
475
 
482
476
  loop do
483
477
  case cur_state
484
- when State::NEXT_VALUES
478
+ when :next_values
485
479
  chars = items[index]
486
480
  break unless chars
487
481
  items[index] = chars[1..-1]
488
- cur_state = State::IN_VALUE
489
- when State::IN_VALUE
482
+ cur_state = :in_value
483
+ when :in_value
490
484
  chars = items[index]
491
485
  index += 1
492
486
  if chars.start_with?("'")
487
+ chars_size = chars.size
493
488
  # single item (not last item)
494
489
  # size check added below otherwise end_with? matches the single quote which was also used by start_with?
495
- if chars.size > 1 and chars.end_with?("'") and !last_char_escaped?(chars)
496
- @values << replace_escape_char(chars[1..-2])
490
+ if chars_size > 1 and chars.end_with?("'") and !(chars.end_with?("\\'") and last_char_escaped?(chars))
491
+ values << (chars[1..-2]).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
497
492
  # single item (last item)
498
493
  # size check added below otherwise end_with? matches the single quote which was also used by start_with?
499
- elsif chars.size > 2 and chars.end_with?("')") and !last_char_escaped?(chars[0..-2])
500
- @values << replace_escape_char(chars[1..-3])
501
- @values_set << @values
502
- @values = []
503
- cur_state = State::NEXT_VALUES
494
+ elsif chars_size > 2 and chars.end_with?("')") and !(chars[0..-2].end_with?("\\'") and last_char_escaped?(chars[0..-2]))
495
+ values << (chars[1..-3]).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
496
+ values_set << values
497
+ values = []
498
+ cur_state = :next_values
504
499
  # multi items
505
500
  else
506
501
  cur_value = chars[1..-1]
507
502
  loop do
508
503
  next_chars = items[index]
509
504
  index += 1
510
- if next_chars.end_with?('\'') and !last_char_escaped?(next_chars)
505
+ if next_chars.end_with?('\'') and !(next_chars.end_with?("\\'") and last_char_escaped?(next_chars))
511
506
  cur_value << ','
512
507
  cur_value << next_chars[0..-2]
513
- @values << replace_escape_char(cur_value)
508
+ values << (cur_value).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
514
509
  break
515
- elsif next_chars.end_with?("')") and !last_char_escaped?(next_chars[0..-2])
510
+ elsif next_chars.end_with?("')") and !(next_chars[0..-2].end_with?("\\'") and last_char_escaped?(next_chars[0..-2]))
516
511
  cur_value << ','
517
512
  cur_value << next_chars[0..-3]
518
- @values << replace_escape_char(cur_value)
519
- @values_set << @values
520
- @values = []
521
- cur_state = State::NEXT_VALUES
513
+ values << (cur_value).gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
514
+ values_set << values
515
+ values = []
516
+ cur_state = :next_values
522
517
  break
523
518
  else
524
519
  cur_value << ','
@@ -529,26 +524,28 @@ EOS
529
524
  else
530
525
  if chars.end_with?(')')
531
526
  chars = chars[0..-2]
532
- @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
533
- @values_set << @values
534
- @values = []
535
- cur_state = State::NEXT_VALUES
527
+ values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
528
+ values_set << values
529
+ values = []
530
+ cur_state = :next_values
536
531
  else
537
- @values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
532
+ values << (chars == 'NULL' ? nil : remove_leading_zeros(chars))
538
533
  end
539
534
  end
540
535
  else
541
536
  raise "Invalid state: #{cur_state}"
542
537
  end
543
538
  end
544
- return @values_set
539
+ return values_set
545
540
  end
546
541
 
542
+ ESCAPE_REGEXP = /\\\\|\\'|\\"|\\n|\\r/
547
543
  ESCAPE_HASH_TABLE = {"\\\\" => "\\", "\\'" => "'", "\\\"" => "\"", "\\n" => "\n", "\\r" => "\r"}
548
544
 
549
- def replace_escape_char(original)
550
- original.gsub(/\\\\|\\'|\\"|\\n|\\r/, ESCAPE_HASH_TABLE)
551
- end
545
+ # For the peformance, this function is put inline.
546
+ #def replace_escape_char(original)
547
+ # original.gsub(ESCAPE_REGEXP, ESCAPE_HASH_TABLE)
548
+ #end
552
549
 
553
550
  # This method assume that the last character is '(single quotation)
554
551
  # abcd\' -> true
@@ -54,39 +54,46 @@ grammar MysqlAlterTable
54
54
  }]
55
55
  end
56
56
  }
57
- /
58
- alter_specs (sp partition_opts)? {
57
+ / ( partition_rules ) 1..1 { # This does not exist in sql_yacc.yy
59
58
  def actions
60
- alter_specs.actions
59
+ action = elements[0].action
60
+ action[:query] = text_value
61
+ action[:support_level] = :nonbreaking
62
+ [action]
63
+ end
64
+ }
65
+ / alter_list (sp partition_opts)? {
66
+ def actions
67
+ alter_list.actions
61
68
  end
62
69
  }
63
70
  end
64
71
 
65
- rule alter_specs
66
- alter_spec ( comma alter_spec )* {
72
+ rule alter_list
73
+ alter_list_item ( comma alter_list_item )* {
67
74
  def actions
68
75
  ret = []
69
- ret.concat(actions_with_query(alter_spec))
76
+ ret.concat(actions_with_query(alter_list_item))
70
77
  0.upto(elements[1].elements.size - 1) do |i|
71
78
  ret.concat(actions_with_query(elements[1].elements[i].elements[1]))
72
79
  end
73
80
  ret
74
81
  end
75
82
 
76
- def actions_with_query(alter_spec)
77
- action = alter_spec.action
83
+ def actions_with_query(alter_list_item)
84
+ action = alter_list_item.action
78
85
  if action.kind_of?(Array)
79
86
  actions = action
80
87
  else
81
88
  actions = [action]
82
89
  end
83
- actions.each {|a| a[:query] = alter_spec.text_value }
90
+ actions.each {|a| a[:query] = alter_list_item.text_value }
84
91
  actions
85
92
  end
86
93
  }
87
94
  end
88
95
 
89
- rule alter_spec
96
+ rule alter_list_item
90
97
  add_key sp key_def {
91
98
  def action
92
99
  action_hash = key_def.action
@@ -730,6 +737,10 @@ grammar MysqlAlterTable
730
737
  [0-9]+ # TODO May not be the correct definition
731
738
  end
732
739
 
740
+ rule real_ulong_num
741
+ [0-9]+ # TODO - May have to expand later
742
+ end
743
+
733
744
  rule auto_increment_opt
734
745
  'auto_increment'i
735
746
  end
@@ -1018,4 +1029,273 @@ grammar MysqlAlterTable
1018
1029
  rule dot
1019
1030
  '.'
1020
1031
  end
1032
+
1033
+ ## Partition related rules
1034
+
1035
+ rule partition_rules
1036
+ add_partition_rule {
1037
+ def action
1038
+ { action: :add_partition }
1039
+ end
1040
+ }
1041
+ / drop_sym sp partition_sym sp alt_part_name_list {
1042
+ def action
1043
+ { action: :drop_partition }
1044
+ end
1045
+ }
1046
+ / rebuild_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
1047
+ def action
1048
+ { action: :rebuild_partition }
1049
+ end
1050
+ }
1051
+ / optimize_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
1052
+ def action
1053
+ { action: :optimize_partition }
1054
+ end
1055
+ }
1056
+ / analyze_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list {
1057
+ def action
1058
+ { action: :analyze_partition }
1059
+ end
1060
+ }
1061
+ / check_sym sp partition_sym sp all_or_alt_part_name_list opt_mi_check_type {
1062
+ def action
1063
+ { action: :check_partition }
1064
+ end
1065
+ }
1066
+ / repair_sym sp partition_sym opt_no_write_to_binlog sp all_or_alt_part_name_list opt_mi_repair_type {
1067
+ def action
1068
+ { action: :repair_partition }
1069
+ end
1070
+ }
1071
+ / coalesce_sym sp partition_sym opt_no_write_to_binlog sp real_ulong_num {
1072
+ def action
1073
+ { action: :coalesce_partition }
1074
+ end
1075
+ }
1076
+ / truncate_sym sp partition_sym sp all_or_alt_part_name_list {
1077
+ def action
1078
+ { action: :truncate_partition }
1079
+ end
1080
+ }
1081
+ / reorg_partition_rule {
1082
+ def action
1083
+ { action: :reorganize_partition }
1084
+ end
1085
+ }
1086
+ / exchange_sym sp partition_sym sp alt_part_name_item sp
1087
+ with_sym sp table_sym sp table_ident opt_validation {
1088
+ def action
1089
+ { action: :exchange_partition }
1090
+ end
1091
+ }
1092
+ / discard_sym sp partition_sym sp all_or_alt_part_name_list sp tablespace_sym {
1093
+ def action
1094
+ { action: :discard_partition_tablespace }
1095
+ end
1096
+ }
1097
+ / import_sym sp partition_sym sp all_or_alt_part_name_list sp tablespace_sym {
1098
+ def action
1099
+ { action: :import_partition_tablespace }
1100
+ end
1101
+ }
1102
+ / remove_partitioning {
1103
+ def action
1104
+ { action: :remove_partitioning }
1105
+ end
1106
+ }
1107
+ end
1108
+
1109
+ rule remove_partitioning
1110
+ remove_sym sp partitioning_sym
1111
+ end
1112
+
1113
+ rule add_partition_rule
1114
+ add_sym sp partition_sym opt_no_write_to_binlog nsp add_part_extra
1115
+ end
1116
+
1117
+ rule reorg_partition_rule
1118
+ reorganize_sym sp partition_sym opt_no_write_to_binlog nsp reorg_parts_rule
1119
+ end
1120
+
1121
+ rule opt_validation
1122
+ ( sp (with_sym / without_sym) sp validation_sym )?
1123
+ end
1124
+
1125
+ rule remove_sym
1126
+ 'remove'i ![A-Za-z0-9_]
1127
+ end
1128
+
1129
+ rule all_sym
1130
+ 'all'i ![A-Za-z0-9_]
1131
+ end
1132
+
1133
+ rule check_sym
1134
+ 'check'i ![A-Za-z0-9_]
1135
+ end
1136
+
1137
+ rule table_sym
1138
+ 'table'i ![A-Za-z0-9_]
1139
+ end
1140
+
1141
+ rule with_sym
1142
+ 'with'i ![A-Za-z0-9_]
1143
+ end
1144
+
1145
+ rule without_sym
1146
+ 'without'i ![A-Za-z0-9_]
1147
+ end
1148
+
1149
+ rule validation_sym
1150
+ 'validation'i ![A-Za-z0-9_]
1151
+ end
1152
+
1153
+ rule exchange_sym
1154
+ 'exchange'i ![A-Za-z0-9_]
1155
+ end
1156
+
1157
+ rule reorganize_sym
1158
+ 'reorganize'i ![A-Za-z0-9_]
1159
+ end
1160
+
1161
+ rule truncate_sym
1162
+ 'truncate'i ![A-Za-z0-9_]
1163
+ end
1164
+
1165
+ rule coalesce_sym
1166
+ 'coalesce'i ![A-Za-z0-9_]
1167
+ end
1168
+
1169
+ rule repair_sym
1170
+ 'repair'i ![A-Za-z0-9_]
1171
+ end
1172
+
1173
+ rule analyze_sym
1174
+ 'analyze'i ![A-Za-z0-9_]
1175
+ end
1176
+
1177
+ rule optimize_sym
1178
+ 'optimize'i ![A-Za-z0-9_]
1179
+ end
1180
+
1181
+ rule rebuild_sym
1182
+ 'rebuild'i ![A-Za-z0-9_]
1183
+ end
1184
+
1185
+ rule drop_sym
1186
+ 'drop'i ![A-Za-z0-9_]
1187
+ end
1188
+
1189
+ rule add_sym
1190
+ 'add'i ![A-Za-z0-9_]
1191
+ end
1192
+
1193
+ rule partition_sym
1194
+ 'partition'i ![A-Za-z0-9_]
1195
+ end
1196
+
1197
+ rule partitioning_sym
1198
+ 'partitioning'i ![A-Za-z0-9_]
1199
+ end
1200
+
1201
+ rule local_sym
1202
+ 'local'i ![A-Za-z0-9_]
1203
+ end
1204
+
1205
+ rule for_sym
1206
+ 'for'i ![A-Za-z0-9_]
1207
+ end
1208
+
1209
+ rule upgrade_sym
1210
+ 'upgrade'i ![A-Za-z0-9_]
1211
+ end
1212
+
1213
+ rule changed_sym
1214
+ 'changed'i ![A-Za-z0-9_]
1215
+ end
1216
+
1217
+ rule extended_sym
1218
+ 'extended'i ![A-Za-z0-9_]
1219
+ end
1220
+
1221
+ rule medium_sym
1222
+ 'medium'i ![A-Za-z0-9_]
1223
+ end
1224
+
1225
+ rule fast_sym
1226
+ 'fast'i ![A-Za-z0-9_]
1227
+ end
1228
+
1229
+ rule quick_sym
1230
+ 'quick'i ![A-Za-z0-9_]
1231
+ end
1232
+
1233
+ rule use_frm_sym
1234
+ 'use_frm'i ![A-Za-z0-9_]
1235
+ end
1236
+
1237
+ # query we parse may never have this, including for the sake of completeness
1238
+ rule no_write_to_binlog_sym
1239
+ 'no_write_to_binlog'i ![A-Za-z0-9_]
1240
+ end
1241
+
1242
+ rule opt_no_write_to_binlog
1243
+ ( sp (no_write_to_binlog_sym / local_sym) )?
1244
+ end
1245
+
1246
+ # Generic implementation(matches anything). Should be modified when required
1247
+ rule add_part_extra
1248
+ .*
1249
+ end
1250
+
1251
+ # Generic implementation (matches anything). Should be modified when required
1252
+ rule reorg_parts_rule
1253
+ .*
1254
+ end
1255
+
1256
+ rule all_or_alt_part_name_list
1257
+ all_sym / alt_part_name_list
1258
+ end
1259
+
1260
+ rule alt_part_name_list
1261
+ alt_part_name_item comma alt_part_name_list
1262
+ / alt_part_name_item
1263
+ end
1264
+
1265
+ rule alt_part_name_item
1266
+ ident
1267
+ end
1268
+
1269
+ rule opt_mi_check_type
1270
+ ( sp mi_check_types )?
1271
+ end
1272
+
1273
+ rule mi_check_types
1274
+ mi_check_type sp mi_check_types
1275
+ / mi_check_type
1276
+ end
1277
+
1278
+ rule mi_check_type
1279
+ quick_sym
1280
+ / fast_sym
1281
+ / medium_sym
1282
+ / extended_sym
1283
+ / changed_sym
1284
+ / for_sym sp upgrade_sym
1285
+ end
1286
+
1287
+ rule opt_mi_repair_type
1288
+ ( sp mi_repair_types )?
1289
+ end
1290
+
1291
+ rule mi_repair_types
1292
+ mi_repair_type sp mi_repair_types
1293
+ / mi_repair_type
1294
+ end
1295
+
1296
+ rule mi_repair_type
1297
+ quick_sym
1298
+ / extended_sym
1299
+ / use_frm_sym
1300
+ end
1021
1301
  end
@@ -119,13 +119,21 @@ EOS
119
119
  line
120
120
  end
121
121
 
122
+ NULL_STR = "NULL"
123
+
122
124
  def self.replace_default_value(type, default_value)
123
- val = default_value ? "'#{default_value}'" : "NULL"
125
+ return NULL_STR if default_value.nil?
124
126
  case type
125
127
  when 'timestamp'
126
- (val.upcase == "'CURRENT_TIMESTAMP'") ? 'SYSDATE' : val
128
+ if default_value.upcase == "CURRENT_TIMESTAMP"
129
+ 'SYSDATE'
130
+ else
131
+ "'#{self.parse_timestamp(default_value)}'"
132
+ end
133
+ when 'date'
134
+ "'#{self.parse_date(default_value)}'"
127
135
  else
128
- val
136
+ "'#{default_value}'"
129
137
  end
130
138
  end
131
139
 
@@ -165,7 +173,7 @@ EOS
165
173
  def self.escape(text)
166
174
  text.gsub("'", "\\\\'")
167
175
  end
168
-
176
+
169
177
  def self.check_and_replace_max(params, max_size_a)
170
178
  final_params = []
171
179
  params.split(",").each_with_index do |param, i|
@@ -174,6 +182,38 @@ EOS
174
182
  end
175
183
  final_params.join(",")
176
184
  end
185
+
186
+ APACHE_TIMESTAMP_REGEXP = Regexp.new('^(?<apache_time_format>\[[0-3]\d\/\D{3}\/[1-2]\d{3}:[0-2]\d:[0-5]\d:[0-5]\d ?[\+\-]\d{2}:?\d{2}\])$')
187
+
188
+ def self.parse_timestamp(value)
189
+ if value.kind_of?(Integer) or /^\d+$/ === value
190
+ # Unix epoch in UTC
191
+ t = DateTime.strptime(value.to_s, '%s')
192
+ elsif APACHE_TIMESTAMP_REGEXP.match(value)
193
+ # apache time format
194
+ t = DateTime.strptime(value, "[%d/%b/%Y:%H:%M:%S %Z]")
195
+ else
196
+ t = DateTime.parse(value)
197
+ end
198
+ t = t.new_offset(0) # Redshift Plug-in uses UTC
199
+ t.strftime('%Y-%m-%d %H:%M:%S.%6N')
200
+ rescue ArgumentError => ae
201
+ # '0000-00-00 00:00:00' is valid for mysql datetime column
202
+ if value.start_with?('0000-00-00 00:00:00')
203
+ return '0001-01-01 00:00:00.000000'
204
+ else
205
+ raise ae
206
+ end
207
+ end
208
+
209
+ def self.parse_date(value)
210
+ dt = Date.parse(value)
211
+ dt.strftime('%Y-%m-%d')
212
+ rescue ArgumentError => ae
213
+ # '0000-00-00' is valid for mysql date column
214
+ return '0001-01-01' if value == '0000-00-00'
215
+ raise ae
216
+ end
177
217
  end
178
218
 
179
219
  end
@@ -225,6 +225,8 @@ EOT
225
225
  setup_initial_flydata_files
226
226
  allow(MysqlBinlogInput::BinlogUtil).to receive(:to_hash) {|e| e}
227
227
  allow(Mysql::BinLogPositionFile).to receive(:new).with(TEST_POSITION_FILE).and_return(binlog_position_file)
228
+ allow(plugin).to receive(:`).with("mysql -V").and_return("mysql Ver 14.14 Distrib 5.5.40")
229
+ allow(plugin).to receive(:`).with(/^echo 'select version\(\);'/).and_return("version()\n5.5.40-0ubuntu0.12.04.1-log")
228
230
  Timecop.freeze(now)
229
231
  end
230
232
 
@@ -860,6 +860,90 @@ describe 'MysqlAlterTableParser' do
860
860
  end
861
861
  end
862
862
  end
863
+ shared_examples "test optional no_write_to_binlog" do |*examples|
864
+ context "with no_write_to_binlog" do
865
+ it_behaves_like *examples do
866
+ let(:no_write_to_binlog){" no_write_to_binlog"}
867
+ end
868
+ end
869
+ context "with local" do
870
+ it_behaves_like *examples do
871
+ let(:no_write_to_binlog){" LOCAL"}
872
+ end
873
+ end
874
+ context "without local or no_write_to_binlog flag" do
875
+ it_behaves_like *examples do
876
+ let(:no_write_to_binlog){""}
877
+ end
878
+ end
879
+ end
880
+ shared_examples "test partition names" do |*examples|
881
+ context "ALL" do
882
+ it_behaves_like *examples do
883
+ let(:partition_names){"ALL"}
884
+ end
885
+ end
886
+ context "ALL lower case" do
887
+ it_behaves_like *examples do
888
+ let(:partition_names){"all"}
889
+ end
890
+ end
891
+ context "single partition" do
892
+ it_behaves_like *examples do
893
+ let(:partition_names){"p0"}
894
+ end
895
+ end
896
+ context "multiple partitions" do
897
+ it_behaves_like *examples do
898
+ let(:partition_names){"p0,P1, p2"}
899
+ end
900
+ end
901
+ end
902
+ shared_examples "test check types" do |*examples|
903
+ ["quick", "FAST", "medium", "EXTENDED", "CHANGED", "for upgrade"].each do |check_type|
904
+ context "check_type is #{check_type}" do
905
+ it_behaves_like *examples do
906
+ let(:check_type) {check_type}
907
+ end
908
+ end
909
+ end
910
+ context "multiple check types" do
911
+ it_behaves_like *examples do
912
+ let(:check_type) {"fast for upgrade"}
913
+ end
914
+ end
915
+ end
916
+ shared_examples "test repair types" do |*examples|
917
+ ["quick", "EXTENDED", "use_frm"].each do |repair_type|
918
+ context "repair_type is #{repair_type}" do
919
+ it_behaves_like *examples do
920
+ let(:repair_type) {repair_type}
921
+ end
922
+ end
923
+ end
924
+ context "multiple repair types" do
925
+ it_behaves_like *examples do
926
+ let(:repair_type) {"quick USE_FRM"}
927
+ end
928
+ end
929
+ end
930
+ shared_examples "test optional validation" do |*examples|
931
+ context "with validation" do
932
+ it_behaves_like *examples do
933
+ let(:validation){" with validation"}
934
+ end
935
+ end
936
+ context "without validation" do
937
+ it_behaves_like *examples do
938
+ let(:validation){" WITHOUT VALIDATION"}
939
+ end
940
+ end
941
+ context "none" do
942
+ it_behaves_like *examples do
943
+ let(:validation){""}
944
+ end
945
+ end
946
+ end
863
947
  context 'add index' do
864
948
  let(:action) { :add_index }
865
949
  context "with a simple add index" do
@@ -1138,5 +1222,116 @@ describe 'MysqlAlterTableParser' do
1138
1222
  let(:action) { :import_tablespace }
1139
1223
  it_behaves_like "a parser parsing a breaking query"
1140
1224
  end
1225
+ context "add partition" do
1226
+ let(:action) { :add_partition }
1227
+ context "simple definition with spaces" do
1228
+ let(:alter_table_action) { "add partition#{no_write_to_binlog} (PARTITION p2)" }
1229
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1230
+ end
1231
+ context "complex partition definition without spaces" do
1232
+ let(:alter_table_action) { "ADD partition#{no_write_to_binlog}(PARTITION p2 VALUES LESS THAN (800))" }
1233
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1234
+ end
1235
+ end
1236
+ context "drop partition" do
1237
+ let(:action) { :drop_partition }
1238
+ context "drop single partition" do
1239
+ let(:alter_table_action) { "DROP partition p0" }
1240
+ it_behaves_like "a parser parsing a nonbreaking query"
1241
+ end
1242
+ context "drop multiple partitions" do
1243
+ context "upcase and spaces between partition names" do
1244
+ let(:alter_table_action) { "DROP PARTITION p0, p1" }
1245
+ it_behaves_like "a parser parsing a nonbreaking query"
1246
+ end
1247
+ context "lowercase and no spaces between partition names" do
1248
+ let(:alter_table_action) { "drop partition p0,p1,p2" }
1249
+ it_behaves_like "a parser parsing a nonbreaking query"
1250
+ end
1251
+ end
1252
+ end
1253
+ context "rebuild, optimize, analyze partitions" do
1254
+ ["rebuild", "optimize", "analyze"].each do |op|
1255
+ context "#{op} partition" do
1256
+ let(:action) { "#{op}_partition".to_sym }
1257
+ context "with upcase op" do
1258
+ let(:alter_table_action) { "#{op.upcase} PARTITION#{no_write_to_binlog} #{partition_names}" }
1259
+ it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1260
+ end
1261
+ context "with lowercase op" do
1262
+ let(:alter_table_action) { "#{op} partition#{no_write_to_binlog} #{partition_names}" }
1263
+ it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1264
+ end
1265
+ end
1266
+ end
1267
+ end
1268
+ context "check partition" do
1269
+ let(:action) { :check_partition }
1270
+ context "without check_type" do
1271
+ let(:alter_table_action) { "CHECK PARTITION #{partition_names}" }
1272
+ it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
1273
+ end
1274
+ context "with check type" do
1275
+ let(:alter_table_action) { "check partition #{partition_names} #{check_type}" }
1276
+ it_behaves_like "test check types", "test partition names", "a parser parsing a nonbreaking query"
1277
+ end
1278
+ end
1279
+ context "repair partition" do
1280
+ let(:action) { :repair_partition }
1281
+ context "without repair_type" do
1282
+ let(:alter_table_action) { "REPAIR PARTITION#{no_write_to_binlog} #{partition_names}" }
1283
+ it_behaves_like "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1284
+ end
1285
+ context "with repair type" do
1286
+ let(:alter_table_action) { "repair partition#{no_write_to_binlog} #{partition_names} #{repair_type}" }
1287
+ it_behaves_like "test repair types", "test partition names", "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1288
+ end
1289
+ end
1290
+ context "coalesce partition" do
1291
+ let(:action) { :coalesce_partition }
1292
+ context "single digit partition number" do
1293
+ let(:alter_table_action) { "coalesce partition#{no_write_to_binlog} 1" }
1294
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1295
+ end
1296
+ context "double digit partition number" do
1297
+ let(:alter_table_action) { "COALESCE PARTITION#{no_write_to_binlog} 22" }
1298
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1299
+ end
1300
+ end
1301
+ context "truncate partition" do
1302
+ let(:action) { :truncate_partition }
1303
+ let(:alter_table_action) { "TRUNCATE PARTITION #{partition_names}" }
1304
+ it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
1305
+ end
1306
+ context "reorganize partition" do
1307
+ let(:action) { :reorganize_partition }
1308
+ context "simple definition without partition names and partition definition" do
1309
+ let(:alter_table_action) { "reorganize partition#{no_write_to_binlog}" }
1310
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1311
+ end
1312
+ context "complex partition definition" do
1313
+ let(:alter_table_action) { "REORGANIZE PARTITION#{no_write_to_binlog} p0 INTO (PARTITION n0 VALUES LESS THAN (1960))" }
1314
+ it_behaves_like "test optional no_write_to_binlog", "a parser parsing a nonbreaking query"
1315
+ end
1316
+ end
1317
+ context "exchange partition" do
1318
+ let(:action) { :exchange_partition }
1319
+ let(:alter_table_action) { "EXCHANGE PARTITION #{ident} with table#{identifier}#{validation}" }
1320
+ it_behaves_like "test table identifier", "test optional validation", "a parser parsing a nonbreaking query"
1321
+ end
1322
+ context "discard and import partition tablespace" do
1323
+ ["discard", "import"].each do |op|
1324
+ context "#{op}" do
1325
+ let(:action) {"#{op}_partition_tablespace".to_sym}
1326
+ let(:alter_table_action) {"#{op} PARTITION #{partition_names} TABLESPACE"}
1327
+ it_behaves_like "test partition names", "a parser parsing a nonbreaking query"
1328
+ end
1329
+ end
1330
+ end
1331
+ context "remove partitioning" do
1332
+ let(:action) {:remove_partitioning}
1333
+ let(:alter_table_action) { "REMOVE PARTITIONING" }
1334
+ it_behaves_like "a parser parsing a nonbreaking query"
1335
+ end
1141
1336
  end
1142
1337
  end
@@ -262,17 +262,52 @@ EOT
262
262
  it_behaves_like *examples
263
263
  end
264
264
 
265
- context 'with datetime column def' do
266
- let(:default_value) { '2014-10-17 22:22:22' }
267
- let(:default_value_sql) { "'#{default_value}'" }
265
+ context 'with date column def' do
266
+ before do
267
+ column[:column] = "value_date"
268
+ column[:type] = "date"
269
+ end
270
+ let(:type_sql) { %Q|"value_date" date| }
268
271
  let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
272
+
273
+ context 'when default_value is normal' do
274
+ let(:default_value) { '2014-10-17' }
275
+ let(:default_value_sql) { "'#{default_value}'" }
276
+ it_behaves_like *examples
277
+ end
278
+
279
+ context 'when default_value is 0000-00-00' do
280
+ let(:default_value) { '0000-00-00' }
281
+ let(:default_value_sql) { "'0001-01-01'" }
282
+ it_behaves_like *examples
283
+ end
284
+ end
285
+
286
+ context 'with datetime column def' do
269
287
  before do
270
288
  column[:column] = "value_datetime"
271
289
  column[:type] = "datetime"
272
290
  end
273
291
  let(:type_sql) { %Q|"value_datetime" timestamp| }
292
+ let(:not_null_default_sql) { " DEFAULT '0000-01-01'" }
274
293
 
275
- it_behaves_like *examples
294
+ context 'when default_value is normal' do
295
+ let(:default_value) { '2014-10-17 22:22:22' }
296
+ let(:default_value_sql) { "'#{default_value}.000000'" }
297
+ it_behaves_like *examples
298
+ end
299
+
300
+ context 'when default_value is CURRENT_TIMESTAMP' do
301
+ let(:default_value) { 'CURRENT_TIMESTAMP' }
302
+ let(:default_value_sql) { "SYSDATE" }
303
+ it_behaves_like *examples
304
+ end
305
+
306
+ context 'when default_value is 0000-00-00 00:00:00' do
307
+ let(:default_value) { '0000-00-00 00:00:00' }
308
+ let(:default_value_sql) { "'0001-01-01 00:00:00.000000'" }
309
+ it_behaves_like *examples
310
+ end
276
311
  end
277
312
 
278
313
  context 'with decimal column def' do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: flydata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.10
4
+ version: 0.2.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Koichi Fujikawa
@@ -12,7 +12,7 @@ authors:
12
12
  autorequire:
13
13
  bindir: bin
14
14
  cert_chain: []
15
- date: 2014-10-27 00:00:00.000000000 Z
15
+ date: 2014-10-31 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: rest-client
@@ -376,6 +376,20 @@ dependencies:
376
376
  - - ~>
377
377
  - !ruby/object:Gem::Version
378
378
  version: '1.0'
379
+ - !ruby/object:Gem::Dependency
380
+ name: pry
381
+ requirement: !ruby/object:Gem::Requirement
382
+ requirements:
383
+ - - '>='
384
+ - !ruby/object:Gem::Version
385
+ version: '0'
386
+ type: :development
387
+ prerelease: false
388
+ version_requirements: !ruby/object:Gem::Requirement
389
+ requirements:
390
+ - - '>='
391
+ - !ruby/object:Gem::Version
392
+ version: '0'
379
393
  description: FlyData Agent
380
394
  email: sysadmin@flydata.com
381
395
  executables:
@@ -391,6 +405,11 @@ files:
391
405
  - Gemfile.lock
392
406
  - Rakefile
393
407
  - VERSION
408
+ - benchmark/benchmark_helper.rb
409
+ - benchmark/data/insert_parser_test_data.sql.gz
410
+ - benchmark/data/insert_parser_test_data_num_only.sql.gz
411
+ - benchmark/insert_parser_bench.rb
412
+ - benchmark/insert_parser_prof.rb
394
413
  - bin/fdmysqldump
395
414
  - bin/flydata
396
415
  - bin/serverinfo
@@ -501,7 +520,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
501
520
  version: '0'
502
521
  requirements: []
503
522
  rubyforge_project:
504
- rubygems_version: 2.2.2
523
+ rubygems_version: 2.0.14
505
524
  signing_key:
506
525
  specification_version: 4
507
526
  summary: FlyData Agent