mysql-parser 0.0.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: acd650741a0f7b5b7631b03b8fe200494a4265df
4
+ data.tar.gz: f543002b4a58e312a8934fd6610d2dbed5a36f59
5
+ SHA512:
6
+ metadata.gz: b3968deb3c2ad1c145263086292524a48661d8e7e5952ce2da4d8117cfe64bd178f563c518d3321ac77bc2c5f369f7da1bcd1b8933229534a96295b5b1be02fc
7
+ data.tar.gz: 8da6a07a5ecf645c069373dc334e35223455d7f64693a1d8de7d69e6ecfe0bfd475bf6bc7908f073c5300ccc828614807af8f0dabd0e84a3c2f46cbebf7eed06
@@ -0,0 +1,131 @@
1
+ MySQLParser
2
+ ===========
3
+
4
+ This is a library to parse SQL commands. The only commands that are currently
5
+ supported are ddl statements, specifically CREATE TABLE, ALTER TABLE, DROP VIEW,
6
+ and DROP TABLE.
7
+
8
+ Installation
9
+ ------------
10
+
11
+ In command line:
12
+
13
+ > gem install mysql-parser.x.x.x.gem
14
+
15
+ Usage
16
+ -----
17
+
18
+ In ruby:
19
+
20
+ > require 'mysql-parser'
21
+ > MySQLParser.new.parse "ALTER TABLE `table` DROP INDEX abc, DROP INDEX def"
22
+ => {:tree=><root: [<S: [" "]>, <r_commands: [<r_ALTER_TABLE: ["ALTER", <S: [" "]>, <r_ONLINE_OFFLINE: []>, <r_opt_IGNORE: []>, "TABLE", <S: [" "]>, <r_tbl_name: [<r_tbl_name_int: [<ident: ["`", <opt_ident_in_backtick: [<opt_ident_in_backtick: []>, "table"]>, "`", <S: [" "]>]>]>]>, <r_opt_alter_commands: [<r_comma_separated_alter_specification: [<r_comma_separated_alter_specification: [<r_alter_specification: ["DROP", <S: [" "]>, <r_INDEX_or_KEY: ["INDEX", <S: [" "]>]>, <r_index_name: [<ident: [<raw_ident: ["abc", <S: [" "]>]>]>]>]>]>, <comma: [",", <S: [" "]>]>, <r_alter_specification: ["DROP", <S: [" "]>, <r_INDEX_or_KEY: ["INDEX", <S: [" "]>]>, <r_index_name: [<ident: [<raw_ident: ["def", <S: [" "]>]>]>]>]>]>]>, <r_opt_after_alter: []>, <r_opt_PARTITION_options: []>]>]>]>, :state=>{}}
23
+
24
+ Files
25
+ -----
26
+
27
+ ### mysql.rex.rb
28
+ This file is a lexer. It determines that `DROP` is a command, and
29
+ that `'abcdef'` is a string. Most of this file is auto-generated
30
+ by `bin/generate-literal` which reads `mysql.y.rb` and generates the required
31
+ literals automatically. Therefore this file will not normally need to be edited.
32
+ The following reasons might be reason to edit it:
33
+
34
+ 1. Creating a synonym
35
+ 2. Creating a long literal[1]
36
+ 3. Creating a literal which doesn't exist in the parser already
37
+
38
+ [1] a long literal is a literal which is needed for a special purpose, for example,
39
+ because it consists of spaces and needs a synonym to be assigned to it.
40
+ Normally, however, we do not need them.
41
+
42
+ #### Convention
43
+
44
+ 1. `S_...` means some symbol
45
+ 2. `A_...` means some state
46
+ 3. `L_...` means long literal
47
+ 4. Everything else is literal
48
+
49
+ ### mysql.y.rb
50
+ This is the main file of this library. It contains all grammar and associated
51
+ actions.
52
+
53
+ #### Grammar
54
+
55
+ The original MySQL grammar can be found at
56
+ https://github.com/twitter/mysql/blob/master/sql/sql_yacc.yy. The documentation
57
+ can be found under https://dev.mysql.com/doc/refman/5.6/en/
58
+
59
+ ##### Conflict
60
+ `sql_yacc.yy` is not perfect. It contains a lot of conflicts. When translating
61
+ to `mysql.y.rb`, please resolve those conflicts so that the rules can be
62
+ predictable.
63
+
64
+ #### Debugging
65
+
66
+ Debugging `mysql.y.rb` can be done by setting `@yydebug` to `true` in
67
+ `initialize`.
68
+
69
+ #### Literals
70
+
71
+ Introduction of a new literals can be done by just _using_ it. `bin/generate-literal.rb`
72
+ will read `mysql.y.rb` and make sure that new literals are created.
73
+ Similarly, removing any literal can be done by just _not using_ it.
74
+
75
+ #### Convention
76
+
77
+ Following is general convention.
78
+
79
+ 1. Space is `S`
80
+ 2. `opt_...` means that the rule is optional. The first branch should be
81
+ empty.
82
+ 3. `{comma, space}_separated_...` means a collection of items separated by
83
+ a separator (comma or space)
84
+
85
+ ### bin/generate-literal.rb
86
+
87
+ This script scans `mysql.y.rb` (or actually, `parser.output` which is
88
+ generated from `mysql.y.rb`) and adds new literals that don't exist, and
89
+ removes literals that are unused.
90
+
91
+ ### bin/runner
92
+
93
+ This is a REPL. Just input whatever and send an end-of-file character
94
+ (`ctrl-d`) to let the script process the input. To exit, just terminate
95
+ using `ctrl-c`.
96
+
97
+ > DROP TABLE
98
+
99
+
100
+ table
101
+ ^D
102
+ parse error on value "table" (TABLE)
103
+ >
104
+
105
+ ### bin/sanity_check
106
+
107
+ This script makes sure that there is no skipped action and that all action
108
+ names are correct.
109
+
110
+ Development
111
+ -----------
112
+
113
+ After changing `mysql.rex.rb` or `mysql.y.rb`, run `rake generate` to
114
+ generate the real lexer and parser. Run `rake spec` to run all test cases
115
+
116
+ License
117
+ =======
118
+
119
+ Copyright 2015 Square, Inc.
120
+
121
+ Licensed under the Apache License, Version 2.0 (the "License");
122
+ you may not use this file except in compliance with the License.
123
+ You may obtain a copy of the License at
124
+
125
+ http://www.apache.org/licenses/LICENSE-2.0
126
+
127
+ Unless required by applicable law or agreed to in writing, software
128
+ distributed under the License is distributed on an "AS IS" BASIS,
129
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
130
+ See the License for the specific language governing permissions and
131
+ limitations under the License.
@@ -0,0 +1,72 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require 'set'
4
+ require_relative '../lib/helper'
5
+
6
+ parser_filename = 'mysql.y.rb'
7
+ lexer_filename = 'mysql.rex.rb'
8
+ output_filename = 'parser.output'
9
+
10
+ START = 0
11
+ STOP = -1
12
+
13
+ def prepare_autogen(filename)
14
+ FileUtil.read_three_parts(
15
+ filename,
16
+ '# BEGIN LITERAL (AUTO-GENERATED)',
17
+ '# END LITERAL (AUTO-GENERATED)'
18
+ )
19
+ end
20
+
21
+ def diff(new_list, old_list)
22
+ new_set = new_list.to_set
23
+ old_set = old_list.to_set
24
+ added = new_set - old_set
25
+ removed = old_set - new_set
26
+ if !added.empty?
27
+ puts "\n\n======= ADDED ======="
28
+ added.each do |t|
29
+ puts t
30
+ end
31
+ puts "\n\n"
32
+ end
33
+ if !removed.empty?
34
+ puts "\n\n======= REMOVED ======="
35
+ removed.each do |t|
36
+ puts t
37
+ end
38
+ puts "\n\n"
39
+ end
40
+ end
41
+
42
+ old_lex, lines_before_lex, lines_after_lex = prepare_autogen(lexer_filename)
43
+ old_parse, lines_before_parse, lines_after_parse = prepare_autogen(parser_filename)
44
+ arr = FileUtil.read_three_parts(
45
+ output_filename,
46
+ '**Terminals, with rules where they appear',
47
+ '--------- State ---------'
48
+ )[0].map { |line|
49
+ result = /\s+(\w+)\s+\(/.match(line)
50
+ result ? result[1] : ''
51
+ }.select { |line| !line.empty? }.sort.reverse.reject { |w| w == 'error' }
52
+
53
+ literals = arr.reject { |w|
54
+ w == "S" || w.start_with?("S_") || w.start_with?("L_")
55
+ }
56
+
57
+ File.open(lexer_filename, 'w') do |f|
58
+ new_lex = literals.map{ |t| ":A_NIL #{t}\\b { [:#{t}, text] }\n" }
59
+ diff(new_lex, old_lex)
60
+ f.write((lines_before_lex + new_lex + lines_after_lex).join)
61
+ end
62
+
63
+ File.open(parser_filename, 'w') do |f|
64
+ name = 'dot'
65
+ new_parse = ([" #{name} :\n",
66
+ " #{arr[0]} { call(:#{name}, :#{arr[0]}, val) }\n"] +
67
+ arr[START+1..STOP].map{ |t|
68
+ " | #{t} { call(:#{name}, :#{t}, val) }\n"
69
+ })
70
+ diff(new_parse, old_parse)
71
+ f.write((lines_before_parse + new_parse + lines_after_parse).join)
72
+ end
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require_relative '../lib/mysql-parser'
4
+
5
+ while true do
6
+ begin
7
+ print "> "
8
+ puts (MySQLParser.new.parse ARGF.read)
9
+ rescue => e
10
+ puts e
11
+ puts e.backtrace
12
+ end
13
+ end
@@ -0,0 +1,73 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require_relative '../lib/helper'
4
+
5
+ parser_filename = 'mysql.y.rb'
6
+ output_filename = 'parser.output'
7
+
8
+ arr = FileUtil.read_three_parts(
9
+ output_filename,
10
+ '**Terminals, with rules where they appear',
11
+ '--------- State ---------'
12
+ )[0].map { |line|
13
+ result = /\s+(\w+)\s+\(/.match(line)
14
+ result ? result[1] : ''
15
+ }.select { |line| !line.empty? }.sort.reverse.reject { |w| w == 'error' }
16
+
17
+ literals = arr.reject { |w| /^[A-Z\d_]+$/ =~ w }
18
+ if !literals.empty?
19
+ $stderr.puts literals
20
+ raise 'unrecognized literals'
21
+ end
22
+
23
+ lines = []
24
+
25
+ File.open(parser_filename, 'r') do |f|
26
+ f.each_line do |line|
27
+ lines << line
28
+ end
29
+ end
30
+
31
+ raw_content = lines.join
32
+ .gsub(/^---- header.*/m, '') # elimate code section
33
+
34
+ content = raw_content
35
+ content = content.gsub(/#.*?$/m, '') # eliminate comments
36
+
37
+ res = /:\s[^{]*?\|/m.match(content)
38
+ if res
39
+ $stderr.puts res
40
+ raise 'First case should have action'
41
+ end
42
+
43
+ res = /\|[^{]*?\|/m.match(content)
44
+ if res
45
+ $stderr.puts res
46
+ raise 'Middle case should have action'
47
+ end
48
+
49
+ res = /\|[^{]*?:\s/m.match(content)
50
+ if res
51
+ $stderr.puts res
52
+ raise 'Last case should have action'
53
+ end
54
+
55
+ res = /:\s[^{]*?:/m.match(content)
56
+ if res
57
+ $stderr.puts res
58
+ raise 'One case should have action'
59
+ end
60
+
61
+ content = content.gsub(/call\(:(.*?), (.*?), val\)/, '\1')
62
+
63
+ # this scan will not match "dot" which is the last rule
64
+ # this is okay as "dot" is auto generated and is always correct.
65
+ content.scan(/(?=^\s*(\S*?)(\s*?:\s[^:]+))/m).each do |group|
66
+ all_branches = group[1].count "{"
67
+ count_matched = group[1].scan(/\{\s*#{group[0]}/).count
68
+ count_raise = group[1].scan(/\{\s*raise/).count
69
+ if all_branches != count_matched + count_raise
70
+ $stderr.puts "#{group}"
71
+ raise 'name does not match action symbol'
72
+ end
73
+ end
@@ -0,0 +1,110 @@
1
+ class AST
2
+ def initialize(a_name, a_subname, a_val)
3
+ @name = a_name
4
+ @subname = a_subname
5
+ @val = a_val
6
+ end
7
+
8
+ def update(a_name, a_subname, a_val)
9
+ initialize(a_name, a_subname, a_val)
10
+ end
11
+
12
+ def match(options)
13
+ (
14
+ {top: true}.merge(options)[:top] &&
15
+ @name == options[:name] &&
16
+ (options[:subname].nil? || @subname == options[:subname])
17
+ )
18
+ end
19
+
20
+ def find_all(options={})
21
+ ret = []
22
+ ret << self if match options
23
+ sub_options = options.merge top: true
24
+ @val.each do |v|
25
+ if v.is_a? AST
26
+ ret.concat (v.find_all sub_options)
27
+ end
28
+ end
29
+ ret
30
+ end
31
+
32
+ def find_left(options={})
33
+ return self if match options
34
+ sub_options = options.merge top: true
35
+ @val.each do |v|
36
+ if v.is_a? AST
37
+ ret = v.find_left sub_options
38
+ return ret if ret
39
+ end
40
+ end
41
+ nil
42
+ end
43
+
44
+ def find_top(options={})
45
+ return self if match options
46
+ sub_options = options.merge top: true
47
+ @val.each do |v|
48
+ if v.is_a? AST
49
+ return v if v.match sub_options
50
+ end
51
+ end
52
+ nil
53
+ end
54
+
55
+ def name
56
+ @name
57
+ end
58
+
59
+ def subname
60
+ @subname
61
+ end
62
+
63
+ def val
64
+ @val
65
+ end
66
+
67
+ def val=(a_val)
68
+ @val = a_val
69
+ end
70
+
71
+ def eval
72
+ to_s.to_f
73
+ end
74
+
75
+ def to_list
76
+ to_list_helper.flatten
77
+ end
78
+
79
+ def to_s
80
+ @val.map { |v| v.to_s }.join
81
+ end
82
+
83
+ def norm_name
84
+ s_all = to_s.strip
85
+ node = find_left(name: :r_tbl_name) || find_left(name: :ident)
86
+ s = node.to_s.strip
87
+ raise 'Internal Error: trying to normalize not-a-name' if s_all != s
88
+ if s.start_with? '`'
89
+ s[1..-2] # strip ` from both beginning and end
90
+ else
91
+ s
92
+ end
93
+ end
94
+
95
+ def inspect
96
+ "<#{@name}: #{@val}>"
97
+ end
98
+
99
+ protected
100
+
101
+ def to_list_helper
102
+ @val.map { |v|
103
+ if v.is_a?(AST) && v.name == @name
104
+ v.to_list_helper
105
+ else
106
+ [v]
107
+ end
108
+ }
109
+ end
110
+ end
@@ -0,0 +1,32 @@
1
+ module FileUtil
2
+ def self.read_three_parts(filename, begin_middle, begin_after)
3
+ lines_before = []
4
+ lines_middle = []
5
+ lines_after = []
6
+ File.open(filename, 'r') do |f|
7
+ state = :BEFORE
8
+ f.each_line do |line|
9
+ case line.strip
10
+ when begin_middle
11
+ lines_before << line
12
+ state = :MIDDLE
13
+ next
14
+ when begin_after
15
+ lines_after << line
16
+ state = :AFTER
17
+ next
18
+ end
19
+ case state
20
+ when :BEFORE
21
+ lines_before << line
22
+ when :MIDDLE
23
+ lines_middle << line
24
+ when :AFTER
25
+ lines_after << line
26
+ end
27
+ end
28
+ end
29
+
30
+ return lines_middle, lines_before, lines_after
31
+ end
32
+ end
@@ -0,0 +1,724 @@
1
+ #--
2
+ # DO NOT MODIFY!!!!
3
+ # This file is automatically generated by rex 1.0.5
4
+ # from lexical definition file "mysql.rex.rb".
5
+ #++
6
+
7
+ require 'racc/parser'
8
+ class MySQLParser < Racc::Parser
9
+ require 'strscan'
10
+
11
+ class ScanError < StandardError ; end
12
+
13
+ attr_reader :lineno
14
+ attr_reader :filename
15
+ attr_accessor :state
16
+
17
+ def scan_setup(str)
18
+ @ss = StringScanner.new(str)
19
+ @lineno = 1
20
+ @state = nil
21
+ end
22
+
23
+ def action
24
+ yield
25
+ end
26
+
27
+ def scan_str(str)
28
+ scan_setup(str)
29
+ do_parse
30
+ end
31
+ alias :scan :scan_str
32
+
33
+ def load_file( filename )
34
+ @filename = filename
35
+ open(filename, "r") do |f|
36
+ scan_setup(f.read)
37
+ end
38
+ end
39
+
40
+ def scan_file( filename )
41
+ load_file(filename)
42
+ do_parse
43
+ end
44
+
45
+
46
+ def next_token
47
+ return if @ss.eos?
48
+
49
+ # skips empty actions
50
+ until token = _next_token or @ss.eos?; end
51
+ token
52
+ end
53
+
54
+ def _next_token
55
+ text = @ss.peek(1)
56
+ @lineno += 1 if text == "\n"
57
+ token = case @state
58
+ when nil
59
+ case
60
+ when (text = @ss.scan(/\A/i))
61
+ action { @state = :A_NIL; nil }
62
+
63
+ else
64
+ text = @ss.string[@ss.pos .. -1]
65
+ raise ScanError, "can not match: '" + text + "'"
66
+ end # if
67
+
68
+ when :A_NIL
69
+ case
70
+ when (text = @ss.scan(/ *\/\*/i))
71
+ action { @state = :A_REM_MULTI; [:S_REM_IN, [text, ' /* ']] }
72
+
73
+ when (text = @ss.scan(/ *(\#|--)/i))
74
+ action { @state = :A_REM_INLINE; [:S_REM_IN, '-- '] }
75
+
76
+ when (text = @ss.scan(/`/i))
77
+ action { @state = :A_BACKTICK; [:S_BACKTICK_IN, text] }
78
+
79
+ when (text = @ss.scan(/"/i))
80
+ action { @state = :A_DOUBLEQUOTE; [:S_DOUBLEQUOTE_IN, text] }
81
+
82
+ when (text = @ss.scan(/'/i))
83
+ action { @state = :A_SINGLEQUOTE; [:S_SINGLEQUOTE_IN, text] }
84
+
85
+ when (text = @ss.scan(/TRUE\b/i))
86
+ action { [:S_ONE, text] }
87
+
88
+ when (text = @ss.scan(/FALSE\b/i))
89
+ action { [:S_ZERO, text] }
90
+
91
+ when (text = @ss.scan(/BOOLEAN\b/i))
92
+ action { [:TINYINT, text] }
93
+
94
+ when (text = @ss.scan(/CHARSET\b/i))
95
+ action { [:L_CHARACTER_SET, text] }
96
+
97
+ when (text = @ss.scan(/CHARACTER[ \t\n]+SET\b/i))
98
+ action { [:L_CHARACTER_SET, text] }
99
+
100
+ when (text = @ss.scan(/FROM\b/i))
101
+ action { [:FROM, text] }
102
+
103
+ when (text = @ss.scan(/WHERE\b/i))
104
+ action { [:WHERE, text] }
105
+
106
+ when (text = @ss.scan(/ZEROFILL\b/i))
107
+ action { [:ZEROFILL, text] }
108
+
109
+ when (text = @ss.scan(/YEAR\b/i))
110
+ action { [:YEAR, text] }
111
+
112
+ when (text = @ss.scan(/WITH\b/i))
113
+ action { [:WITH, text] }
114
+
115
+ when (text = @ss.scan(/VIEW\b/i))
116
+ action { [:VIEW, text] }
117
+
118
+ when (text = @ss.scan(/VARCHAR\b/i))
119
+ action { [:VARCHAR, text] }
120
+
121
+ when (text = @ss.scan(/VARBINARY\b/i))
122
+ action { [:VARBINARY, text] }
123
+
124
+ when (text = @ss.scan(/VALUES\b/i))
125
+ action { [:VALUES, text] }
126
+
127
+ when (text = @ss.scan(/UTF8MB4\b/i))
128
+ action { [:UTF8MB4, text] }
129
+
130
+ when (text = @ss.scan(/UTF8MB3\b/i))
131
+ action { [:UTF8MB3, text] }
132
+
133
+ when (text = @ss.scan(/UTF8\b/i))
134
+ action { [:UTF8, text] }
135
+
136
+ when (text = @ss.scan(/USING\b/i))
137
+ action { [:USING, text] }
138
+
139
+ when (text = @ss.scan(/UPDATE\b/i))
140
+ action { [:UPDATE, text] }
141
+
142
+ when (text = @ss.scan(/UNSIGNED\b/i))
143
+ action { [:UNSIGNED, text] }
144
+
145
+ when (text = @ss.scan(/UNIQUE\b/i))
146
+ action { [:UNIQUE, text] }
147
+
148
+ when (text = @ss.scan(/UNION\b/i))
149
+ action { [:UNION, text] }
150
+
151
+ when (text = @ss.scan(/UNDEFINED\b/i))
152
+ action { [:UNDEFINED, text] }
153
+
154
+ when (text = @ss.scan(/TRUNCATE\b/i))
155
+ action { [:TRUNCATE, text] }
156
+
157
+ when (text = @ss.scan(/TO\b/i))
158
+ action { [:TO, text] }
159
+
160
+ when (text = @ss.scan(/TINYTEXT\b/i))
161
+ action { [:TINYTEXT, text] }
162
+
163
+ when (text = @ss.scan(/TINYINT\b/i))
164
+ action { [:TINYINT, text] }
165
+
166
+ when (text = @ss.scan(/TINYBLOB\b/i))
167
+ action { [:TINYBLOB, text] }
168
+
169
+ when (text = @ss.scan(/TIMESTAMP\b/i))
170
+ action { [:TIMESTAMP, text] }
171
+
172
+ when (text = @ss.scan(/TIME\b/i))
173
+ action { [:TIME, text] }
174
+
175
+ when (text = @ss.scan(/THAN\b/i))
176
+ action { [:THAN, text] }
177
+
178
+ when (text = @ss.scan(/TEXT\b/i))
179
+ action { [:TEXT, text] }
180
+
181
+ when (text = @ss.scan(/TEMPTABLE\b/i))
182
+ action { [:TEMPTABLE, text] }
183
+
184
+ when (text = @ss.scan(/TEMPORARY\b/i))
185
+ action { [:TEMPORARY, text] }
186
+
187
+ when (text = @ss.scan(/TABLESPACE\b/i))
188
+ action { [:TABLESPACE, text] }
189
+
190
+ when (text = @ss.scan(/TABLE\b/i))
191
+ action { [:TABLE, text] }
192
+
193
+ when (text = @ss.scan(/SUBPARTITION\b/i))
194
+ action { [:SUBPARTITION, text] }
195
+
196
+ when (text = @ss.scan(/STORAGE\b/i))
197
+ action { [:STORAGE, text] }
198
+
199
+ when (text = @ss.scan(/SQL\b/i))
200
+ action { [:SQL, text] }
201
+
202
+ when (text = @ss.scan(/SPATIAL\b/i))
203
+ action { [:SPATIAL, text] }
204
+
205
+ when (text = @ss.scan(/SMALLINT\b/i))
206
+ action { [:SMALLINT, text] }
207
+
208
+ when (text = @ss.scan(/SIMPLE\b/i))
209
+ action { [:SIMPLE, text] }
210
+
211
+ when (text = @ss.scan(/SET\b/i))
212
+ action { [:SET, text] }
213
+
214
+ when (text = @ss.scan(/SELECT\b/i))
215
+ action { [:SELECT, text] }
216
+
217
+ when (text = @ss.scan(/SECURITY\b/i))
218
+ action { [:SECURITY, text] }
219
+
220
+ when (text = @ss.scan(/ROW_FORMAT\b/i))
221
+ action { [:ROW_FORMAT, text] }
222
+
223
+ when (text = @ss.scan(/RESTRICT\b/i))
224
+ action { [:RESTRICT, text] }
225
+
226
+ when (text = @ss.scan(/REPLACE\b/i))
227
+ action { [:REPLACE, text] }
228
+
229
+ when (text = @ss.scan(/REPAIR\b/i))
230
+ action { [:REPAIR, text] }
231
+
232
+ when (text = @ss.scan(/REORGANIZE\b/i))
233
+ action { [:REORGANIZE, text] }
234
+
235
+ when (text = @ss.scan(/RENAME\b/i))
236
+ action { [:RENAME, text] }
237
+
238
+ when (text = @ss.scan(/REMOVE\b/i))
239
+ action { [:REMOVE, text] }
240
+
241
+ when (text = @ss.scan(/REFERENCES\b/i))
242
+ action { [:REFERENCES, text] }
243
+
244
+ when (text = @ss.scan(/REDUNDANT\b/i))
245
+ action { [:REDUNDANT, text] }
246
+
247
+ when (text = @ss.scan(/REBUILD\b/i))
248
+ action { [:REBUILD, text] }
249
+
250
+ when (text = @ss.scan(/REAL\b/i))
251
+ action { [:REAL, text] }
252
+
253
+ when (text = @ss.scan(/PRIMARY\b/i))
254
+ action { [:PRIMARY, text] }
255
+
256
+ when (text = @ss.scan(/PASSWORD\b/i))
257
+ action { [:PASSWORD, text] }
258
+
259
+ when (text = @ss.scan(/PARTITIONING\b/i))
260
+ action { [:PARTITIONING, text] }
261
+
262
+ when (text = @ss.scan(/PARTITION\b/i))
263
+ action { [:PARTITION, text] }
264
+
265
+ when (text = @ss.scan(/PARTIAL\b/i))
266
+ action { [:PARTIAL, text] }
267
+
268
+ when (text = @ss.scan(/PARSER\b/i))
269
+ action { [:PARSER, text] }
270
+
271
+ when (text = @ss.scan(/PACK_KEYS\b/i))
272
+ action { [:PACK_KEYS, text] }
273
+
274
+ when (text = @ss.scan(/ORDER\b/i))
275
+ action { [:ORDER, text] }
276
+
277
+ when (text = @ss.scan(/OR\b/i))
278
+ action { [:OR, text] }
279
+
280
+ when (text = @ss.scan(/OPTION\b/i))
281
+ action { [:OPTION, text] }
282
+
283
+ when (text = @ss.scan(/OPTIMIZE\b/i))
284
+ action { [:OPTIMIZE, text] }
285
+
286
+ when (text = @ss.scan(/ONLINE\b/i))
287
+ action { [:ONLINE, text] }
288
+
289
+ when (text = @ss.scan(/ON\b/i))
290
+ action { [:ON, text] }
291
+
292
+ when (text = @ss.scan(/OFFLINE\b/i))
293
+ action { [:OFFLINE, text] }
294
+
295
+ when (text = @ss.scan(/NUMERIC\b/i))
296
+ action { [:NUMERIC, text] }
297
+
298
+ when (text = @ss.scan(/NULL\b/i))
299
+ action { [:NULL, text] }
300
+
301
+ when (text = @ss.scan(/NOT\b/i))
302
+ action { [:NOT, text] }
303
+
304
+ when (text = @ss.scan(/NODEGROUP\b/i))
305
+ action { [:NODEGROUP, text] }
306
+
307
+ when (text = @ss.scan(/NO\b/i))
308
+ action { [:NO, text] }
309
+
310
+ when (text = @ss.scan(/MODIFY\b/i))
311
+ action { [:MODIFY, text] }
312
+
313
+ when (text = @ss.scan(/MIN_ROWS\b/i))
314
+ action { [:MIN_ROWS, text] }
315
+
316
+ when (text = @ss.scan(/MERGE\b/i))
317
+ action { [:MERGE, text] }
318
+
319
+ when (text = @ss.scan(/MEMORY\b/i))
320
+ action { [:MEMORY, text] }
321
+
322
+ when (text = @ss.scan(/MEDIUMTEXT\b/i))
323
+ action { [:MEDIUMTEXT, text] }
324
+
325
+ when (text = @ss.scan(/MEDIUMINT\b/i))
326
+ action { [:MEDIUMINT, text] }
327
+
328
+ when (text = @ss.scan(/MEDIUMBLOB\b/i))
329
+ action { [:MEDIUMBLOB, text] }
330
+
331
+ when (text = @ss.scan(/MAX_ROWS\b/i))
332
+ action { [:MAX_ROWS, text] }
333
+
334
+ when (text = @ss.scan(/MAXVALUE\b/i))
335
+ action { [:MAXVALUE, text] }
336
+
337
+ when (text = @ss.scan(/MATCH\b/i))
338
+ action { [:MATCH, text] }
339
+
340
+ when (text = @ss.scan(/LONGTEXT\b/i))
341
+ action { [:LONGTEXT, text] }
342
+
343
+ when (text = @ss.scan(/LONGBLOB\b/i))
344
+ action { [:LONGBLOB, text] }
345
+
346
+ when (text = @ss.scan(/LOCAL\b/i))
347
+ action { [:LOCAL, text] }
348
+
349
+ when (text = @ss.scan(/LIKE\b/i))
350
+ action { [:LIKE, text] }
351
+
352
+ when (text = @ss.scan(/LESS\b/i))
353
+ action { [:LESS, text] }
354
+
355
+ when (text = @ss.scan(/LATIN1\b/i))
356
+ action { [:LATIN1, text] }
357
+
358
+ when (text = @ss.scan(/LAST\b/i))
359
+ action { [:LAST, text] }
360
+
361
+ when (text = @ss.scan(/KEY_BLOCK_SIZE\b/i))
362
+ action { [:KEY_BLOCK_SIZE, text] }
363
+
364
+ when (text = @ss.scan(/KEYS\b/i))
365
+ action { [:KEYS, text] }
366
+
367
+ when (text = @ss.scan(/KEY\b/i))
368
+ action { [:KEY, text] }
369
+
370
+ when (text = @ss.scan(/INVOKER\b/i))
371
+ action { [:INVOKER, text] }
372
+
373
+ when (text = @ss.scan(/INTO\b/i))
374
+ action { [:INTO, text] }
375
+
376
+ when (text = @ss.scan(/INTEGER\b/i))
377
+ action { [:INTEGER, text] }
378
+
379
+ when (text = @ss.scan(/INT\b/i))
380
+ action { [:INT, text] }
381
+
382
+ when (text = @ss.scan(/INSERT_METHOD\b/i))
383
+ action { [:INSERT_METHOD, text] }
384
+
385
+ when (text = @ss.scan(/INNODB\b/i))
386
+ action { [:INNODB, text] }
387
+
388
+ when (text = @ss.scan(/INDEX\b/i))
389
+ action { [:INDEX, text] }
390
+
391
+ when (text = @ss.scan(/IN\b/i))
392
+ action { [:IN, text] }
393
+
394
+ when (text = @ss.scan(/IMPORT\b/i))
395
+ action { [:IMPORT, text] }
396
+
397
+ when (text = @ss.scan(/IGNORE\b/i))
398
+ action { [:IGNORE, text] }
399
+
400
+ when (text = @ss.scan(/IF\b/i))
401
+ action { [:IF, text] }
402
+
403
+ when (text = @ss.scan(/HASH\b/i))
404
+ action { [:HASH, text] }
405
+
406
+ when (text = @ss.scan(/FULLTEXT\b/i))
407
+ action { [:FULLTEXT, text] }
408
+
409
+ when (text = @ss.scan(/FULL\b/i))
410
+ action { [:FULL, text] }
411
+
412
+ when (text = @ss.scan(/FOREIGN\b/i))
413
+ action { [:FOREIGN, text] }
414
+
415
+ when (text = @ss.scan(/FLOAT\b/i))
416
+ action { [:FLOAT, text] }
417
+
418
+ when (text = @ss.scan(/FIXED\b/i))
419
+ action { [:FIXED, text] }
420
+
421
+ when (text = @ss.scan(/FIRST\b/i))
422
+ action { [:FIRST, text] }
423
+
424
+ when (text = @ss.scan(/EXISTS\b/i))
425
+ action { [:EXISTS, text] }
426
+
427
+ when (text = @ss.scan(/ENUM\b/i))
428
+ action { [:ENUM, text] }
429
+
430
+ when (text = @ss.scan(/ENGINE\b/i))
431
+ action { [:ENGINE, text] }
432
+
433
+ when (text = @ss.scan(/ENABLE\b/i))
434
+ action { [:ENABLE, text] }
435
+
436
+ when (text = @ss.scan(/DYNAMIC\b/i))
437
+ action { [:DYNAMIC, text] }
438
+
439
+ when (text = @ss.scan(/DROP\b/i))
440
+ action { [:DROP, text] }
441
+
442
+ when (text = @ss.scan(/DOUBLE\b/i))
443
+ action { [:DOUBLE, text] }
444
+
445
+ when (text = @ss.scan(/DISK\b/i))
446
+ action { [:DISK, text] }
447
+
448
+ when (text = @ss.scan(/DISCARD\b/i))
449
+ action { [:DISCARD, text] }
450
+
451
+ when (text = @ss.scan(/DISABLE\b/i))
452
+ action { [:DISABLE, text] }
453
+
454
+ when (text = @ss.scan(/DIRECTORY\b/i))
455
+ action { [:DIRECTORY, text] }
456
+
457
+ when (text = @ss.scan(/DESC\b/i))
458
+ action { [:DESC, text] }
459
+
460
+ when (text = @ss.scan(/DELETE\b/i))
461
+ action { [:DELETE, text] }
462
+
463
+ when (text = @ss.scan(/DELAY_KEY_WRITE\b/i))
464
+ action { [:DELAY_KEY_WRITE, text] }
465
+
466
+ when (text = @ss.scan(/DEFINER\b/i))
467
+ action { [:DEFINER, text] }
468
+
469
+ when (text = @ss.scan(/DEFAULT\b/i))
470
+ action { [:DEFAULT, text] }
471
+
472
+ when (text = @ss.scan(/DECIMAL\b/i))
473
+ action { [:DECIMAL, text] }
474
+
475
+ when (text = @ss.scan(/DATETIME\b/i))
476
+ action { [:DATETIME, text] }
477
+
478
+ when (text = @ss.scan(/DATE\b/i))
479
+ action { [:DATE, text] }
480
+
481
+ when (text = @ss.scan(/DATA\b/i))
482
+ action { [:DATA, text] }
483
+
484
+ when (text = @ss.scan(/CURRENT_USER\b/i))
485
+ action { [:CURRENT_USER, text] }
486
+
487
+ when (text = @ss.scan(/CURRENT_TIMESTAMP\b/i))
488
+ action { [:CURRENT_TIMESTAMP, text] }
489
+
490
+ when (text = @ss.scan(/CREATE\b/i))
491
+ action { [:CREATE, text] }
492
+
493
+ when (text = @ss.scan(/CONVERT\b/i))
494
+ action { [:CONVERT, text] }
495
+
496
+ when (text = @ss.scan(/CONSTRAINT\b/i))
497
+ action { [:CONSTRAINT, text] }
498
+
499
+ when (text = @ss.scan(/CONNECTION\b/i))
500
+ action { [:CONNECTION, text] }
501
+
502
+ when (text = @ss.scan(/COMPRESSED\b/i))
503
+ action { [:COMPRESSED, text] }
504
+
505
+ when (text = @ss.scan(/COMPACT\b/i))
506
+ action { [:COMPACT, text] }
507
+
508
+ when (text = @ss.scan(/COMMENT\b/i))
509
+ action { [:COMMENT, text] }
510
+
511
+ when (text = @ss.scan(/COLUMN_FORMAT\b/i))
512
+ action { [:COLUMN_FORMAT, text] }
513
+
514
+ when (text = @ss.scan(/COLUMN\b/i))
515
+ action { [:COLUMN, text] }
516
+
517
+ when (text = @ss.scan(/COLLATE\b/i))
518
+ action { [:COLLATE, text] }
519
+
520
+ when (text = @ss.scan(/COALESCE\b/i))
521
+ action { [:COALESCE, text] }
522
+
523
+ when (text = @ss.scan(/CHECKSUM\b/i))
524
+ action { [:CHECKSUM, text] }
525
+
526
+ when (text = @ss.scan(/CHECK\b/i))
527
+ action { [:CHECK, text] }
528
+
529
+ when (text = @ss.scan(/CHAR\b/i))
530
+ action { [:CHAR, text] }
531
+
532
+ when (text = @ss.scan(/CHANGE\b/i))
533
+ action { [:CHANGE, text] }
534
+
535
+ when (text = @ss.scan(/CASCADED\b/i))
536
+ action { [:CASCADED, text] }
537
+
538
+ when (text = @ss.scan(/CASCADE\b/i))
539
+ action { [:CASCADE, text] }
540
+
541
+ when (text = @ss.scan(/BY\b/i))
542
+ action { [:BY, text] }
543
+
544
+ when (text = @ss.scan(/BTREE\b/i))
545
+ action { [:BTREE, text] }
546
+
547
+ when (text = @ss.scan(/BLOB\b/i))
548
+ action { [:BLOB, text] }
549
+
550
+ when (text = @ss.scan(/BIT\b/i))
551
+ action { [:BIT, text] }
552
+
553
+ when (text = @ss.scan(/BINARY\b/i))
554
+ action { [:BINARY, text] }
555
+
556
+ when (text = @ss.scan(/BIGINT\b/i))
557
+ action { [:BIGINT, text] }
558
+
559
+ when (text = @ss.scan(/AVG_ROW_LENGTH\b/i))
560
+ action { [:AVG_ROW_LENGTH, text] }
561
+
562
+ when (text = @ss.scan(/AUTO_INCREMENT\b/i))
563
+ action { [:AUTO_INCREMENT, text] }
564
+
565
+ when (text = @ss.scan(/ASC\b/i))
566
+ action { [:ASC, text] }
567
+
568
+ when (text = @ss.scan(/AS\b/i))
569
+ action { [:AS, text] }
570
+
571
+ when (text = @ss.scan(/ANALYZE\b/i))
572
+ action { [:ANALYZE, text] }
573
+
574
+ when (text = @ss.scan(/ALTER\b/i))
575
+ action { [:ALTER, text] }
576
+
577
+ when (text = @ss.scan(/ALL\b/i))
578
+ action { [:ALL, text] }
579
+
580
+ when (text = @ss.scan(/ALGORITHM\b/i))
581
+ action { [:ALGORITHM, text] }
582
+
583
+ when (text = @ss.scan(/AFTER\b/i))
584
+ action { [:AFTER, text] }
585
+
586
+ when (text = @ss.scan(/ADD\b/i))
587
+ action { [:ADD, text] }
588
+
589
+ when (text = @ss.scan(/ACTION\b/i))
590
+ action { [:ACTION, text] }
591
+
592
+ when (text = @ss.scan(/,/i))
593
+ action { [:S_COMMA , text] }
594
+
595
+ when (text = @ss.scan(/@/i))
596
+ action { [:S_AT , text] }
597
+
598
+ when (text = @ss.scan(/0+\b/i))
599
+ action { [:S_ZERO , text] } # this must come before S_NAT
600
+
601
+ when (text = @ss.scan(/1\b/i))
602
+ action { [:S_ONE , text] } # this must come before S_NAT
603
+
604
+ when (text = @ss.scan(/\d+/i))
605
+ action { [:S_NAT , text] } # definitely not 0, 1
606
+
607
+ when (text = @ss.scan(/-?\d+\.\d+/i))
608
+ action { [:S_FLOAT , text] } # this must come before S_DOT
609
+
610
+ when (text = @ss.scan(/[\$a-zA-Z0-9_]+/i))
611
+ action { [:S_IDENT_NORMAL , text] }
612
+
613
+ when (text = @ss.scan(/=/i))
614
+ action { [:S_EQUAL , text] }
615
+
616
+ when (text = @ss.scan(/\(/i))
617
+ action { [:S_LEFT_PAREN , text] }
618
+
619
+ when (text = @ss.scan(/\)/i))
620
+ action { [:S_RIGHT_PAREN , text] }
621
+
622
+ when (text = @ss.scan(/-/i))
623
+ action { [:S_MINUS , text] }
624
+
625
+ when (text = @ss.scan(/\./i))
626
+ action { [:S_DOT , text] }
627
+
628
+ when (text = @ss.scan(/[ \t\n]+/i))
629
+ action { [:S_SPACE, ' '] } # set to one space
630
+
631
+ else
632
+ text = @ss.string[@ss.pos .. -1]
633
+ raise ScanError, "can not match: '" + text + "'"
634
+ end # if
635
+
636
+ when :A_REM_MULTI
637
+ case
638
+ when (text = @ss.scan(/\*\/ */i))
639
+ action { @state = :A_NIL; [:S_REM_OUT, ' */ '] }
640
+
641
+ when (text = @ss.scan(/(.+)(?=\*\/ *)/i))
642
+ action { [:S_COMMENT, text] }
643
+
644
+ else
645
+ text = @ss.string[@ss.pos .. -1]
646
+ raise ScanError, "can not match: '" + text + "'"
647
+ end # if
648
+
649
+ when :A_REM_INLINE
650
+ case
651
+ when (text = @ss.scan(/\n/i))
652
+ action { @state = :A_NIL; [:S_REM_OUT, text] }
653
+
654
+ when (text = @ss.scan(/.*(?=$)/i))
655
+ action { [:S_COMMENT, text] }
656
+
657
+ else
658
+ text = @ss.string[@ss.pos .. -1]
659
+ raise ScanError, "can not match: '" + text + "'"
660
+ end # if
661
+
662
+ when :A_BACKTICK
663
+ case
664
+ when (text = @ss.scan(/``/i))
665
+ action { [:S_IDENT_IN_BACKTICK, text] }
666
+
667
+ when (text = @ss.scan(/`/i))
668
+ action { @state = :A_NIL; [:S_BACKTICK_OUT, text] }
669
+
670
+ when (text = @ss.scan(/[^`]+/i))
671
+ action { [:S_IDENT_IN_BACKTICK, text] }
672
+
673
+ else
674
+ text = @ss.string[@ss.pos .. -1]
675
+ raise ScanError, "can not match: '" + text + "'"
676
+ end # if
677
+
678
+ when :A_DOUBLEQUOTE
679
+ case
680
+ when (text = @ss.scan(/""/i))
681
+ action { [:S_STRING_IN_QUOTE, text] }
682
+
683
+ when (text = @ss.scan(/"/i))
684
+ action { @state = :A_NIL; [:S_DOUBLEQUOTE_OUT, text] }
685
+
686
+ when (text = @ss.scan(/[^"]*/i))
687
+ action { [:S_STRING_IN_QUOTE, text] }
688
+
689
+ else
690
+ text = @ss.string[@ss.pos .. -1]
691
+ raise ScanError, "can not match: '" + text + "'"
692
+ end # if
693
+
694
+ when :A_SINGLEQUOTE
695
+ case
696
+ when (text = @ss.scan(/''/i))
697
+ action { [:S_STRING_IN_QUOTE, text] }
698
+
699
+ when (text = @ss.scan(/'/i))
700
+ action { @state = :A_NIL; [:S_SINGLEQUOTE_OUT, text] }
701
+
702
+ when (text = @ss.scan(/[^']*/i))
703
+ action { [:S_STRING_IN_QUOTE, text] }
704
+
705
+ else
706
+ text = @ss.string[@ss.pos .. -1]
707
+ raise ScanError, "can not match: '" + text + "'"
708
+ end # if
709
+
710
+ else
711
+ raise ScanError, "undefined state: '" + state.to_s + "'"
712
+ end # case state
713
+ token
714
+ end # def _next_token
715
+
716
+ def tokenize(code)
717
+ scan_setup(code)
718
+ tokens = []
719
+ while token = next_token
720
+ tokens << token
721
+ end
722
+ tokens
723
+ end
724
+ end # class