wvanbergen-sql_tree 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +6 -0
- data/Rakefile +3 -0
- data/lib/sql_tree.rb +14 -0
- data/lib/sql_tree/generator.rb +3 -0
- data/lib/sql_tree/node.rb +22 -0
- data/lib/sql_tree/parser.rb +214 -0
- data/lib/sql_tree/token.rb +63 -0
- data/lib/sql_tree/tokenizer.rb +99 -0
- data/spec/integration/api_spec.rb +5 -0
- data/spec/lib/matchers.rb +80 -0
- data/spec/spec_helper.rb +11 -0
- data/spec/unit/parser_spec.rb +81 -0
- data/spec/unit/tokenizer_spec.rb +76 -0
- data/tasks/github-gem.rake +250 -0
- metadata +82 -0
data/README.rdoc
ADDED
data/Rakefile
ADDED
data/lib/sql_tree.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
module SQLTree
|
2
|
+
|
3
|
+
end
|
4
|
+
|
5
|
+
require 'sql_tree/token'
|
6
|
+
require 'sql_tree/tokenizer'
|
7
|
+
require 'sql_tree/parser'
|
8
|
+
|
9
|
+
require 'sql_tree/node'
|
10
|
+
require 'sql_tree/node/select_query'
|
11
|
+
require 'sql_tree/node/expression'
|
12
|
+
require 'sql_tree/node/leafs'
|
13
|
+
|
14
|
+
require 'sql_tree/generator'
|
@@ -0,0 +1,22 @@
|
|
1
|
+
class SQLTree::Node
|
2
|
+
|
3
|
+
def inspect
|
4
|
+
"#{self.class.name}[#{self.to_sql}]"
|
5
|
+
end
|
6
|
+
|
7
|
+
def quote_var(name)
|
8
|
+
"\"#{name}\""
|
9
|
+
end
|
10
|
+
|
11
|
+
def quote_str(str)
|
12
|
+
"'#{str.gsub(/\'/, "''")}'"
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.[](arg)
|
16
|
+
case arg
|
17
|
+
when Symbol; Variable.new(arg.to_s)
|
18
|
+
else; Value.new(arg)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -0,0 +1,214 @@
|
|
1
|
+
class SQLTree::Parser
|
2
|
+
|
3
|
+
def current_token
|
4
|
+
@current_token
|
5
|
+
end
|
6
|
+
|
7
|
+
def next_token
|
8
|
+
@current_token = @tokens.shift
|
9
|
+
end
|
10
|
+
|
11
|
+
def consume(check)
|
12
|
+
error(current_token) unless check == next_token
|
13
|
+
end
|
14
|
+
|
15
|
+
def peek_token(distance = 1)
|
16
|
+
@tokens[distance - 1]
|
17
|
+
end
|
18
|
+
|
19
|
+
def error(token)
|
20
|
+
raise "Unexpected token: #{token.inspect}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def debug
|
24
|
+
p @tokens.inspect
|
25
|
+
end
|
26
|
+
|
27
|
+
def parse(tokens, options = {:as => :query})
|
28
|
+
if tokens.kind_of?(String)
|
29
|
+
tokenizer = SQLTree::Tokenizer.new
|
30
|
+
@tokens = tokenizer.tokenize(tokens)
|
31
|
+
else
|
32
|
+
@tokens = tokens
|
33
|
+
end
|
34
|
+
|
35
|
+
send("parse_#{options[:as]}".to_sym)
|
36
|
+
end
|
37
|
+
|
38
|
+
def parse_query
|
39
|
+
case peek_token
|
40
|
+
when SQLTree::Token::SELECT; parse_select_query
|
41
|
+
else raise "Could not parse query"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def parse_select_query
|
46
|
+
select_node = SQLTree::Node::SelectQuery.new
|
47
|
+
consume(SQLTree::Token::SELECT)
|
48
|
+
|
49
|
+
if peek_token == SQLTree::Token::DISTINCT
|
50
|
+
consume(SQLTree::Token::DISTINCT)
|
51
|
+
select_node.distinct = true
|
52
|
+
end
|
53
|
+
|
54
|
+
select_node.select = parse_select_clause
|
55
|
+
select_node.from = parse_from_clause if peek_token == SQLTree::Token::FROM
|
56
|
+
select_node.where = parse_where_clause if peek_token == SQLTree::Token::WHERE
|
57
|
+
|
58
|
+
return select_node
|
59
|
+
end
|
60
|
+
|
61
|
+
def parse_select_clause
|
62
|
+
expressions = [parse_select_expression]
|
63
|
+
while peek_token == SQLTree::Token::COMMA
|
64
|
+
consume(SQLTree::Token::COMMA)
|
65
|
+
expressions << parse_select_expression
|
66
|
+
end
|
67
|
+
return expressions
|
68
|
+
end
|
69
|
+
|
70
|
+
def parse_select_expression
|
71
|
+
if peek_token == SQLTree::Token::MULTIPLY
|
72
|
+
consume(SQLTree::Token::MULTIPLY)
|
73
|
+
return SQLTree::Node::ALL_FIELDS
|
74
|
+
else
|
75
|
+
expr = SQLTree::Node::SelectExpression.new(parse_expression)
|
76
|
+
if peek_token == SQLTree::Token::AS
|
77
|
+
consume(SQLTree::Token::AS)
|
78
|
+
expr.variable = parse_variable_name
|
79
|
+
end
|
80
|
+
return expr
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def parse_single_expression
|
85
|
+
if SQLTree::Token::LPAREN === peek_token(1)
|
86
|
+
consume(SQLTree::Token::LPAREN)
|
87
|
+
expr = parse_expression
|
88
|
+
consume(SQLTree::Token::RPAREN)
|
89
|
+
return expr
|
90
|
+
elsif SQLTree::Token::Variable === peek_token(1) && peek_token(2) == SQLTree::Token::LPAREN
|
91
|
+
return parse_function_call
|
92
|
+
elsif SQLTree::Token::Variable === peek_token(1)
|
93
|
+
return parse_variable
|
94
|
+
else
|
95
|
+
return parse_value
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def parse_expression
|
100
|
+
parse_logical_expression
|
101
|
+
end
|
102
|
+
|
103
|
+
def parse_secondary_arithmetic_expression
|
104
|
+
expr = parse_single_expression
|
105
|
+
while [SQLTree::Token::PLUS, SQLTree::Token::MINUS].include?(peek_token)
|
106
|
+
expr = SQLTree::Node::ArithmeticExpression.new(next_token.literal, expr, parse_single_expression)
|
107
|
+
end
|
108
|
+
return expr
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
def parse_primary_arithmetic_expression
|
113
|
+
expr = parse_secondary_arithmetic_expression
|
114
|
+
while [SQLTree::Token::PLUS, SQLTree::Token::MINUS].include?(peek_token)
|
115
|
+
expr = SQLTree::Node::ArithmeticExpression.new(next_token.literal, expr, parse_secondary_arithmetic_expression)
|
116
|
+
end
|
117
|
+
return expr
|
118
|
+
end
|
119
|
+
|
120
|
+
def parse_comparison_expression
|
121
|
+
expr = parse_primary_arithmetic_expression
|
122
|
+
while [SQLTree::Token::EQ, SQLTree::Token::NE, SQLTree::Token::GT, SQLTree::Token::GTE, SQLTree::Token::LT, SQLTree::Token::LTE].include?(peek_token)
|
123
|
+
expr = SQLTree::Node::ComparisonExpression.new(next_token.literal, expr, parse_primary_arithmetic_expression)
|
124
|
+
end
|
125
|
+
return expr
|
126
|
+
end
|
127
|
+
|
128
|
+
def parse_logical_expression
|
129
|
+
expr = parse_comparison_expression
|
130
|
+
while [SQLTree::Token::AND, SQLTree::Token::OR].include?(peek_token)
|
131
|
+
expr = SQLTree::Node::LogicalExpression.new(next_token.literal, [expr, parse_comparison_expression])
|
132
|
+
end
|
133
|
+
return expr
|
134
|
+
end
|
135
|
+
|
136
|
+
def parse_logical_not_expression
|
137
|
+
|
138
|
+
end
|
139
|
+
|
140
|
+
def parse_function_call
|
141
|
+
|
142
|
+
expr = SQLTree::Node::FunctionExpression.new(next_token.literal)
|
143
|
+
consume(SQLTree::Token::LPAREN)
|
144
|
+
until peek_token == SQLTree::Token::RPAREN
|
145
|
+
expr.arguments << parse_expression
|
146
|
+
consume(SQLTree::Token::COMMA) if peek_token == SQLTree::Token::COMMA
|
147
|
+
end
|
148
|
+
consume(SQLTree::Token::RPAREN)
|
149
|
+
return expr
|
150
|
+
end
|
151
|
+
|
152
|
+
def parse_from_clause
|
153
|
+
consume(SQLTree::Token::FROM)
|
154
|
+
from_expressions = [parse_from_expression]
|
155
|
+
while peek_token == SQLTree::Token::COMMA
|
156
|
+
consume(SQLTree::Token::COMMA)
|
157
|
+
from_expressions << parse_from_expression
|
158
|
+
end
|
159
|
+
|
160
|
+
return from_expressions
|
161
|
+
end
|
162
|
+
|
163
|
+
def parse_from_expression
|
164
|
+
from_expression = case peek_token
|
165
|
+
when SQLTree::Token::Variable; parse_table_import
|
166
|
+
else; error(peek_token)
|
167
|
+
end
|
168
|
+
return from_expression
|
169
|
+
end
|
170
|
+
|
171
|
+
def parse_table_import
|
172
|
+
table_import = SQLTree::Node::TableImport.new(next_token.literal)
|
173
|
+
if peek_token == SQLTree::Token::AS || SQLTree::Token::Variable === peek_token
|
174
|
+
consume(SQLTree::Token::AS) if peek_token == SQLTree::Token::AS
|
175
|
+
table_import.variable = parse_variable
|
176
|
+
end
|
177
|
+
return table_import
|
178
|
+
end
|
179
|
+
|
180
|
+
def parse_field
|
181
|
+
lhs = next_token
|
182
|
+
lhs = (lhs == SQLTree::Token::MULTIPLY) ? :all : lhs.literal
|
183
|
+
|
184
|
+
if peek_token == SQLTree::Token::DOT
|
185
|
+
consume(SQLTree::Token::DOT)
|
186
|
+
rhs = next_token
|
187
|
+
rhs = (rhs == SQLTree::Token::MULTIPLY) ? :all : rhs.literal
|
188
|
+
SQLTree::Node::Field.new(rhs, lhs)
|
189
|
+
else
|
190
|
+
SQLTree::Node::Field.new(lhs)
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def parse_variable_name
|
195
|
+
return SQLTree::Node::Variable.new(next_token.literal)
|
196
|
+
end
|
197
|
+
|
198
|
+
def parse_variable
|
199
|
+
if peek_token(2) == SQLTree::Token::DOT
|
200
|
+
parse_field
|
201
|
+
else
|
202
|
+
parse_variable_name
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def parse_value
|
207
|
+
SQLTree::Node::Value.new(next_token.literal)
|
208
|
+
end
|
209
|
+
|
210
|
+
|
211
|
+
def parse_where_clause
|
212
|
+
end
|
213
|
+
|
214
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
class SQLTree::Token
|
2
|
+
|
3
|
+
attr_accessor :literal
|
4
|
+
|
5
|
+
def initialize(literal)
|
6
|
+
@literal = literal
|
7
|
+
end
|
8
|
+
|
9
|
+
def ==(other)
|
10
|
+
other.class == self.class && @literal == other.literal
|
11
|
+
end
|
12
|
+
|
13
|
+
alias :eql? :==
|
14
|
+
|
15
|
+
def inspect
|
16
|
+
literal
|
17
|
+
end
|
18
|
+
|
19
|
+
# Token types
|
20
|
+
|
21
|
+
class Value < SQLTree::Token
|
22
|
+
end
|
23
|
+
|
24
|
+
class Variable < SQLTree::Token::Value
|
25
|
+
end
|
26
|
+
|
27
|
+
class String < SQLTree::Token::Value
|
28
|
+
end
|
29
|
+
|
30
|
+
class Number < SQLTree::Token::Value
|
31
|
+
end
|
32
|
+
|
33
|
+
class Keyword < SQLTree::Token
|
34
|
+
def inspect
|
35
|
+
":#{literal.downcase}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
class Operator < SQLTree::Token
|
40
|
+
def inspect
|
41
|
+
OPERATORS[literal].inspect
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
LPAREN = Class.new(SQLTree::Token).new('(')
|
46
|
+
RPAREN = Class.new(SQLTree::Token).new(')')
|
47
|
+
DOT = Class.new(SQLTree::Token).new('.')
|
48
|
+
COMMA = Class.new(SQLTree::Token).new(',')
|
49
|
+
|
50
|
+
KEYWORDS = %w{select from where group having order distinct left right inner outer join and or not as}
|
51
|
+
KEYWORDS.each do |kw|
|
52
|
+
self.const_set(kw.upcase, Class.new(SQLTree::Token::Keyword).new(kw.upcase))
|
53
|
+
end
|
54
|
+
|
55
|
+
ARITHMETHIC_OPERATORS = { '+' => :plus, '-' => :minus, '*' => :multiply, '/' => :divide, '%' => :modulo }
|
56
|
+
LOGICAL_OPERATORS = { '=' => :eq, '!=' => :ne, '<>' => :ne, '>' => :gt, '<' => :lt, '>=' => :gte, '<=' => :lte }
|
57
|
+
|
58
|
+
OPERATORS = ARITHMETHIC_OPERATORS.merge(LOGICAL_OPERATORS)
|
59
|
+
OPERATORS.each do |literal, symbol|
|
60
|
+
self.const_set(symbol.to_s.upcase, Class.new(SQLTree::Token::Operator).new(literal)) unless self.const_defined?(symbol.to_s.upcase)
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
class SQLTree::Tokenizer
|
2
|
+
|
3
|
+
include Enumerable
|
4
|
+
|
5
|
+
def tokenize(string)
|
6
|
+
@string = string
|
7
|
+
@current_char_pos = -1
|
8
|
+
to_a
|
9
|
+
end
|
10
|
+
|
11
|
+
def current_char
|
12
|
+
@current_char
|
13
|
+
end
|
14
|
+
|
15
|
+
def peek_char(amount = 1)
|
16
|
+
@string[@current_char_pos + amount, 1]
|
17
|
+
end
|
18
|
+
|
19
|
+
def next_char
|
20
|
+
@current_char_pos += 1
|
21
|
+
@current_char = @string[@current_char_pos, 1]
|
22
|
+
end
|
23
|
+
|
24
|
+
OPERATOR_CHARS = /\=|<|>|!|\-|\+|\/|\*|\%/
|
25
|
+
|
26
|
+
def each_token(&block)
|
27
|
+
while next_char
|
28
|
+
case current_char
|
29
|
+
when /^\s?$/; # whitespace, go to next token
|
30
|
+
when '('; yield(SQLTree::Token::LPAREN)
|
31
|
+
when ')'; yield(SQLTree::Token::RPAREN)
|
32
|
+
when '.'; yield(SQLTree::Token::DOT)
|
33
|
+
when '.'; yield(SQLTree::Token::COMMA)
|
34
|
+
when /\d/; tokenize_number(&block)
|
35
|
+
when "'"; tokenize_quoted_string(&block)
|
36
|
+
when OPERATOR_CHARS; tokenize_operator(&block)
|
37
|
+
when /\w/; tokenize_literal(&block)
|
38
|
+
when '"'; tokenize_quoted_literal(&block) # TODO: allow MySQL quoting mode
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
alias :each :each_token
|
44
|
+
|
45
|
+
def tokenize_literal(&block)
|
46
|
+
literal = current_char
|
47
|
+
literal << next_char while /[\w]/ =~ peek_char
|
48
|
+
|
49
|
+
if SQLTree::Token::KEYWORDS.include?(literal.downcase)
|
50
|
+
yield(SQLTree::Token.const_get(literal.upcase))
|
51
|
+
else
|
52
|
+
yield(SQLTree::Token::Variable.new(literal))
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def tokenize_number(&block)
|
57
|
+
number = current_char
|
58
|
+
dot_encountered = false
|
59
|
+
while /\d/ =~ peek_char || (peek_char == '.' && !dot_encountered)
|
60
|
+
dot_encountered = true if peek_char == '.'
|
61
|
+
number << next_char
|
62
|
+
end
|
63
|
+
|
64
|
+
if dot_encountered
|
65
|
+
yield(SQLTree::Token::Number.new(number.to_f))
|
66
|
+
else
|
67
|
+
yield(SQLTree::Token::Number.new(number.to_i))
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def tokenize_quoted_string(&block)
|
72
|
+
string = ''
|
73
|
+
until next_char.nil? || current_char == "'"
|
74
|
+
string << (current_char == "\\" ? next_char : current_char)
|
75
|
+
end
|
76
|
+
yield(SQLTree::Token::String.new(string))
|
77
|
+
end
|
78
|
+
|
79
|
+
def tokenize_quoted_literal(&block)
|
80
|
+
literal = ''
|
81
|
+
until next_char.nil? || current_char == '"' # TODO: allow MySQL quoting mode
|
82
|
+
literal << (current_char == "\\" ? next_char : current_char)
|
83
|
+
end
|
84
|
+
yield(SQLTree::Token::Variable.new(literal))
|
85
|
+
end
|
86
|
+
|
87
|
+
|
88
|
+
def tokenize_operator(&block)
|
89
|
+
operator = current_char
|
90
|
+
if operator == '-' && /[\d\.]/ =~ peek_char
|
91
|
+
tokenize_number(&block)
|
92
|
+
else
|
93
|
+
operator << next_char if SQLTree::Token::OPERATORS.has_key?(operator + peek_char)
|
94
|
+
yield(SQLTree::Token.const_get(SQLTree::Token::OPERATORS[operator].to_s.upcase))
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
class ParseAs
|
2
|
+
|
3
|
+
def initialize(expected_tree)
|
4
|
+
@expected_tree = expected_tree
|
5
|
+
end
|
6
|
+
|
7
|
+
def matches?(found_tree)
|
8
|
+
@found_tree = found_tree.to_tree
|
9
|
+
return @found_tree.eql?(@expected_tree)
|
10
|
+
end
|
11
|
+
|
12
|
+
def description
|
13
|
+
"expected to parse to #{@expected_tree.inspect}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def failure_message
|
17
|
+
" #{@expected_tree.inspect} expected, but found #{@found_tree.inspect}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def negative_failure_message
|
21
|
+
" expected not to be tokenized to #{@expected_tree.inspect}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def parse_as(tree)
|
26
|
+
ParseAs.new(tree)
|
27
|
+
end
|
28
|
+
|
29
|
+
class TokenizeTo
|
30
|
+
|
31
|
+
def initialize(expected_tokens)
|
32
|
+
@expected_tokens = expected_tokens.map do |t|
|
33
|
+
case t
|
34
|
+
when SQLTree::Token; t
|
35
|
+
when String; SQLTree::Token::String.new(t)
|
36
|
+
when Integer; SQLTree::Token::Number.new(t)
|
37
|
+
when Float; SQLTree::Token::Number.new(t)
|
38
|
+
when Symbol; SQLTree::Token.const_get(t.to_s.upcase)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def matches?(found_tokens)
|
44
|
+
@found_tokens = found_tokens
|
45
|
+
return @found_tokens.eql?(@expected_tokens)
|
46
|
+
end
|
47
|
+
|
48
|
+
def description
|
49
|
+
"expected to tokenized to #{@expected_tokens.inspect}"
|
50
|
+
end
|
51
|
+
|
52
|
+
def failure_message
|
53
|
+
" #{@expected_tokens.inspect} expected, but found #{@found_tokens.inspect}"
|
54
|
+
end
|
55
|
+
|
56
|
+
def negative_failure_message
|
57
|
+
" expected not to be tokenized to #{@expected_tokens.inspect}"
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
def tokenize_to(*expected_tokens)
|
63
|
+
TokenizeTo.new(expected_tokens)
|
64
|
+
end
|
65
|
+
|
66
|
+
def sql_var(name)
|
67
|
+
SQLTree::Token::Variable.new(name.to_s)
|
68
|
+
end
|
69
|
+
|
70
|
+
def dot
|
71
|
+
SQLTree::Token::DOT
|
72
|
+
end
|
73
|
+
|
74
|
+
def lparen
|
75
|
+
SQLTree::Token::LPAREN
|
76
|
+
end
|
77
|
+
|
78
|
+
def rparen
|
79
|
+
SQLTree::Token::RPAREN
|
80
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
require "#{File.dirname(__FILE__)}/../spec_helper"
|
2
|
+
|
3
|
+
describe SQLTree::Parser do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
@parser = SQLTree::Parser.new
|
7
|
+
end
|
8
|
+
|
9
|
+
context :expressions do
|
10
|
+
|
11
|
+
it "should parse a variable" do
|
12
|
+
@parser.parse("field", :as => :expression).should eql(SQLTree::Node[:field])
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should parse a table field" do
|
16
|
+
@parser.parse('tbl.field', :as => :expression).should eql(SQLTree::Node::Field[:tbl, :field])
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
it "should parse a number" do
|
21
|
+
@parser.parse("1.0", :as => :expression).should eql(SQLTree::Node[1.0])
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should parse a string" do
|
25
|
+
@parser.parse("'str'", :as => :expression).should eql(SQLTree::Node['str'])
|
26
|
+
end
|
27
|
+
|
28
|
+
it "should parse a function call without arguments" do
|
29
|
+
@parser.parse("NOW()", :as => :expression).should parse_as([:NOW])
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should parse a function call with a single numeric argument" do
|
33
|
+
@parser.parse("MD5('abc')", :as => :expression).should parse_as([:MD5, 'abc'])
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should parse a function call with multiple arguments" do
|
37
|
+
@parser.parse("CONCAT('Mr. ', last_name)", :as => :expression).should parse_as([:CONCAT, 'Mr. ', :last_name])
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should parse a simple arithmetic operator" do
|
41
|
+
@parser.parse("1 + 2", :as => :expression).should parse_as([:plus, 1, 2])
|
42
|
+
end
|
43
|
+
|
44
|
+
it "should parse a simple comparison" do
|
45
|
+
@parser.parse("1 < 2", :as => :expression).should parse_as([:lt, 1, 2])
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should parse a simple subexpressions with parenthesis on the start" do
|
49
|
+
@parser.parse("(1 + 1) = 2", :as => :expression).should parse_as([:eq, [:plus, 1, 1], 2])
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should parse a simple subexpressions with parenthesis on the end" do
|
53
|
+
@parser.parse("1 + (1 = 2)", :as => :expression).should parse_as([:plus, 1, [:eq, 1, 2]])
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should parse arithmetic operations before comparisons" do
|
57
|
+
@parser.parse("1 + 1 = 2", :as => :expression).should parse_as([:eq, [:plus, 1, 1], 2])
|
58
|
+
end
|
59
|
+
|
60
|
+
it "should parse arithmetic operations before comparisons" do
|
61
|
+
@parser.parse("1 + 2 - 3", :as => :expression).should parse_as([:minus, [:plus, 1, 2], 3])
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
it "should parse arithmetic operations before comparisons" do
|
66
|
+
@parser.parse("a and b", :as => :expression).should parse_as([:and, :a, :b])
|
67
|
+
end
|
68
|
+
|
69
|
+
it "should parse arithmetic operations before comparisons" do
|
70
|
+
@parser.parse("a and b and c", :as => :expression).should parse_as([:and, [:and, :a, :b], :c])
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should parse arithmetic operations before comparisons" do
|
74
|
+
@parser.parse("a and b or c", :as => :expression).should parse_as([:or, [:and, :a, :b], :c])
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
|
81
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "#{File.dirname(__FILE__)}/../spec_helper"
|
2
|
+
|
3
|
+
describe SQLTree::Tokenizer do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
@tokenizer = SQLTree::Tokenizer.new
|
7
|
+
end
|
8
|
+
|
9
|
+
context "recognizing single tokens" do
|
10
|
+
it "should tokenize SQL query keywords" do
|
11
|
+
@tokenizer.tokenize('WHERE').should tokenize_to(:where)
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should tokenize expression keywords" do
|
15
|
+
@tokenizer.tokenize('and').should tokenize_to(:and)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "should tokenize muliple keywords" do
|
19
|
+
@tokenizer.tokenize('SELECT DISTINCT').should tokenize_to(:select, :distinct)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should ignore excessive whitespace" do
|
23
|
+
@tokenizer.tokenize("\tSELECT DISTINCT \r\r").should tokenize_to(:select, :distinct)
|
24
|
+
end
|
25
|
+
|
26
|
+
it "should tokenize variables" do
|
27
|
+
@tokenizer.tokenize("var").should tokenize_to(sql_var('var'))
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should tokenize quoted variables" do
|
31
|
+
@tokenizer.tokenize('"var"').should tokenize_to(sql_var('var'))
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should tokenize strings" do
|
35
|
+
@tokenizer.tokenize("'hello' ' world '").should tokenize_to('hello', ' world ')
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should tokenize numbers" do
|
39
|
+
@tokenizer.tokenize("1 -2 3.14 -4.0").should tokenize_to(1, -2, 3.14, -4.0)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should tokenize logical operators" do
|
43
|
+
@tokenizer.tokenize("< = <> >=").should tokenize_to(:lt, :eq, :ne, :gte)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should tokenize arithmetic operators" do
|
47
|
+
@tokenizer.tokenize("+ - / * %").should tokenize_to(:plus, :minus, :divide, :multiply, :modulo)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should tokenize parentheses" do
|
51
|
+
@tokenizer.tokenize("(a)").should tokenize_to(lparen, sql_var('a'), rparen)
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should tokenize dots" do
|
55
|
+
@tokenizer.tokenize('a."b"').should tokenize_to(sql_var('a'), dot, sql_var('b'))
|
56
|
+
end
|
57
|
+
|
58
|
+
it "should tokenize commas" do
|
59
|
+
@tokenizer.tokenize('a,"b"').should tokenize_to(sql_var('a'), comma, sql_var('b'))
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
end
|
64
|
+
|
65
|
+
context "combined tokens" do
|
66
|
+
it "should tokenize a full sql_query" do
|
67
|
+
@tokenizer.tokenize("SELECT a.* FROM a_table AS a WHERE a.id > 1").should tokenize_to(
|
68
|
+
:select, sql_var('a'), dot, :multiply, :from, sql_var('a_table'), :as, sql_var('a'), :where, sql_var('a'), dot, sql_var('id'), :gt, 1)
|
69
|
+
end
|
70
|
+
|
71
|
+
it "should tokenize a function call" do
|
72
|
+
@tokenizer.tokenize("MD5('test')").should tokenize_to(sql_var('MD5'), lparen, 'test', rparen)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
@@ -0,0 +1,250 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rubyforge'
|
3
|
+
require 'rake'
|
4
|
+
require 'rake/tasklib'
|
5
|
+
require 'date'
|
6
|
+
|
7
|
+
module Rake
|
8
|
+
|
9
|
+
class GithubGem < TaskLib
|
10
|
+
|
11
|
+
attr_accessor :name
|
12
|
+
attr_accessor :specification
|
13
|
+
|
14
|
+
def self.define_tasks!
|
15
|
+
gem_task_builder = Rake::GithubGem.new
|
16
|
+
gem_task_builder.register_all_tasks!
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
reload_gemspec!
|
22
|
+
end
|
23
|
+
|
24
|
+
def register_all_tasks!
|
25
|
+
namespace(:gem) do
|
26
|
+
desc "Updates the file lists for this gem"
|
27
|
+
task(:manifest) { manifest_task }
|
28
|
+
|
29
|
+
desc "Releases a new version of #{@name}"
|
30
|
+
task(:build => [:manifest]) { build_task }
|
31
|
+
|
32
|
+
|
33
|
+
release_dependencies = [:check_clean_master_branch, :version, :build, :create_tag]
|
34
|
+
release_dependencies.push 'doc:publish' if has_rdoc?
|
35
|
+
release_dependencies.unshift 'test' if has_tests?
|
36
|
+
release_dependencies.unshift 'spec' if has_specs?
|
37
|
+
|
38
|
+
desc "Releases a new version of #{@name}"
|
39
|
+
task(:release => release_dependencies) { release_task }
|
40
|
+
|
41
|
+
# helper task for releasing
|
42
|
+
task(:check_clean_master_branch) { verify_clean_status('master') }
|
43
|
+
task(:check_version) { verify_version(ENV['VERSION'] || @specification.version) }
|
44
|
+
task(:version => [:check_version]) { set_gem_version! }
|
45
|
+
task(:create_tag) { create_version_tag! }
|
46
|
+
end
|
47
|
+
|
48
|
+
# Register RDoc tasks
|
49
|
+
if has_rdoc?
|
50
|
+
require 'rake/rdoctask'
|
51
|
+
|
52
|
+
namespace(:doc) do
|
53
|
+
desc 'Generate documentation for request-log-analyzer'
|
54
|
+
Rake::RDocTask.new(:compile) do |rdoc|
|
55
|
+
rdoc.rdoc_dir = 'doc'
|
56
|
+
rdoc.title = @name
|
57
|
+
rdoc.options += @specification.rdoc_options
|
58
|
+
rdoc.rdoc_files.include(@specification.extra_rdoc_files)
|
59
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
60
|
+
end
|
61
|
+
|
62
|
+
desc "Publish RDoc files for #{@name} to Github"
|
63
|
+
task(:publish => :compile) do
|
64
|
+
sh 'git checkout gh-pages'
|
65
|
+
sh 'git pull origin gh-pages'
|
66
|
+
sh 'cp -rf doc/* .'
|
67
|
+
sh "git commit -am \"Publishing newest RDoc documentation for #{@name}\""
|
68
|
+
sh "git push origin gh-pages"
|
69
|
+
sh "git checkout master"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# Setup :spec task if RSpec files exist
|
75
|
+
if has_specs?
|
76
|
+
require 'spec/rake/spectask'
|
77
|
+
|
78
|
+
desc "Run all specs for #{@name}"
|
79
|
+
Spec::Rake::SpecTask.new(:spec) do |t|
|
80
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
# Setup :test task if unit test files exist
|
85
|
+
if has_tests?
|
86
|
+
require 'rake/testtask'
|
87
|
+
|
88
|
+
desc "Run all unit tests for #{@name}"
|
89
|
+
Rake::TestTask.new(:test) do |t|
|
90
|
+
t.pattern = 'test/**/*_test.rb'
|
91
|
+
t.verbose = true
|
92
|
+
t.libs << 'test'
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
protected
|
98
|
+
|
99
|
+
def has_rdoc?
|
100
|
+
@specification.has_rdoc
|
101
|
+
end
|
102
|
+
|
103
|
+
def has_specs?
|
104
|
+
Dir['spec/**/*_spec.rb'].any?
|
105
|
+
end
|
106
|
+
|
107
|
+
def has_tests?
|
108
|
+
Dir['test/**/*_test.rb'].any?
|
109
|
+
end
|
110
|
+
|
111
|
+
def reload_gemspec!
|
112
|
+
raise "No gemspec file found!" if gemspec_file.nil?
|
113
|
+
spec = File.read(gemspec_file)
|
114
|
+
@specification = eval(spec)
|
115
|
+
@name = specification.name
|
116
|
+
end
|
117
|
+
|
118
|
+
def run_command(command)
|
119
|
+
lines = []
|
120
|
+
IO.popen(command) { |f| lines = f.readlines }
|
121
|
+
return lines
|
122
|
+
end
|
123
|
+
|
124
|
+
def git_modified?(file)
|
125
|
+
return !run_command('git status').detect { |line| Regexp.new(Regexp.quote(file)) =~ line }.nil?
|
126
|
+
end
|
127
|
+
|
128
|
+
def git_commit_file(file, message, branch = nil)
|
129
|
+
verify_current_branch(branch) unless branch.nil?
|
130
|
+
if git_modified?(file)
|
131
|
+
sh "git add #{file}"
|
132
|
+
sh "git commit -m \"#{message}\""
|
133
|
+
else
|
134
|
+
raise "#{file} is not modified and cannot be committed!"
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def git_create_tag(tag_name, message)
|
139
|
+
sh "git tag -a \"#{tag_name}\" -m \"#{message}\""
|
140
|
+
end
|
141
|
+
|
142
|
+
def git_push(remote = 'origin', branch = 'master', options = [])
|
143
|
+
verify_clean_status(branch)
|
144
|
+
options_str = options.map { |o| "--#{o}"}.join(' ')
|
145
|
+
sh "git push #{options_str} #{remote} #{branch}"
|
146
|
+
end
|
147
|
+
|
148
|
+
def gemspec_version=(new_version)
|
149
|
+
spec = File.read(gemspec_file)
|
150
|
+
spec.gsub!(/^(\s*s\.version\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_version}'#{$5}" }
|
151
|
+
spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{Date.today.strftime('%Y-%m-%d')}'#{$5}" }
|
152
|
+
File.open(gemspec_file, 'w') { |f| f << spec }
|
153
|
+
reload_gemspec!
|
154
|
+
end
|
155
|
+
|
156
|
+
def gemspec_date=(new_date)
|
157
|
+
spec = File.read(gemspec_file)
|
158
|
+
spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_date.strftime('%Y-%m-%d')}'#{$5}" }
|
159
|
+
File.open(gemspec_file, 'w') { |f| f << spec }
|
160
|
+
reload_gemspec!
|
161
|
+
end
|
162
|
+
|
163
|
+
def gemspec_file
|
164
|
+
@gemspec_file ||= Dir['*.gemspec'].first
|
165
|
+
end
|
166
|
+
|
167
|
+
def verify_current_branch(branch)
|
168
|
+
run_command('git branch').detect { |line| /^\* (.+)/ =~ line }
|
169
|
+
raise "You are currently not working in the master branch!" unless branch == $1
|
170
|
+
end
|
171
|
+
|
172
|
+
def verify_clean_status(on_branch = nil)
|
173
|
+
sh "git fetch"
|
174
|
+
lines = run_command('git status')
|
175
|
+
raise "You don't have the most recent version available. Run git pull first." if /^\# Your branch is behind/ =~ lines[1]
|
176
|
+
raise "You are currently not working in the #{on_branch} branch!" unless on_branch.nil? || (/^\# On branch (.+)/ =~ lines.first && $1 == on_branch)
|
177
|
+
raise "Your master branch contains modifications!" unless /^nothing to commit \(working directory clean\)/ =~ lines.last
|
178
|
+
end
|
179
|
+
|
180
|
+
def verify_version(new_version)
|
181
|
+
newest_version = run_command('git tag').map { |tag| tag.split(name + '-').last }.compact.map { |v| Gem::Version.new(v) }.max
|
182
|
+
raise "This version number (#{new_version}) is not higher than the highest tagged version (#{newest_version})" if !newest_version.nil? && newest_version >= Gem::Version.new(new_version.to_s)
|
183
|
+
end
|
184
|
+
|
185
|
+
def set_gem_version!
|
186
|
+
# update gemspec file
|
187
|
+
self.gemspec_version = ENV['VERSION'] if Gem::Version.correct?(ENV['VERSION'])
|
188
|
+
self.gemspec_date = Date.today
|
189
|
+
end
|
190
|
+
|
191
|
+
def manifest_task
|
192
|
+
verify_current_branch('master')
|
193
|
+
|
194
|
+
list = Dir['**/*'].sort
|
195
|
+
list -= [gemspec_file]
|
196
|
+
|
197
|
+
if File.exist?('.gitignore')
|
198
|
+
File.read('.gitignore').each_line do |glob|
|
199
|
+
glob = glob.chomp.sub(/^\//, '')
|
200
|
+
list -= Dir[glob]
|
201
|
+
list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
# update the spec file
|
206
|
+
spec = File.read(gemspec_file)
|
207
|
+
spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
|
208
|
+
assignment = $1
|
209
|
+
bunch = $2 ? list.grep(/^(test.*_test\.rb|spec.*_spec.rb)$/) : list
|
210
|
+
'%s%%w(%s)' % [assignment, bunch.join(' ')]
|
211
|
+
end
|
212
|
+
|
213
|
+
File.open(gemspec_file, 'w') { |f| f << spec }
|
214
|
+
reload_gemspec!
|
215
|
+
end
|
216
|
+
|
217
|
+
def build_task
|
218
|
+
sh "gem build #{gemspec_file}"
|
219
|
+
Dir.mkdir('pkg') unless File.exist?('pkg')
|
220
|
+
sh "mv #{name}-#{specification.version}.gem pkg/#{name}-#{specification.version}.gem"
|
221
|
+
end
|
222
|
+
|
223
|
+
def install_task
|
224
|
+
raise "#{name} .gem file not found" unless File.exist?("pkg/#{name}-#{specification.version}.gem")
|
225
|
+
sh "gem install pkg/#{name}-#{specification.version}.gem"
|
226
|
+
end
|
227
|
+
|
228
|
+
def uninstall_task
|
229
|
+
raise "#{name} .gem file not found" unless File.exist?("pkg/#{name}-#{specification.version}.gem")
|
230
|
+
sh "gem uninstall #{name}"
|
231
|
+
end
|
232
|
+
|
233
|
+
def create_version_tag!
|
234
|
+
# commit the gemspec file
|
235
|
+
git_commit_file(gemspec_file, "Updated #{gemspec_file} for release of version #{@specification.version}") if git_modified?(gemspec_file)
|
236
|
+
|
237
|
+
# create tag and push changes
|
238
|
+
git_create_tag("#{@name}-#{@specification.version}", "Tagged version #{@specification.version}")
|
239
|
+
git_push('origin', 'master', [:tags])
|
240
|
+
end
|
241
|
+
|
242
|
+
def release_task
|
243
|
+
puts
|
244
|
+
puts '------------------------------------------------------------'
|
245
|
+
puts "Released #{@name} - version #{@specification.version}"
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
Rake::GithubGem.define_tasks!
|
metadata
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: wvanbergen-sql_tree
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Willem van Bergen
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-08-02 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: To make it easier to build and manipulate SQL queries, sql_tree can parse an SQL query to represent it as a tree of nodes and can generate an SQL query given a tree as input
|
17
|
+
email: willem@vanbergen.org
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files:
|
23
|
+
- README.rdoc
|
24
|
+
files:
|
25
|
+
- README.rdoc
|
26
|
+
- Rakefile
|
27
|
+
- lib
|
28
|
+
- lib/sql_tree
|
29
|
+
- lib/sql_tree.rb
|
30
|
+
- lib/sql_tree/generator.rb
|
31
|
+
- lib/sql_tree/node
|
32
|
+
- lib/sql_tree/node.rb
|
33
|
+
- lib/sql_tree/parser.rb
|
34
|
+
- lib/sql_tree/token.rb
|
35
|
+
- lib/sql_tree/tokenizer.rb
|
36
|
+
- spec
|
37
|
+
- spec/integration
|
38
|
+
- spec/integration/api_spec.rb
|
39
|
+
- spec/lib
|
40
|
+
- spec/lib/matchers.rb
|
41
|
+
- spec/spec_helper.rb
|
42
|
+
- spec/unit
|
43
|
+
- spec/unit/parser_spec.rb
|
44
|
+
- spec/unit/tokenizer_spec.rb
|
45
|
+
- tasks
|
46
|
+
- tasks/github-gem.rake
|
47
|
+
has_rdoc: true
|
48
|
+
homepage: http://wiki.github.com/wvanbergen/sql_tree
|
49
|
+
licenses:
|
50
|
+
post_install_message:
|
51
|
+
rdoc_options:
|
52
|
+
- --title
|
53
|
+
- sql_tree
|
54
|
+
- --main
|
55
|
+
- README.rdoc
|
56
|
+
- --line-numbers
|
57
|
+
- --inline-source
|
58
|
+
require_paths:
|
59
|
+
- lib
|
60
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
61
|
+
requirements:
|
62
|
+
- - ">="
|
63
|
+
- !ruby/object:Gem::Version
|
64
|
+
version: "0"
|
65
|
+
version:
|
66
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: "0"
|
71
|
+
version:
|
72
|
+
requirements: []
|
73
|
+
|
74
|
+
rubyforge_project:
|
75
|
+
rubygems_version: 1.3.5
|
76
|
+
signing_key:
|
77
|
+
specification_version: 2
|
78
|
+
summary: A pure Ruby library to represent SQL queries as trees of nodes.
|
79
|
+
test_files:
|
80
|
+
- spec/integration/api_spec.rb
|
81
|
+
- spec/unit/parser_spec.rb
|
82
|
+
- spec/unit/tokenizer_spec.rb
|