sql_tree 0.1.0 → 0.1.1

data/lib/sql_tree/parser.rb

@@ -1,10 +1,23 @@
+# The <tt>SQLTree::Parser</tt> class is used to construct a syntax tree
+# using <tt>SQLTree::Node</tt> instances from a tree of tokens.
+#
+# This class does only kickstart the parsing process and manages the
+# token stream that is being parsed. The actual parsing of the nodes
+# occurs in the <tt>parse</tt> class method of the different node classes.
 class SQLTree::Parser
 
+  # The <tt>SQLTree::Parser::UnexpectedToken</tt> exception is thrown
+  # when the parser meets a token that it not expect. 
+  #
+  # This exceptions usually means that an SQL syntax error has been found, 
+  # however it can also mean that the SQL construct that is being used is
+  # not (yet) supported by this library. Please create an issue on Github
+  # if the latter is the case.
   class UnexpectedToken < StandardError
 
     attr_reader :expected_token, :actual_token
 
-    def initialize(actual_token, expected_token = nil)
+    def initialize(actual_token, expected_token = nil) # :nodoc:
       @expected_token, @actual_token = expected_token, actual_token
       message =  "Unexpected token: found #{actual_token.inspect}"
       message << ", but expected #{expected_token.inspect}" if expected_token
@@ -12,48 +25,91 @@ class SQLTree::Parser
       super(message)
     end
   end
-
+  
+  # Kickstarts the parser by creating a new instance with the provided
+  # string, and calling the <tt>parse!</tt> method on this instance.
+  #
+  # Do not use this method directly, but use the <tt>SQLTree.[]</tt>
+  # method instead to parse SQL strings.
+  #
+  # <tt>sql_string</tt>:: The string to parse
+  # <tt>options</tt>:: Options to pass to the parser
   def self.parse(sql_string, options = {})
     self.new(sql_string, options).parse!
   end
 
+  # Hash for parser options.
   attr_reader :options
 
-  def initialize(tokens, options)
-    if tokens.kind_of?(String)
-      @tokens = SQLTree::Tokenizer.new.tokenize(tokens)
-    else
-      @tokens = tokens
-    end
+  # Initializes the parser instance.
+  # <tt>tokens</tt>:: The stream of tokens to turn into a syntax tree. If a 
+  #                   string is given, it is tokenized automatically.
+  # <tt>options</tt>:: An optional hash of parser options.
+  def initialize(tokens, options = {})
+    @tokens  = tokens.kind_of?(String) ? SQLTree::Tokenizer.tokenize(tokens) : tokens
     @options = options
   end
 
+  # Returns the current token that is being parsed.
   def current
     @current_token
   end
 
+  # Returns the next token on the token queue, and moves the token queue
+  # one position forward. This will update the result of the
+  # <tt>SQLTree::Parser#current</tt> method.
   def next
     @current_token = @tokens.shift
   end
 
-  def consume(*checks)
+  # Consumes the current token(s), which will make the parser continue to the
+  # next token (see <tt>SQLTree::Parser#next</tt>).
+  #
+  # This method will also check if the consumed token is of the expected type.
+  # It will raise a <tt>SQLTree::Parser::UnexpectedToken</tt> exception if the
+  # consumed token is not of the expected type
+  #
+  # <tt>*checks</tt>:: a list of token types to consume.
+  def consume(*checks) # :raises: SQLTree::Parser::UnexpectedToken
     checks.each do |check|
-      raise UnexpectedToken.new(self.current, check) unless check == self.next
+      raise UnexpectedToken.new(self.current, check) unless check === self.next
     end
   end
 
-  def peek(distance = 1)
-    @tokens[distance - 1]
+  # Looks at the next token on the token queue without consuming it. 
+  #
+  # The token queue will not be adjusted, will is the case when using 
+  # <tt>SQLTree::Parser#next</tt>.
+  #
+  # <tt>lookahead</tt>:: the number of positions to look ahead. Defaults to 1.
+  def peek(lookahead = 1)
+    @tokens[lookahead - 1]
   end
 
-  def peek_tokens(amount)
-    @tokens[0, amount]
+  # Peeks multiple tokens at the same time.
+  #
+  # This method will return an array of the requested number of tokens,
+  # except for when the token stream is nearing its end. In this case, the 
+  # number of tokens returned can be less than requested.
+  # <tt>lookahead_amount</tt>:: the amount of tokens to return from the 
+  #                             front of the token queue.
+  def peek_multiple(lookahead_amount)
+    @tokens[0, lookahead_amount]
   end
 
+  # Prints the current list of tokens to $stdout for inspection.
   def debug
     puts @tokens.inspect
   end
 
+  # Parser a complete SQL query into a tree of <tt>SQLTree::Node</tt> instances.
+  #
+  # Currently, SELECT, INSERT, UPDATE and DELETE queries are supported for
+  # the most part. This emthod should not be called directly, but is called
+  # by the <tt>SQLTree.[]</tt> method, e.g.:
+  #
+  #   tree = SQLTree['SELECT * FROM table WHERE 1=1']
+  #
   def parse!
     case self.peek
     when SQLTree::Token::SELECT then SQLTree::Node::SelectQuery.parse(self)

data/lib/sql_tree/token.rb

@@ -30,24 +30,50 @@ class SQLTree::Token
     literal
   end
 
+  def not?
+    SQLTree::Token::NOT === self
+  end
+
+  def optional_not_prefix?
+    [SQLTree::Token::LIKE, SQLTree::Token::ILIKE, SQLTree::Token::IN].include?(self.class)
+  end
+  
+  def optional_not_suffix?
+    [SQLTree::Token::IS].include?(self.class)
+  end
+  
+  # Returns true if this is a JOIN token?
   def join?
     [SQLTree::Token::JOIN, SQLTree::Token::LEFT, SQLTree::Token::RIGHT,
       SQLTree::Token::INNER, SQLTree::Token::OUTER, SQLTree::Token::NATURAL,
-      SQLTree::Token::FULL].include?(self)
+      SQLTree::Token::FULL].include?(self.class)
+  end
+  
+  def prefix_operator?
+    SQLTree::Node::Expression::PrefixOperator::TOKENS.include?(self.class)
+  end
+  
+  def variable?
+    self.kind_of?(SQLTree::Token::Identifier)
+  end
+  
+  def value?
+    self.kind_of?(SQLTree::Token::LiteralValue)
   end
 
+  # Returns true if this is an order direction token.
   def direction?
-    [SQLTree::Token::ASC, SQLTree::Token::DESC].include?(self)
+    [SQLTree::Token::ASC, SQLTree::Token::DESC].include?(self.class)
   end
 
   ###################################################################
   # DYNAMIC TOKEN TYPES
   ###################################################################
 
-  # The <tt>SQLTree::Token::Value</tt> class is the base class for
+  # The <tt>SQLTree::Token::DynamicToken</tt> class is the base class for
   # every dynamic token. A dynamic token is a token for which the
   # literal value used remains impoirtant during parsing.
-  class Value < SQLTree::Token
+  class DynamicToken < SQLTree::Token
 
     def inspect # :nodoc:
       "#<#{self.class.name.split('::').last}:#{literal.inspect}>"
@@ -59,22 +85,27 @@ class SQLTree::Token
       other.class == self.class && @literal == other.literal
     end
   end
-
-  # The <tt>SQLTree::Token::Variable</tt> class represents SQL
+  
+  # The <tt>SQLTree::Token::Identifier</tt> class represents SQL
   # variables. The variable name is stored in the literal as string,
   # without quotes if they were present.
-  class Variable < SQLTree::Token::Value
+  class Identifier < DynamicToken
+  end
+  
+  # The <tt>SQLTree::Token::LiteralVakue</tt> class represents literal values
+  # like strings and numbers that occur in SQL.
+  class LiteralValue < DynamicToken
   end
 
   # The <tt>SQLTree::Token::String</tt> class represents strings.
   # The actual string is stored in the literal as string without quotes.
-  class String < SQLTree::Token::Value
+  class String < LiteralValue
   end
 
   # The <tt>SQLTree::Token::Keyword</tt> class represents numbers.
   # The actual number is stored as an integer or float in the token's
   # literal.
-  class Number < SQLTree::Token::Value
+  class Number < LiteralValue
   end
 
   ###################################################################
@@ -115,21 +146,16 @@ class SQLTree::Token
                 AND OR NOT AS ON IS NULL BY LIKE ILIKE BETWEEN IN ASC DESC INSERT INTO VALUES DELETE UPDATE SET}
 
   # Create a token for all the reserved keywords in SQL
-  KEYWORDS.each { |kw| const_set(kw, Class.new(SQLTree::Token::Keyword).new(kw)) }
+  KEYWORDS.each { |kw| const_set(kw, Class.new(SQLTree::Token::Keyword)) }
 
-  # A list of keywords that aways occur in fixed combinations. Register these as separate keywords.
-  KEYWORD_COMBINATIONS = [%w{IS NOT}, %w{NOT LIKE}, %w{NOT BETWEEN}, %w{NOT ILIKE}]
-  KEYWORD_COMBINATIONS.each { |kw| const_set(kw.join('_'), Class.new(SQLTree::Token::Keyword).new(kw.join(' '))) }
+  OPERATORS_HASH = { '+' => :plus, '-' => :minus, '*' => :multiply, 
+      '/' => :divide, '%' => :modulo, '||' => :concat, '|' => :binary_or, 
+      '&' => :binary_and, '<<' => :lshift, '>>' => :rshift, '=' => :eq, 
+      '!=' => :ne, '<>' => :ne, '>' => :gt, '<' => :lt, '>=' => :gte, 
+      '<=' => :lte, '==' => :eq }
 
-  ARITHMETHIC_OPERATORS_HASH = { '+' => :plus, '-' => :minus, '*' => :multiply, '/' => :divide, '%' => :modulo }
-  COMPARISON_OPERATORS_HASH  = { '=' => :eq, '!=' => :ne, '<>' => :ne, '>' => :gt, '<' => :lt, '>=' => :gte, '<=' => :lte }
-
-  # Register a token class and single instance for every token.
-  OPERATORS_HASH = ARITHMETHIC_OPERATORS_HASH.merge(COMPARISON_OPERATORS_HASH)
+  # Register a token class and single instance for every operator.
   OPERATORS_HASH.each_pair do |literal, symbol|
-    self.const_set(symbol.to_s.upcase, Class.new(SQLTree::Token::Operator).new(literal)) unless self.const_defined?(symbol.to_s.upcase)
+    self.const_set(symbol.to_s.upcase, Class.new(SQLTree::Token::Operator)) unless self.const_defined?(symbol.to_s.upcase)
   end
-
-  COMPARISON_OPERATORS = COMPARISON_OPERATORS_HASH.map { |(literal, symbol)| const_get(symbol.to_s.upcase) } +
-    [SQLTree::Token::IN, SQLTree::Token::IS, SQLTree::Token::BETWEEN, SQLTree::Token::LIKE, SQLTree::Token::ILIKE, SQLTree::Token::NOT]
 end

data/lib/sql_tree/tokenizer.rb

@@ -14,19 +14,24 @@ class SQLTree::Tokenizer
 
   include Enumerable
 
+  # Returns an array of tokens for the given string.
+  # <tt>string</tt>:: the string to tokenize
+  def self.tokenize(string)
+    self.new(string).tokens
+  end
+
   # The keyword queue, on which kywords are placed before they are yielded
   # to the parser, to enable keyword combining (e.g. NOT LIKE)
   attr_reader :keyword_queue
 
-  def initialize # :nodoc:
+  def initialize(string) # :nodoc:
+    @string = string
+    @current_char_pos = -1
     @keyword_queue = []
   end
 
-  # Returns an array of tokens for the given string.
-  # <tt>string</tt>:: the string to tokenize
-  def tokenize(string)
-    @string = string
-    @current_char_pos = -1
+  # Tokeinzes the string and returns all tokens as an array
+  def tokens
     self.entries
   end
 
@@ -79,6 +84,7 @@ class SQLTree::Tokenizer
   # This method is aliased to <tt>:each</tt> to make the Enumerable
   # methods work on this method.
   def each_token(&block) # :yields: SQLTree::Token
+    
     while next_char
       case current_char
       when /^\s?$/;        # whitespace, go to next character
@@ -88,9 +94,9 @@ class SQLTree::Tokenizer
       when ',';            handle_token(SQLTree::Token::COMMA, &block)
       when /\d/;           tokenize_number(&block)
       when "'";            tokenize_quoted_string(&block)
-      when OPERATOR_CHARS; tokenize_operator(&block)
       when /\w/;           tokenize_keyword(&block)
-      when '"';            tokenize_quoted_variable(&block)     # TODO: allow MySQL quoting mode
+      when OPERATOR_CHARS; tokenize_operator(&block)
+      when SQLTree.identifier_quote_char; tokenize_quoted_identifier(&block)
       end
     end
 
@@ -109,9 +115,9 @@ class SQLTree::Tokenizer
     literal << next_char while /[\w]/ =~ peek_char
 
     if SQLTree::Token::KEYWORDS.include?(literal.upcase)
-      handle_token(SQLTree::Token.const_get(literal.upcase), &block)
+      handle_token(SQLTree::Token.const_get(literal.upcase).new(literal), &block)
     else
-      handle_token(SQLTree::Token::Variable.new(literal), &block)
+      handle_token(SQLTree::Token::Identifier.new(literal), &block)
     end
   end
 
@@ -145,20 +151,20 @@ class SQLTree::Tokenizer
   end
 
   # Tokenize a quoted variable from the SQL stream. This method will
-  # yield an SQLTree::Token::Variable when to closing quote is found.
+  # yield an SQLTree::Token::Identifier when to closing quote is found.
   #
   # The actual quote character that is used depends on the DBMS. For now,
   # only the more standard double quote is accepted.
-  def tokenize_quoted_variable(&block) # :yields: SQLTree::Token::Variable
+  def tokenize_quoted_identifier(&block) # :yields: SQLTree::Token::Identifier
     variable = ''
-    until next_char.nil? || current_char == '"' # TODO: allow MySQL quoting mode
+    until next_char.nil? || current_char == SQLTree.identifier_quote_char # TODO: allow MySQL quoting mode
       variable << (current_char == "\\" ? next_char : current_char)
     end
-    handle_token(SQLTree::Token::Variable.new(variable), &block)
+    handle_token(SQLTree::Token::Identifier.new(variable), &block)
   end
 
   # A regular expression that matches all operator characters.
-  OPERATOR_CHARS = /\=|<|>|!|\-|\+|\/|\*|\%/
+  OPERATOR_CHARS = /\=|<|>|!|\-|\+|\/|\*|\%|\||\&/
 
   # Tokenizes an operator in the SQL stream. This method will yield the
   # operator token when the last character of the token is encountered.
@@ -168,7 +174,8 @@ class SQLTree::Tokenizer
       tokenize_number(&block)
     else
       operator << next_char if SQLTree::Token::OPERATORS_HASH.has_key?(operator + peek_char)
-      handle_token(SQLTree::Token.const_get(SQLTree::Token::OPERATORS_HASH[operator].to_s.upcase), &block)
+      operator_class = SQLTree::Token.const_get(SQLTree::Token::OPERATORS_HASH[operator].to_s.upcase)
+      handle_token(operator_class.new(operator), &block)
     end
   end
 end

data/spec/integration/parse_and_generate_spec.rb

@@ -2,9 +2,17 @@ require "#{File.dirname(__FILE__)}/../spec_helper"
 
 describe SQLTree, 'parsing and generating SQL' do
 
+  before(:each) { SQLTree.identifier_quote_char = '"' }
+
   it "should parse an generate q query without FROM" do
     SQLTree['SELECT 1'].to_sql.should == 'SELECT 1'
   end
+  
+  it "should parse and generate MySQL type identifier quotes" do
+    SQLTree.identifier_quote_char = "`"
+    SQLTree['SELECT `field` FROM `table`'].to_sql.should == 
+            'SELECT `field` FROM `table`'
+  end
 
   it "should parse and generate SQL fo a simple list query" do
     SQLTree["SELECT * FROM table"].to_sql.should == 'SELECT * FROM "table"'

data/spec/lib/matchers.rb

@@ -14,7 +14,8 @@ class TokenizeTo
 
   def matches?(found_tokens)
     @found_tokens = found_tokens
-    return @found_tokens == @expected_tokens
+    return @found_tokens.length == @expected_tokens.length &&
+        @found_tokens.zip(@expected_tokens).all? { |(f, e)| e === f }
   end
 
   def description
@@ -36,7 +37,7 @@ def tokenize_to(*expected_tokens)
 end
 
 def sql_var(name)
-  SQLTree::Token::Variable.new(name.to_s)
+  SQLTree::Token::Identifier.new(name.to_s)
 end
 
 def dot

data/spec/unit/delete_query_spec.rb

@@ -4,13 +4,13 @@ describe SQLTree::Node::DeleteQuery do
 
   it "should parse a delete query without WHERE clause correctly" do
     delete = SQLTree::Node::DeleteQuery["DELETE FROM table"]
-    delete.table.should == 'table'
+    delete.table.should == SQLTree::Node::TableReference.new("table")
     delete.where.should be_nil
   end
 
   it "should parse a delete query without WHERE clause correctly" do
     delete = SQLTree::Node::DeleteQuery["DELETE FROM table WHERE 1 = 1"]
-    delete.table.should == 'table'
+    delete.table.should == SQLTree::Node::TableReference.new("table")
     delete.where.should be_kind_of(SQLTree::Node::Expression)
   end
 end

data/spec/unit/expression_node_spec.rb

@@ -4,7 +4,7 @@ describe SQLTree::Node::Expression do
 
   describe '.parse' do
     it "shoud parse a value correctly" do
-      SQLTree::Node::Expression['123'].should == SQLTree::Node::Value.new(123)
+      SQLTree::Node::Expression['123'].should == SQLTree::Node::Expression::Value.new(123)
     end
 
     it "shoud parse a function call without arguments correctly" do
@@ -16,19 +16,21 @@ describe SQLTree::Node::Expression do
     it "shoud parse a function call with arguments correctly" do
       function = SQLTree::Node::Expression["MD5('string')"]
       function.function.should == 'MD5'
-      function.arguments.should == [SQLTree::Node::Value.new('string')]
+      function.arguments.should == [SQLTree::Node::Expression::Value.new('string')]
     end
 
     it "should parse a logical OR expression correctly" do
       logical = SQLTree::Node::Expression["'this' OR 'that"]
-      logical.operator.should    == :or
-      logical.expressions.should == [SQLTree::Node::Value.new('this'), SQLTree::Node::Value.new('that')]
+      logical.operator.should == 'OR'
+      logical.lhs.should      == SQLTree::Node::Expression::Value.new('this')
+      logical.rhs.should      == SQLTree::Node::Expression::Value.new('that')
     end
 
     it "should parse a logical AND expression correctly" do
       logical = SQLTree::Node::Expression['1 AND 2']
-      logical.operator.should    == :and
-      logical.expressions        == [SQLTree::Node::Value.new(1), SQLTree::Node::Value.new(2)]
+      logical.operator.should    == 'AND'
+      logical.lhs.should         == SQLTree::Node::Expression::Value.new(1)
+      logical.rhs.should         == SQLTree::Node::Expression::Value.new(2)
     end
 
     it "should nest a logical AND expression correctly" do
@@ -42,61 +44,60 @@ describe SQLTree::Node::Expression do
     end
 
     it "should parse a NOT expression without parenteheses correctly" do
-      SQLTree::Node::Expression['NOT 1'].should == SQLTree::Node::LogicalNotExpression.new(SQLTree::Node::Value.new(1))
+      SQLTree::Node::Expression['NOT 1'].should == SQLTree::Node::Expression::PrefixOperator.new(:operator => 'NOT', :rhs => SQLTree::Node::Expression::Value.new(1))
     end
 
     it "should parse a NOT expression without parenteheses correctly" do
-      SQLTree::Node::Expression['NOT(1)'].should == SQLTree::Node::LogicalNotExpression.new(SQLTree::Node::Value.new(1))
+      SQLTree::Node::Expression['NOT(1)'].should == SQLTree::Node::Expression::PrefixOperator.new(:operator => 'NOT', :rhs => SQLTree::Node::Expression::Value.new(1))
     end
 
     it "should parse a comparison expression correctly" do
       comparison = SQLTree::Node::Expression['1 < 2']
       comparison.operator.should == '<'
-      comparison.lhs.should      == SQLTree::Node::Value.new(1)
-      comparison.rhs.should      == SQLTree::Node::Value.new(2)
+      comparison.lhs.should      == SQLTree::Node::Expression::Value.new(1)
+      comparison.rhs.should      == SQLTree::Node::Expression::Value.new(2)
     end
 
     it "should parse an IS NULL expression corectly" do
       comparison = SQLTree::Node::Expression['field IS NULL']
       comparison.operator.should == 'IS'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should == SQLTree::Node::Value.new(nil)
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Expression::Value.new(nil)
     end
 
     it "should parse an IS NOT NULL expression corectly" do
       comparison = SQLTree::Node::Expression['field IS NOT NULL']
       comparison.operator.should == 'IS NOT'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should == SQLTree::Node::Value.new(nil)
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Expression::Value.new(nil)
     end
 
     it "should parse a LIKE expression corectly" do
       comparison = SQLTree::Node::Expression["field LIKE '%search%"]
       comparison.operator.should == 'LIKE'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should == SQLTree::Node::Value.new('%search%')
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Expression::Value.new('%search%')
     end
 
     it "should parse a NOT ILIKE expression corectly" do
       comparison = SQLTree::Node::Expression["field NOT ILIKE '%search%"]
       comparison.operator.should == 'NOT ILIKE'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should == SQLTree::Node::Value.new('%search%')
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Expression::Value.new('%search%')
     end
 
     it "should parse an IN expression correctly" do
       comparison = SQLTree::Node::Expression["field IN (1,2,3,4)"]
       comparison.operator.should == 'IN'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should be_kind_of(SQLTree::Node::SetExpression)
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should be_kind_of(SQLTree::Node::Expression::List)
     end
 
     it "should parse a NOT IN expression correctly" do
       comparison = SQLTree::Node::Expression["field NOT IN (1>2, 3+6, 99)"]
       comparison.operator.should == 'NOT IN'
-      comparison.lhs.should == SQLTree::Node::Variable.new('field')
-      comparison.rhs.should be_kind_of(SQLTree::Node::SetExpression)
+      comparison.lhs.should == SQLTree::Node::Expression::Variable.new('field')
+      comparison.rhs.should be_kind_of(SQLTree::Node::Expression::List)
     end
-
   end
 end