sql_tree 0.0.1

data/lib/sql_tree/tokenizer.rb

@@ -0,0 +1,174 @@
+# The <tt>SQLTree::Tokenizer</tt> class transforms a string or stream of
+# characters into a enumeration of tokens, that are more appropriate for
+# the SQL parser to work with.
+#
+# An example:
+#
+#   >> SQLTree::Tokenizer.new.tokenize('SELECT * FROM table')
+#   => [:select, :all, :from, Variable('table')]
+#
+# The <tt>tokenize</tt> method will return an array of tokens, while
+# the <tt>each_token</tt> (aliased to <tt>each</tt>) will yield every
+# token one by one.
+class SQLTree::Tokenizer
+  
+  include Enumerable
+  
+  # The keyword queue, on which kywords are placed before they are yielded
+  # to the parser, to enable keyword combining (e.g. NOT LIKE)
+  attr_reader :keyword_queue
+  
+  def initialize # :nodoc:
+    @keyword_queue = []
+  end
+  
+  # Returns an array of tokens for the given string.
+  # <tt>string</tt>:: the string to tokenize
+  def tokenize(string)
+    @string = string
+    @current_char_pos = -1
+    self.entries
+  end
+  
+  # Returns the current character that is being tokenized
+  def current_char
+    @current_char
+  end
+
+  # Returns the next character to tokenize, but does not move
+  # the pointer of the current character forward.
+  # <tt>lookahead</tt>:: how many positions forward to peek.
+  def peek_char(lookahead = 1)
+    @string[@current_char_pos + lookahead, 1]
+  end
+
+  # Returns the next character to tokenize, and moves the pointer
+  # of the current character one position forward.
+  def next_char
+    @current_char_pos += 1
+    @current_char = @string[@current_char_pos, 1]
+  end
+
+  # Combines several tokens to a single token if possible, and
+  # yields teh result, or yields every single token if they cannot
+  # be combined. 
+  # <tt>token</tt>:: the token to yield or combine
+  # <tt>block</tt>:: the block to yield tokens and combined tokens to.
+  def handle_token(token, &block) # :yields: SQLTree::Token
+    if token.kind_of?(SQLTree::Token::Keyword)
+      keyword_queue.push(token)
+    else
+      empty_keyword_queue!(&block)
+      block.call(token)
+    end
+  end
+  
+  # This method ensures that every keyword currently in the queue is
+  # yielded. This method get called by <tt>handle_token</tt> when it
+  # knows for sure that the keywords on the queue cannot be combined
+  # into a single keyword.
+  # <tt>block</tt>:: the block to yield the tokens on the queue to.
+  def empty_keyword_queue!(&block) # :yields: SQLTree::Token
+    block.call(@keyword_queue.shift) until @keyword_queue.empty?
+  end
+
+  # Iterator method that yields each token that is encountered in the
+  # SQL stream. These tokens are passed to the SQL parser to construct 
+  # a syntax tree for the SQL query.
+  #
+  # This method is aliased to <tt>:each</tt> to make the Enumerable 
+  # methods work on this method.
+  def each_token(&block) # :yields: SQLTree::Token
+    while next_char
+      case current_char
+      when /^\s?$/;        # whitespace, go to next character
+      when '(';            handle_token(SQLTree::Token::LPAREN, &block)
+      when ')';            handle_token(SQLTree::Token::RPAREN, &block)
+      when '.';            handle_token(SQLTree::Token::DOT, &block)
+      when ',';            handle_token(SQLTree::Token::COMMA, &block)
+      when /\d/;           tokenize_number(&block)
+      when "'";            tokenize_quoted_string(&block)
+      when OPERATOR_CHARS; tokenize_operator(&block)
+      when /\w/;           tokenize_keyword(&block)
+      when '"';            tokenize_quoted_variable(&block)     # TODO: allow MySQL quoting mode
+      end
+    end
+    
+    # Make sure to yield any tokens that are still stashed on the queue.
+    empty_keyword_queue!(&block)
+  end
+  
+  alias :each :each_token
+  
+  # Tokenizes a eyword in the code. This can either be a reserved SQL keyword
+  # or a variable. This method will yield variables directly. Keywords will be
+  # yielded with a delay, because they may need to be combined with other
+  # keywords in the <tt>handle_token</tt> method.
+  def tokenize_keyword(&block) # :yields: SQLTree::Token
+    literal = current_char
+    literal << next_char while /[\w]/ =~ peek_char
+      
+    if SQLTree::Token::KEYWORDS.include?(literal.upcase)
+      handle_token(SQLTree::Token.const_get(literal.upcase), &block)
+    else
+      handle_token(SQLTree::Token::Variable.new(literal), &block)
+    end
+  end
+
+  # Tokenizes a number (either an integer or float) in the SQL stream.
+  # This method will yield the token after the last digit of the number
+  # has been encountered.
+  def tokenize_number(&block) # :yields: SQLTree::Token::Number
+    number = current_char
+    dot_encountered = false
+    while /\d/ =~ peek_char || (peek_char == '.' && !dot_encountered)
+      dot_encountered = true if peek_char == '.'
+      number << next_char
+    end
+    
+    if dot_encountered
+      handle_token(SQLTree::Token::Number.new(number.to_f), &block)
+    else
+      handle_token(SQLTree::Token::Number.new(number.to_i), &block)
+    end
+  end
+
+  # Reads a quoted string token from the SQL stream. This method will
+  # yield an SQLTree::Token::String when the closing quote character is
+  # encountered.
+  def tokenize_quoted_string(&block) # :yields: SQLTree::Token::String
+    string = ''
+    until next_char.nil? || current_char == "'"
+      string << (current_char == "\\" ? next_char : current_char)
+    end
+    handle_token(SQLTree::Token::String.new(string), &block)
+  end
+
+  # Tokenize a quoted variable from the SQL stream. This method will
+  # yield an SQLTree::Token::Variable when to closing quote is found.
+  #
+  # The actual quote character that is used depends on the DBMS. For now,
+  # only the more standard double quote is accepted.
+  def tokenize_quoted_variable(&block) # :yields: SQLTree::Token::Variable
+    variable = ''
+    until next_char.nil? || current_char == '"' # TODO: allow MySQL quoting mode
+      variable << (current_char == "\\" ? next_char : current_char)
+    end
+    handle_token(SQLTree::Token::Variable.new(variable), &block)
+  end
+
+  # A regular expression that matches all operator characters.
+  OPERATOR_CHARS = /\=|<|>|!|\-|\+|\/|\*|\%/
+
+  # Tokenizes an operator in the SQL stream. This method will yield the 
+  # operator token when the last character of the token is encountered.
+  def tokenize_operator(&block) # :yields: SQLTree::Token
+    operator = current_char
+    if operator == '-' && /[\d\.]/ =~ peek_char
+      tokenize_number(&block)
+    else
+      operator << next_char if SQLTree::Token::OPERATORS_HASH.has_key?(operator + peek_char)
+      handle_token(SQLTree::Token.const_get(SQLTree::Token::OPERATORS_HASH[operator].to_s.upcase), &block)
+    end
+  end
+end

data/spec/integration/api_spec.rb

@@ -0,0 +1,5 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe SQLTree, :API do
+  
+end

data/spec/integration/full_queries_spec.rb

@@ -0,0 +1,21 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe SQLTree, 'parsing and generating SQL' do
+
+  it "should parse and generate SQL fo a simple list query" do
+    SQLTree["SELECT * FROM table"].to_sql.should == 'SELECT * FROM "table"'
+  end
+  
+  it "should parse and generate the DISTINCT keyword" do
+    SQLTree["SELECT DISTINCT * FROM table"].to_sql.should == 'SELECT DISTINCT * FROM "table"'
+  end
+  
+  it 'should parse and generate table aliases' do
+    SQLTree["SELECT a.* FROM table AS a"].to_sql.should == 'SELECT "a".* FROM "table" AS "a"'
+  end
+  
+  it "parse and generate a complex SQL query" do
+    SQLTree['SELECT a.*,   MD5( a.name )   AS checksum   FROM  table  AS  a , other   WHERE   other.timestamp    >  a.timestamp'].to_sql.should ==
+                  'SELECT "a".*, MD5("a"."name") AS "checksum" FROM "table" AS "a", "other" WHERE ("other"."timestamp" > "a"."timestamp")'
+  end
+end

data/spec/lib/matchers.rb

@@ -0,0 +1,84 @@
+class ParseAs
+  
+  def initialize(expected_tree)
+    @expected_tree = expected_tree
+  end 
+  
+  def matches?(found_tree)
+    @found_tree = found_tree.to_tree
+    return @found_tree == @expected_tree
+  end
+  
+  def description
+    "expected to parse to #{@expected_tree.inspect}"
+  end
+  
+  def failure_message
+    " #{@expected_tree.inspect} expected, but found #{@found_tree.inspect}"
+  end
+  
+  def negative_failure_message
+    " expected not to be tokenized to #{@expected_tree.inspect}"
+  end  
+end
+
+def parse_as(tree)
+  ParseAs.new(tree)
+end
+
+class TokenizeTo
+  
+  def initialize(expected_tokens)
+    @expected_tokens = expected_tokens.map do |t|
+      case t
+        when SQLTree::Token then t
+        when String         then SQLTree::Token::String.new(t)
+        when Numeric        then SQLTree::Token::Number.new(t)
+        when Symbol         then SQLTree::Token.const_get(t.to_s.upcase)
+        else "Cannot check for this token: #{t.inspect}!"
+      end
+    end
+  end
+  
+  def matches?(found_tokens)
+    @found_tokens = found_tokens
+    return @found_tokens == @expected_tokens
+  end
+  
+  def description
+    "expected to tokenized to #{@expected_tokens.inspect}"
+  end
+  
+  def failure_message
+    " #{@expected_tokens.inspect} expected, but found #{@found_tokens.inspect}"
+  end
+  
+  def negative_failure_message
+    " expected not to be tokenized to #{@expected_tokens.inspect}"
+  end  
+  
+end
+
+def tokenize_to(*expected_tokens)
+  TokenizeTo.new(expected_tokens)
+end
+
+def sql_var(name)
+  SQLTree::Token::Variable.new(name.to_s)
+end
+
+def dot
+  SQLTree::Token::DOT
+end
+
+def comma
+  SQLTree::Token::COMMA
+end
+
+def lparen
+  SQLTree::Token::LPAREN
+end
+
+def rparen
+  SQLTree::Token::RPAREN
+end

data/spec/spec_helper.rb

@@ -0,0 +1,26 @@
+$:.reject! { |e| e.include? 'TextMate' }
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'spec'
+require 'sql_tree'
+
+module SQLTree::Spec
+  module NodeLoader
+    def self.const_missing(const)
+      SQLTree::Node.const_get(const)
+    end
+  end
+  
+  module TokenLoader
+    def self.const_missing(const)
+      SQLTree::Token.const_get(const)
+    end
+  end  
+end
+
+Spec::Runner.configure do |config|
+  
+end
+
+require "#{File.dirname(__FILE__)}/lib/matchers"

data/spec/unit/expression_node_spec.rb

@@ -0,0 +1,102 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe SQLTree::Node::Expression do
+
+  describe '.parse' do
+    it "shoud parse a value correctly" do
+      SQLTree::Node::Expression['123'].should == SQLTree::Node::Value.new(123)
+    end
+
+    it "shoud parse a function call without arguments correctly" do
+      function = SQLTree::Node::Expression['NOW()']
+      function.function.should == 'NOW'
+      function.arguments.should be_empty
+    end
+
+    it "shoud parse a function call with arguments correctly" do
+      function = SQLTree::Node::Expression["MD5('string')"]
+      function.function.should == 'MD5'
+      function.arguments.should == [SQLTree::Node::Value.new('string')]
+    end
+
+    it "should parse a logical OR expression correctly" do
+      logical = SQLTree::Node::Expression["'this' OR 'that"]
+      logical.operator.should    == :or
+      logical.expressions.should == [SQLTree::Node::Value.new('this'), SQLTree::Node::Value.new('that')]
+    end
+
+    it "should parse a logical AND expression correctly" do
+      logical = SQLTree::Node::Expression['1 AND 2']
+      logical.operator.should    == :and
+      logical.expressions        == [SQLTree::Node::Value.new(1), SQLTree::Node::Value.new(2)]
+    end
+
+    it "should nest a logical AND expression correctly" do
+      logical = SQLTree::Node::Expression['1 AND 2 AND 3']
+      logical.should == SQLTree::Node::Expression['(1 AND 2) AND 3']
+    end
+  
+    it "should nest expressions correctly when parentheses are used" do
+      logical = SQLTree::Node::Expression['1 AND (2 AND 3)']
+      logical.should_not == SQLTree::Node::Expression['(1 AND 2) AND 3']
+    end
+    
+    it "should parse a NOT expression without parenteheses correctly" do
+      SQLTree::Node::Expression['NOT 1'].should == SQLTree::Node::LogicalNotExpression.new(SQLTree::Node::Value.new(1))
+    end
+
+    it "should parse a NOT expression without parenteheses correctly" do
+      SQLTree::Node::Expression['NOT(1)'].should == SQLTree::Node::LogicalNotExpression.new(SQLTree::Node::Value.new(1))
+    end
+
+    it "should parse a comparison expression correctly" do
+      comparison = SQLTree::Node::Expression['1 < 2']
+      comparison.operator.should == '<'
+      comparison.lhs.should      == SQLTree::Node::Value.new(1)
+      comparison.rhs.should      == SQLTree::Node::Value.new(2)
+    end
+    
+    it "should parse an IS NULL expression corectly" do
+      comparison = SQLTree::Node::Expression['field IS NULL']
+      comparison.operator.should == 'IS'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Value.new(nil)
+    end
+    
+    it "should parse an IS NOT NULL expression corectly" do
+      comparison = SQLTree::Node::Expression['field IS NOT NULL']
+      comparison.operator.should == 'IS NOT'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Value.new(nil)
+    end
+    
+    it "should parse a LIKE expression corectly" do
+      comparison = SQLTree::Node::Expression["field LIKE '%search%"]
+      comparison.operator.should == 'LIKE'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Value.new('%search%')
+    end
+
+    it "should parse a NOT ILIKE expression corectly" do
+      comparison = SQLTree::Node::Expression["field NOT ILIKE '%search%"]
+      comparison.operator.should == 'NOT ILIKE'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should == SQLTree::Node::Value.new('%search%')
+    end
+    
+    it "should parse an IN expression correctly" do
+      comparison = SQLTree::Node::Expression["field IN (1,2,3,4)"]
+      comparison.operator.should == 'IN'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should be_kind_of(SQLTree::Node::SetExpression)
+    end
+    
+    it "should parse a NOT IN expression correctly" do
+      comparison = SQLTree::Node::Expression["field NOT IN (1>2, 3+6, 99)"]
+      comparison.operator.should == 'NOT IN'
+      comparison.lhs.should == SQLTree::Node::Variable.new('field')
+      comparison.rhs.should be_kind_of(SQLTree::Node::SetExpression)
+    end
+    
+  end
+end

data/spec/unit/leaf_node_spec.rb

@@ -0,0 +1,84 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe SQLTree::Node::Value do
+  
+  describe '.parse' do
+    it "should not parse a field name" do
+      lambda { SQLTree::Node::Value['field_name'] }.should raise_error(SQLTree::Parser::UnexpectedToken)
+    end
+
+    it "should parse an integer value correctly" do
+      SQLTree::Node::Value['123'].value.should == 123
+    end
+
+    it "should parse a string correctly" do
+      SQLTree::Node::Value["'123'"].value.should == '123'
+    end
+    
+    it "should parse a NULL value correctly" do
+      SQLTree::Node::Value['NULL'].value.should == nil
+    end
+    
+  end
+end
+
+describe SQLTree::Node::Variable do
+
+  describe '.parse' do
+    it "should parse a variable name correctly" do
+      SQLTree::Node::Field['variable'].name.should == 'variable'
+    end
+
+    it "should parse a quoted variable name correctly" do
+      SQLTree::Node::Field['"variable"'].name.should == 'variable'
+    end
+
+    it "should raise an error when parsing a reserved keyword as variable" do
+      lambda { SQLTree::Node::Field['select'] }.should raise_error(SQLTree::Parser::UnexpectedToken)
+    end
+
+    it "should parse a quoted reserved keyword as variable name correctly" do
+      SQLTree::Node::Field['"select"'].name.should == 'select'
+    end
+  end
+end
+
+describe SQLTree::Node::Field do
+  describe '.parse' do
+    it "should parse a field name with table name correclty" do
+      field = SQLTree::Node::Field['table.field']
+      field.table.should == 'table'
+      field.name.should  == 'field'
+    end
+
+    it "should parse a field name without table name correclty" do
+      field = SQLTree::Node::Field['field']
+      field.table.should be_nil
+      field.name.should == 'field'
+    end
+
+    it "should parse a quoted field name without table name correclty" do
+      field = SQLTree::Node::Field['"field"']
+      field.table.should be_nil
+      field.name.should == 'field'
+    end
+
+    it "should parse a quoted field name with quoted table name correclty" do
+      field = SQLTree::Node::Field['"table"."field"']
+      field.table.should == 'table'
+      field.name.should  == 'field'
+    end
+
+    it "should parse a quoted field name with non-quoted table name correclty" do
+      field = SQLTree::Node::Field['table."field"']
+      field.table.should == 'table'
+      field.name.should  == 'field'
+    end
+
+    it "should parse a non-quoted field name with quoted table name correclty" do
+      field = SQLTree::Node::Field['"table".field']
+      field.table.should == 'table'
+      field.name.should  == 'field'
+    end
+  end
+end