tsql_shparser 0.0.1

data/lib/tsql_shparser.rb

@@ -0,0 +1,198 @@
+#
+# tsql_shparser.rb: Shallow Parser for t-SQL
+# Copyright (c) 2005-2006 Shashank Date (shanko_date@yahoo.com)
+#
+# License: Ruby's
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+#
+require 'tsql_shparser/tsql_tokenizer'
+require 'tsql_shparser/tsql_stmt'
+
+## Shallow Parser for t-SQL: Creates an array of arrays of Token objects for each 
+## t-SQL Statement parsed as input and generates an array of TSQLStmt objects.
+class TSQL_ShParser
+  VERSION = "0.0.1"
+  
+  @@start = ["SELECT","INSERT","UPDATE","DELETE","DROP","ALTER","CREATE","TRUNCATE"]
+  @@first = ["ALTER", "BACKUP", "BEGIN", "BREAK", "CHECKPOINT", "CONTINUE", "DENY", "GRANT", "REVOKE", 
+             "BULK", "CLOSE", "COMMIT", "CREATE","DBCC", "DEALLOCATE",
+             "DECLARE", "DELETE","DROP", "DUMP", "ERRLVL", "EXEC", "EXECUTE", "EXIT", "FETCH",
+             "GO", "GOTO", "IF", "INSERT","KILL", "LOAD", "OPEN", "OPENDATASOURCE", 
+             "OPENQUERY", "OPENROWSET","PRINT","RAISERROR", "READTEXT", "RECONFIGURE", 
+             "RESTORE","RETURN", "ROLLBACK", "SAVE", "SELECT", "SET", "SETUSER", "SHUTDOWN",
+             "STATISTICS","TRUNCATE", "UPDATE", "UPDATETEXT", "USE","WAITFOR", "WHILE", "WRITETEXT"]
+  
+  def initialize(file=nil)
+   @input_file = file
+   @tok = Tokenizer.new(file)
+   @tok.tokenize_file  if file
+  end
+
+  ## Return nil if it is not a sub-select
+  ## Return relative index (w.r.t SELECT) of the token  
+  ## prior to the leftmost LEFT_PARAN otherwise
+  def is_sub_select?
+  
+    ss = nil
+    
+    # Make sure that we are indeed looking at the sub-SELECT
+    # The SELECT token is already consumed, so we have to look back to verify
+    curr = @tok.look_back(1)    
+    return ss unless (curr.token_value == 'SELECT')
+    
+    # If there is no token before SELECT then this is not a sub-SELECT
+    prev = @tok.look_back(2)
+    return ss unless prev     
+
+    n = 3
+    
+    # Take care of the arbitrary/redundant nesting of expressions in ( )    
+    while(prev && (prev.token_value == LEFT_PARAN))
+      prev = @tok.look_back(n)
+      n += 1
+    end 
+
+    # Check the token before the left most LEFT_PARAN
+    follow = ['SELECT','DISTINCT','PERCENT','JOIN','WHERE',
+              'BY','IN','ANY','ALL','EXISTS','UNION','FROM',
+              '<','>','=','>=','<=','<>',',']  
+
+    prev_prev = @tok.look_back(n)
+
+    ss = n if prev && follow.include?(prev.token_value) 
+
+    # Consider the case: SELECT TOP 10 (select ...)   
+    
+    if (prev && (prev.token_value =~ /^\d+$/))    
+      ss = n if (prev_prev && (prev_prev.token_value == 'TOP'))
+    end
+
+    # Consider the case UNION ALL
+    
+    if (prev && (prev.token_value == 'ALL'))
+      ss = n if (prev_prev && (prev_prev.token_value == 'UNION'))
+    end
+    
+    ## Finally, consider the case of SELECT being part of the INSERT statement
+    #
+    # skip the (optional) list of column names that may 
+    # follow the name of the table in an INSERT, for e.g:
+    #     INSERT INTO Table1 (COL1, COL2, COL3) 
+    #     SELECT 'A1','B2','C3';      
+    if (prev && prev.token_value == RIGHT_PARAN) 
+
+      prev = @tok.look_back(n)
+ 
+      # Rewind to the matching LEFT_PARAN
+      # We are more strict here and verify that all tokens
+      # are either Id or Comma: not considering the possibility of a Dot
+      while (prev and ((prev.token_type == :Id) or (prev.token_type == :Comma)))
+        prev = @tok.look_back(n)
+        n += 1        
+      end
+      
+      # This must be the LEFT_PARAN
+      prev = @tok.look_back(n) if (prev and (prev.token_value == LEFT_PARAN))
+    end
+       
+    while (prev and ((prev.token_type == :Id) or (prev.token_type == :Dot)))
+      prev = @tok.look_back(n)
+      n += 1      
+    end
+ 
+    # ??? prev = @tok.look_back(n) ???
+    ss = n if (prev and prev.token_value == 'INSERT')
+
+    prev_prev = @tok.look_back(n)
+    ss = n+1 if (prev and (prev.token_value == 'INTO') and prev_prev and (prev_prev.token_value == 'INSERT'))      
+ 
+    ss
+    
+  end
+  
+  ## Keep consuming tokens till you detect the end of the t-SQL statement.
+  ## It is not very accurate about determining the end especially if the
+  ## statements are redundantly nested inside paranthesis or are immediately
+  ## followed by user-defined functions or stored procs.
+  ## It takes in as its argument a category which is one of the symbols in 
+  ## the @@start array. It returns an array (possibly empty) of Token objects.
+  def parse_stmt(category="SELECT")
+
+    stmt = []
+    
+    curr = @tok.get_next_token
+    return stmt if (curr.token_value != category)     
+            
+    stmt << curr
+    
+    found_set = false if category == 'UPDATE'
+    case_end  = 0    
+    
+    prev = curr.token_value
+    
+    # Loop to find the end of the stmt
+    loop do 
+
+      curr = @tok.get_next_token
+      break unless curr
+
+      curr_tok  = curr.token_value
+      case_end += 1 if curr_tok == 'CASE'
+      case_end -= 1 if curr_tok == 'END'
+      
+      if ((not is_sub_select?) and (@@first.include?(curr_tok) or (curr.token_type == :Label) or (curr_tok == ';') or ((case_end < 0) and (curr_tok == 'END'))))
+        # Handle TRIGGER Syntax
+        unless (['OF','FOR','AFTER', ',','IF','AND','OR'].include?(prev) and ['INSERT','UPDATE','DELETE'].include?(curr_tok))        
+          if ((category == 'UPDATE') and (not found_set) and (curr_tok == 'SET'))
+            found_set = true
+          else
+            @tok.unget_token 
+            break
+          end
+        end
+      end
+      
+      stmt << curr    
+      prev = curr_tok
+    end 
+    
+    stmt
+    
+  end
+  
+  ## Parse the string passed as the argument (str). If no string is passed, 
+  ## it is assumed that the parser constructor was invoked with a file-name
+  ## which was used to open an existing file and was parsed successfully.
+  ## If no file-name or string was given, an empty array is returned.
+  ## Otherwise an array of TSQLStmt objects is returned, one object per t-SQL
+  ## statement that starts with the symbol in @@start.
+  def parse(str=nil)
+  
+    stmts = []
+
+    @tok.tokenize_string(str) if str
+    t = @tok.get_next_token
+    while t
+
+      category = t.token_value
+      
+      if @@start.include?(category)
+          @tok.unget_token
+          s = parse_stmt(category)
+          stmts << s unless s == []
+      end
+      
+      t = @tok.get_next_token
+
+    end
+    
+    stmts.map{|st| TSQLStmt.new(st)}
+    
+  end
+  
+end
+

data/lib/tsql_shparser/tsql_stmt.rb

@@ -0,0 +1,673 @@
+#
+# tsql_stmt.rb: Shallow Parser for t-SQL
+# Copyright (c) 2005-2006 Shashank Date (shanko_date@yahoo.com)
+#
+# License: Ruby's
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+#
+
+LEFT_PARAN  = "("
+RIGHT_PARAN = ")"
+
+## A facade class which adds helper methods to the Array class
+## and converts it into a TokenStream
+class TokenStream < Array
+  VERSION = "0.0.1"
+  def initialize(arr=[])
+    @arr = super(arr)
+  end
+
+  def +(arr);       TokenStream.new(super);  end  
+  def -(arr);       TokenStream.new(super);  end  
+  def slice(range); TokenStream.new(super);  end
+  
+  def to_arr
+    (@arr ? @arr.collect{|t| t.token_value} : @arr)
+  end  
+      
+  def find_matching_paran(from)
+      
+      list = self.slice(from..-1).to_arr
+
+      m = level = 0
+      if (list[0] == LEFT_PARAN)
+          list.each_index { |i|
+              level += 1 if (list[i] == LEFT_PARAN)
+              level -= 1 if (list[i] == RIGHT_PARAN)
+              (m = i; break) if (level == 0)
+          } 
+      end
+      
+      return m
+  end
+  
+  def find_matching_word(words,from=0)
+    list = self.slice(from..-1).to_arr
+    pos = 0 # assume that the word will NEVER be the first word in the list
+    words.each{|word|
+      m = level = 0
+      list.each_index { |i|    
+          level += 1 if list[i] == LEFT_PARAN
+          level -= 1 if list[i] == RIGHT_PARAN
+          (m = i; break) if ((list[i] == word) and (level == 0))  
+          p [i,level,m] if $DEBUG
+      }
+      (pos = m; break) if m > 0
+    }
+    pos
+  end
+  
+end
+
+## The main class which identifies the various clauses for the t-SQL Statement.
+## It constructs the TSQLStmt object from the array of tokens parsed by the Parser.
+## Generates a TokenStream object out of this array and provides methods to 
+## query the TokenStream.
+class TSQLStmt
+  VERSION = "0.0.1"
+  ## Hash used in determining which clauses follow the clause indicated by the key
+  @@follow = {   
+    'SELECT' => ['INTO','FROM','JOIN','WHERE','GROUP','HAVING','UNION','ORDER'],
+    'FROM'   => ['JOIN','WHERE','GROUP','HAVING','UNION','ORDER'],
+    'JOIN'   => ['WHERE','GROUP','HAVING','UNION','ORDER'],
+    'WHERE'  => ['GROUP','HAVING','UNION','ORDER'],
+    'GROUP'  => ['HAVING','UNION','ORDER'],
+    'HAVING' => ['UNION','ORDER'],
+    'UNION'  => ['ORDER'],
+    'SET'    => ['FROM','JOIN','WHERE','UNION'],  # Part of the UPDATE
+  }
+  
+  attr_reader :typ
+  attr_reader :stmt
+  attr_reader :line
+  attr_reader :col
+  
+  ## Constructor: takes in array of tokens created by the ShParser. 
+  ## Each array has tokens of exactly one SQL statement. Sometimes the array
+  ## may have some extra tokens towards the end, since the parser could not
+  ## correctly identify the end of the statement. 
+  def initialize(st)  
+    @stmt = st
+    if (@stmt and @stmt[0])
+      @typ  = @stmt[0].token_value
+      @typ  = 'INTO' if ((@typ == 'SELECT') and (TokenStream.new(@stmt).find_matching_word('INTO') > 0))
+      @line = @stmt[0].line
+      @col  = @stmt[0].col
+    end    
+  end
+
+# ---------------------
+  def hdr(start)
+    hdr = TokenStream.new
+    return hdr unless (self.typ == start)
+
+    @stmt.each_with_index{|tok,i|
+        break  unless (tok.token_type == :KeyWord)
+        hdr += [tok]
+    }
+    
+    hdr  
+  end
+
+private :hdr
+
+# ---------------------
+
+  def name_picker(list,msg="",&block)
+
+    cols,i,n  = [], 0, list.length
+    while (i < n)
+      tok = list[i]
+      if tok.token_value == 'SELECT'
+
+        if ((i > 0) and (list[i-1].token_value == LEFT_PARAN))  
+          j = list.find_matching_paran(i-1)
+          st = list.slice(i...(i+j-1))
+          i  += j
+        else
+          st = list.slice(i..-1)
+          i = n
+        end
+        
+        cols += TSQLStmt.new(st).send(msg) if msg.length > 0
+
+      else
+       
+        cols << tok if ((tok.token_type == :Id) and 
+                  ((list[i+1] == nil) or (not [LEFT_PARAN,"."].include?(list[i+1].token_value))) and
+                  ((block_given?) ? block.call(i,list[i-1],list[i+1]) : true))                  
+        i += 1
+      end
+    end
+    TokenStream.new(cols)
+  end
+
+private :name_picker  
+
+# ---------------------
+
+  def clause(keyword,follow=nil,st=nil)
+    klause = TokenStream.new  
+    return [klause,0] unless @stmt
+    
+    follow ||= @@follow[keyword]
+    st     ||= self.stmt
+      
+    n = st.find_matching_word(keyword)
+
+    if n > 0      
+      m = st.find_matching_word(follow,n)
+      klause = st.slice(n..((m == 0) ? -1: n+m-1))
+    end
+    
+    [klause,n]
+  end
+
+private :clause
+
+# ---------------------
+
+  ## Convert the array of tokens to a TokenStream object
+  def stmt;    TokenStream.new(@stmt);  end  
+
+  ## List all the expressions in the SELECT part of the statement
+  ## Returns an array of TokenStream objects. Each object in the
+  ## array is exactly one expression of the SELECT.
+  def list_select_expressions
+    columns,list = [],[]
+    return unless (@stmt and ((self.typ == 'SELECT') or (self.typ == 'INTO')))
+
+    # Skip the header part of the SELECT
+    prev,n = nil,nil
+    @stmt.each_with_index{|tok,i|
+      
+      (prev = tok; next) if ["SELECT","ALL", "DISTINCT", "TOP", "PERCENT", "WITH", "TIES"].include?(tok.token_value)
+      next if ((tok.token_value =~ /^\d+$/) and prev and (prev.token_value == 'TOP'))
+      n = i
+      break
+    }
+    
+    # Find the end of the expression list. 
+    if n       
+       m   = self.stmt.find_matching_word(@@follow['SELECT'],n)  
+      list = self.stmt.slice(n..((m == 0) ? -1: n+m-1))     
+    end
+
+    i = 0
+    column = TokenStream.new
+    
+    while i < list.length            
+        tok = list[i]        
+        k = ((tok.token_value == LEFT_PARAN) ? list.find_matching_paran(i) : 1)                
+        raise "I cannot find matching paran in:\n #{list.slice(i..-1).to_arr.inspect}\n" if k == 0
+        
+        column += list.slice(i...(i+k))       
+        if (list[i+k-1] and (list[i+k-1].token_value == ","))
+          columns << column
+          column = TokenStream.new
+        end
+        i += k         
+    end
+    columns << column
+  rescue 
+    puts $!.to_s
+  ensure 
+    return columns
+  end
+
+  def hdr_select
+    hdr = TokenStream.new
+    return hdr unless ((self.typ == 'SELECT') or (self.typ == 'INTO'))
+    prev = nil
+    @stmt.each{|tok|
+      if (tok.token_type == :KeyWord)
+        hdr += [tok]
+      elsif (tok.token_type == :Number) and (prev.token_value == 'TOP') 
+        hdr += [tok]
+      else 
+        break 
+      end
+      prev = tok
+    }
+    hdr
+  end
+  
+  
+  def hdr_insert; hdr('INSERT');  end
+  def hdr_update; hdr('UPDATE');  end
+  def hdr_delete; hdr('DELETE');  end
+   
+
+  def columns_delete(all=false)
+    cols = TokenStream.new
+    return cols unless self.typ == 'DELETE'  
+    
+    i = ((@stmt[1].token_value == 'FROM') ? 3 : 2)   
+    
+    cols
+  end
+  
+  def columns_insert(all=false)
+    cols = TokenStream.new
+    return cols unless self.typ == 'INSERT'
+    
+    j = 0
+    i = ((@stmt[1].token_value == 'INTO') ? 3 : 2)  
+
+    return cols if ((@stmt[i] == nil) or (@stmt[i].token_value == 'EXECUTE') or (@stmt[i].token_value == 'EXEC'))
+    
+    if (@stmt[i].token_value == LEFT_PARAN)
+      j = self.stmt.find_matching_paran(i)
+    end
+    j += i
+
+    cols += name_picker(self.stmt.slice(i..j)) if j > i
+    cols += name_picker(self.stmt.slice((j+1)..-1),"columns_all") if all
+    cols
+  end
+  
+  
+  def columns_select(all=false)
+    cols = TokenStream.new
+    self.list_select_expressions.each{|list|
+      cols += name_picker(list,(all ? "columns_all" :"columns_select")){|i,prev,nxt| 
+                    ((nxt == nil) or (nxt.token_value != '=')) and
+                    ((i < 1) or (not ['AS',RIGHT_PARAN,'END'].include?(prev.token_value))) and
+                    ((i < 1) or (not [:Id,:String,:Number,:HostVariable].include?(prev.token_type)))
+               }
+    }
+    cols
+  end
+  
+  def columns_all
+    all_cols = TokenStream.new
+    return all_cols if ['DROP','TRUNCATE'].include?(self.typ)
+
+    all_cols += columns_select(true)   + columns_insert(true) +
+                columns_update(true)   + columns_from(true)   +
+                columns_join_on(true)  + columns_where(true)  + 
+                columns_group_by(true) + columns_having(true) + 
+                columns_order_by() 
+
+    list_union.each{|st|
+      n = 0
+      st.each_with_index{|tok,i| (n=i; break) if tok.token_value == 'SELECT'}
+      ss = TSQLStmt.new(st.slice(n..-1))
+      all_cols += ss.columns_select(true)   + ss.columns_from(true)   +
+                  ss.columns_join_on(true)  + ss.columns_where(true)  + 
+                  ss.columns_group_by(true) + ss.columns_having(true) + 
+                  ss.columns_order_by() 
+    }
+    all_cols 
+  end
+  
+  def table_into;   TokenStream.new((self.typ == 'INTO') ? [clause_into[1]] : []);  end
+  def table_update; TokenStream.new((self.typ == 'UPDATE') ? [@stmt[1]] : []); end
+  def table_truncate; TokenStream.new((self.typ == 'TRUNCATE') ? [@stmt[2]] : []); end
+  def table_create; TokenStream.new(((self.typ == 'CREATE') and 
+                                    (self.typ == 'TABLE')) ? [@stmt[2]] : []); end
+  def table_alter; TokenStream.new(((self.typ == 'ALTER') and 
+                                    (@stmt[1].token_value == 'TABLE')) ? [@stmt[2]] : []); end
+  def tables_drop
+    tbls = TokenStream.new
+    tbls = TokenStream.new(@stmt.select{|tok| tok.token_type == :Id}) if ((self.typ == 'DROP') and @stmt[1] and (@stmt[1].token_value == 'TABLE')) 
+    tbls
+  end
+
+  def tables_from(all=false)
+    tbls = name_picker(clause_from,(all ? "tables_all" : "tables_from")){|i,prev| 
+                ((i < 1) or (not ['AS',RIGHT_PARAN].include?(prev.token_value))) and
+                ((i < 1) or (prev.token_type != :Id))
+           } 
+    tbls
+  end
+  
+  def tables_where(all=false)
+    tbls = TokenStream.new
+    list_sub_selects(clause_where).each{|ss|
+      tbls += TSQLStmt.new(ss.slice(1...-1)).tables_all
+    }
+    tbls
+  end
+  
+  def table_insert 
+    tbl = []
+    if (self.typ == 'INSERT')
+      tbl = ((@stmt[1].token_value == 'INTO') ? [@stmt[2]] : [@stmt[1]])
+    end
+    TokenStream.new(tbl)
+  end
+  
+  def table_delete    
+    tbl = []
+    if (self.typ == 'DELETE')
+      tbl = ((@stmt[1].token_value == 'FROM') ? [@stmt[2]] : [@stmt[1]])
+    end
+    TokenStream.new(tbl)
+  end
+
+  def tables_join(all=false)
+    tbls = TokenStream.new
+    clause_join.each{|list|
+   
+    tbls += name_picker(list,(all ? "tables_all" : "tables_join")){|i,prev,nxt| 
+                ((i < 1) or (not ['AS',RIGHT_PARAN].include?(prev.token_value))) and
+                ((i < 1) or (prev.token_type != :Id)) and 
+                (i < list.to_arr.index('ON'))
+
+           } 
+    }
+    tbls
+  end
+  
+  def tables_all(kind=nil)
+    mod_tbls = table_insert + table_delete + table_update + table_into + table_create +
+               tables_drop  + table_truncate + table_alter
+    ref_tbls = tables_where + tables_from(true) + tables_join(true)
+    list_union.each{|st|
+      n = 0
+      st.each_with_index{|tok,i| (n=i; break) if tok.token_value == 'SELECT'}
+      ss = TSQLStmt.new(st.slice(n..-1))
+      ref_tbls += ss.tables_from(true) + ss.tables_join(true)
+    }
+    all_tbls = mod_tbls + ref_tbls
+    (kind ? ((kind == 'mod') ? mod_tbls : ref_tbls) : all_tbls)
+  end
+  
+  def tables_referred;    tables_all('ref');  end
+  def tables_modified;   tables_all('mod');  end
+  
+  def clause_into
+
+    into = TokenStream.new
+
+    if @stmt
+      n = nil
+      @stmt.each_with_index{|tok,i|
+        (n = i; break) if tok.token_value == 'INTO'
+      }
+      into = self.stmt.slice(n..(n+1)) if n       
+    end
+    
+    into
+    
+  end
+
+  def clause_order_by
+    order_by = TokenStream.new
+    return order_by  unless @stmt
+    
+    n = self.stmt.find_matching_word('ORDER')
+    order_by = self.stmt.slice(n..-1) if n > 0 
+    order_by
+  end
+   
+  def clause_from     
+    k,n = clause('FROM')
+    if n > 0
+      k.pop if k[-1].token_value == 'OUTER'
+      k.pop if ['CROSS','INNER','LEFT','RIGHT','FULL'].include?(k[-1].token_value)
+    end
+    k
+  end
+
+  def columns_from(all=false)
+    name_picker(clause_from,(all ? "columns_all" : "")){|i,prev| false } 
+  end
+
+  def alias_columns(all=false)
+    cols = TokenStream.new
+    self.list_select_expressions.each{|list|
+      cols += name_picker(list)
+    }
+    cols - columns_select
+  end
+    
+  
+  def alias_from(all=false)
+    tbls = name_picker(clause_from)
+    tbls - tables_from(all)
+  end
+
+  def alias_hash(of="ALL")
+    case of.upcase
+    when "FROM"
+      name_alias  = name_picker(clause_from).to_arr
+      names = tables_from.to_arr
+    when "JOIN"
+      name_alias = []
+      clause_join.each{|list|
+        name_alias  += name_picker(list).to_arr
+      }
+      names = tables_join.to_arr    
+    when "ALL"
+      name_alias  = name_picker(clause_from).to_arr 
+      clause_join.each{|list|
+        name_alias += name_picker(list).to_arr
+      }
+      names = (tables_from + tables_join).to_arr    
+    end
+    
+    a_hash={}
+
+    names.each{|nm|
+      i = name_alias.index(nm)
+      a = nil
+      a = name_alias[i+1] if i and (not names.include?(name_alias[i+1]))
+      a_hash[nm] = a if a
+    }
+    
+    a_hash
+  end
+  
+  def list_join(join_cl,prev)
+  
+    join_lyst = []
+    return join_lyst if join_cl.length == 0
+    
+    n = join_cl.find_matching_word('JOIN',1)
+    
+    while n > 0
+      j = join_cl.slice(0..n)
+      if ((prev.length > 0) and (prev[-1].token_value == 'OUTER'))
+        j.unshift(prev[-1])
+        prev = prev.slice(0...-1)
+      end
+      
+      if ((prev.length > 0) and (['INNER','LEFT','RIGHT','CROSS','FULL'].include?(prev[-1].token_value)))
+        j.unshift(prev[-1])
+      end
+
+      prev = j
+      
+      j = j.slice(0...-1) if (j[-1].token_value == 'OUTER')
+      j = j.slice(0...-1) if (['INNER','LEFT','RIGHT','CROSS','FULL'].include?(j[-1].token_value))
+      join_lyst << TokenStream.new(j) 
+       
+
+      join_cl = join_cl.slice((n+1)..-1)
+      n = join_cl.find_matching_word('JOIN',1)
+    end
+
+    if ((prev.length > 0) and (prev[-1].token_value == 'OUTER'))
+      join_cl.unshift(prev[-1])
+      prev = prev.slice(0...-1)
+    end
+
+    if ((prev.length > 0) and (['INNER','LEFT','RIGHT','CROSS','FULL'].include?(prev[-1].token_value)))
+      join_cl.unshift(prev[-1])
+    end
+    
+    join_lyst << join_cl
+    join_lyst
+    
+  end
+  
+  def clause_join
+  
+    return [] unless @stmt
+   
+    join,n = clause('JOIN')
+    prev = ((n > 1) ? @stmt.slice((n-2)..(n-1)) : [])
+    list_join(join,prev)
+
+  end
+
+  def columns_join_on(all=false)
+    cols = TokenStream.new
+    clause_join.each{|list|
+      cols += name_picker(list,(all ? "columns_all" : "columns_join_on")){|i,prev| 
+                    ((i < 1) or (not ['JOIN','AS',RIGHT_PARAN].include?(prev.token_value))) and
+                    ((i < 1) or (prev.token_type != :Id))  and
+                    (i > list.to_arr.index('ON'))
+               }
+    }
+    cols
+  end
+
+ 
+  def alias_join(all=false)
+    tbls = TokenStream.new
+    clause_join.each{|list| tbls += name_picker(list)}
+    tbls - tables_join(all) - columns_join_on(all)
+  end
+   
+  def clause_where;    k,n = clause('WHERE');  k; end
+  def columns_where(all=false); name_picker(clause_where,(all ? "columns_all" : "columns_where"));   end
+  
+  def clause_group_by;  k,n = clause('GROUP'); k; end
+  def columns_group_by(all=false); name_picker(clause_group_by,(all ? "columns_all" : "")) end
+  
+  def clause_having; k,n = clause('HAVING');   k; end
+  def columns_having(all=false); name_picker(clause_having,(all ? "columns_all" : "")) end
+
+  def clause_set;    k,n = clause('SET');      k; end
+  
+  def columns_update(all=false)
+    cols = TokenStream.new
+    return cols unless self.typ == 'UPDATE'
+    if all
+      cols += name_picker(clause_set,"columns_all")
+    else
+      cols += name_picker(clause_set,"columns_update"){|i,prev,nxt|
+         ((i < 1) or (['SET',','].include?(prev.token_value))) and
+         ((nxt == nil) or (nxt.token_value == '='))
+      }
+    end
+
+    cols
+  end
+
+  def list_union
+  
+    return [] unless @stmt
+    
+    union_lyst = []
+    
+    n = self.stmt.find_matching_word('UNION')
+    return union_lyst if n == 0
+    
+    union_cl = self.stmt.slice(n..-1)
+    n = union_cl.find_matching_word('UNION',1)
+    while n > 0
+      u = union_cl.slice(0..n)
+      union_lyst << u
+      union_cl = union_cl.slice((n+1)..-1)
+      n = union_cl.find_matching_word('UNION',1)
+    end
+    
+    n = union_cl.find_matching_word('ORDER',1)
+    union_lyst << ((n > 0) ? union_cl.slice(0..n) : union_cl)
+    
+    union_lyst
+    
+  end
+  
+  
+  def columns_order_by
+    # Does not handle case ... end 
+    clause_order_by.select{|tok| 
+      (not ['ORDER','BY',',','ASC','DESC'].include?(tok.token_value)) and (tok.token_type == :Id)
+      #((tok.token_type == :Number) or (tok.token_type == :Id))
+    }
+  end
+
+  def list_sub_selects(st=nil)
+    return [] unless @stmt
+    
+    sub_selects = []
+    
+    ss = st || self.stmt
+    begin
+      n,prev = nil,nil
+      
+      # Find the first occurance of SELECT
+      ss.each_with_index{|tok,i|
+        (n = i; break) if ((tok.token_value == 'SELECT') and prev and (prev.token_value == LEFT_PARAN))
+        prev = tok
+      }
+
+      if n
+        m = ss.find_matching_paran(n-1)
+        sub_selects << ss.slice((n-1)...(n+m))
+        ss = ss.slice((n+m)..-1)
+      end
+
+    end while n
+    
+    sub_selects
+  end
+  
+  def method_missing(methId,*args)
+    str = methId.id2name
+    str = ((str[-1] == ?s) ? str.chop : (str + 's'))
+    self.send(str,*args) if self.respond_to?(str)
+  end
+  alias column_aliases alias_columns
+  alias from_aliases   alias_from
+  alias join_aliases   alias_join
+  
+  alias into     clause_into
+  alias from     clause_from
+  alias join_on  clause_join
+  alias where    clause_where
+  alias group_by clause_group_by
+  alias having   clause_having
+  alias order_by clause_order_by
+  alias set      clause_set
+  
+  alias into_table     table_into
+  alias delete_table   table_delete
+  alias insert_table   table_insert
+  alias update_table   table_update
+  alias truncate_table table_truncate
+  alias create_table   table_create
+  alias alter_table    table_alter
+  
+  alias drop_tables  tables_drop
+  alias from_tables  tables_from
+  alias join_tables  tables_join
+  alias all_tables   tables_all
+  alias where_tables tables_where
+  
+  alias referred_tables tables_referred
+  alias modified_tables tables_modified
+  
+  alias all_columns      columns_all
+  alias select_columns   columns_select
+  alias where_columns    columns_where
+  alias group_by_columns columns_group_by
+  alias having_columns   columns_having
+  alias order_by_columns columns_order_by
+  alias join_columns     columns_join_on
+  alias insert_columns   columns_insert
+  alias update_columns   columns_update
+end
+
+
+
+