vsql_parser 0.2

data/MIT_LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010 LeadTune
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/formatter.rb

@@ -0,0 +1,81 @@
+require_relative "./vsql_node_extensions"
+
+module VSql
+  module Formatter
+    extend self
+
+    def indent(value, amount = 2, first_line = false)
+      spacer = (" " * amount)
+      (first_line ? spacer : "") + value.gsub("\n", "\n#{spacer}")
+    end
+    
+    def eager_indent(value, amount = 2)
+      indent(value, amount, true)
+    end
+    
+    def quote_alias_if_needed(a)
+      if a.match(/[^a-z0-9_]/)
+        '"' + a + '"'
+      else
+        a
+      end
+    end
+
+    DEFAULT_FORMATTER = lambda { |n|
+      n.pieces.map { |p|
+        p.is_a?(String) ? p : format_node(p)
+      }.join
+    }
+    
+    NODE_FORMATTERS = {
+      SelectExpression => lambda { |n|
+        formatted_expr = format_node(n.elements[0])
+        if n.alias_node
+          expr_alias = n.alias_node.text_value
+          [formatted_expr, " AS ", quote_alias_if_needed(expr_alias)].join
+        else
+          formatted_expr
+        end
+      },
+      SelectStatement => lambda { |n|
+        statements = n.match(SelectExpression, Query)
+        "\nSELECT\n" + eager_indent(statements.map { |s| format_node(s) }.join(",\n"))
+      },
+      LimitStatement => lambda { |n|
+        "\nLIMIT" + n.elements[1..-1].map(&:text_value).join
+      },
+      WhereStatement  => lambda { |n|
+        exprs = n.elements.select {|e| e.is_a?(Expression) }
+        "\nWHERE " + indent(exprs.map {|e| DEFAULT_FORMATTER[e] }.join)
+      },
+      FromStatement   => lambda { |n|
+        expressions = n.match(FromExpression, Query)
+        "\nFROM " + indent(expressions.map {|e| format_node(e) }.join("\n"))
+      },
+      JoinStatement   => lambda { |n|
+        join_keyword = n.match(JoinKeyword, Query)[0]
+        from, criteria = n.elements.select {|e| e.is_a?(Expression) }
+        ["\n",
+         join_keyword.text_value.upcase,
+         " ",
+         indent(format_node(from)),
+         " ON ",
+         indent(format_node(criteria))].join
+      },
+      OrderByStatement   => lambda { |n|
+        exprs = n.elements.select {|e| e.is_a?(OrderByExpression) }
+        "\nORDER BY " + indent(exprs.map {|e| DEFAULT_FORMATTER[e] }.join)
+      }
+    }
+
+    FORMATTERS = Hash.new(DEFAULT_FORMATTER).update(NODE_FORMATTERS)
+
+    def format_node(node)
+      FORMATTERS[node.class][node]
+    end
+
+    def format(node)
+      format_node(node).split("\n").map(&:rstrip).join("\n").strip + "\n"
+    end
+  end
+end

data/lib/test_chamber.rb

@@ -0,0 +1,63 @@
+# In file parser.rb
+require_relative './vsql_parser.rb'
+
+module TestChamber
+  module Helpers
+    NORMAL_COLOR ||= 37
+    def colorize(color, output)
+      "\e[0;#{color}m#{output}\e[0;#{NORMAL_COLOR}m"
+    end
+  end
+
+  include Helpers
+  extend self
+
+  def pparse(sql, output_errors = true)
+    parse(sql, output_errors).tap { |q| q.prune! }
+  end
+
+  def parse(sql, output_errors = true)
+    parser = ::VSqlParser.parser
+    VSqlParser.parse(sql).tap do |tree|
+      # If the AST is nil then there was an error during parsing
+      # we need to report a simple error message to help the user
+      if tree.nil?
+        output_error(sql, parser) if output_errors
+        raise Exception, parser.failure_reason
+      end
+    end
+  end
+
+  private
+
+  def output_error(sql, parser)
+    fail_index = parser.max_terminal_failure_index
+    STDERR.flush
+    STDOUT.flush
+    STDERR.puts( "\n" +
+                 ((fail_index > 0) ? colorize(42, sql[0..(fail_index - 1)]) : "") +
+                 colorize(41, sql[(fail_index)..-1]) +
+                 "\n\n")
+
+    STDERR.flush
+  end
+
+  def clean_tree(root_node)
+    return if(root_node.elements.nil?)
+    root_node.elements.delete_if{|node| node.class.name == "Treetop::Runtime::SyntaxNode" }
+    root_node.elements.each {|node| self.clean_tree(node) }
+  end
+
+  def reload
+    Object.send(:remove_const, :SqlParser) rescue nil
+    Object.send(:remove_const, :Sql) rescue nil
+    Object.send(:remove_const, :VSql) rescue nil
+    TestChamber.send(:remove_const, :PARSER) rescue nil
+
+    load(File.join(VSQLPARSER_BASE_PATH, 'vsql_node_extensions.rb'))
+    load(File.join(VSQLPARSER_BASE_PATH, 'formatter.rb'))
+    Treetop.load(File.join(VSQLPARSER_BASE_PATH, 'vsql_parser.treetop'))
+    VSqlParser.extend(VSqlParserHelpers)
+    load(__FILE__)
+  end
+end

data/lib/vsql_node_extensions.rb

@@ -0,0 +1,246 @@
+module ScanHelpers
+  extend self
+  require 'strscan'
+  def gsub_replacements(string, pattern, replacement)
+    pattern = Regexp.new(Regexp.escape(pattern)) if pattern.is_a?(String)
+    [].tap do |matches|
+      scanner = StringScanner.new(string)
+      until scanner.eos?
+        return matches unless scanner.scan_until(pattern)
+        matches.push([(scanner.pos - scanner.matched_size)..(scanner.pos - 1),
+                      replacement.size - scanner.matched_size,
+                      replacement])
+      end
+    end
+  end
+end
+
+module Replaceability
+  def index_of(str)
+    e.text_value.index(str) + e.interval.first
+  end
+
+  def adjust_intervals!(idx, delta)
+    case
+    when @interval.include?(idx)
+      @interval = (@interval.first)...[@interval.first, @interval.last + delta].max
+    when @interval.first > idx
+      @interval = (@interval.first + delta)...(@interval.last + delta)
+    end
+    elements && elements.each { |e| e.adjust_intervals!(idx, delta) }
+    true
+  end
+
+  def gsub!(pattern, replacement)
+    ScanHelpers.gsub_replacements(text_value, pattern, replacement).reverse.each do |(range, delta, rep_str)|
+      end_idx = (@interval.min + range.max)
+      @input[(@interval.min + range.min)..end_idx] = rep_str
+      root.adjust_intervals!(end_idx, delta)
+    end
+  end
+end
+
+class Treetop::Runtime::SyntaxNode
+  def _pieces_with_gaps(cursor, elements, results = [])
+    return [cursor, results] if elements.nil? || elements.empty?
+    element, interval, next_elements = elements[0], elements[0].interval, elements[1..-1]
+    next_results = [*results,
+                    *(input[cursor...interval.first] if cursor != elements.first.interval.first),
+                    element]
+    _pieces_with_gaps(interval.last,
+                      next_elements,
+                      next_results)
+  end
+
+  def pieces
+    last_pos, pieces = _pieces_with_gaps(interval.first, elements)
+    if last_pos != interval.last
+      [input[last_pos...(interval.last)], *pieces]
+    else
+      pieces
+    end
+  end
+
+  def match(klass = Treetop::Runtime::SyntaxNode, skip = nil)
+    VSql::Helpers.find_elements(self, klass, skip)
+  end
+
+  def find(klass)
+    match(klass).first
+  end
+
+  def delete!
+    parent.elements.delete(self)
+  end
+
+  def vanilla?
+    (self.class == Treetop::Runtime::SyntaxNode) &&
+      (parent && parent.class == Treetop::Runtime::SyntaxNode || text_value.length == 0) &&
+      (elements.nil? || elements.all?(&:vanilla?))
+  end
+
+  def prune_if!(&block)
+    delete! if yield(self)
+  end
+
+  def prune!
+    es = match(Treetop::Runtime::SyntaxNode)
+    es.reverse.each do |e|
+      e.prune_if!(&:vanilla?)
+    end
+    self
+  end
+
+  def root
+    parent ? parent.root : self
+  end
+
+  def match_nearest(klass)
+    case
+    when parent.nil?
+      nil
+    when parent.is_a?(klass)
+      parent
+    else
+      parent.match_nearest(klass)
+    end
+  end
+
+  include Replaceability
+end
+
+module VSql
+  module Helpers
+    def self.find_elements(node, klass, skip_klass = nil)
+      results = []
+      return results unless node.elements
+      node.elements.each do |e|
+        case
+        when e.is_a?(klass)
+          results << e
+          results.concat(find_elements(e, klass, skip_klass))
+        when skip_klass && e.is_a?(skip_klass)
+          next
+        else
+          results.concat(find_elements(e, klass, skip_klass))
+        end
+      end
+      results
+    end
+  end
+
+  class VSqlSyntaxNode < Treetop::Runtime::SyntaxNode
+  end
+
+  class Operator < VSqlSyntaxNode
+  end
+
+  class Statement < VSqlSyntaxNode
+  end
+
+  class SelectStatement < Statement
+    def expressions
+      Helpers.find_elements(self, SelectExpression)
+    end
+  end
+
+  class SelectExpression < VSqlSyntaxNode
+    def expression_sql
+    end
+
+    def alias_node
+      @alias_node ||= Helpers.find_elements(self, Alias, Query).first
+    end
+
+    def root_nodes
+      elements[0].elements.select { |e| ! e.text_value.empty? }
+    end
+
+    def name
+      case
+      when alias_node
+        alias_node.text_value
+      when root_nodes.length == 1 && root_nodes.first.is_a?(Function)
+        root_nodes.first.name
+      when root_nodes.length == 1 && root_nodes.first.is_a?(FieldRef)
+        element =
+          Helpers.find_elements(self, FieldGlob).last ||
+          Helpers.find_elements(self, Name).last
+        element.text_value
+      else "?column?"
+      end
+    end
+  end
+
+  class NameExpression < VSqlSyntaxNode
+  end
+
+  class FromStatement < Statement
+  end
+
+  class FromExpression < VSqlSyntaxNode
+  end
+
+  class JoinStatement < Statement
+  end
+
+  class JoinKeyword < VSqlSyntaxNode
+  end
+
+  class WhereStatement < Statement
+  end
+
+  class OrderByStatement < Statement
+  end
+
+  class LimitStatement < Statement
+  end
+
+  class OrderByExpression < VSqlSyntaxNode
+  end
+
+  class Name < VSqlSyntaxNode
+  end
+
+  class FieldRef < VSqlSyntaxNode
+  end
+
+  class TablePart < VSqlSyntaxNode
+  end
+
+  class FieldGlob < VSqlSyntaxNode
+  end
+
+  class Alias < VSqlSyntaxNode
+  end
+
+  class Function < VSqlSyntaxNode
+    def name
+      elements[0].text_value
+    end
+  end
+
+  class Entity < VSqlSyntaxNode
+    # def to_array
+    #   return self.elements[0].to_array
+    # end
+  end
+
+  class Expression < VSqlSyntaxNode
+  end
+
+  class QuotedEntity < Entity
+  end
+
+  class Query < VSqlSyntaxNode
+    # def to_array
+    #   return self.elements.map {|x| x.to_array}
+    # end
+    # def select_statement
+    #   elements.detect { |e| e.is_a?(SelectStatement) }
+    # end
+  end
+
+  class SubQuery < VSqlSyntaxNode
+  end
+end

data/lib/vsql_parser.rb

@@ -0,0 +1,25 @@
+# In file parser.rb
+require 'treetop'
+require_relative './vsql_node_extensions.rb'
+require_relative './formatter.rb'
+
+VSQLPARSER_BASE_PATH ||= File.expand_path(File.dirname(__FILE__))
+
+module VSqlParserHelpers
+  def parser
+    @parser ||= VSqlParser.new
+  end
+
+  def parse(sql)
+    d_sql = sql.downcase
+    parser.parse(d_sql).tap do
+      d_sql.replace(sql)
+    end
+  end
+end
+
+# Find out what our base path is
+Treetop.load(File.join(VSQLPARSER_BASE_PATH, 'vsql_parser.treetop')) # <- This creates the VSqlParser class
+
+VSqlParser.extend(VSqlParserHelpers)
+

data/lib/vsql_parser.treetop

@@ -0,0 +1,237 @@
+grammar VSql
+
+  rule query
+    space*
+    select_statement
+    (space from_statement (space join_statement)*
+      (space where_statement)?
+      (space group_by_statement)?
+      (space having_statement)?
+      (space window_statement)*
+      (space order_by_statement)?
+      (space limit_statement)?
+      )?
+    (space* 'union' space query)?
+    (space / ';')*
+    <Query>
+  end
+
+  rule select_statement
+    'select' space (distinct_predicate space)? select_expressions <SelectStatement>
+  end
+
+  rule join_keyword
+    ('left' / 'outer' / 'inner' / 'right' / 'full' / space)* 'join' <JoinKeyword>
+  end
+
+  rule join_statement
+    join_keyword space expression (space alias)? space 'on' space expression <JoinStatement>
+  end
+
+  rule where_statement
+    'where' space expression <WhereStatement>
+  end
+
+  rule having_statement
+    'having' space expression
+  end
+
+  rule window_statement
+    'window' space name_expression space 'as' space window
+  end
+
+  rule group_by_statement
+    'group by' space expression (expression_separator expression)* <Statement>
+  end
+
+  rule order_by_statement
+    'order by' space order_by_expression (expression_separator order_by_expression)* <OrderByStatement>
+  end
+
+  rule limit_statement
+    'limit' space [0-9]+ <LimitStatement>
+  end
+
+  rule select_expressions
+    select_expression (expression_separator select_expression)* <Entity>
+  end
+
+  rule expression_separator
+    space* ',' space* <Entity>
+  end
+
+  rule order_by_expression
+    expression (space ('desc' / 'asc'))? <OrderByExpression>
+  end
+
+  rule select_expression
+    expression (space alias)? <SelectExpression>
+  end
+
+  rule alias
+    ('as' space)?
+    (
+      '"' ([^\"]+ <Alias>) '"'
+      /
+      !(keyword word_boundary) [\w]+ <Alias>
+    )
+  end
+
+  rule expression
+    prefix_modifier?
+    (
+      '(' query ')' <Query>
+      /
+      '(' space? expression space? ')' <Entity>
+      /
+      sub_expression)
+    (space? inline_window)?
+    ('::' [\w]+)? # cast
+    (space? set_operator space? (set))*
+    (space? operator space? expression)? <Expression>
+  end
+
+  rule prefix_modifier
+    'not' space
+    /
+    '-' space?
+  end
+
+  rule inline_window
+    'over' space (window / name_expression)
+  end
+
+  # poorman window parsing right now... we'll need to beef this up if we use windows with parentheses
+  rule window
+    '(' [^\)]* ')'
+  end
+
+  rule set
+    set_literal
+    /
+    '(' query ')'
+  end
+
+  rule distinct_predicate
+    'distinct' / 'all'
+  end
+
+  rule set_literal
+    '(' space? primitive_literal (space? ',' space? primitive_literal)* space? ')'
+  end
+
+  rule primitive_literal
+    interval_literal / numeric_literal / string_literal / date_literal
+  end
+
+  rule interval_literal
+    'interval' space "'" [\w ]+ "'"
+  end
+
+  rule date_literal
+    '{d' space+ "'" [^\'\}]+ "'}"
+  end
+
+  rule numeric_literal
+    [0-9]+ ("." [0-9]+)?
+  end
+
+  rule sub_expression
+    case_statement
+    /
+    function
+    /
+    primitive_literal
+    /
+    field_reference
+  end
+
+  rule set_operator
+    'not in' / 'in' &space
+  end
+
+  rule operator
+    [+\-/=\|><!]+
+    /
+    ('is not' / 'is' / 'like' / 'between' / 'and' / 'or') &space
+  end
+
+  rule case_statement
+    'case'
+    (space !('when') expression)?
+    (space 'when' space expression space 'then' space expression)*
+    (space 'else' space expression)?
+    space 'end' <Entity>
+
+  end
+
+  rule function
+    [\w]+ space* '(' (distinct_predicate space)? (expression / ',' / space)* ')' <Function>
+  end
+
+  rule string_literal
+    "'" [^\']+ "'"
+  end
+
+  rule name_expression
+    '"' ([^\"]+ <Name>) '"' <NameExpression>
+    /
+    ([\w]+ <Name>) <NameExpression>
+  end
+
+  rule field_glob
+    '*' <FieldGlob>
+  end
+
+  rule field_reference
+    (name_expression "." ' '* <TablePart>)? (name_expression / field_glob) <FieldRef>
+  end
+
+  rule from_statement
+    'from' space+ from_expression (expression_separator from_expression)* <FromStatement>
+  end
+
+  rule from_expression
+    ( name_expression
+      /
+      '(' query ')' <SubQuery>) (space alias)? <FromExpression>
+  end
+
+  # matches at least one space. Handles comments, too.
+  rule space
+    ([\s]+ / ('--' (!"\n" .)+ ))+
+  end
+
+  rule word_boundary
+    ![\w]
+  end
+
+  rule keyword
+    # 'select' / 'from' / 'inner' / 'outer' / 'full' / 'left' / 'right' / 'join' / 'on' / 'where' / 'group by' / 'order by' / 'having' / 'limit' / 'union'
+    (
+      'all' / 'analyse' / 'analyze' / 'and' / 'any' / 'array' / 'asc' / 'as' /
+      'binary' / 'both' /
+      'case' / 'cast' / 'check' / 'column' / 'constraint' / 'correlation' / 'create' / 'current_database' / 'current_date' / 'current_schema' / 'current_timestamp' / 'current_time' / 'current_user' /
+      'default' / 'deferrable' / 'desc' / 'distinct' / 'do' /
+      'else' / 'encoded' / 'end' / 'except' /
+      'false' / 'foreign' / 'for' / 'from' /
+      'grant' / 'grouped' / 'group' /
+      'having' /
+      'initially' / 'intersect' / 'intervalym' / 'interval' / 'into' / 'in' /
+      'join' /
+      'ksafe' /
+      'leading' / 'left' / 'limit' / 'localtimestamp' / 'localtime'
+      'match' /
+      'new' / 'not' / 'nullsequal' / 'null' /
+      'offset' / 'off' / 'old' / 'only' / 'on' / 'order' / 'or' /
+      'pinned' / 'placing' / 'primary' / 'projection' /
+      'references' /
+      'schema' / 'segmented' / 'select' / 'session_user' / 'some' / 'sysdate' /
+      'table' / 'then' / 'timeseries' / 'to' / 'trailing' / 'true' /
+      'unbounded' / 'union' / 'unique' / 'unsegmented' / 'user' / 'using' /
+      'when' / 'where' / 'window' / 'within' / 'with'
+    )
+    word_boundary
+  end
+
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: vsql_parser
+version: !ruby/object:Gem::Version
+  version: '0.2'
+  prerelease: 
+platform: ruby
+authors:
+- Tim Harper
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-04 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: &70201796873120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70201796873120
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &70201796872740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70201796872740
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70201796872140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 2.10.0
+  type: :development
+  prerelease: false
+  version_requirements: *70201796872140
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: &70201796871700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70201796871700
+description: Provides treetop grammar for Vertica SQL. (Most likely works with Postgres
+  SQL as well)
+email:
+- tim@redbrainlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/formatter.rb
+- lib/test_chamber.rb
+- lib/vsql_node_extensions.rb
+- lib/vsql_parser.rb
+- lib/vsql_parser.treetop
+- MIT_LICENSE
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.7
+signing_key: 
+specification_version: 3
+summary: Vertica SQL Parser
+test_files: []
+has_rdoc: