sparkql 0.1.8

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+lib/sparkql/*.output
+pkg/*

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use ree-1.8.7-2011.03
+

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://gems.dev.fbsdata.com/public" 
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in sparkapi_parser.gemspec
+gemspec

data/README.md

@@ -0,0 +1,54 @@
+SparkQL query language parser
+=====================
+This gem contains the syntax parser for processing spark api filter queries into manageable 
+expressions. To get an overview of the language syntax-wise, refer to the following files:
+
+* lib/sparkql/parser.y   # BNF Grammar
+* lib/sparkql/token.rb   # Token matching rules
+
+Installation
+-------------
+
+Add the gem to your gemfile:
+
+Gemfile
+	gem 'sparkql', '~> 0.0.1' 
+
+When completed, run 'bundle install'.
+
+
+Usage
+-------------
+See test/unit/parser_test.rb for generic parsing examples. In most cases an extended parser is 
+needed to do anything of significance, such as the postgres and db2 search implementations in the
+API.
+
+Here is a basic example:
+
+	expressions = Parser.new.parse("Hello Eq 'World')
+
+The return value will be an array with one expression element containing the query information:
+
+	{
+	  :field => "Hello",
+	  :type => :character,
+	  :value => "'World'",
+	  :operator => 'Eq'
+	  # ...
+	}
+
+
+Development
+-------------
+The parser is based on racc, a yacc like LR parser that is a part of the ruby runtime.  The grammar 
+is located at lib/sparkql/parser.y and is compiled as part of the test process.  Refer to the 
+Rakefile for details. When modifying the grammar, please checkin BOTH the parser.y and parser.rb 
+files.
+
+Debugging grammar issues can be done by hand using the "racc" command. For example, a dump of the 
+parser states (and conflicts) can be generated via
+
+	racc -o lib/sparkql/parser.rb lib/sparkql/parser.y -v  # see lib/sparkql/parser.output
+
+The rails/journey project was an inspiration for this gem. Look it up on github for reference.
+

data/Rakefile

@@ -0,0 +1,18 @@
+require "rubygems"
+require 'rubygems/user_interaction'
+require 'flexmls_gems/tasks'
+require 'flexmls_gems/tasks/test_unit'
+require 'flexmls_gems/tasks/rdoc'
+
+rule '.rb' => '.y' do |t|
+  sh "racc -l -o #{t.name} #{t.source}"
+end
+
+desc "Compile the racc parser from the grammar"
+task :compile => "lib/sparkql/parser.rb"
+
+Rake::Task[:test].prerequisites.unshift "lib/sparkql/parser.rb"
+
+desc 'Default: run unit tests.'
+task :default => :test
+

data/VERSION

@@ -0,0 +1 @@
+0.1.8

data/lib/sparkql/errors.rb

@@ -0,0 +1,81 @@
+module Sparkql
+
+class ErrorsProcessor
+  attr_accessor :errors
+
+  def initialize( errors )
+    @errors = errors || []
+  end
+
+  # true if the error stack contains at least one error
+  def errors?
+    @errors.size > 0
+  end
+
+  # true if there is at least one error of status :status in the error stack
+  def errors_by_status?( status )
+    @errors.each do | error |
+      return true if status == error.status
+    end
+    false
+  end
+
+  # true if there is at least one :fatal error in the error stack
+  def fatal_errors?
+    errors_by_status? :fatal
+  end
+
+  # true if there is at least one :dropped error in the error stack
+  def dropped_errors?
+    errors_by_status? :dropped
+  end
+
+  # true if there is at least one :recovered error in the error stack
+  def recovered_errors?
+    errors_by_status? :recovered
+  end
+
+end
+
+class ParserError
+  attr_accessor :token, :expression, :message, :status, :recovered_as
+
+  def initialize(error_hash=nil)
+    error_hash = {} if error_hash.nil?
+    @token = error_hash[:token]
+    @expression = error_hash[:expression]
+    @message = error_hash[:message]
+    @status = error_hash[:status]
+    @recovered_as = error_hash[:recovered_as]
+    self.syntax= error_hash[:syntax] == false ? false : true
+  end
+  
+  def syntax=(syntax_error)
+    @syntax = syntax_error
+  end
+
+  def syntax?
+    @syntax
+  end
+
+  def to_s
+    str = case @status
+          # Do nothing. Dropping the expressions isn't special
+          when :dropped then "Dropped: " 
+          # Fatal errors cannot be recovered from, and should cause anaylisis or
+          # compilation to stop.
+          when :fatal then "Fatal: "
+          # Recovered errors are those that are syntatically
+          # or symantically incorrect, but are ones that we could "guess" at the
+          # intention
+          when :recovered then
+            "Recovered as #{@recovered_as}: "
+          else ""
+          end
+    str += "<#{@token}> in " unless @token.nil?
+    str += "<#{@expression}>: #{@message}."
+    str
+  end
+end
+
+end

data/lib/sparkql/expression_state.rb

@@ -0,0 +1,20 @@
+# Custom fields need to add a table join to the customfieldsearch table when AND'd together, 
+# but not when they are OR'd. This class maintains the state for all custom field expressions
+# lets the parser know when to do either.
+class Sparkql::ExpressionState
+  
+  def initialize
+    @expressions = []
+    @last_conjunction = "And" # always start with a join
+  end
+  
+  def push(expression)
+    @expressions << expression
+    @last_conjunction = expression[:conjunction]
+  end
+  
+  def needs_join?
+    return @expressions.size == 1 || "And" == @last_conjunction
+  end
+  
+end

data/lib/sparkql/function_resolver.rb

@@ -0,0 +1,106 @@
+require 'time'
+
+# Binding class to all supported function calls in the parser. Current support requires that the 
+# resolution of function calls to happen on the fly at parsing time at which point a value and 
+# value type is required, just as literals would be returned to the expression tokenization level.
+#
+# Name and argument requirements for the function should match the function declaration in 
+# SUPPORTED_FUNCTIONS which will run validation on the function syntax prior to execution.
+class Sparkql::FunctionResolver
+  SECONDS_IN_DAY = 60 * 60 * 24
+  
+  SUPPORTED_FUNCTIONS = {
+    :days => {
+      :args => [:integer],
+      :return_type => :datetime
+    }, 
+    :now => { 
+      :args => [],
+      :return_type => :datetime
+    }
+  }
+  
+  # Construct a resolver instance for a function
+  # name: function name (String)
+  # args: array of literal hashes of the format {:type=><literal_type>, :value=><escaped_literal_value>}.
+  #       Empty arry for functions that have no arguments.
+  def initialize(name, args)
+    @name = name
+    @args = args
+    @errors = []
+  end
+  
+  # Validate the function instance prior to calling it. All validation failures will show up in the
+  # errors array. 
+  def validate()
+    name = @name.to_sym
+    unless support.has_key?(name)
+      @errors << Sparkql::ParserError.new(:token => @name, 
+        :message => "Unsupported function call '#{@name}' for expression",
+        :status => :fatal )
+      return
+    end
+    required_args = support[name][:args]
+    unless required_args.size == @args.size
+      @errors << Sparkql::ParserError.new(:token => @name, 
+        :message => "Function call '#{@name}' requires #{required_args.size} arguments",
+        :status => :fatal )
+      return
+    end
+    
+    count = 0
+    @args.each do |arg|
+      unless arg[:type] == required_args[count]
+        @errors << Sparkql::ParserError.new(:token => @name, 
+          :message => "Function call '#{@name}' has an invalid argument at #{arg[:value]}",
+          :status => :fatal )
+      end
+      count +=1
+    end
+  end
+  
+  def return_type
+    supported[@name.to_sym][:return_type]
+  end
+  
+  def errors
+    @errors
+  end
+  
+  def errors?
+    @errors.size > 0
+  end
+  
+  def support
+    SUPPORTED_FUNCTIONS
+  end
+  
+  # Execute the function
+  def call()
+    real_vals = @args.map { |i| i[:value]}
+    self.send(@name.to_sym, *real_vals)
+  end
+  
+  protected 
+  
+  # Supported function calls
+  
+  # Offset the current timestamp by a number of days
+  def days(num)
+    # date calculated as the offset from midnight tommorrow. Zero will provide values for all times 
+    # today.
+    d = Date.today + num
+    {
+      :type => :date,
+      :value => d.to_s
+    }
+  end
+  
+  # The current timestamp
+  def now()
+    {
+      :type => :datetime,
+      :value => Time.now.iso8601
+    }
+  end
+end

data/lib/sparkql/lexer.rb

@@ -0,0 +1,114 @@
+class Sparkql::Lexer < StringScanner
+  include Sparkql::Token
+
+  def initialize(str)
+    str.freeze
+    super(str, false) # DO NOT dup str
+    @level = 0
+    @block_group_identifier = 0
+    @expression_count = 0
+  end
+  
+  # Lookup the next matching token
+  # 
+  # TODO the old implementation did value type detection conversion at a later date, we can perform
+  # this at parse time if we want!!!!
+  def shift
+    token = case
+      when value = scan(SPACE)
+        [:SPACE, value]
+      when value = scan(LPAREN)
+        levelup
+        [:LPAREN, value]
+      when value = scan(RPAREN)
+        # leveldown do this after parsing group
+        [:RPAREN, value]
+      when value = scan(/\,/)
+        [:COMMA,value]
+      when value = scan(NULL)
+        literal :NULL, "NULL"
+      when value = scan(STANDARD_FIELD)
+        check_standard_fields(value)
+      when value = scan(DATETIME)
+        literal :DATETIME, value
+      when value = scan(DATE)
+        literal :DATE, value
+      when value = scan(DECIMAL)
+        literal :DECIMAL, value
+      when value = scan(INTEGER)
+        literal :INTEGER, value
+      when value = scan(CHARACTER)
+        literal :CHARACTER, value
+      when value = scan(BOOLEAN)
+        literal :BOOLEAN, value
+      when value = scan(KEYWORD)
+        check_keywords(value)
+      when value = scan(CUSTOM_FIELD)
+        [:CUSTOM_FIELD,value]
+      when empty?
+        [false, false] # end of file, \Z don't work with StringScanner
+      else
+        [:UNKNOWN, "ERROR: '#{self.string}'"]
+    end
+    #value.freeze
+    token.freeze
+  end
+  
+  def check_reserved_words(value)
+    u_value = value.capitalize
+    if OPERATORS.include?(u_value)
+      [:OPERATOR,u_value]
+    elsif CONJUNCTIONS.include?(u_value)
+      [:CONJUNCTION,u_value]
+    else
+      [:UNKNOWN, "ERROR: '#{self.string}'"]
+    end
+  end
+  
+  def check_standard_fields(value)
+    result = check_reserved_words(value)
+    if result.first == :UNKNOWN
+      @last_field = value
+      result = [:STANDARD_FIELD,value]
+    end
+    result
+  end
+
+  def check_keywords(value)
+    result = check_reserved_words(value)
+    if result.first == :UNKNOWN
+      result = [:KEYWORD,value]
+    end
+    result
+  end
+  
+  def level
+    @level
+  end
+
+  def block_group_identifier
+    @block_group_identifier
+  end
+  
+  def levelup
+    @level += 1
+    @block_group_identifier += 1
+  end
+  
+  def leveldown
+    @level -= 1
+  end
+  
+  def literal(symbol, value)
+    node = {
+      :type => symbol.to_s.downcase.to_sym,
+      :value => value
+    }
+    [symbol, node]
+  end
+  
+  def last_field
+    @last_field
+  end
+  
+end

data/lib/sparkql/parser.rb

@@ -0,0 +1,268 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by Racc 1.4.8
+# from Racc grammer file "".
+#
+
+require 'racc/parser.rb'
+
+# $Id$
+module Sparkql
+  class Parser < Racc::Parser
+
+  include Sparkql::ParserTools
+  include Sparkql::ParserCompatibility
+  
+##### State transition tables begin ###
+
+racc_action_table = [
+    35,     4,    31,     5,     6,    14,    15,    16,    18,    19,
+    21,    23,    26,    32,    14,    15,    16,    18,    19,    21,
+    23,    14,    15,    16,    18,    19,    21,    23,    39,    13,
+    12,     4,    40,     5,     6,     4,    10,     5,     6,    13,
+    29,    28,    14,    15,    16 ]
+
+racc_action_check = [
+    32,    13,    22,    13,    13,    32,    32,    32,    32,    32,
+    32,    32,    10,    25,    10,    10,    10,    10,    10,    10,
+    10,    40,    40,    40,    40,    40,    40,    40,    36,     8,
+     7,     4,    36,     4,     4,     0,     2,     0,     0,    11,
+    12,    11,    31,    31,    31 ]
+
+racc_action_pointer = [
+    30,   nil,    33,   nil,    26,   nil,   nil,    30,    25,   nil,
+     3,    35,    40,    -4,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,    -8,   nil,   nil,     8,   nil,   nil,   nil,   nil,
+   nil,    31,    -6,   nil,   nil,   nil,    22,   nil,   nil,   nil,
+    10,   nil ]
+
+racc_action_default = [
+    -2,    -4,   -30,    -6,   -30,    -9,   -10,   -30,    -1,    -3,
+   -30,   -30,   -30,   -30,   -23,   -24,   -25,    -5,   -26,   -27,
+   -11,   -28,   -12,   -29,   -13,   -30,   -16,   -21,    -8,    42,
+    -7,   -30,   -30,   -22,   -19,   -14,   -30,   -17,   -20,   -15,
+   -30,   -18 ]
+
+racc_goto_table = [
+    27,    37,     8,    22,    20,    17,    11,    24,    25,    41,
+    36,    30,     7,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,    33 ]
+
+racc_goto_check = [
+    14,    13,     2,     9,     8,     6,     2,    10,    11,    13,
+    12,     3,     1,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,    14 ]
+
+racc_goto_pointer = [
+   nil,    12,     2,    -2,   nil,   nil,    -5,   nil,    -6,    -7,
+    -3,    -2,   -22,   -31,   -10 ]
+
+racc_goto_default = [
+   nil,   nil,   nil,     9,     1,     2,   nil,     3,    34,   nil,
+   nil,   nil,   nil,   nil,    38 ]
+
+racc_reduce_table = [
+  0, 0, :racc_error,
+  1, 19, :_reduce_none,
+  0, 19, :_reduce_2,
+  1, 20, :_reduce_none,
+  1, 20, :_reduce_none,
+  3, 21, :_reduce_5,
+  1, 21, :_reduce_none,
+  3, 22, :_reduce_7,
+  3, 25, :_reduce_8,
+  1, 23, :_reduce_none,
+  1, 23, :_reduce_none,
+  1, 24, :_reduce_none,
+  1, 24, :_reduce_none,
+  1, 24, :_reduce_none,
+  3, 28, :_reduce_14,
+  4, 28, :_reduce_15,
+  1, 29, :_reduce_none,
+  1, 30, :_reduce_none,
+  3, 30, :_reduce_none,
+  1, 31, :_reduce_none,
+  1, 31, :_reduce_none,
+  1, 27, :_reduce_none,
+  3, 27, :_reduce_22,
+  1, 32, :_reduce_none,
+  1, 32, :_reduce_none,
+  1, 32, :_reduce_none,
+  1, 26, :_reduce_none,
+  1, 26, :_reduce_none,
+  1, 26, :_reduce_none,
+  1, 26, :_reduce_none ]
+
+racc_reduce_n = 30
+
+racc_shift_n = 42
+
+racc_token_table = {
+  false => 0,
+  :error => 1,
+  :UMINUS => 2,
+  :OPERATOR => 3,
+  :CONJUNCTION => 4,
+  :LPAREN => 5,
+  :RPAREN => 6,
+  :STANDARD_FIELD => 7,
+  :CUSTOM_FIELD => 8,
+  :KEYWORD => 9,
+  :COMMA => 10,
+  :INTEGER => 11,
+  :DECIMAL => 12,
+  :CHARACTER => 13,
+  :DATE => 14,
+  :DATETIME => 15,
+  :BOOLEAN => 16,
+  :NULL => 17 }
+
+racc_nt_base = 18
+
+racc_use_result_var = true
+
+Racc_arg = [
+  racc_action_table,
+  racc_action_check,
+  racc_action_default,
+  racc_action_pointer,
+  racc_goto_table,
+  racc_goto_check,
+  racc_goto_default,
+  racc_goto_pointer,
+  racc_nt_base,
+  racc_reduce_table,
+  racc_token_table,
+  racc_shift_n,
+  racc_reduce_n,
+  racc_use_result_var ]
+
+Racc_token_to_s_table = [
+  "$end",
+  "error",
+  "UMINUS",
+  "OPERATOR",
+  "CONJUNCTION",
+  "LPAREN",
+  "RPAREN",
+  "STANDARD_FIELD",
+  "CUSTOM_FIELD",
+  "KEYWORD",
+  "COMMA",
+  "INTEGER",
+  "DECIMAL",
+  "CHARACTER",
+  "DATE",
+  "DATETIME",
+  "BOOLEAN",
+  "NULL",
+  "$start",
+  "target",
+  "expressions",
+  "expression",
+  "conjunction",
+  "field",
+  "condition",
+  "group",
+  "literal",
+  "literal_list",
+  "function",
+  "function_name",
+  "function_args",
+  "function_arg",
+  "literals" ]
+
+Racc_debug_parser = false
+
+##### State transition tables end #####
+
+# reduce 0 omitted
+
+# reduce 1 omitted
+
+def _reduce_2(val, _values, result)
+ result = 0 
+    result
+end
+
+# reduce 3 omitted
+
+# reduce 4 omitted
+
+def _reduce_5(val, _values, result)
+ result = tokenize_expression(val[0], val[1],val[2]) 
+    result
+end
+
+# reduce 6 omitted
+
+def _reduce_7(val, _values, result)
+ result = tokenize_conjunction(val[0], val[1],val[2]) 
+    result
+end
+
+def _reduce_8(val, _values, result)
+ result = tokenize_group(val[1]) 
+    result
+end
+
+# reduce 9 omitted
+
+# reduce 10 omitted
+
+# reduce 11 omitted
+
+# reduce 12 omitted
+
+# reduce 13 omitted
+
+def _reduce_14(val, _values, result)
+ result = tokenize_function(val[0], []) 
+    result
+end
+
+def _reduce_15(val, _values, result)
+ result = tokenize_function(val[0], val[2]) 
+    result
+end
+
+# reduce 16 omitted
+
+# reduce 17 omitted
+
+# reduce 18 omitted
+
+# reduce 19 omitted
+
+# reduce 20 omitted
+
+# reduce 21 omitted
+
+def _reduce_22(val, _values, result)
+ result = tokenize_multiple(val[0], val[2]) 
+    result
+end
+
+# reduce 23 omitted
+
+# reduce 24 omitted
+
+# reduce 25 omitted
+
+# reduce 26 omitted
+
+# reduce 27 omitted
+
+# reduce 28 omitted
+
+# reduce 29 omitted
+
+def _reduce_none(val, _values, result)
+  val[0]
+end
+
+  end   # class Parser
+  end   # module Sparkql
+
+
+# END PARSER