sparkql 0.1.8

data/lib/sparkql/parser.y

@@ -0,0 +1,93 @@
+# $Id$
+#
+# SparkQL grammar
+
+class Sparkql::Parser
+  prechigh
+    nonassoc UMINUS
+  preclow
+rule
+  target
+    : expressions
+    | /* none */ { result = 0 }
+    ;
+
+  expressions
+    : expression
+    | conjunction
+    ;
+    
+  expression
+    : field OPERATOR condition { result = tokenize_expression(val[0], val[1],val[2]) }
+    | group
+    ;
+  
+  conjunction
+    : expressions CONJUNCTION expression { result = tokenize_conjunction(val[0], val[1],val[2]) }
+    ;
+  
+  group
+  	: LPAREN expressions RPAREN { result = tokenize_group(val[1]) }
+  	;
+
+  field
+  	: STANDARD_FIELD
+  	| CUSTOM_FIELD
+  	;
+  
+  condition
+    : literal
+    | literal_list
+    | function
+    ;
+    
+  function
+    : function_name LPAREN RPAREN { result = tokenize_function(val[0], []) }
+    | function_name LPAREN function_args RPAREN { result = tokenize_function(val[0], val[2]) }
+    ;
+    
+  function_name
+    : KEYWORD
+    ;
+    
+  function_args
+    : function_arg
+    | function_args COMMA function_arg
+    ; 
+    
+  function_arg
+    : literal
+    | literals
+    ;
+    
+  literal_list
+    : literals
+    | literal_list COMMA literals { result = tokenize_multiple(val[0], val[2]) }
+    ;
+    
+  # Literals that support multiple
+  literals
+    : INTEGER
+    | DECIMAL
+    | CHARACTER
+    ;
+  
+  # Literals that support single only
+  literal
+    : DATE
+    | DATETIME
+    | BOOLEAN
+    | NULL
+    ;
+    
+end
+
+---- header
+# $Id$
+---- inner
+  include Sparkql::ParserTools
+  include Sparkql::ParserCompatibility
+  
+---- footer
+
+# END PARSER

data/lib/sparkql/parser_compatibility.rb

@@ -0,0 +1,231 @@
+# Required interface for existing parser implementations
+module Sparkql::ParserCompatibility
+  
+  MAXIMUM_MULTIPLE_VALUES = 25
+  MAXIMUM_EXPRESSIONS = 50
+  MAXIMUM_LEVEL_DEPTH = 2
+  
+  # TODO I Really don't think this is required anymore
+  # Ordered by precedence.
+  FILTER_VALUES = [
+    {
+      :type => :datetime,
+      :regex => /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}\:[0-9]{2}\:[0-9]{2}\.[0-9]{6}$/,
+      :operators => Sparkql::Token::OPERATORS
+    },
+    {
+      :type => :date,
+      :regex => /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/,
+      :operators => Sparkql::Token::OPERATORS
+    },
+    {
+      :type => :character,
+      :regex => /^'([^'\\]*(\\.[^'\\]*)*)'$/, # Strings must be single quoted.  Any inside single quotes must be escaped.
+      :multiple => /^'([^'\\]*(\\.[^'\\]*)*)'/,
+      :operators => Sparkql::Token::EQUALITY_OPERATORS
+    },
+    {
+      :type => :integer,
+      :regex => /^\-?[0-9]+$/,
+      :multiple => /^\-?[0-9]+/,
+      :operators => Sparkql::Token::OPERATORS
+    },
+    {
+      :type => :decimal,
+      :regex => /^\-?[0-9]+\.[0-9]+$/,
+      :multiple => /^\-?[0-9]+\.[0-9]+/,
+      :operators => Sparkql::Token::OPERATORS
+    },
+    {
+      :type => :boolean,
+      :regex => /^true|false$/,
+      :operators => Sparkql::Token::EQUALITY_OPERATORS
+    },
+    {
+      :type => :null,
+      :regex => /^NULL|Null|null$/,
+      :operators => Sparkql::Token::EQUALITY_OPERATORS
+    }
+  ]
+  
+  OPERATORS_SUPPORTING_MULTIPLES = ["Eq","Ne"]
+  
+  # To be implemented by child class.
+  # Shall return a valid query string for the respective database,
+  # or nil if the source could not be processed.  It may be possible to return a valid
+  # SQL string AND have errors ( as checked by errors? ), but this will be left
+  # to the discretion of the child class.
+  def compile( source, mapper )
+   raise NotImplementedError
+  end
+  
+  # Returns a list of expressions tokenized in the following format:
+  # [{ :field => IdentifierName, :operator => "Eq", :value => "'Fargo'", :type => :character, :conjunction => "And" }]
+  # This step will set errors if source is not syntactically correct.
+  def tokenize( source )
+    raise ArgumentError, "You must supply a source string to tokenize!" unless source.is_a?(String)
+
+    # Reset the parser error stack
+    @errors = []
+      
+    expressions = self.parse(source)
+    expressions
+  end
+  
+  # Returns an array of errors.  This is an array of ParserError objects
+  def errors
+    @errors = [] unless defined?(@errors)
+    @errors
+  end
+  
+  # Delegator for methods to process the error list.
+  def process_errors
+    Sparkql::ErrorsProcessor.new(@errors)
+  end
+  
+  # delegate :errors?, :fatal_errors?, :dropped_errors?, :recovered_errors?, :to => :process_errors
+  # Since I don't have rails delegate...
+  def errors?
+    process_errors.errors?
+  end
+  def fatal_errors?
+    process_errors.fatal_errors?
+  end
+  def dropped_errors?
+    process_errors.dropped_errors?
+  end
+  def recovered_errors?
+    process_errors.recovered_errors?
+  end
+  
+  def escape_value_list( expression )
+    final_list = []
+    expression[:value].each do | value |
+      new_exp = {
+        :value => value,
+        :type => expression[:type]
+      }
+      final_list << escape_value(new_exp)
+    end
+    expression[:value] = final_list
+  end
+  
+  def escape_value( expression )
+    if expression[:value].is_a? Array
+      return escape_value_list( expression )
+    end
+    case expression[:type]
+    when :character
+      return character_escape(expression[:value])
+    when :integer
+      return integer_escape(expression[:value])
+    when :decimal
+      return decimal_escape(expression[:value])
+    when :date
+      return date_escape(expression[:value])
+    when :datetime
+      return datetime_escape(expression[:value])
+    when :boolean
+      return boolean_escape(expression[:value])
+    when :null
+      return nil
+    end
+    expression[:value]
+  end
+
+  # processes escape characters for a given string.  May be overridden by
+  # child classes.
+  def character_escape( string )
+    string.gsub(/^\'/,'').gsub(/\'$/,'').gsub(/\\'/, "'")
+  end
+
+  def integer_escape( string )
+    string.to_i
+  end
+
+  def decimal_escape( string )
+    string.to_f
+  end
+
+  def date_escape(string)
+    Date.parse(string)
+  end
+
+  def datetime_escape(string)
+    DateTime.parse(string)
+  end
+
+  def boolean_escape(string)
+    "true" == string
+  end
+  
+  # Returns the rule hash for a given type
+  def rules_for_type( type )
+    FILTER_VALUES.each do |rule|
+      return rule if rule[:type] == type
+    end
+    nil
+  end
+  
+  # true if a given type supports multiple values
+  def supports_multiple?( type )
+    rules_for_type(type).include?( :multiple )
+  end
+  
+  # Maximum supported nesting level for the parser filters
+  def max_level_depth
+    MAXIMUM_LEVEL_DEPTH
+  end
+  
+  private
+  
+  def tokenizer_error( error_hash )
+    self.errors << Sparkql::ParserError.new( error_hash )
+  end
+  alias :compile_error :tokenizer_error
+  
+  # Checks the type of an expression with what is expected.
+  def check_type!(expression, expected, supports_nulls = true)
+    if expected == expression[:type] || (supports_nulls && expression[:type] == :null)
+      return true
+    elsif expected == :datetime &&  expression[:type] == :date
+      expression[:type] = :datetime
+      expression[:cast] = :date
+      return true
+    end
+    type_error(expression, expected)
+    false
+  end
+  
+  def type_error( expression, expected )
+      compile_error(:token => expression[:field], :expression => expression,
+            :message => "expected #{expected} but found #{expression[:type]}",
+            :status => :fatal )
+  end
+  
+  # Builds the correct operator based on the type and the value.
+  # default should be the operator provided in the actual filter string
+  def get_operator(expression, default )
+    f = rules_for_type(expression[:type])
+    if f[:operators].include?(default)
+      if f[:multiple] && multiple_values?( expression[:value])
+        return nil unless operator_supports_multiples?(default)
+        return default == "Ne" ? "Not In" : "In"
+      elsif default == "Ne"
+        return "Not Eq"
+      end
+      return default
+    else
+      return nil
+    end
+  end
+  
+  def multiple_values?(value)
+    Array(value).size > 1
+  end
+  
+  def operator_supports_multiples?(operator)
+    OPERATORS_SUPPORTING_MULTIPLES.include?(operator)
+  end
+  
+end

data/lib/sparkql/parser_tools.rb

@@ -0,0 +1,93 @@
+# This is the guts of the parser internals and is mixed into the parser for organization.
+module Sparkql::ParserTools
+  
+  def parse(str)
+    @lexer = Sparkql::Lexer.new(str)
+    results = do_parse
+    max = Sparkql::ParserCompatibility::MAXIMUM_EXPRESSIONS
+    return if results.nil?
+    results.size > max ? results[0,max] : results
+  end
+
+  def next_token
+    t = @lexer.shift
+    while t[0] == :SPACE or t[0] == :NEWLINE
+      t = @lexer.shift
+    end
+    t
+  end
+  
+  def tokenize_expression(field, op, val)
+    operator = get_operator(val,op) unless val.nil?
+    custom_field = field.start_with?('"')
+    block_group = (@lexer.level == 0) ? 0 : @lexer.block_group_identifier
+    expression = {:field => field, :operator => operator, :conjunction => 'And', 
+      :level => @lexer.level, :block_group => block_group, :custom_field => custom_field}
+    expression = val.merge(expression) unless val.nil?
+    if @lexer.level > max_level_depth
+      compile_error(:token => "(", :expression => expression,
+            :message => "You have exceeded the maximum nesting level.  Please nest no more than #{max_level_depth} levels deep.",
+            :status => :fatal, :syntax => false )
+    end
+    if operator.nil?
+      tokenizer_error(:token => op, :expression => expression,
+        :message => "Operator not supported for this type and value string", :status => :fatal )
+    end
+    [expression]
+  end
+
+  def tokenize_conjunction(exp1, conj, exp2)
+    exp2.first[:conjunction] = conj
+    exp1 + exp2
+  end
+  
+  def tokenize_group(expressions)
+    @lexer.leveldown
+    expressions
+  end
+
+  def tokenize_multiple(lit1, lit2)
+    if lit1[:type] != lit2[:type]
+      tokenizer_error(:token => @lexer.last_field, 
+                      :message => "Type mismatch in field list.",
+                      :status => :fatal, 
+                      :syntax => true)    
+    end
+    array = Array(lit1[:value])
+    unless array.size >= Sparkql::ParserCompatibility::MAXIMUM_MULTIPLE_VALUES
+      array << lit2[:value]
+    end
+    {
+      :type => lit1[:type],
+      :value => array,
+      :multiple => "true" # TODO ?
+    }
+  end
+  
+  def tokenize_function(name, f_args)
+    args = f_args.instance_of?(Array) ? f_args : [f_args]
+    args.each do |arg|
+      arg[:value] = escape_value(arg)
+    end
+    resolver = Sparkql::FunctionResolver.new(name, args)
+    
+    resolver.validate
+    if(resolver.errors?)
+      errors += resolver.errors
+      return nil
+    else
+      return resolver.call()
+    end
+  end
+  
+  def on_error(error_token_id, error_value, value_stack)
+    token_name = token_to_str(error_token_id)
+    token_name.downcase!
+    token = error_value.to_s.inspect
+    tokenizer_error(:token => @lexer.last_field, 
+                    :message => "Error parsing token #{token_name}",
+                    :status => :fatal, 
+                    :syntax => true)    
+  end  
+
+end

data/lib/sparkql/token.rb

@@ -0,0 +1,21 @@
+module Sparkql::Token
+  SPACE = /[\t ]+/
+  NEWLINE = /\r\n|\n\r|\r|\n/
+  LPAREN = /\(/
+  RPAREN = /\)/
+  KEYWORD = /[A-Za-z]+/
+  STANDARD_FIELD = /[A-Z]+[A-Za-z]*/
+  CUSTOM_FIELD = /^(\"([^$."][^."]+)\".\"([^$."][^."]+)\")/
+  INTEGER = /^\-?[0-9]+/
+  DECIMAL = /^\-?[0-9]+\.[0-9]+/
+  CHARACTER = /^'([^'\\]*(\\.[^'\\]*)*)'/
+  DATE = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}/
+  DATETIME = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}\:[0-9]{2}\:[0-9]{2}\.[0-9]{6}/
+  BOOLEAN = /^true|false/
+  NULL = /NULL|null|Null/
+  # Reserved words
+  EQUALITY_OPERATORS = ['Eq','Ne']
+  OPERATORS = ['Eq','Ne','Gt','Ge','Lt','Le'] + EQUALITY_OPERATORS
+  CONJUNCTIONS = ['And','Or']
+
+end

data/lib/sparkql/version.rb

@@ -0,0 +1,3 @@
+module Sparkql
+  VERSION = File.read(File.dirname(__FILE__) + "/../../VERSION").chomp
+end

data/lib/sparkql.rb

@@ -0,0 +1,13 @@
+require "sparkql/version"
+require "sparkql/token"
+require "sparkql/errors"
+require "sparkql/expression_state"
+require "sparkql/lexer"
+require "sparkql/function_resolver"
+require "sparkql/parser_tools"
+require "sparkql/parser_compatibility"
+require "sparkql/parser"
+
+module Sparkql
+  # I AM A SPARKQLING MODULE!!!
+end

data/sparkql.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "sparkql/version"
+
+Gem::Specification.new do |s|
+  s.name        = "sparkql"
+  s.version     = Sparkql::VERSION
+  s.authors     = ["Wade McEwen"]
+  s.email       = ["wade@fbsdata.com"]
+  s.homepage    = ""
+  s.summary     = %q{API Parser engine for filter searching}
+  s.description = %q{Specification and base implementation of the Spark API parsing system.}
+
+  s.rubyforge_project = "sparkql"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency 'racc', '1.4.8'
+  s.add_development_dependency 'flexmls_gems', '~> 0.2.9'
+  s.add_development_dependency 'rake', '~> 0.9.2'
+  s.add_development_dependency 'test-unit', '~> 2.1.0'
+  s.add_development_dependency 'ci_reporter', '~> 1.6'
+  s.add_development_dependency 'rcov', '~> 0.9.9'
+
+end

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'test/unit'
+require 'sparkql'

data/test/unit/expression_state_test.rb

@@ -0,0 +1,57 @@
+require 'test_helper'
+
+class ExpressionStateTest < Test::Unit::TestCase
+  include Sparkql
+
+  def setup
+    @subject = ExpressionState.new
+    @parser = Parser.new
+  end
+  
+  def test_needs_join
+    filter = '"General Property Description"."Taxes" Lt 500.0'
+    process(filter)
+    assert @subject.needs_join?
+  end
+  
+  def test_or
+    filter = '"General Property Description"."Taxes" Lt 500.0 Or "General Property Description"."Taxes" Gt 400.0'
+    process(filter)
+    assert !@subject.needs_join?, "#{@subject.inspect} Expressions:#{ @expressions.inspect}"
+  end
+
+  def test_and
+    filter = '"General Property Description"."Taxes" Lt 500.0 And "General Property Description"."Taxes2" Eq 1.0'
+    process(filter)
+    assert @subject.needs_join?
+  end
+
+  def test_and_or
+    filter = '"General Property Description"."Taxes" Lt 500.0 And "General Property Description"."Taxes2" ' + 
+      'Eq 1.0 Or "General Property Description"."Taxes" Gt 400.0'
+    process(filter)
+    assert !@subject.needs_join?
+  end
+
+  def test_or_and
+    filter = '"General Property Description"."Taxes" Lt 500.0 Or "General Property Description"."Taxes" ' +
+      'Gt 400.0 And "General Property Description"."Taxes2" Eq 1.0'
+    process(filter)
+    assert @subject.needs_join?
+  end
+  
+  def test_or_with_standard_field
+    filter = 'Test Eq 0.0 Or "General Property Description"."Taxes" Lt 500.0'
+    process(filter)
+    assert @subject.needs_join?
+  end
+  
+  def process(filter)
+    @expressions = @parser.parse(filter)
+    @expressions.each do |ex|
+      @subject.push(ex) if ex[:custom_field] == true
+    end
+    @expressions
+  end
+    
+end

data/test/unit/function_resolver_test.rb

@@ -0,0 +1,50 @@
+require 'test_helper'
+
+class ParserTest < Test::Unit::TestCase
+  include Sparkql
+
+  def test_now
+    start = Time.now
+    f = FunctionResolver.new('now', [])
+    f.validate
+    assert !f.errors?, "Errors #{f.errors.inspect}"
+    value = f.call
+    assert_equal :datetime, value[:type]
+    test_time = Time.parse(value[:value])
+    assert (-5 < test_time - start && 5 > test_time - start), "Time range off by more than five seconds #{test_time - start} '#{test_time} - #{start}'"
+  end
+  
+  def test_day
+    d = Date.today
+    dt = DateTime.new(d.year, d.month,d.day, 0,0,0, DateTime.now.offset)
+    start = Time.parse(dt.to_s)
+    f = FunctionResolver.new('days', [{:type=>:integer, :value =>7}])
+    f.validate
+    assert !f.errors?, "Errors #{f.errors.inspect}"
+    value = f.call
+    assert_equal :date, value[:type]
+    test_time = Time.parse(value[:value])
+    assert (605000 > test_time - start && 604000 < test_time - start), "Time range off by more than five seconds #{test_time - start} '#{test_time} - #{start}'"
+  end
+  
+  def test_invalid_param
+    f = FunctionResolver.new('now', [{:type => :character, :value=>'bad value'}])
+    f.validate
+    assert f.errors?, "'now' function does not support parameters"
+    
+    f = FunctionResolver.new('days', [])
+    f.validate
+    assert f.errors?, "'days' function requires one parameter"
+      
+    f = FunctionResolver.new('days', [{:type => :character, :value=>'bad value'}])
+    f.validate
+    assert f.errors?, "'days' function needs integer parameter"
+  end
+  
+  def test_invalid_function
+    f = FunctionResolver.new('then', [])
+    f.validate
+    assert f.errors?, "'then' is not a function"
+  end
+  
+end

data/test/unit/lexer_test.rb

@@ -0,0 +1,29 @@
+require 'test_helper'
+
+class LexerTest < Test::Unit::TestCase
+  include Sparkql
+  
+  def test_check_reserved_words_standard_fields
+    ["OrOrOr Eq true", "Equador Eq true", "Oregon Ge 10"].each do |standard_field|
+      @lexer = Lexer.new(standard_field)
+      token = @lexer.shift
+      assert_equal :STANDARD_FIELD, token.first, standard_field
+    end
+  end
+  def test_check_reserved_words_conjunctions
+    ['And Derp', 'Or 123'].each do |conjunction|
+      @lexer = Lexer.new(conjunction)
+      token = @lexer.shift
+      assert_equal :CONJUNCTION, token.first, conjunction
+    end
+  end
+
+  def test_check_reserved_words_operators
+    ['Eq Derp', 'Gt 123'].each do |op|
+      @lexer = Lexer.new(op)
+      token = @lexer.shift
+      assert_equal :OPERATOR, token.first, op
+    end
+  end
+      
+end