sparkql 0.1.8 → 0.3.2

data/lib/sparkql/parser_compatibility.rb

@@ -1,22 +1,22 @@
 # Required interface for existing parser implementations
 module Sparkql::ParserCompatibility
-  
-  MAXIMUM_MULTIPLE_VALUES = 25
+
+  MAXIMUM_MULTIPLE_VALUES = 200
   MAXIMUM_EXPRESSIONS = 50
   MAXIMUM_LEVEL_DEPTH = 2
-  
+
   # TODO I Really don't think this is required anymore
   # Ordered by precedence.
   FILTER_VALUES = [
     {
       :type => :datetime,
       :regex => /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}\:[0-9]{2}\:[0-9]{2}\.[0-9]{6}$/,
-      :operators => Sparkql::Token::OPERATORS
+      :operators => Sparkql::Token::OPERATORS + [Sparkql::Token::RANGE_OPERATOR]
     },
     {
       :type => :date,
       :regex => /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$/,
-      :operators => Sparkql::Token::OPERATORS
+      :operators => Sparkql::Token::OPERATORS + [Sparkql::Token::RANGE_OPERATOR]
     },
     {
       :type => :character,
@@ -28,13 +28,18 @@ module Sparkql::ParserCompatibility
       :type => :integer,
       :regex => /^\-?[0-9]+$/,
       :multiple => /^\-?[0-9]+/,
-      :operators => Sparkql::Token::OPERATORS
+      :operators => Sparkql::Token::OPERATORS + [Sparkql::Token::RANGE_OPERATOR]
     },
     {
       :type => :decimal,
       :regex => /^\-?[0-9]+\.[0-9]+$/,
       :multiple => /^\-?[0-9]+\.[0-9]+/,
-      :operators => Sparkql::Token::OPERATORS
+      :operators => Sparkql::Token::OPERATORS + [Sparkql::Token::RANGE_OPERATOR]
+    },
+    {
+      :type => :shape,
+      # This type is not parseable, so no regex
+      :operators => Sparkql::Token::EQUALITY_OPERATORS
     },
     {
       :type => :boolean,
@@ -47,9 +52,9 @@ module Sparkql::ParserCompatibility
       :operators => Sparkql::Token::EQUALITY_OPERATORS
     }
   ]
-  
+
   OPERATORS_SUPPORTING_MULTIPLES = ["Eq","Ne"]
-  
+
   # To be implemented by child class.
   # Shall return a valid query string for the respective database,
   # or nil if the source could not be processed.  It may be possible to return a valid
@@ -58,7 +63,7 @@ module Sparkql::ParserCompatibility
   def compile( source, mapper )
    raise NotImplementedError
   end
-  
+
   # Returns a list of expressions tokenized in the following format:
   # [{ :field => IdentifierName, :operator => "Eq", :value => "'Fargo'", :type => :character, :conjunction => "And" }]
   # This step will set errors if source is not syntactically correct.
@@ -67,22 +72,22 @@ module Sparkql::ParserCompatibility
 
     # Reset the parser error stack
     @errors = []
-      
+
     expressions = self.parse(source)
     expressions
   end
-  
+
   # Returns an array of errors.  This is an array of ParserError objects
   def errors
     @errors = [] unless defined?(@errors)
     @errors
   end
-  
+
   # Delegator for methods to process the error list.
   def process_errors
     Sparkql::ErrorsProcessor.new(@errors)
   end
-  
+
   # delegate :errors?, :fatal_errors?, :dropped_errors?, :recovered_errors?, :to => :process_errors
   # Since I don't have rails delegate...
   def errors?
@@ -97,7 +102,7 @@ module Sparkql::ParserCompatibility
   def recovered_errors?
     process_errors.recovered_errors?
   end
-  
+
   def escape_value_list( expression )
     final_list = []
     expression[:value].each do | value |
@@ -109,7 +114,7 @@ module Sparkql::ParserCompatibility
     end
     expression[:value] = final_list
   end
-  
+
   def escape_value( expression )
     if expression[:value].is_a? Array
       return escape_value_list( expression )
@@ -158,7 +163,7 @@ module Sparkql::ParserCompatibility
   def boolean_escape(string)
     "true" == string
   end
-  
+
   # Returns the rule hash for a given type
   def rules_for_type( type )
     FILTER_VALUES.each do |rule|
@@ -166,49 +171,72 @@ module Sparkql::ParserCompatibility
     end
     nil
   end
-  
+
   # true if a given type supports multiple values
   def supports_multiple?( type )
     rules_for_type(type).include?( :multiple )
   end
-  
+
   # Maximum supported nesting level for the parser filters
   def max_level_depth
     MAXIMUM_LEVEL_DEPTH
   end
-  
+
+  def max_expressions
+    MAXIMUM_EXPRESSIONS
+  end
+
+  def max_values
+    MAXIMUM_MULTIPLE_VALUES
+  end
+
   private
-  
+
   def tokenizer_error( error_hash )
     self.errors << Sparkql::ParserError.new( error_hash )
   end
   alias :compile_error :tokenizer_error
-  
+
   # Checks the type of an expression with what is expected.
   def check_type!(expression, expected, supports_nulls = true)
     if expected == expression[:type] || (supports_nulls && expression[:type] == :null)
       return true
-    elsif expected == :datetime &&  expression[:type] == :date
+    elsif expected == :datetime && expression[:type] == :date
       expression[:type] = :datetime
       expression[:cast] = :date
       return true
+    elsif expected == :date && expression[:type] == :datetime 
+      expression[:type] = :date
+      expression[:cast] = :datetime
+      if multiple_values?(expression[:value])
+        expression[:value].map!{ |val| coerce_datetime val }
+      else
+        expression[:value] = coerce_datetime expression[:value]
+      end
+      return true
+    elsif expected == :decimal && expression[:type] == :integer
+      expression[:type] = :decimal
+      expression[:cast] = :integer
+      return true
     end
     type_error(expression, expected)
     false
   end
-  
+
   def type_error( expression, expected )
       compile_error(:token => expression[:field], :expression => expression,
             :message => "expected #{expected} but found #{expression[:type]}",
             :status => :fatal )
   end
-  
+
   # Builds the correct operator based on the type and the value.
   # default should be the operator provided in the actual filter string
   def get_operator(expression, default )
     f = rules_for_type(expression[:type])
     if f[:operators].include?(default)
-      if f[:multiple] && multiple_values?( expression[:value])
+      if f[:multiple] && range?(expression[:value]) && default == 'Bt'
+        return "Bt"
+      elsif f[:multiple] && multiple_values?(expression[:value])
         return nil unless operator_supports_multiples?(default)
         return default == "Ne" ? "Not In" : "In"
       elsif default == "Ne"
@@ -219,13 +247,25 @@ module Sparkql::ParserCompatibility
       return nil
     end
   end
-  
+
   def multiple_values?(value)
     Array(value).size > 1
   end
-  
+
+  def range?(value)
+    Array(value).size == 2
+  end
+
   def operator_supports_multiples?(operator)
     OPERATORS_SUPPORTING_MULTIPLES.include?(operator)
   end
   
+  def coerce_datetime datetime
+    if datestr = datetime.match(/^(\d{4}-\d{2}-\d{2})/)
+      datestr[0]
+    else
+      datetime
+    end
+  end
+
 end

data/lib/sparkql/parser_tools.rb

@@ -1,12 +1,16 @@
 # This is the guts of the parser internals and is mixed into the parser for organization.
 module Sparkql::ParserTools
+
+  # Coercible types from highest precision to lowest
+  DATE_TYPES = [:datetime, :date]
+  NUMBER_TYPES = [:decimal, :integer]
   
   def parse(str)
     @lexer = Sparkql::Lexer.new(str)
     results = do_parse
-    max = Sparkql::ParserCompatibility::MAXIMUM_EXPRESSIONS
     return if results.nil?
-    results.size > max ? results[0,max] : results
+    validate_expressions results
+    results
   end
 
   def next_token
@@ -24,59 +28,89 @@ module Sparkql::ParserTools
     expression = {:field => field, :operator => operator, :conjunction => 'And', 
       :level => @lexer.level, :block_group => block_group, :custom_field => custom_field}
     expression = val.merge(expression) unless val.nil?
-    if @lexer.level > max_level_depth
-      compile_error(:token => "(", :expression => expression,
-            :message => "You have exceeded the maximum nesting level.  Please nest no more than #{max_level_depth} levels deep.",
-            :status => :fatal, :syntax => false )
-    end
+    validate_level_depth expression
     if operator.nil?
       tokenizer_error(:token => op, :expression => expression,
         :message => "Operator not supported for this type and value string", :status => :fatal )
     end
     [expression]
   end
-
+  
   def tokenize_conjunction(exp1, conj, exp2)
     exp2.first[:conjunction] = conj
+    exp2.first[:conjunction_level] = @lexer.level
     exp1 + exp2
   end
-  
+
+  def tokenize_unary_conjunction(conj, exp)
+    exp.first[:unary] = conj
+    exp.first[:unary_level] = @lexer.level
+    exp
+  end
+    
   def tokenize_group(expressions)
     @lexer.leveldown
     expressions
   end
 
+  def tokenize_list(list)
+    validate_multiple_values list[:value]
+    list[:condition] ||= list[:value]
+    list
+  end
+
   def tokenize_multiple(lit1, lit2)
+    final_type = lit1[:type]
     if lit1[:type] != lit2[:type]
-      tokenizer_error(:token => @lexer.last_field, 
-                      :message => "Type mismatch in field list.",
-                      :status => :fatal, 
-                      :syntax => true)    
+      final_type = coercible_types(lit1[:type],lit2[:type])
+      if final_type.nil?
+        final_type = lit1[:type]
+        tokenizer_error(:token => @lexer.last_field, 
+                        :message => "Type mismatch in field list.",
+                        :status => :fatal, 
+                        :syntax => true)
+      end
     end
     array = Array(lit1[:value])
-    unless array.size >= Sparkql::ParserCompatibility::MAXIMUM_MULTIPLE_VALUES
-      array << lit2[:value]
-    end
+    condition = lit1[:condition] || lit1[:value] 
+    array << lit2[:value]
     {
-      :type => lit1[:type],
+      :type => final_type ,
       :value => array,
-      :multiple => "true" # TODO ?
+      :multiple => "true",
+      :condition => condition + "," + (lit2[:condition] || lit2[:value])
     }
   end
   
+  def tokenize_function_args(lit1, lit2)
+    array = lit1.kind_of?(Array) ? lit1 : [lit1]
+    array << lit2
+    array
+  end
+  
   def tokenize_function(name, f_args)
+    @lexer.leveldown
+    @lexer.block_group_identifier -= 1
+
     args = f_args.instance_of?(Array) ? f_args : [f_args]
+    validate_multiple_arguments args
+    condition_list = []
     args.each do |arg|
+      condition_list << arg[:value] # Needs to be pure string value
       arg[:value] = escape_value(arg)
     end
     resolver = Sparkql::FunctionResolver.new(name, args)
     
     resolver.validate
     if(resolver.errors?)
-      errors += resolver.errors
+      tokenizer_error(:token => @lexer.last_field, 
+                      :message => "Error parsing function #{resolver.errors.join(',')}",
+                      :status => :fatal, 
+                      :syntax => true)    
       return nil
     else
-      return resolver.call()
+      result = resolver.call()
+      return result.nil? ? result : result.merge(:condition => "#{name}(#{condition_list.join(',')})")
     end
   end
   
@@ -90,4 +124,54 @@ module Sparkql::ParserTools
                     :syntax => true)    
   end  
 
+  def validate_level_depth expression
+    if @lexer.level > max_level_depth
+      compile_error(:token => "(", :expression => expression,
+            :message => "You have exceeded the maximum nesting level.  Please nest no more than #{max_level_depth} levels deep.",
+            :status => :fatal, :syntax => false, :constraint => true )
+    end
+  end
+  
+  def validate_expressions results
+    if results.size > max_expressions 
+      compile_error(:token => results[max_expressions][:field], :expression => results[max_expressions],
+            :message => "You have exceeded the maximum expression count.  Please limit to no more than #{max_expressions} expressions in a filter.",
+            :status => :fatal, :syntax => false, :constraint => true )
+      results.slice!(max_expressions..-1)
+    end
+  end
+  
+  def validate_multiple_values values
+    values = Array(values)
+    if values.size > max_values 
+      compile_error(:token => values[max_values],
+            :message => "You have exceeded the maximum value count.  Please limit to #{max_values} values in a single expression.",
+            :status => :fatal, :syntax => false, :constraint => true )
+      values.slice!(max_values..-1)
+    end
+  end
+  
+  def validate_multiple_arguments args
+    args = Array(args)
+    if args.size > max_values 
+      compile_error(:token => args[max_values],
+            :message => "You have exceeded the maximum parameter count.  Please limit to #{max_values} parameters to a single function.",
+            :status => :fatal, :syntax => false, :constraint => true )
+      args.slice!(max_values..-1)
+    end
+  end
+  
+  # If both types support coercion with eachother, always selects the highest 
+  # precision type to return as a reflection of the two. Any type that doesn't
+  # support coercion with the other type returns nil
+  def coercible_types type1, type2
+    if DATE_TYPES.include?(type1) && DATE_TYPES.include?(type2)
+      DATE_TYPES.first
+    elsif NUMBER_TYPES.include?(type1) && NUMBER_TYPES.include?(type2)
+      NUMBER_TYPES.first
+    else
+      nil
+    end
+  end
+
 end

data/lib/sparkql/token.rb

@@ -4,18 +4,20 @@ module Sparkql::Token
   LPAREN = /\(/
   RPAREN = /\)/
   KEYWORD = /[A-Za-z]+/
-  STANDARD_FIELD = /[A-Z]+[A-Za-z]*/
-  CUSTOM_FIELD = /^(\"([^$."][^."]+)\".\"([^$."][^."]+)\")/
+  STANDARD_FIELD = /[A-Z]+[A-Za-z0-9]*/
+  CUSTOM_FIELD = /^(\"([^$."][^."]+)\".\"([^$."][^."]*)\")/
   INTEGER = /^\-?[0-9]+/
   DECIMAL = /^\-?[0-9]+\.[0-9]+/
   CHARACTER = /^'([^'\\]*(\\.[^'\\]*)*)'/
   DATE = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}/
-  DATETIME = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}\:[0-9]{2}\:[0-9]{2}\.[0-9]{6}/
+  DATETIME = /^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}\:[0-9]{2}(\:[0-9]{2})?(\.[0-9]{1,50})?(((\+|-)[0-9]{2}\:?[0-9]{2})|Z)?/
   BOOLEAN = /^true|false/
   NULL = /NULL|null|Null/
   # Reserved words
+  RANGE_OPERATOR = 'Bt'
   EQUALITY_OPERATORS = ['Eq','Ne']
-  OPERATORS = ['Eq','Ne','Gt','Ge','Lt','Le'] + EQUALITY_OPERATORS
+  OPERATORS = ['Gt','Ge','Lt','Le'] + EQUALITY_OPERATORS
+  UNARY_CONJUNCTIONS = ['Not']
   CONJUNCTIONS = ['And','Or']
 
-end
+end

data/script/bootstrap

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+ruby -v
+echo "==> Installing gems..."
+bundle check --path .bundle 2>&1 > /dev/null || {
+  bundle install  --quiet --path .bundle
+}

data/script/ci_build

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -e
+
+script/bootstrap
+
+echo "==> Running tests..."
+bundle exec rake ci:setup:testunit test

data/script/markdownify.rb

@@ -0,0 +1,63 @@
+# Parses the grammar into a fancy markdown document.
+
+class Markdownify
+  
+  def initialize file
+    @file = file
+    @line_num = 0
+    @markdowning = false
+    @codeblock = false
+  end
+  
+  def format!
+    line_num=0
+    markdowning = false
+    File.open(@file).each do |line|
+      if line =~ /^\#STOP_MARKDOWN/
+        @markdowning = false
+      end
+      if markdowning? && !(line =~ /^\s+$/)
+        print format_line(line)
+      end
+      if line =~ /^\#START_MARKDOWN/
+        @markdowning = true
+      end
+    end
+    finish_code_block if @codeblock
+  end
+  
+  def markdowning?
+    @markdowning
+  end
+  
+  def format_line(line)
+    if line =~ /\s*\#/
+      finish_code_block if @codeblock
+      @codeblock = false
+      format_doc line
+    else
+      start_code_block unless @codeblock
+      @codeblock = true
+      format_bnf line
+    end
+  end
+  
+  def format_doc line
+    line.sub(/\s*\#\s*/, '')
+  end
+  
+  def format_bnf line
+    bnf = line.gsub(/\{.+\}/, '')
+    "   #{bnf}"
+  end
+
+  def start_code_block
+    print "\n\n```\n"
+  end
+  
+  def finish_code_block 
+    print "```\n\n"
+  end
+end
+
+Markdownify.new('lib/sparkql/parser.y').format!

data/script/release

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -e
+
+rm -rf pkg/*
+bundle exec rake build
+gem push pkg/* $@

data/sparkql.gemspec

@@ -12,17 +12,22 @@ Gem::Specification.new do |s|
   s.description = %q{Specification and base implementation of the Spark API parsing system.}
 
   s.rubyforge_project = "sparkql"
-
+  
+  s.license       = 'Apache 2.0'
+ 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
+  # georuby 2.1.x adds ruby 1.9-only syntax, so that's 
+  # a no-go for us at the moment
+  s.add_dependency             'georuby', '~> 2.0.0'
   s.add_development_dependency 'racc', '1.4.8'
-  s.add_development_dependency 'flexmls_gems', '~> 0.2.9'
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'test-unit', '~> 2.1.0'
   s.add_development_dependency 'ci_reporter', '~> 1.6'
+  s.add_development_dependency 'mocha', '~> 0.12.0'
   s.add_development_dependency 'rcov', '~> 0.9.9'
 
 end

data/test/test_helper.rb

@@ -1,2 +1,3 @@
 require 'test/unit'
-require 'sparkql'
+require 'mocha'
+require 'sparkql'

data/test/unit/errors_test.rb

@@ -0,0 +1,30 @@
+require 'test_helper'
+
+class ParserTest < Test::Unit::TestCase
+  include Sparkql
+
+  def test_error_defaults
+    errors = ParserError.new
+    assert errors.syntax?
+    assert !errors.constraint?
+  end
+
+  def test_error_constraint
+    errors = ParserError.new(:constraint => true, :syntax => false)
+    assert !errors.syntax?
+    assert errors.constraint?
+  end
+  
+  def test_process_fatal_errors
+    p = ErrorsProcessor.new(ParserError.new(:status => :fatal))
+    assert p.fatal_errors?
+    assert !p.dropped_errors?
+  end
+
+  def test_process_dropped_errors
+    p = ErrorsProcessor.new(ParserError.new(:status => :dropped))
+    assert p.dropped_errors?
+    assert !p.fatal_errors?
+  end
+
+end

data/test/unit/expression_state_test.rb

@@ -20,6 +20,12 @@ class ExpressionStateTest < Test::Unit::TestCase
     assert !@subject.needs_join?, "#{@subject.inspect} Expressions:#{ @expressions.inspect}"
   end
 
+  def test_not
+    filter = '"General Property Description"."Taxes" Lt 500.0 Not "General Property Description"."Taxes2" Eq 1.0'
+    process(filter)
+    assert @subject.needs_join?
+  end
+
   def test_and
     filter = '"General Property Description"."Taxes" Lt 500.0 And "General Property Description"."Taxes2" Eq 1.0'
     process(filter)
@@ -46,6 +52,38 @@ class ExpressionStateTest < Test::Unit::TestCase
     assert @subject.needs_join?
   end
   
+  # Nesting
+  def test_nested_or
+    parse '"General Property Description"."Taxes" Lt 5.0 Or ("General Property Description"."Taxes" Gt 4.0)'
+    @expressions.each do |ex|
+      @subject.push(ex)
+      assert @subject.needs_join?, "#{@subject.inspect} Expression:#{ ex.inspect}"
+    end
+  end
+
+  def test_nested_ors
+    parse '"Tax"."Taxes" Lt 5.0 Or ("Tax"."Taxes" Gt 4.0 Or "Tax"."Taxes" Gt 2.0)'
+    @subject.push(@expressions[0])
+    assert @subject.needs_join?
+    @subject.push(@expressions[1])
+    assert @subject.needs_join?
+    @subject.push(@expressions[2])
+    assert !@subject.needs_join?
+  end
+  
+  # Nesting
+  def test_nested_and
+    parse '"Tax"."Taxes" Lt 5.0 Or ("Tax"."Taxes" Gt 4.0 And "Tax"."Taxes" Gt 2.0)'
+    @expressions.each do |ex|
+      @subject.push(ex)
+      assert @subject.needs_join?, "#{@subject.inspect} Expression:#{ ex.inspect}"
+    end
+  end
+  
+  def parse(filter)
+    @expressions = @parser.parse(filter)
+  end
+  
   def process(filter)
     @expressions = @parser.parse(filter)
     @expressions.each do |ex|