sparkql 0.1.8 → 0.3.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OTgxODlmNjI2NzVkYTBmNjE1Njg3MmYwZmU0M2UyZjE4MDQ5Yzg5MA==
+  data.tar.gz: !binary |-
+    MDRlZTM2ODFjNWIxNDBiZTUyOGFhY2FlMjQ5OTVjZDJmMGUxZWRjZA==
+SHA512:
+  metadata.gz: !binary |-
+    ZTFjYTAxZGRlYmFiODY2ZTFmYTgwN2MxYWNlMWJiMWFjMWYzYmIzOGUwZmYy
+    NmY0M2YwMmE3ZTNkMmNiMmQzMDRhOTExNWIyZDI3MmEwMTQzNDU2MjRkMmZm
+    MzIyNWI5OWNjNmUzZjI0MTQ5ZmM3YWIyY2Q3MDVlMzQwMGIwOTQ=
+  data.tar.gz: !binary |-
+    ZjYzMjgyZjc0NzI2YzE0MmUzOWUwOGI4NjVkZWRlODNkYTM5ZjU1ZWIxYTVi
+    ZWYyOWM3ODFjMWY1ZDVlMDg0ZDIwMzRiYjc2NDJkMmQ4MTdiOWVmMmYzYWFl
+    YmY0MWMwNmVjODc1N2E4ODAyMDBjMzU5OTdkMGQ3NjY4MzgxN2Y=

data/.gitignore

@@ -3,3 +3,5 @@
 Gemfile.lock
 lib/sparkql/*.output
 pkg/*
+*.swp
+test/reports

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+
+v0.3.2, 2015-04-14 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.18...v0.3.2))
+-------------------
+
+  * [BUGFIX] Allow seconds for ISO-8601
+
+v0.3.18, 2015-04-10 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.17...v0.3.18))
+-------------------
+
+  * [BUGFIX] Better support for ISO-8601
+

data/GRAMMAR.md

@@ -0,0 +1,208 @@
+## SparkQL BNF Grammar
+This document explains the rules for the Spark API filter language syntax and
+is a living document generated from the reference implementation at 
+https://github.com/sparkapi/sparkql.
+### Precedence Rules
+Unless otherwise specified, SparkQL follows SQL precendence conventions for 
+operators and conjunctions.
+Unary minus is always tied to value, such as for negative numbers.
+
+
+```
+   prechigh
+     nonassoc UMINUS
+   preclow
+```
+
+### Grammar Rules
+A filter (target) is a composition of filter basic filter expressions.
+
+
+```
+   rule
+     target
+       : expressions
+       | /* none */ 
+       ;
+```
+
+#### Expressions
+One or more expressions
+
+
+```
+     expressions
+       : expression
+       | conjunction
+       | unary_conjunction
+       ;
+```
+
+#### Expression
+The core of the filtering system, the expression requires a field, a condition 
+and criteria for comparing the value of the field to the value(s) of the 
+condition. The result of evaluating the expression on a resource is a true of 
+false for matching the criteria.
+
+
+```
+     expression
+       : field OPERATOR condition 
+       | field RANGE_OPERATOR range 
+       | group
+       ;
+```
+
+#### Unary Conjunction
+Some conjunctions don't need to expression at all times (e.g. 'NOT'). 
+
+
+```
+     unary_conjunction
+       : UNARY_CONJUNCTION expression 
+       ;  
+```
+
+#### Conjunction
+Two expressions joined together using a supported conjunction
+
+
+```
+     conjunction
+       : expressions CONJUNCTION expression 
+       | expressions UNARY_CONJUNCTION expression 
+       ;
+```
+
+#### Group
+One or more expressions encased in parenthesis. There are limitations on nesting depth at the time of this writing.
+
+
+```
+     group
+     	: LPAREN expressions RPAREN 
+     	;
+```
+
+#### Field
+Keyword for searching on, these fields should be discovered using the metadata 
+rules. In general, Keywords that cannot be found will be dropped from the 
+filter.
+
+
+```
+     field
+     	: STANDARD_FIELD
+     	| CUSTOM_FIELD
+     	;
+```
+
+#### Condition
+The determinant of the filter, this is typically a value or set of values of 
+a type that the field supports (review the field meta data for support). 
+Functions are also supported on some field types, and provide more flexibility
+on filtering values
+
+
+```
+     condition
+       : literal
+       | function
+       | literal_list 
+       ;
+```
+
+#### Function
+Functions may replace static values for conditions with supported field 
+types. Functions may have parameters that match types supported by 
+fields.
+
+
+```
+     function
+       : function_name LPAREN RPAREN 
+       | function_name LPAREN function_args RPAREN 
+       ;
+     function_name
+       : KEYWORD
+       ;
+```
+
+#### Function Arguments
+Functions may optionally have a comma delimited list of parameters.
+
+
+```
+     function_args
+       : function_arg
+       | function_args COMMA function_arg 
+       ; 
+     function_arg
+       : literal
+       | literals
+       ;
+```
+
+#### Literal List
+A comma delimited list of functions and values.
+
+
+```
+     literal_list
+       : literals
+       | function
+       | literal_list COMMA literals 
+       | literal_list COMMA function 
+       ;
+```
+
+#### Range List
+A comma delimited list of values that support ranges for the Between operator 
+(see rangeable).
+
+
+```
+     range                                                                             
+       : rangeable COMMA rangeable 
+       ;
+```
+
+#### Literals
+Literals that support multiple values in a list for a condition
+
+
+```
+     literals
+       : INTEGER
+       | DECIMAL
+       | CHARACTER
+       ;
+```
+
+#### Literal
+Literals only support a single value in a condition
+
+
+```
+     literal
+       : DATE
+       | DATETIME
+       | BOOLEAN
+       | NULL
+       ;
+```
+
+#### Range List
+Functions, and literals that can be used in a range                                                       
+
+
+```
+     rangeable
+       : INTEGER
+       | DECIMAL
+       | DATE
+       | DATETIME
+       | function
+       ;
+```
+

data/Gemfile

@@ -1,4 +1,4 @@
-source "http://gems.dev.fbsdata.com/public" 
+source "http://gems.flexmls.com/" 
 source "http://rubygems.org"
 
 # Specify your gem's dependencies in sparkapi_parser.gemspec

data/README.md

@@ -25,7 +25,7 @@ API.
 
 Here is a basic example:
 
-	expressions = Parser.new.parse("Hello Eq 'World')
+	expressions = Parser.new.parse("Hello Eq 'World'")
 
 The return value will be an array with one expression element containing the query information:
 
@@ -50,5 +50,5 @@ parser states (and conflicts) can be generated via
 
 	racc -o lib/sparkql/parser.rb lib/sparkql/parser.y -v  # see lib/sparkql/parser.output
 
-The rails/journey project was an inspiration for this gem. Look it up on github for reference.
+The [rails/journey](https://github.com/rails/journey) project was an inspiration for this gem. Look it up on github for reference.
 

data/Rakefile

@@ -1,17 +1,30 @@
 require "rubygems"
 require 'rubygems/user_interaction'
-require 'flexmls_gems/tasks'
-require 'flexmls_gems/tasks/test_unit'
-require 'flexmls_gems/tasks/rdoc'
+require 'rake/testtask'
+require 'ci/reporter/rake/test_unit'
+require 'bundler/gem_tasks'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
 
 rule '.rb' => '.y' do |t|
   sh "racc -l -o #{t.name} #{t.source}"
 end
 
 desc "Compile the racc parser from the grammar"
-task :compile => "lib/sparkql/parser.rb"
+task :compile => ["lib/sparkql/parser.rb", "grammar"]
+
+desc "Generate grammar Documenation"
+task :grammar do
+  puts "Generating grammar documentation..."
+  sh "ruby script/markdownify.rb > GRAMMAR.md"
+end
 
 Rake::Task[:test].prerequisites.unshift "lib/sparkql/parser.rb"
+Rake::Task[:test].prerequisites.unshift "grammar"
 
 desc 'Default: run unit tests.'
 task :default => :test

data/VERSION

@@ -1 +1 @@
-0.1.8
+0.3.2

data/lib/sparkql/errors.rb

@@ -3,8 +3,8 @@ module Sparkql
 class ErrorsProcessor
   attr_accessor :errors
 
-  def initialize( errors )
-    @errors = errors || []
+  def initialize( errors = [] )
+    @errors = Array(errors)
   end
 
   # true if the error stack contains at least one error
@@ -39,24 +39,26 @@ end
 
 class ParserError
   attr_accessor :token, :expression, :message, :status, :recovered_as
+  attr_writer :syntax, :constraint
 
-  def initialize(error_hash=nil)
-    error_hash = {} if error_hash.nil?
+  def initialize(error_hash={})
     @token = error_hash[:token]
     @expression = error_hash[:expression]
     @message = error_hash[:message]
     @status = error_hash[:status]
     @recovered_as = error_hash[:recovered_as]
+    @recovered_as = error_hash[:recovered_as]
     self.syntax= error_hash[:syntax] == false ? false : true
+    self.constraint= error_hash[:constraint] == true
   end
   
-  def syntax=(syntax_error)
-    @syntax = syntax_error
-  end
-
   def syntax?
     @syntax
   end
+  
+  def constraint?
+    @constraint
+  end
 
   def to_s
     str = case @status

data/lib/sparkql/expression_state.rb

@@ -1,20 +1,23 @@
 # Custom fields need to add a table join to the customfieldsearch table when AND'd together, 
-# but not when they are OR'd. This class maintains the state for all custom field expressions
+# but not when they are OR'd or nested. This class maintains the state for all custom field expressions
 # lets the parser know when to do either.
 class Sparkql::ExpressionState
   
   def initialize
-    @expressions = []
+    @expressions = {0=>[]}
     @last_conjunction = "And" # always start with a join
+    @block_group = 0
   end
   
   def push(expression)
-    @expressions << expression
+    @block_group = expression[:block_group]
+    @expressions[@block_group] ||= [] 
+    @expressions[@block_group] << expression
     @last_conjunction = expression[:conjunction]
   end
   
   def needs_join?
-    return @expressions.size == 1 || "And" == @last_conjunction
+    return @expressions[@block_group].size == 1 || ["Not", "And"].include?(@last_conjunction)
   end
   
 end

data/lib/sparkql/function_resolver.rb

@@ -1,4 +1,6 @@
 require 'time'
+require 'geo_ruby'
+require 'sparkql/geo'
 
 # Binding class to all supported function calls in the parser. Current support requires that the 
 # resolution of function calls to happen on the fly at parsing time at which point a value and 
@@ -8,12 +10,37 @@ require 'time'
 # SUPPORTED_FUNCTIONS which will run validation on the function syntax prior to execution.
 class Sparkql::FunctionResolver
   SECONDS_IN_DAY = 60 * 60 * 24
+  STRFTIME_FORMAT = '%Y-%m-%d'
   
   SUPPORTED_FUNCTIONS = {
+    :polygon => {
+      :args => [:character],
+      :return_type => :shape
+    }, 
+    :rectangle => {
+      :args => [:character],
+      :return_type => :shape
+    }, 
+    :radius => {
+      :args => [:character, :decimal],
+      :return_type => :shape
+    },
+    :linestring => {
+      :args => [:character],
+      :return_type => :shape
+    },
     :days => {
       :args => [:integer],
       :return_type => :datetime
     }, 
+    :months => {
+      :args => [:integer],
+      :return_type => :datetime
+    },
+    :years => {
+      :args => [:integer],
+      :return_type => :datetime
+    },
     :now => { 
       :args => [],
       :return_type => :datetime
@@ -78,7 +105,14 @@ class Sparkql::FunctionResolver
   # Execute the function
   def call()
     real_vals = @args.map { |i| i[:value]}
-    self.send(@name.to_sym, *real_vals)
+    v = self.send(@name.to_sym, *real_vals)
+
+    unless v.nil?
+      v[:function_name] = @name
+      v[:function_parameters] = real_vals
+    end
+
+    v
   end
   
   protected 
@@ -92,7 +126,7 @@ class Sparkql::FunctionResolver
     d = Date.today + num
     {
       :type => :date,
-      :value => d.to_s
+      :value => d.strftime(STRFTIME_FORMAT)
     }
   end
   
@@ -103,4 +137,139 @@ class Sparkql::FunctionResolver
       :value => Time.now.iso8601
     }
   end
-end
+
+  def months num_months
+    d = DateTime.now >> num_months
+    {
+      :type => :date,
+      :value => d.strftime(STRFTIME_FORMAT)
+    }
+  end
+
+  def years num_years
+    d = DateTime.now >> (num_years * 12)
+    {
+      :type => :date,
+      :value => d.strftime(STRFTIME_FORMAT)
+    }
+  end
+  
+  # TODO Donuts: to extend, we'd just replace (coords) param with (linear_ring1,linear_ring2, ...)
+  def polygon(coords)
+    new_coords = parse_coordinates(coords)
+    unless new_coords.size > 2
+      @errors << Sparkql::ParserError.new(:token => coords, 
+        :message => "Function call 'polygon' requires at least three coordinates",
+        :status => :fatal )
+      return
+    end
+
+    # auto close the polygon if it's open
+    unless new_coords.first == new_coords.last
+      new_coords << new_coords.first.clone
+    end
+    
+    shape = GeoRuby::SimpleFeatures::Polygon.from_coordinates([new_coords])
+    {
+      :type => :shape,
+      :value => shape
+    }
+  end
+
+  def linestring(coords)
+    new_coords = parse_coordinates(coords)
+    unless new_coords.size > 1
+      @errors << Sparkql::ParserError.new(:token => coords,
+        :message => "Function call 'linestring' requires at least two coordinates",
+        :status => :fatal )
+      return
+    end
+
+    shape = GeoRuby::SimpleFeatures::LineString.from_coordinates(new_coords)
+    {
+      :type => :shape,
+      :value => shape
+    }
+  end
+    
+  def rectangle(coords)
+    bounding_box = parse_coordinates(coords)
+    unless bounding_box.size == 2
+      @errors << Sparkql::ParserError.new(:token => coords, 
+        :message => "Function call 'rectangle' requires two coordinates for the bounding box",
+        :status => :fatal )
+      return
+    end
+    poly_coords = [
+                    bounding_box.first, 
+                    [bounding_box.last.first, bounding_box.first.last],
+                    bounding_box.last,
+                    [bounding_box.first.first, bounding_box.last.last],
+                    bounding_box.first.clone, 
+                  ]
+    shape = GeoRuby::SimpleFeatures::Polygon.from_coordinates([poly_coords])
+    {
+      :type => :shape,
+      :value => shape
+    }
+  end
+  
+  def radius(coords, length)
+
+    unless length > 0
+      @errors << Sparkql::ParserError.new(:token => length, 
+        :message => "Function call 'radius' length must be positive",
+        :status => :fatal )
+      return
+    end
+
+    # The radius() function is overloaded to allow an identifier
+    # to be specified over lat/lon.  This identifier should specify a
+    # record that, in turn, references a lat/lon. Naturally, this won't be
+    # validated here.
+    shape_error = false
+    shape = if is_coords?(coords)
+              new_coords = parse_coordinates(coords)
+              if new_coords.size != 1
+                shape_error = true
+              else
+                GeoRuby::SimpleFeatures::Circle.from_coordinates(new_coords.first, length);
+              end
+            elsif Sparkql::Geo::RecordRadius.valid_record_id?(coords)
+              Sparkql::Geo::RecordRadius.new(coords, length)
+            else
+              shape_error = true
+            end
+
+    if shape_error
+      @errors << Sparkql::ParserError.new(:token => coords, 
+        :message => "Function call 'radius' requires one coordinate for the center",
+        :status => :fatal )
+      return
+    end
+
+    {
+      :type => :shape,
+      :value => shape 
+    }
+  end
+  
+  private
+
+  def is_coords?(coord_string)
+    coord_string.split(" ").size > 1
+  end
+
+  def parse_coordinates coord_string
+    terms = coord_string.strip.split(',')
+    coords = terms.map do |term|
+      term.strip.split(/\s+/).reverse.map { |i| i.to_f }
+    end
+    coords
+  rescue => e
+    @errors << Sparkql::ParserError.new(:token => coord_string, 
+      :message => "Unable to parse coordinate string.",
+      :status => :fatal )
+  end
+  
+end

data/lib/sparkql/geo/record_circle.rb

@@ -0,0 +1,18 @@
+module Sparkql
+  module Geo
+    class RecordRadius
+      RECORD_ID_REGEX = /\A[0-9]{26}\z/
+
+      attr_accessor :record_id, :radius
+
+      def self.valid_record_id?(record_id)
+        record_id =~ RECORD_ID_REGEX
+      end
+
+      def initialize(record_id, radius)
+        self.record_id = record_id
+        self.radius = radius
+      end
+    end
+  end
+end

data/lib/sparkql/geo.rb

@@ -0,0 +1 @@
+require 'sparkql/geo/record_circle'

data/lib/sparkql/lexer.rb

@@ -1,5 +1,9 @@
 class Sparkql::Lexer < StringScanner
   include Sparkql::Token
+  
+  attr_accessor :level, :block_group_identifier
+  
+  attr_reader :last_field
 
   def initialize(str)
     str.freeze
@@ -21,7 +25,7 @@ class Sparkql::Lexer < StringScanner
         levelup
         [:LPAREN, value]
       when value = scan(RPAREN)
-        # leveldown do this after parsing group
+        # leveldown: do this after parsing group
         [:RPAREN, value]
       when value = scan(/\,/)
         [:COMMA,value]
@@ -58,8 +62,12 @@ class Sparkql::Lexer < StringScanner
     u_value = value.capitalize
     if OPERATORS.include?(u_value)
       [:OPERATOR,u_value]
+    elsif RANGE_OPERATOR == u_value
+      [:RANGE_OPERATOR,u_value]
     elsif CONJUNCTIONS.include?(u_value)
       [:CONJUNCTION,u_value]
+    elsif UNARY_CONJUNCTIONS.include?(u_value)
+      [:UNARY_CONJUNCTION,u_value]
     else
       [:UNKNOWN, "ERROR: '#{self.string}'"]
     end
@@ -82,14 +90,6 @@ class Sparkql::Lexer < StringScanner
     result
   end
   
-  def level
-    @level
-  end
-
-  def block_group_identifier
-    @block_group_identifier
-  end
-  
   def levelup
     @level += 1
     @block_group_identifier += 1
@@ -107,8 +107,4 @@ class Sparkql::Lexer < StringScanner
     [symbol, node]
   end
   
-  def last_field
-    @last_field
-  end
-  
-end
+end