sparkql 1.2.3 → 1.2.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTIxMGFmY2I2YTljYzEwNGNlMzMzOGFkMDg1NjM0YmY1MDZjZTliOA==
+    YmIyYTljOGIyZmFjZTEyZmY1MzU2ODZjYTg4ZWU1NDI2NWNkOWRkYw==
   data.tar.gz: !binary |-
-    YmVkZGFkODQ5OWE1Mzc4MDdmNGNjZjM0NzM2YWI5OWRlOTk0MmM3Nw==
+    NDkwMjczNDY2N2JlOGRmMzA5MDQ1YmI4Njg3ZTM5MTJlMzBiODVhMQ==
 SHA512:
   metadata.gz: !binary |-
-    ZjgxMWJlMDFlODY5NGJhZGE4NzU2ZjE5NmI2YzI1MWM2MDU3MzgwZTc1YzA3
-    MWQ1Njg4MTI0MjBjNWRlZTU0ZWU5NTc2MjUwYmU1ZTVjMDkxMDQ4ZjMxYjFh
-    OWFkMDU1OWVlODg4NTZhZDQ1NjY4ODU4MDI1NTgxNjlkZWQ0NTQ=
+    OTk2NTgwODMyNjExMjBiZThmODdjNzczZmM4M2MzMmY0MWNiNTkxODU5NThk
+    Y2UwYzY0ZTQ1NGRhNmFjYjI1ODIwYzBiMmMyMDM2NzFmNmNhNTI1OGNkYjNj
+    NzJhZTRjMDk4MzVmYjJlNDM4YTE2ZWFjYTVjNzlmMGZlNzM4NjQ=
   data.tar.gz: !binary |-
-    YjZkMDdlYTZkZWZlZDgxMzkyZjQ5MWY5MDA4ZjRmNGQ2OGIxN2U5YWRlZTMw
-    NTMyYmFjM2E0M2FhOGY2NGI4NDFiZDUzZGEwZGNmMGIyMmY0NzA3OTQzMzI0
-    MTAzODg2ODU4NzA1NTY1MjFhMzdmZWMzN2EyYmZlMjA3MDk5NmY=
+    NjhmNWU0NmZjNDVkMmUzMDJhMGM2ZTY0ZWVjMTk3ZGUwZGI3MzM2OTQwN2M4
+    MTUzYTk3N2E1NzI2YWFmMTA1YWIxZWY2MTliZTRiOGNjOTU5NjEwMGFlODkz
+    YjRmYmVmN2M4NTJkMGUyM2NlNjhmODA0ZGFkYWYzM2Q0OTNlMDU=

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+v1.2.4, 2018-12-13
+-------------------
+  * [IMPROVEMENT] Support decimal arithmetic
+  * [BUGFIX] Correctly handle type checking with arithmetic
+
 v1.2.3, 2018-12-05
 -------------------
   * [IMPROVEMENT] Support Arithmetic Grouping and Negation

data/VERSION

@@ -1 +1 @@
-1.2.3
+1.2.4

data/lib/sparkql/parser_compatibility.rb

@@ -161,7 +161,7 @@ module Sparkql::ParserCompatibility
   def datetime_escape(string)
     DateTime.parse(string)
   end
-  
+
   def time_escape(string)
     DateTime.parse(string)
   end
@@ -214,7 +214,8 @@ module Sparkql::ParserCompatibility
 
   # Checks the type of an expression with what is expected.
   def check_type!(expression, expected, supports_nulls = true)
-    if expected == expression[:type] || check_function_type?(expression, expected) ||
+    if (expected == expression[:type] && !expression.key?(:field_manipulations)) ||
+        (expression.key?(:field_manipulations) && check_function_type?(expression, expected)) ||
       (supports_nulls && expression[:type] == :null)
       return true
     # If the field will be passed into a function,
@@ -230,7 +231,7 @@ module Sparkql::ParserCompatibility
       expression[:type] = :datetime
       expression[:cast] = :date
       return true
-    elsif expected == :date && expression[:type] == :datetime 
+    elsif expected == :date && expression[:type] == :datetime
       expression[:type] = :date
       expression[:cast] = :datetime
       if multiple_values?(expression[:value])
@@ -253,26 +254,44 @@ module Sparkql::ParserCompatibility
             :message => "expected #{expected} but found #{expression[:type]}",
             :status => :fatal )
   end
-  
+
   # If a function is being applied to a field, we check that the return type of
   # the function matches what is expected, and that the function supports the
   # field type as the first argument.
   def check_function_type?(expression, expected)
-    return false unless expression.key?(:field_manipulations) && expression[:field_manipulations][:return_type] == expression[:type]
-    # Lookup the function arguments
-    function = Sparkql::FunctionResolver::SUPPORTED_FUNCTIONS[deepest_function(expression[:field_manipulations])[:function_name].to_sym]
-    return false if function.nil?
-
-    Array(function[:args].first).include?(expected)
+    validate_manipulation_types(expression[:field_manipulations], expected)
   end
 
+  def validate_manipulation_types(field_manipulations, expected)
+    if field_manipulations[:type] == :function
+      function = Sparkql::FunctionResolver::SUPPORTED_FUNCTIONS[field_manipulations[:function_name].to_sym]
+      return false if function.nil?
+      field_manipulations[:args].each_with_index do |arg, index|
+        if arg[:type] == :field
+          return false unless function[:args][index].include?(:field)
+        end
+      end
+    elsif field_manipulations[:type] == :arithmetic
+      lhs = field_manipulations[:lhs]
+      return false unless validate_side(lhs, expected)
 
-  def deepest_function(function)
-    if function[:args].first[:type] == :function
-      deepest_function(function[:args].first)
-    else
-      function
+      rhs = field_manipulations[:rhs]
+      return false unless rhs.nil? || validate_side(rhs, expected)
+    end
+    true
+  end
+
+  def validate_side(side, expected)
+    if side[:type] == :arithmetic
+      return validate_manipulation_types(side, expected)
+    elsif side[:type] == :field
+      return false unless [:decimal, :integer].include?(expected)
+    elsif side[:type] == :function
+      return false unless [:decimal, :integer].include?(side[:return_type])
+    elsif ![:decimal, :integer].include?(side[:type])
+      return false
     end
+    true
   end
 
   # Builds the correct operator based on the type and the value.
@@ -305,7 +324,7 @@ module Sparkql::ParserCompatibility
   def operator_supports_multiples?(operator)
     OPERATORS_SUPPORTING_MULTIPLES.include?(operator)
   end
-  
+
   def coerce_datetime datetime
     if datestr = datetime.match(/^(\d{4}-\d{2}-\d{2})/)
       datestr[0]

data/lib/sparkql/parser_tools.rb

@@ -1,4 +1,6 @@
 # This is the guts of the parser internals and is mixed into the parser for organization.
+require 'bigdecimal'
+
 module Sparkql::ParserTools
 
   # Coercible types from highest precision to lowest
@@ -264,19 +266,22 @@ module Sparkql::ParserTools
   def add_fold(n1, n2)
     return if arithmetic_error?(n1) || arithmetic_error?(n2)
 
-    { type: arithmetic_type(n1, n2), value: (escape_value(n1) + escape_value(n2)).to_s }
+    value = escape_arithmetic_value(n1) + escape_arithmetic_value(n2)
+    { type: arithmetic_type(n1, n2), value: unescape_arithmetic(value) }
   end
 
   def sub_fold(n1, n2)
     return if arithmetic_error?(n1) || arithmetic_error?(n2)
 
-    { type: arithmetic_type(n1, n2), value: (escape_value(n1) - escape_value(n2)).to_s }
+    value = escape_arithmetic_value(n1) - escape_arithmetic_value(n2)
+    { type: arithmetic_type(n1, n2), value: unescape_arithmetic(value) }
   end
 
   def mul_fold(n1, n2)
     return if arithmetic_error?(n1) || arithmetic_error?(n2)
 
-    { type: arithmetic_type(n1, n2), value: (escape_value(n1) * escape_value(n2)).to_s }
+    value = escape_arithmetic_value(n1) * escape_arithmetic_value(n2)
+    { type: arithmetic_type(n1, n2), value: unescape_arithmetic(value) }
   end
 
   def div_fold(n1, n2)
@@ -284,7 +289,8 @@ module Sparkql::ParserTools
       arithmetic_error?(n2) ||
       zero_error?(n2)
 
-    { type: arithmetic_type(n1, n2), value: (escape_value(n1) / escape_value(n2)).to_s }
+    value = escape_arithmetic_value(n1) / escape_arithmetic_value(n2)
+    { type: arithmetic_type(n1, n2), value: unescape_arithmetic(value) }
   end
 
   def mod_fold(n1, n2)
@@ -292,7 +298,8 @@ module Sparkql::ParserTools
       arithmetic_error?(n2) ||
       zero_error?(n2)
 
-    { type: arithmetic_type(n1, n2), value: (escape_value(n1) % escape_value(n2)).to_s }
+    value = escape_arithmetic_value(n1) % escape_arithmetic_value(n2)
+    { type: arithmetic_type(n1, n2), value: unescape_arithmetic(value) }
   end
 
   def arithmetic_type(num1, num2)
@@ -303,6 +310,23 @@ module Sparkql::ParserTools
     end
   end
 
+  def escape_arithmetic_value(expression)
+    case expression[:type]
+    when :decimal
+      BigDecimal.new(expression[:value])
+    else
+      escape_value(expression)
+    end
+  end
+
+  def unescape_arithmetic(value)
+    if value.is_a?(BigDecimal)
+      value.round(20).to_s('F')
+    else
+      value.to_s
+    end
+  end
+
   def zero_error?(number)
     return unless escape_value(number) == 0
 

data/test/unit/parser_compatability_test.rb

@@ -565,4 +565,19 @@ class ParserCompatabilityTest < Test::Unit::TestCase
     assert parser.send(:check_type!, expression, :datetime)
     assert_equal '3', parser.escape_value(expression)
   end
+
+  test "function with field and arithmetic" do
+    filter = "year(CloseDate) add 1 Eq 2017"
+    parser = Parser.new
+    expression = parser.tokenize(filter).first
+    assert parser.send(:check_type!, expression, :datetime)
+  end
+
+  test "Cannot perform arithmetic on a String field" do
+    filter = "City Add 3.0 Eq 'Fargo'"
+    parser = Parser.new
+    expression = parser.tokenize(filter).first
+    # Type mismatch
+    assert !parser.send(:check_type!, expression, :datetime)
+  end
 end

data/test/unit/parser_test.rb

@@ -232,6 +232,54 @@ class ParserTest < Test::Unit::TestCase
     assert_equal 'Mod', field_manipulations[:op]
   end
 
+  test 'Mod returns decimal precision' do
+    @parser = Parser.new
+    filter = "Baths Eq 32.7 Mod 20.7"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '12.0', expressions.first[:value]
+  end
+
+  test 'Adding returns decimal precision' do
+    @parser = Parser.new
+    filter = "Baths Eq 0.1 Add 0.2"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '0.3', expressions.first[:value]
+  end
+
+  test 'Subtracting returns decimal precision' do
+    @parser = Parser.new
+    filter = "Baths Eq 0.3 Sub 0.1"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '0.2', expressions.first[:value]
+  end
+
+  test 'Division returns decimal precision' do
+    @parser = Parser.new
+    filter = "Baths Eq 0.6 Div 0.2"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '3.0', expressions.first[:value]
+  end
+
+  test 'Arithmetic rounds to 20 decimal places' do
+    @parser = Parser.new
+    filter = "Baths Eq 7 Div 10.1"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '0.69306930693069306931', expressions.first[:value]
+  end
+
+  test 'Multiplication returns decimal precision' do
+    @parser = Parser.new
+    filter = "Baths Eq 7 Mul 0.1"
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, @parser.errors.inspect
+    assert_equal '0.7', expressions.first[:value]
+  end
+
   test 'arithmetic with field function' do
     @parser = Parser.new
     filter = "floor(Baths) Add 2 Eq 1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sparkql
 version: !ruby/object:Gem::Version
-  version: 1.2.3
+  version: 1.2.4
 platform: ruby
 authors:
 - Wade McEwen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-05 00:00:00.000000000 Z
+date: 2018-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: georuby