dentaku 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93d875177f2fb6231220227687ce2fa8cfb01a83
-  data.tar.gz: 2bfd4229e677bacb4838f101180574def2d1941b
+  metadata.gz: 8e51005fddbd5c542da8892bb3db03358e7d4cdf
+  data.tar.gz: 7c13a0a048a14e5bb40151224a95f7a2635ae9fa
 SHA512:
-  metadata.gz: 95daa0fa7ff7bc708492f0747ef33601c555b32a441f5b50b13c8201edbc3f76d0d606cdd8057ee408e271a9869f1ff1a34ca8219d19e2bd00fc5119c52ea311
-  data.tar.gz: 25e9184f6f077d7af863e434dbdbd87e505cf44d4fd3baf9c1a69400de77912f31b39ff48ad94f412ac3e759acd3a5ead9943837484b8555cc83d9e333a3a6e4
+  metadata.gz: 46026444a4b39bdc8df5f8913422619ac54b52d671e030b5599aa2db9642beabfdfd0c4e79ac9f40dc211de34e3ec791420aed2e162f86dd373c460229e6350a
+  data.tar.gz: 14409916459c8b1481e6f9e1005bd7c2bf49f9c84e7b29a3a97cc2161d92bf0615bde9b3b08005e177e9fcd188d76298324041fd54b36f966961e52ee4ecdf4f

data/.travis.yml

@@ -1,3 +1,8 @@
 language: ruby
 rvm:
-  - "1.9.3"
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - 2.1.1
+  - jruby-19mode
+  - rbx-2

data/README.md

@@ -10,7 +10,8 @@ DESCRIPTION
 
 Dentaku is a parser and evaluator for a mathematical and logical formula
 language that allows run-time binding of values to variables referenced in the
-formulas.
+formulas.  It is intended to safely evaluate untrusted expressions without
+opening security holes.
 
 EXAMPLE
 -------
@@ -129,7 +130,9 @@ and the exponent, so the token list could be defined as: `[:numeric,
 :numeric]`.  Other functions might be variadic -- consider `max`, a function
 that takes any number of numeric inputs and returns the largest one.  Its token
 list could be defined as: `[:non_close_plus]` (one or more tokens that are not
-closing parentheses.
+closing parentheses).  See the
+[rules definitions](https://github.com/rubysolo/dentaku/blob/master/lib/dentaku/token_matcher.rb#L61)
+for the names of token patterns you can use.
 
 Functions can be added individually using Calculator#add_function, or en masse using
 Calculator#add_functions.
@@ -169,7 +172,17 @@ THANKS
 ------
 
 Big thanks to [ElkStone Basements](http://www.elkstonebasements.com/) for
-allowing me to extract and open source this code.
+allowing me to extract and open source this code.  Thanks also to all the
+contributors:
+
+* [CraigCottingham](https://github.com/CraigCottingham)
+* [arnaudl](https://github.com/arnaudl)
+* [thbar](https://github.com/thbar) / [BoxCar](https://www.boxcar.io)
+* [antonversal](https://github.com/antonversal)
+* [mvbrocato](https://github.com/mvbrocato)
+* [brixen](https://github.com/brixen)
+* [0xCCD](https://github.com/0xCCD)
+
 
 LICENSE
 -------

data/lib/dentaku.rb

@@ -1,3 +1,4 @@
+require "bigdecimal"
 require "dentaku/calculator"
 require "dentaku/version"
 
@@ -6,6 +7,14 @@ module Dentaku
     calculator.evaluate(expression, data)
   end
 
+  class UnboundVariableError < StandardError
+    attr_reader :unbound_variables
+
+    def initialize(unbound_variables)
+      @unbound_variables = unbound_variables
+    end
+  end
+
   private
 
   def self.calculator

data/lib/dentaku/binary_operation.rb

@@ -15,7 +15,7 @@ module Dentaku
     def divide
       quotient, remainder = left.divmod(right)
       return [:numeric, quotient] if remainder == 0
-      [:numeric, left.to_f / right.to_f]
+      [:numeric, BigDecimal.new(left.to_s) / BigDecimal.new(right.to_s)]
     end
 
     def mod;      [:numeric, left % right]; end

data/lib/dentaku/calculator.rb

@@ -1,7 +1,7 @@
 require 'dentaku/evaluator'
+require 'dentaku/expression'
 require 'dentaku/rules'
 require 'dentaku/token'
-require 'dentaku/tokenizer'
 
 module Dentaku
   class Calculator
@@ -22,19 +22,20 @@ module Dentaku
     end
 
     def evaluate(expression, data={})
-      @tokenizer ||= Tokenizer.new
-      @tokens = @tokenizer.tokenize(expression)
+      evaluate!(expression, data)
+    rescue UnboundVariableError
+      yield expression if block_given?
+    end
 
+    def evaluate!(expression, data={})
       store(data) do
+        expr = Expression.new(expression, @memory)
+        raise UnboundVariableError.new(expr.identifiers) if expr.unbound?
         @evaluator ||= Evaluator.new
-        @result = @evaluator.evaluate(replace_identifiers_with_values)
+        @result = @evaluator.evaluate(expr.tokens)
       end
     end
 
-    def memory(key=nil)
-      key ? @memory[key.to_sym] : @memory
-    end
-
     def store(key_or_hash, value=nil)
       restore = @memory.dup
 
@@ -63,28 +64,5 @@ module Dentaku
     def empty?
       @memory.empty?
     end
-
-    private
-
-    def replace_identifiers_with_values
-      @tokens.map do |token|
-        if token.is?(:identifier)
-          value = memory(token.value)
-          type  = type_for_value(value)
-
-          Token.new(type, value)
-        else
-          token
-        end
-      end
-    end
-
-    def type_for_value(value)
-      case value
-      when String then :string
-      when TrueClass, FalseClass then :logical
-      else :numeric
-      end
-    end
   end
 end

data/lib/dentaku/evaluator.rb

@@ -10,7 +10,7 @@ module Dentaku
     def evaluate_token_stream(tokens)
       while tokens.length > 1
         matched, tokens = match_rule_pattern(tokens)
-        raise "no rule matched #{ tokens.map(&:category).inspect }" unless matched
+        raise "no rule matched {{#{ inspect_tokens(tokens) }}}" unless matched
       end
 
       tokens << Token.new(:numeric, 0) if tokens.empty?
@@ -18,6 +18,10 @@ module Dentaku
       tokens.first
     end
 
+    def inspect_tokens(tokens)
+      tokens.map { |t| t.to_s }.join(' ')
+    end
+
     def match_rule_pattern(tokens)
       matched = false
       Rules.each do |pattern, evaluator|
@@ -41,8 +45,8 @@ module Dentaku
         matched = true
 
         pattern.each do |matcher|
-          match = matcher.match(token_stream, position + matches.length)
-          matched &&= match.matched?
+          _matched, match = matcher.match(token_stream, position + matches.length)
+          matched &&= _matched
           matches += match
         end
 
@@ -54,19 +58,23 @@ module Dentaku
     end
 
     def evaluate_step(token_stream, start, length, evaluator)
-      expr = token_stream.slice!(start, length)
+      substream = token_stream.slice!(start, length)
 
       if self.respond_to?(evaluator)
-        token_stream.insert start, *self.send(evaluator, *expr)
+        token_stream.insert start, *self.send(evaluator, *substream)
       else
-        func = Rules.func(evaluator)
-        raise "unknown evaluator '#{evaluator.to_s}'" if func.nil?
+        result = user_defined_function(evaluator, substream)
+        token_stream.insert start, result
+      end
+    end
 
-        arguments = extract_arguments_from_function_call(expr).map { |t| t.value }
-        return_value = func.body.call(*arguments)
+    def user_defined_function(evaluator, tokens)
+      function = Rules.func(evaluator)
+      raise "unknown function '#{ evaluator }'" unless function
 
-        token_stream.insert start, Token.new(func.type, return_value)
-      end
+      arguments = extract_arguments_from_function_call(tokens).map { |t| t.value }
+      return_value = function.body.call(*arguments)
+      Token.new(function.type, return_value)
     end
 
     def extract_arguments_from_function_call(tokens)

data/lib/dentaku/expression.rb

@@ -0,0 +1,55 @@
+require 'dentaku/tokenizer'
+
+module Dentaku
+  class Expression
+    attr_reader :tokens
+
+    def initialize(string, variables={})
+      @raw = string
+      @tokenizer ||= Tokenizer.new
+      @tokens = @tokenizer.tokenize(@raw)
+      replace_identifiers_with_values(variables)
+    end
+
+    def identifiers
+      @tokens.select { |t| t.category == :identifier }.map { |t| t.value }
+    end
+
+    def unbound?
+      identifiers.any?
+    end
+
+    private
+
+    def replace_identifiers_with_values(variables)
+      @tokens.map! do |token|
+        if token.is?(:identifier)
+          replace_identifier_with_value(token, variables)
+        else
+          token
+        end
+      end
+    end
+
+    def replace_identifier_with_value(token, variables)
+      key = token.value.to_sym
+
+      if variables.key? key
+        value = variables[key]
+        type  = type_for_value(value)
+
+        Token.new(type, value)
+      else
+        token
+      end
+    end
+
+    def type_for_value(value)
+      case value
+      when String then :string
+      when TrueClass, FalseClass then :logical
+      else :numeric
+      end
+    end
+  end
+end

data/lib/dentaku/token.rb

@@ -8,6 +8,10 @@ module Dentaku
       @raw_value = raw_value
     end
 
+    def to_s
+      raw_value || value
+    end
+
     def length
       raw_value.to_s.length
     end

data/lib/dentaku/token_matcher.rb

@@ -7,8 +7,12 @@ module Dentaku
       @values     = [values].compact.flatten
       @invert     = false
 
+      @categories_hash = Hash[@categories.map { |cat| [cat, 1] }]
+      @values_hash = Hash[@values.map { |value| [value, 1] }]
+
       @min = 1
       @max = 1
+      @range = (@min..@max)
     end
 
     def invert
@@ -23,39 +27,40 @@ module Dentaku
 
     def match(token_stream, offset=0)
       matched_tokens = []
+      matched = false
 
       while self == token_stream[matched_tokens.length + offset] && matched_tokens.length < @max
         matched_tokens << token_stream[matched_tokens.length + offset]
       end
 
-      if (@min..@max).include? matched_tokens.length
-        def matched_tokens.matched?() true end
-      else
-        def matched_tokens.matched?() false end
+      if @range.cover?(matched_tokens.length)
+        matched = true
       end
 
-      matched_tokens
+      [matched, matched_tokens]
     end
 
     def star
       @min = 0
       @max = Float::INFINITY
+      @range = (@min..@max)
       self
     end
 
     def plus
       @max = Float::INFINITY
+      @range = (@min..@max)
       self
     end
 
     private
 
     def category_match(category)
-      @categories.empty? || @categories.include?(category)
+      @categories_hash.empty? || @categories_hash.key?(category)
     end
 
     def value_match(value)
-      @values.empty? || @values.include?(value)
+      @values.empty? || @values_hash.key?(value)
     end
 
     def self.numeric;        new(:numeric);                        end
@@ -92,5 +97,4 @@ module Dentaku
     end
 
   end
-end
-
+end

data/lib/dentaku/token_scanner.rb

@@ -40,7 +40,7 @@ module Dentaku
       end
 
       def numeric
-        new(:numeric, '(\d+(\.\d+)?|\.\d+)\b', lambda { |raw| raw =~ /\./ ? raw.to_f : raw.to_i })
+        new(:numeric, '(\d+(\.\d+)?|\.\d+)\b', lambda { |raw| raw =~ /\./ ? BigDecimal.new(raw) : raw.to_i })
       end
 
       def double_quoted_string
@@ -63,8 +63,8 @@ module Dentaku
 
       def comparator
         names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert
-        alternate = { ne: '<>' }.invert
-        new(:comparator, '<=|>=|!=|<>|<|>|=', lambda { |raw| names[raw] || alternate[raw] })
+        alternate = { ne: '<>', eq: '==' }.invert
+        new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] })
       end
 
       def combinator

data/lib/dentaku/version.rb

@@ -1,3 +1,3 @@
 module Dentaku
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/spec/binary_operation_spec.rb

@@ -5,41 +5,41 @@ describe Dentaku::BinaryOperation do
   let(:logical)   { described_class.new(true, false) }
 
   it 'raises a number to a power' do
-    operation.pow.should eq [:numeric, 8]
+    expect(operation.pow).to eq [:numeric, 8]
   end
 
   it 'adds two numbers' do
-    operation.add.should eq [:numeric, 5]
+    expect(operation.add).to eq [:numeric, 5]
   end
 
   it 'subtracts two numbers' do
-    operation.subtract.should eq [:numeric, -1]
+    expect(operation.subtract).to eq [:numeric, -1]
   end
 
   it 'multiplies two numbers' do
-    operation.multiply.should eq [:numeric, 6]
+    expect(operation.multiply).to eq [:numeric, 6]
   end
 
   it 'divides two numbers' do
-    operation.divide.should eq [:numeric, (2.0/3.0)]
+    expect(operation.divide).to eq [:numeric, (BigDecimal.new('2.0')/BigDecimal.new('3.0'))]
   end
 
   it 'compares two numbers' do
-    operation.le.should eq [:logical, true]
-    operation.lt.should eq [:logical, true]
-    operation.ne.should eq [:logical, true]
+    expect(operation.le).to eq [:logical, true]
+    expect(operation.lt).to eq [:logical, true]
+    expect(operation.ne).to eq [:logical, true]
 
-    operation.ge.should eq [:logical, false]
-    operation.gt.should eq [:logical, false]
-    operation.eq.should eq [:logical, false]
+    expect(operation.ge).to eq [:logical, false]
+    expect(operation.gt).to eq [:logical, false]
+    expect(operation.eq).to eq [:logical, false]
   end
 
   it 'performs logical AND and OR' do
-    logical.and.should eq [:logical, false]
-    logical.or.should  eq [:logical, true]
+    expect(logical.and).to eq [:logical, false]
+    expect(logical.or).to  eq [:logical, true]
   end
 
   it 'mods two numbers' do
-    operation.mod.should eq [:numeric, 2%3]
+    expect(operation.mod).to eq [:numeric, 2%3]
   end
 end

data/spec/calculator_spec.rb

@@ -5,102 +5,102 @@ describe Dentaku::Calculator do
   let(:with_memory) { described_class.new.store(:apples => 3) }
 
   it 'evaluates an expression' do
-    calculator.evaluate('7+3').should eq(10)
+    expect(calculator.evaluate('7+3')).to eq(10)
   end
 
   describe 'memory' do
-    it { calculator.should be_empty }
-    it { with_memory.should_not be_empty   }
-    it { with_memory.clear.should be_empty }
+    it { expect(calculator).to be_empty }
+    it { expect(with_memory).not_to be_empty   }
+    it { expect(with_memory.clear).to be_empty }
 
-    it { with_memory.memory(:apples).should eq(3) }
-    it { with_memory.memory('apples').should eq(3) }
-
-    it { calculator.store(:apples, 3).memory('apples').should eq(3) }
-    it { calculator.store('apples', 3).memory(:apples).should eq(3) }
-
-    it 'should discard local values' do
-      calculator.evaluate('pears * 2', :pears => 5).should eq(10)
-      calculator.should be_empty
-      lambda { calculator.tokenize('pears * 2') }.should raise_error
+    it 'discards local values' do
+      expect(calculator.evaluate('pears * 2', :pears => 5)).to eq(10)
+      expect(calculator).to be_empty
     end
   end
 
-  it 'should evaluate a statement with no variables' do
-    calculator.evaluate('5+3').should eq(8)
-    calculator.evaluate('(1+1+1)/3*100').should eq(100)
+  it 'evaluates a statement with no variables' do
+    expect(calculator.evaluate('5+3')).to eq(8)
+    expect(calculator.evaluate('(1+1+1)/3*100')).to eq(100)
   end
 
-  it 'should fail to evaluate unbound statements' do
-    lambda { calculator.evaluate('foo * 1.5') }.should raise_error
+  it 'fails to evaluate unbound statements' do
+    unbound = 'foo * 1.5'
+    expect { calculator.evaluate!(unbound) }.to raise_error(Dentaku::UnboundVariableError)
+    expect { calculator.evaluate!(unbound) }.to raise_error do |error|
+      expect(error.unbound_variables).to eq [:foo]
+    end
+    expect(calculator.evaluate(unbound)).to be_nil
+    expect(calculator.evaluate(unbound) { :bar }).to eq :bar
+    expect(calculator.evaluate(unbound) { |e| e }).to eq unbound
   end
 
-  it 'should evaluate unbound statements given a binding in memory' do
-    calculator.evaluate('foo * 1.5', :foo => 2).should eq(3)
-    calculator.bind(:monkeys => 3).evaluate('monkeys < 7').should be_true
-    calculator.evaluate('monkeys / 1.5').should eq(2)
+  it 'evaluates unbound statements given a binding in memory' do
+    expect(calculator.evaluate('foo * 1.5', :foo => 2)).to eq(3)
+    expect(calculator.bind(:monkeys => 3).evaluate('monkeys < 7')).to be_truthy
+    expect(calculator.evaluate('monkeys / 1.5')).to eq(2)
   end
 
-  it 'should rebind for each evaluation' do
-    calculator.evaluate('foo * 2', :foo => 2).should eq(4)
-    calculator.evaluate('foo * 2', :foo => 4).should eq(8)
+  it 'rebinds for each evaluation' do
+    expect(calculator.evaluate('foo * 2', :foo => 2)).to eq(4)
+    expect(calculator.evaluate('foo * 2', :foo => 4)).to eq(8)
   end
 
-  it 'should accept strings or symbols for binding keys' do
-    calculator.evaluate('foo * 2', :foo => 2).should eq(4)
-    calculator.evaluate('foo * 2', 'foo' => 4).should eq(8)
+  it 'accepts strings or symbols for binding keys' do
+    expect(calculator.evaluate('foo * 2', :foo => 2)).to eq(4)
+    expect(calculator.evaluate('foo * 2', 'foo' => 4)).to eq(8)
   end
 
-  it 'should accept digits in identifiers' do
-    calculator.evaluate('foo1 * 2', :foo1 => 2).should eq(4)
-    calculator.evaluate('foo1 * 2', 'foo1' => 4).should eq(8)
-    calculator.evaluate('1foo * 2', '1foo' => 2).should eq(4)
-    calculator.evaluate('fo1o * 2', :fo1o => 4).should eq(8)
+  it 'accepts digits in identifiers' do
+    expect(calculator.evaluate('foo1 * 2', :foo1 => 2)).to eq(4)
+    expect(calculator.evaluate('foo1 * 2', 'foo1' => 4)).to eq(8)
+    expect(calculator.evaluate('1foo * 2', '1foo' => 2)).to eq(4)
+    expect(calculator.evaluate('fo1o * 2', :fo1o => 4)).to eq(8)
   end
 
-  it 'should compare string literals with string variables' do
-    calculator.evaluate('fruit = "apple"', :fruit => 'apple').should be_true
-    calculator.evaluate('fruit = "apple"', :fruit => 'pear').should be_false
+  it 'compares string literals with string variables' do
+    expect(calculator.evaluate('fruit = "apple"', :fruit => 'apple')).to be_truthy
+    expect(calculator.evaluate('fruit = "apple"', :fruit => 'pear')).to be_falsey
   end
 
-  it 'should do case-sensitive comparison' do
-    calculator.evaluate('fruit = "Apple"', :fruit => 'apple').should be_false
-    calculator.evaluate('fruit = "Apple"', :fruit => 'Apple').should be_true
+  it 'performs case-sensitive comparison' do
+    expect(calculator.evaluate('fruit = "Apple"', :fruit => 'apple')).to be_falsey
+    expect(calculator.evaluate('fruit = "Apple"', :fruit => 'Apple')).to be_truthy
   end
 
-  it 'should allow binding logical values' do
-    calculator.evaluate('some_boolean AND 7 > 5', :some_boolean => true).should be_true
-    calculator.evaluate('some_boolean AND 7 < 5', :some_boolean => true).should be_false
-    calculator.evaluate('some_boolean AND 7 > 5', :some_boolean => false).should be_false
+  it 'allows binding logical values' do
+    expect(calculator.evaluate('some_boolean AND 7 > 5', :some_boolean => true)).to be_truthy
+    expect(calculator.evaluate('some_boolean AND 7 < 5', :some_boolean => true)).to be_falsey
+    expect(calculator.evaluate('some_boolean AND 7 > 5', :some_boolean => false)).to be_falsey
 
-    calculator.evaluate('some_boolean OR 7 > 5', :some_boolean => true).should be_true
-    calculator.evaluate('some_boolean OR 7 < 5', :some_boolean => true).should be_true
-    calculator.evaluate('some_boolean OR 7 < 5', :some_boolean => false).should be_false
+    expect(calculator.evaluate('some_boolean OR 7 > 5', :some_boolean => true)).to be_truthy
+    expect(calculator.evaluate('some_boolean OR 7 < 5', :some_boolean => true)).to be_truthy
+    expect(calculator.evaluate('some_boolean OR 7 < 5', :some_boolean => false)).to be_falsey
 
   end
 
   describe 'functions' do
-    it 'should include IF' do
-      calculator.evaluate('if(foo < 8, 10, 20)', :foo => 2).should eq(10)
-      calculator.evaluate('if(foo < 8, 10, 20)', :foo => 9).should eq(20)
-      calculator.evaluate('if (foo < 8, 10, 20)', :foo => 2).should eq(10)
-      calculator.evaluate('if (foo < 8, 10, 20)', :foo => 9).should eq(20)
+    it 'include IF' do
+      expect(calculator.evaluate('if(foo < 8, 10, 20)', :foo => 2)).to eq(10)
+      expect(calculator.evaluate('if(foo < 8, 10, 20)', :foo => 9)).to eq(20)
+      expect(calculator.evaluate('if (foo < 8, 10, 20)', :foo => 2)).to eq(10)
+      expect(calculator.evaluate('if (foo < 8, 10, 20)', :foo => 9)).to eq(20)
     end
 
-    it 'should include ROUND' do
-      calculator.evaluate('round(8.2)').should eq(8)
-      calculator.evaluate('round(8.8)').should eq(9)
-      calculator.evaluate('round(8.75, 1)').should eq(8.8)
+    it 'include ROUND' do
+      expect(calculator.evaluate('round(8.2)')).to eq(8)
+      expect(calculator.evaluate('round(8.8)')).to eq(9)
+      expect(calculator.evaluate('round(8.75, 1)')).to eq(BigDecimal.new('8.8'))
 
-      calculator.evaluate('ROUND(apples * 0.93)', { :apples => 10 }).should eq(9)
+      expect(calculator.evaluate('ROUND(apples * 0.93)', { :apples => 10 })).to eq(9)
     end
 
-    it 'should include NOT' do
-      calculator.evaluate('NOT(some_boolean)', :some_boolean => true).should be_false
-      calculator.evaluate('NOT(some_boolean)', :some_boolean => false).should be_true
+    it 'include NOT' do
+      expect(calculator.evaluate('NOT(some_boolean)', :some_boolean => true)).to be_falsey
+      expect(calculator.evaluate('NOT(some_boolean)', :some_boolean => false)).to be_truthy
 
-      calculator.evaluate('NOT(some_boolean) AND 7 > 5', :some_boolean => true).should be_false
-      calculator.evaluate('NOT(some_boolean) OR 7 < 5', :some_boolean => false).should be_true
+      expect(calculator.evaluate('NOT(some_boolean) AND 7 > 5', :some_boolean => true)).to be_falsey
+      expect(calculator.evaluate('NOT(some_boolean) OR 7 < 5', :some_boolean => false)).to be_truthy
     end
   end
 end