dentaku 1.2.6 → 2.0.0

data/spec/tokenizer_spec.rb

@@ -12,7 +12,33 @@ describe Dentaku::Tokenizer do
     expect(tokens.map(&:category)).to eq([:numeric, :operator, :numeric])
     expect(tokens.map(&:value)).to eq([1, :add, 1])
   end
-  
+
+  it 'tokenizes unary minus' do
+    tokens = tokenizer.tokenize('-5')
+    expect(tokens.map(&:category)).to eq([:operator, :numeric])
+    expect(tokens.map(&:value)).to eq([:negate, 5])
+
+    tokens = tokenizer.tokenize('(-5)')
+    expect(tokens.map(&:category)).to eq([:grouping, :operator, :numeric, :grouping])
+    expect(tokens.map(&:value)).to eq([:open, :negate, 5, :close])
+
+    tokens = tokenizer.tokenize('if(-5 > x, -7, -8) - 9')
+    expect(tokens.map(&:category)).to eq([
+      :function, :grouping,                                      # if(
+      :operator, :numeric, :comparator, :identifier, :grouping,  # -5 > x,
+      :operator, :numeric, :grouping,                            # -7,
+      :operator, :numeric, :grouping,                            # -8)
+      :operator, :numeric                                        # - 9
+    ])
+    expect(tokens.map(&:value)).to eq([
+      :if, :fopen,                  # if(
+      :negate, 5, :gt, 'x', :comma, # -5 > x,
+      :negate, 7, :comma,           # -7,
+      :negate, 8, :close,           # -8)
+      :subtract, 9                  # - 9
+    ])
+  end
+
   it 'tokenizes comparison with =' do
     tokens = tokenizer.tokenize('number = 5')
     expect(tokens.map(&:category)).to eq([:identifier, :comparator, :numeric])
@@ -46,7 +72,7 @@ describe Dentaku::Tokenizer do
   it 'tokenizes power operations' do
     tokens = tokenizer.tokenize('0 * 10 ^ -5')
     expect(tokens.map(&:category)).to eq([:numeric, :operator, :numeric, :operator, :operator, :numeric])
-    expect(tokens.map(&:value)).to eq([0, :multiply, 10, :pow, :subtract, 5])
+    expect(tokens.map(&:value)).to eq([0, :multiply, 10, :pow, :negate, 5])
   end
 
   it 'handles floating point' do
@@ -62,9 +88,9 @@ describe Dentaku::Tokenizer do
   end
 
   it 'accepts arbitrary identifiers' do
-    tokens = tokenizer.tokenize('monkeys > 1500')
+    tokens = tokenizer.tokenize('sea_monkeys > 1500')
     expect(tokens.map(&:category)).to eq([:identifier, :comparator, :numeric])
-    expect(tokens.map(&:value)).to eq(['monkeys', :gt, 1500])
+    expect(tokens.map(&:value)).to eq(['sea_monkeys', :gt, 1500])
   end
 
   it 'recognizes double-quoted strings' do
@@ -88,13 +114,13 @@ describe Dentaku::Tokenizer do
   it 'recognizes unary minus operator' do
     tokens = tokenizer.tokenize('-2 + 3')
     expect(tokens.map(&:category)).to eq([:operator, :numeric, :operator, :numeric])
-    expect(tokens.map(&:value)).to eq([:subtract, 2, :add, 3])
+    expect(tokens.map(&:value)).to eq([:negate, 2, :add, 3])
   end
 
   it 'recognizes unary minus operator' do
     tokens = tokenizer.tokenize('2 - -3')
     expect(tokens.map(&:category)).to eq([:numeric, :operator, :operator, :numeric])
-    expect(tokens.map(&:value)).to eq([2, :subtract, :subtract, 3])
+    expect(tokens.map(&:value)).to eq([2, :subtract, :negate, 3])
   end
 
   it 'matches "<=" before "<"' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dentaku
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 2.0.0
 platform: ruby
 authors:
 - Solomon White
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-30 00:00:00.000000000 Z
+date: 2015-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -68,29 +68,50 @@ files:
 - Rakefile
 - dentaku.gemspec
 - lib/dentaku.rb
-- lib/dentaku/binary_operation.rb
+- lib/dentaku/ast.rb
+- lib/dentaku/ast/addition.rb
+- lib/dentaku/ast/combinators.rb
+- lib/dentaku/ast/comparators.rb
+- lib/dentaku/ast/division.rb
+- lib/dentaku/ast/exponentiation.rb
+- lib/dentaku/ast/function.rb
+- lib/dentaku/ast/functions/if.rb
+- lib/dentaku/ast/functions/max.rb
+- lib/dentaku/ast/functions/min.rb
+- lib/dentaku/ast/functions/not.rb
+- lib/dentaku/ast/functions/round.rb
+- lib/dentaku/ast/functions/rounddown.rb
+- lib/dentaku/ast/functions/roundup.rb
+- lib/dentaku/ast/functions/ruby_math.rb
+- lib/dentaku/ast/grouping.rb
+- lib/dentaku/ast/identifier.rb
+- lib/dentaku/ast/multiplication.rb
+- lib/dentaku/ast/negation.rb
+- lib/dentaku/ast/nil.rb
+- lib/dentaku/ast/node.rb
+- lib/dentaku/ast/numeric.rb
+- lib/dentaku/ast/operation.rb
+- lib/dentaku/ast/string.rb
+- lib/dentaku/ast/subtraction.rb
 - lib/dentaku/bulk_expression_solver.rb
 - lib/dentaku/calculator.rb
 - lib/dentaku/dependency_resolver.rb
-- lib/dentaku/evaluator.rb
 - lib/dentaku/exceptions.rb
-- lib/dentaku/expression.rb
-- lib/dentaku/external_function.rb
-- lib/dentaku/rule_set.rb
+- lib/dentaku/parser.rb
 - lib/dentaku/token.rb
 - lib/dentaku/token_matcher.rb
+- lib/dentaku/token_matchers.rb
 - lib/dentaku/token_scanner.rb
 - lib/dentaku/tokenizer.rb
 - lib/dentaku/version.rb
+- spec/ast/function_spec.rb
+- spec/ast/node_spec.rb
 - spec/benchmark.rb
-- spec/binary_operation_spec.rb
 - spec/bulk_expression_solver_spec.rb
 - spec/calculator_spec.rb
 - spec/dentaku_spec.rb
-- spec/evaluator_spec.rb
-- spec/expression_spec.rb
 - spec/external_function_spec.rb
-- spec/rule_set_spec.rb
+- spec/parser_spec.rb
 - spec/spec_helper.rb
 - spec/token_matcher_spec.rb
 - spec/token_scanner_spec.rb
@@ -116,20 +137,19 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: dentaku
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: A formula language parser and evaluator
 test_files:
+- spec/ast/function_spec.rb
+- spec/ast/node_spec.rb
 - spec/benchmark.rb
-- spec/binary_operation_spec.rb
 - spec/bulk_expression_solver_spec.rb
 - spec/calculator_spec.rb
 - spec/dentaku_spec.rb
-- spec/evaluator_spec.rb
-- spec/expression_spec.rb
 - spec/external_function_spec.rb
-- spec/rule_set_spec.rb
+- spec/parser_spec.rb
 - spec/spec_helper.rb
 - spec/token_matcher_spec.rb
 - spec/token_scanner_spec.rb

data/lib/dentaku/binary_operation.rb

@@ -1,35 +0,0 @@
-require 'bigdecimal'
-
-module Dentaku
-  class BinaryOperation
-    attr_reader :left, :right
-
-    def initialize(left, right)
-      @left = left
-      @right = right
-    end
-
-    def pow;      [:numeric, left ** right]; end
-    def add;      [:numeric, left + right];  end
-    def subtract; [:numeric, left - right];  end
-    def multiply; [:numeric, left * right];  end
-
-    def divide
-      quotient, remainder = left.divmod(right)
-      return [:numeric, quotient] if remainder == 0
-      [:numeric, BigDecimal.new(left.to_s) / BigDecimal.new(right.to_s)]
-    end
-
-    def mod;      [:numeric, left % right]; end
-
-    def le;       [:logical, left <= right]; end
-    def ge;       [:logical, left >= right]; end
-    def lt;       [:logical, left <  right]; end
-    def gt;       [:logical, left >  right]; end
-    def ne;       [:logical, left != right]; end
-    def eq;       [:logical, left == right]; end
-
-    def and;      [:logical, left && right]; end
-    def or;       [:logical, left || right]; end
-  end
-end

data/lib/dentaku/evaluator.rb

@@ -1,166 +0,0 @@
-require 'dentaku/rule_set'
-require 'dentaku/binary_operation'
-
-module Dentaku
-  class Evaluator
-    attr_reader :rule_set
-
-    def initialize(rule_set)
-      @rule_set = rule_set
-    end
-
-    def evaluate(tokens)
-      evaluate_token_stream(tokens).value
-    end
-
-    def evaluate_token_stream(tokens)
-      while tokens.length > 1
-        matched, tokens = match_rule_pattern(tokens)
-        raise "no rule matched {{#{ inspect_tokens(tokens) }}}" unless matched
-      end
-
-      tokens << Token.new(:numeric, 0) if tokens.empty?
-
-      tokens.first
-    end
-
-    def inspect_tokens(tokens)
-      tokens.map { |t| t.to_s }.join(' ')
-    end
-
-    def match_rule_pattern(tokens)
-      matched = false
-
-      rule_set.filter(tokens).each do |pattern, evaluator|
-        pos, match = find_rule_match(pattern, tokens)
-
-        if pos
-          tokens = evaluate_step(tokens, pos, match.length, evaluator)
-          matched = true
-          break
-        end
-      end
-
-      [matched, tokens]
-    end
-
-    def find_rule_match(pattern, token_stream)
-      position = 0
-
-      while position <= token_stream.length
-        matches = []
-        matched = true
-
-        pattern.each do |matcher|
-          _matched, match = matcher.match(token_stream, position + matches.length)
-          matched &&= _matched
-          break unless matched
-          matches += match
-        end
-
-        return position, matches if matched
-        return if pattern.first.caret?
-        position += 1
-      end
-
-      nil
-    end
-
-    def evaluate_step(token_stream, start, length, evaluator)
-      substream = token_stream.slice!(start, length)
-
-      if self.respond_to?(evaluator)
-        token_stream.insert start, *self.send(evaluator, *substream)
-      else
-        result = user_defined_function(evaluator, substream)
-        token_stream.insert start, result
-      end
-    end
-
-    def user_defined_function(evaluator, tokens)
-      function = rule_set.function(evaluator)
-      raise "unknown function '#{ evaluator }'" unless function
-
-      arguments = extract_arguments_from_function_call(tokens).map { |t| t.value }
-      return_value = function.body.call(*arguments)
-      Token.new(function.type, return_value)
-    end
-
-    def extract_arguments_from_function_call(tokens)
-      _function_name, _open, *args_and_commas, _close = tokens
-      args_and_commas.reject { |token| token.is?(:grouping) }
-    end
-
-    def evaluate_group(*args)
-      evaluate_token_stream(args[1..-2])
-    end
-
-    def apply(lvalue, operator, rvalue)
-      operation = BinaryOperation.new(lvalue.value, rvalue.value)
-      raise "unknown operation #{ operator.value }" unless operation.respond_to?(operator.value)
-      Token.new(*operation.send(operator.value))
-    end
-
-    def negate(_, token)
-      Token.new(token.category, token.value * -1)
-    end
-
-    def pow_negate(base, _, _, exp)
-      Token.new(base.category, base.value ** (exp.value * -1))
-    end
-
-    def mul_negate(val1, _, _, val2)
-      Token.new(val1.category, val1.value * val2.value * -1)
-    end
-
-    def percentage(token, _)
-      Token.new(token.category, token.value / 100.0)
-    end
-
-    def expand_range(left, oper1, middle, oper2, right)
-      [left, oper1, middle, Token.new(:combinator, :and), middle, oper2, right]
-    end
-
-    def if(*args)
-      _if, _open, condition, _, true_value, _, false_value, _close = args
-
-      if condition.value
-        true_value
-      else
-        false_value
-      end
-    end
-
-    def round(*args)
-      _, _, *tokens, _ = args
-
-      input_tokens, places_tokens = tokens.chunk { |t| t.category == :grouping }.
-                                          reject { |flag, tokens| flag }.
-                                             map { |flag, tokens| tokens }
-
-      input_value  = evaluate_token_stream(input_tokens).value
-      places       = places_tokens ? evaluate_token_stream(places_tokens).value : 0
-
-      value = input_value.round(places)
-
-      Token.new(:numeric, value)
-    end
-
-    def round_int(*args)
-      function, _, *tokens, _ = args
-
-      value = evaluate_token_stream(tokens).value
-      rounded = if function.value == :roundup
-        value.ceil
-      else
-        value.floor
-      end
-
-      Token.new(:numeric, rounded)
-    end
-
-    def not(*args)
-      Token.new(:logical, ! evaluate_token_stream(args[2..-2]).value)
-    end
-  end
-end

data/lib/dentaku/expression.rb

@@ -1,56 +0,0 @@
-require 'dentaku/tokenizer'
-
-module Dentaku
-  class Expression
-    attr_reader :tokens, :variables
-
-    def initialize(string, variables={})
-      @raw = string
-      @tokenizer ||= Tokenizer.new
-      @tokens = @tokenizer.tokenize(@raw)
-      @variables = Hash[variables.map { |k,v| [k.to_s, v] }]
-      replace_identifiers_with_values
-    end
-
-    def identifiers
-      @tokens.select { |t| t.category == :identifier }.map { |t| t.value }
-    end
-
-    def unbound?
-      identifiers.any?
-    end
-
-    private
-
-    def replace_identifiers_with_values
-      @tokens.map! do |token|
-        if token.is?(:identifier)
-          replace_identifier_with_value(token)
-        else
-          token
-        end
-      end
-    end
-
-    def replace_identifier_with_value(token)
-      key = token.value.to_s
-
-      if variables.key? key
-        value = variables[key]
-        type  = type_for_value(value)
-
-        Token.new(type, value)
-      else
-        token
-      end
-    end
-
-    def type_for_value(value)
-      case value
-      when String then :string
-      when TrueClass, FalseClass then :logical
-      else :numeric
-      end
-    end
-  end
-end

data/lib/dentaku/external_function.rb

@@ -1,10 +0,0 @@
-class ExternalFunction < Struct.new(:name, :type, :signature, :body)
-  def initialize(*)
-    super
-    self.name = self.name.to_s
-  end
-
-  def tokens
-    signature.flat_map { |t| [t, :comma] }[0...-1]
-  end
-end

data/lib/dentaku/rule_set.rb

@@ -1,153 +0,0 @@
-require 'dentaku/external_function'
-
-module Dentaku
-  class RuleSet
-    def initialize
-      self.custom_rules = []
-      self.custom_functions = {}
-    end
-
-    def rules
-      custom_rules + core_rules
-    end
-
-    def each
-      rules.each { |r| yield r }
-    end
-
-    def add_function(function)
-      fn = ExternalFunction.new(function[:name], function[:type], function[:signature], function[:body])
-
-      custom_rules.push [
-        function_token_matchers(fn.name, *fn.tokens),
-        fn.name
-      ]
-
-      custom_functions[fn.name] = fn
-      clear_cache
-    end
-
-    def filter(tokens)
-      categories = tokens.map(&:category).uniq
-      values     = tokens.map { |token| token.value.is_a?(Numeric) ? 0 : token.value }
-                         .reject { |token| [:fopen, :close].include?(token) }
-      select(categories, values)
-    end
-
-    def select(categories, values)
-      @cache ||= {}
-      return @cache[categories + values] if @cache.has_key?(categories + values)
-
-      @cache[categories + values] = rules.select do |pattern, _|
-        categories_intersection = matcher_categories[pattern] & categories
-        values_intersection     = matcher_values[pattern] & values
-        categories_intersection.length > 0 && (values_intersection.length > 0 || matcher_values[pattern].empty?)
-      end
-    end
-
-    def function(name)
-      custom_functions.fetch(name)
-    end
-
-    private
-    attr_accessor :custom_rules, :custom_functions
-
-    def matcher_categories
-      @matcher_categories ||= rules.each_with_object({}) do |(pattern, _), h|
-        h[pattern] = pattern.map(&:categories).reduce { |a,b| a.merge(b) }.keys
-      end
-    end
-
-    def matcher_values
-      @matcher_values ||= rules.each_with_object({}) do |(pattern, _), h|
-        h[pattern] = pattern.map(&:values).reduce { |a,b| a.merge(b) }.keys
-      end
-    end
-
-    def clear_cache
-      @cache = nil
-      @matcher_categories = nil
-      @matcher_values = nil
-    end
-
-    def core_rules
-      @core_rules ||= [
-        [ pattern(:if),           :if             ],
-        [ pattern(:round),        :round          ],
-        [ pattern(:roundup),      :round_int      ],
-        [ pattern(:rounddown),    :round_int      ],
-        [ pattern(:not),          :not            ],
-
-        [ pattern(:group),        :evaluate_group ],
-        [ pattern(:start_neg),    :negate         ],
-        [ pattern(:math_pow),     :apply          ],
-        [ pattern(:math_neg_pow), :pow_negate     ],
-        [ pattern(:math_mod),     :apply          ],
-        [ pattern(:math_mul),     :apply          ],
-        [ pattern(:math_neg_mul), :mul_negate     ],
-        [ pattern(:math_add),     :apply          ],
-        [ pattern(:percentage),   :percentage     ],
-        [ pattern(:negation),     :negate         ],
-        [ pattern(:range_asc),    :expand_range   ],
-        [ pattern(:range_desc),   :expand_range   ],
-        [ pattern(:num_comp),     :apply          ],
-        [ pattern(:str_comp),     :apply          ],
-        [ pattern(:combine),      :apply          ]
-      ]
-    end
-
-    def pattern(name)
-      @patterns ||= {
-        group:        token_matchers(:open,     :non_group_star, :close),
-        math_add:     token_matchers(:numeric,  :addsub,         :numeric),
-        math_mul:     token_matchers(:numeric,  :muldiv,         :numeric),
-        math_neg_mul: token_matchers(:numeric,  :muldiv,         :subtract, :numeric),
-        math_pow:     token_matchers(:numeric,  :pow,            :numeric),
-        math_neg_pow: token_matchers(:numeric,  :pow,            :subtract, :numeric),
-        math_mod:     token_matchers(:numeric,  :mod,            :numeric),
-        negation:     token_matchers(:subtract, :numeric),
-        start_neg:    token_matchers(:anchored_minus, :numeric),
-        percentage:   token_matchers(:numeric,  :mod),
-        range_asc:    token_matchers(:numeric,  :comp_lt,        :numeric,  :comp_lt, :numeric),
-        range_desc:   token_matchers(:numeric,  :comp_gt,        :numeric,  :comp_gt, :numeric),
-        num_comp:     token_matchers(:numeric,  :comparator,     :numeric),
-        str_comp:     token_matchers(:string,   :comparator,     :string),
-        combine:      token_matchers(:logical,  :combinator,     :logical),
-
-        if:           function_token_matchers(:if,        :non_group,      :comma, :non_group, :comma, :non_group),
-        round:        function_token_matchers(:round,     :arguments),
-        roundup:      function_token_matchers(:roundup,   :arguments),
-        rounddown:    function_token_matchers(:rounddown, :arguments),
-        not:          function_token_matchers(:not,       :arguments)
-      }
-
-      @patterns[name]
-    end
-
-    def token_matchers(*symbols)
-      symbols.map { |s| matcher(s) }
-    end
-
-    def function_token_matchers(function_name, *symbols)
-      token_matchers(:fopen, *symbols, :close).unshift(
-        TokenMatcher.send(function_name)
-      )
-    end
-
-    def matcher(symbol)
-      @matchers ||= [
-        :numeric, :string, :addsub, :subtract, :muldiv, :pow, :mod,
-        :comparator, :comp_gt, :comp_lt, :fopen, :open, :close, :comma,
-        :non_close_plus, :non_group, :non_group_star, :arguments,
-        :logical, :combinator, :if, :round, :roundup, :rounddown, :not,
-        :anchored_minus, :math_neg_pow, :math_neg_mul
-      ].each_with_object({}) do |name, matchers|
-        matchers[name] = TokenMatcher.send(name)
-      end
-
-      @matchers.fetch(symbol) do
-        raise "Unknown token symbol #{ symbol }"
-      end
-    end
-  end
-end