dentaku_zevo 3.5.2

data/lib/dentaku/token_matcher.rb

@@ -0,0 +1,138 @@
+require 'dentaku/token'
+
+module Dentaku
+  class TokenMatcher
+    attr_reader :children, :categories, :values
+
+    def initialize(categories = nil, values = nil, children = [])
+      # store categories and values as hash to optimize key lookup, h/t @jan-mangs
+      @categories = [categories].compact.flatten.each_with_object({}) { |c, h| h[c] = 1 }
+      @values     = [values].compact.flatten.each_with_object({}) { |v, h| h[v] = 1 }
+      @children   = children.compact
+      @invert     = false
+
+      @min = 1
+      @max = 1
+      @range = (@min..@max)
+    end
+
+    def |(other_matcher)
+      self.class.new(:nomatch, :nomatch, leaf_matchers + other_matcher.leaf_matchers)
+    end
+
+    def invert
+      @invert = ! @invert
+      self
+    end
+
+    def ==(token)
+      leaf_matcher? ? matches_token?(token) : any_child_matches_token?(token)
+    end
+
+    def match(token_stream, offset = 0)
+      matched_tokens = []
+      matched = false
+
+      while self == token_stream[matched_tokens.length + offset] && matched_tokens.length < @max
+        matched_tokens << token_stream[matched_tokens.length + offset]
+      end
+
+      if @range.cover?(matched_tokens.length)
+        matched = true
+      end
+
+      [matched, matched_tokens]
+    end
+
+    def caret
+      @caret = true
+      self
+    end
+
+    def caret?
+      @caret
+    end
+
+    def star
+      @min = 0
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def plus
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def leaf_matcher?
+      children.empty?
+    end
+
+    def leaf_matchers
+      leaf_matcher? ? [self] : children
+    end
+
+    private
+
+    def any_child_matches_token?(token)
+      children.any? { |child| child == token }
+    end
+
+    def matches_token?(token)
+      return false if token.nil?
+      (category_match(token.category) && value_match(token.value)) ^ @invert
+    end
+
+    def category_match(category)
+      @categories.empty? || @categories.key?(category)
+    end
+
+    def value_match(value)
+      @values.empty? || @values.key?(value)
+    end
+
+    def self.datetime;       new(:datetime);                       end
+    def self.numeric;        new(:numeric);                        end
+    def self.string;         new(:string);                         end
+    def self.logical;        new(:logical);                        end
+    def self.value
+      new(:datetime) | new(:numeric) | new(:string) | new(:logical)
+    end
+
+    def self.addsub;         new(:operator, [:add, :subtract]);    end
+    def self.subtract;       new(:operator, :subtract);            end
+    def self.anchored_minus; new(:operator, :subtract).caret;      end
+    def self.muldiv;         new(:operator, [:multiply, :divide]); end
+    def self.pow;            new(:operator, :pow);                 end
+    def self.mod;            new(:operator, :mod);                 end
+    def self.combinator;     new(:combinator);                     end
+
+    def self.comparator;     new(:comparator);                     end
+    def self.comp_gt;        new(:comparator, [:gt, :ge]);         end
+    def self.comp_lt;        new(:comparator, [:lt, :le]);         end
+
+    def self.open;           new(:grouping, :open);                end
+    def self.close;          new(:grouping, :close);               end
+    def self.comma;          new(:grouping, :comma);               end
+    def self.non_group;      new(:grouping).invert;                end
+    def self.non_group_star; new(:grouping).invert.star;           end
+    def self.non_close_plus; new(:grouping, :close).invert.plus;   end
+    def self.arguments;      (value | comma).plus;                 end
+
+    def self.if;             new(:function, :if);                  end
+    def self.round;          new(:function, :round);               end
+    def self.roundup;        new(:function, :roundup);             end
+    def self.rounddown;      new(:function, :rounddown);           end
+    def self.not;            new(:function, :not);                 end
+
+    def self.method_missing(name, *args, &block)
+      new(:function, name)
+    end
+
+    def self.respond_to_missing?(name, include_priv)
+      true
+    end
+  end
+end

data/lib/dentaku/token_matchers.rb

@@ -0,0 +1,29 @@
+module Dentaku
+  module TokenMatchers
+    def self.token_matchers(*symbols)
+      symbols.map { |s| matcher(s) }
+    end
+
+    def self.function_token_matchers(function_name, *symbols)
+      token_matchers(:open, *symbols, :close).unshift(
+        TokenMatcher.send(function_name)
+      )
+    end
+
+    def self.matcher(symbol)
+      @matchers ||= [
+        :datetime, :numeric, :string, :addsub, :subtract, :muldiv, :pow, :mod,
+        :comparator, :comp_gt, :comp_lt, :open, :close, :comma,
+        :non_close_plus, :non_group, :non_group_star, :arguments,
+        :logical, :combinator, :if, :round, :roundup, :rounddown, :not,
+        :anchored_minus, :math_neg_pow, :math_neg_mul
+      ].each_with_object({}) do |name, matchers|
+        matchers[name] = TokenMatcher.send(name)
+      end
+
+      @matchers.fetch(symbol) do
+        raise "Unknown token symbol #{ symbol }"
+      end
+    end
+  end
+end

data/lib/dentaku/token_scanner.rb

@@ -0,0 +1,183 @@
+require 'bigdecimal'
+require 'time'
+require 'dentaku/string_casing'
+require 'dentaku/token'
+
+module Dentaku
+  class TokenScanner
+    extend StringCasing
+
+    def initialize(category, regexp, converter = nil, condition = nil)
+      @category  = category
+      @regexp    = %r{\A(#{ regexp })}i
+      @converter = converter
+      @condition = condition || ->(*) { true }
+    end
+
+    def scan(string, last_token = nil)
+      if (m = @regexp.match(string)) && @condition.call(last_token)
+        value = raw = m.to_s
+        value = @converter.call(raw) if @converter
+
+        return Array(value).map do |v|
+          Token === v ? v : Token.new(@category, v, raw)
+        end
+      end
+
+      false
+    end
+
+    class << self
+      attr_reader :case_sensitive
+
+      def available_scanners
+        [
+          :null,
+          :whitespace,
+          :datetime, # before numeric so it can pick up timestamps
+          :numeric,
+          :hexadecimal,
+          :double_quoted_string,
+          :single_quoted_string,
+          :negate,
+          :combinator,
+          :operator,
+          :grouping,
+          :array,
+          :access,
+          :case_statement,
+          :comparator,
+          :boolean,
+          :function,
+          :identifier
+        ]
+      end
+
+      def register_default_scanners
+        register_scanners(available_scanners)
+      end
+
+      def register_scanners(scanner_ids)
+        @scanners = scanner_ids.each_with_object({}) do |id, scanners|
+          scanners[id] = self.send(id)
+        end
+      end
+
+      def register_scanner(id, scanner)
+        @scanners[id] = scanner
+      end
+
+      def scanners=(scanner_ids)
+        @scanners.select! { |k, v| scanner_ids.include?(k) }
+      end
+
+      def scanners(options = {})
+        @case_sensitive = options.fetch(:case_sensitive, false)
+        @scanners.values
+      end
+
+      def whitespace
+        new(:whitespace, '\s+')
+      end
+
+      def null
+        new(:null, 'null\b')
+      end
+
+      # NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason
+      def datetime
+        new(:datetime, /\d{2}\d{2}?-\d{1,2}-\d{1,2}( \d{1,2}:\d{1,2}:\d{1,2})? ?(Z|((\+|\-)\d{2}\:?\d{2}))?/, lambda { |raw| Time.parse(raw).to_datetime })
+      end
+
+      def numeric
+        new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', lambda { |raw|
+          raw =~ /(\.|e|E)/ ? BigDecimal(raw) : raw.to_i
+        })
+      end
+
+      def hexadecimal
+        new(:numeric, '(0x[0-9a-f]+)\b', lambda { |raw| raw[2..-1].to_i(16) })
+      end
+
+      def double_quoted_string
+        new(:string, '"[^"]*"', lambda { |raw| raw.gsub(/^"|"$/, '') })
+      end
+
+      def single_quoted_string
+        new(:string, "'[^']*'", lambda { |raw| raw.gsub(/^'|'$/, '') })
+      end
+
+      def negate
+        new(:operator, '-', lambda { |raw| :negate }, lambda { |last_token|
+          last_token.nil?             ||
+          last_token.is?(:operator)   ||
+          last_token.is?(:comparator) ||
+          last_token.is?(:combinator) ||
+          last_token.value == :open   ||
+          last_token.value == :comma
+        })
+      end
+
+      def operator
+        names = {
+          pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%', bitor: '|', bitand: '&', bitshiftleft: '<<', bitshiftright: '>>'
+        }.invert
+        new(:operator, '\^|\+|-|\*|\/|%|\||&|<<|>>', lambda { |raw| names[raw] })
+      end
+
+      def grouping
+        names = { open: '(', close: ')', comma: ',' }.invert
+        new(:grouping, '\(|\)|,', lambda { |raw| names[raw] })
+      end
+
+      def array
+        names = { array_start: '{', array_end: '}', }.invert
+        new(:array, '\{|\}|,', lambda { |raw| names[raw] })
+      end
+
+      def access
+        names = { lbracket: '[', rbracket: ']' }.invert
+        new(:access, '\[|\]', lambda { |raw| names[raw] })
+      end
+
+      def case_statement
+        names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert
+        new(:case, '(case|end|then|when|else)\b', lambda { |raw| names[raw.downcase] })
+      end
+
+      def comparator
+        names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert
+        alternate = { ne: '<>', eq: '==' }.invert
+        new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] })
+      end
+
+      def combinator
+        names = { and: '&&', or: '||' }.invert
+        new(:combinator, '(and|or|&&|\|\|)\s', lambda { |raw|
+          norm = raw.strip.downcase
+          names.fetch(norm) { norm.to_sym }
+        })
+      end
+
+      def boolean
+        new(:logical, '(true|false)\b', lambda { |raw| raw.strip.downcase == 'true' })
+      end
+
+      def function
+        new(:function, '\w+!?\s*\(', lambda do |raw|
+          function_name = raw.gsub('(', '')
+          [
+            Token.new(:function, function_name.strip.downcase.to_sym, function_name),
+            Token.new(:grouping, :open, '(')
+          ]
+        end)
+      end
+
+      def identifier
+        new(:identifier, '[[[:word:]]\.]+\b', lambda { |raw| standardize_case(raw.strip) })
+      end
+    end
+
+    register_default_scanners
+  end
+end

data/lib/dentaku/tokenizer.rb

@@ -0,0 +1,110 @@
+require 'dentaku/token'
+require 'dentaku/token_matcher'
+require 'dentaku/token_scanner'
+
+module Dentaku
+  class Tokenizer
+    attr_reader :case_sensitive, :aliases
+
+    LPAREN = TokenMatcher.new(:grouping, :open)
+    RPAREN = TokenMatcher.new(:grouping, :close)
+
+    def tokenize(string, options = {})
+      @nesting = 0
+      @tokens  = []
+      @aliases = options.fetch(:aliases, global_aliases)
+      input    = strip_comments(string.to_s.dup)
+      input    = replace_aliases(input)
+      @case_sensitive = options.fetch(:case_sensitive, false)
+
+      until input.empty?
+        scanned = TokenScanner.scanners(case_sensitive: case_sensitive).any? do |scanner|
+          scanned, input = scan(input, scanner)
+          scanned
+        end
+
+        unless scanned
+          fail! :parse_error, at: input
+        end
+      end
+
+      fail! :too_many_opening_parentheses if @nesting > 0
+
+      @tokens
+    end
+
+    def last_token
+      @tokens.last
+    end
+
+    def scan(string, scanner)
+      if tokens = scanner.scan(string, last_token)
+        tokens.each do |token|
+          if token.empty?
+            fail! :unexpected_zero_width_match,
+                  token_category: token.category, at: string
+          end
+
+          @nesting += 1 if LPAREN == token
+          @nesting -= 1 if RPAREN == token
+          fail! :too_many_closing_parentheses if @nesting < 0
+
+          @tokens << token unless token.is?(:whitespace)
+        end
+
+        match_length = tokens.map(&:length).reduce(:+)
+        [true, string[match_length..-1]]
+      else
+        [false, string]
+      end
+    end
+
+    def strip_comments(input)
+      input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
+    end
+
+    def replace_aliases(string)
+      return string unless @aliases.any?
+
+      string.gsub!(alias_regex) do |match|
+        match_regex = /^#{Regexp.escape(match)}$/i
+
+        @aliases.detect do |(_key, aliases)|
+          !aliases.grep(match_regex).empty?
+        end.first
+      end
+
+      string
+    end
+
+    def alias_regex
+      values = @aliases.values.flatten.join('|')
+      /(?<=\p{Punct}|[[:space:]]|\A)(#{values})(?=\()/i
+    end
+
+    private
+
+    def global_aliases
+      return {} unless Dentaku.respond_to?(:aliases)
+      Dentaku.aliases
+    end
+
+    def fail!(reason, **meta)
+      message =
+        case reason
+        when :parse_error
+          "parse error at: '#{meta.fetch(:at)}'"
+        when :too_many_opening_parentheses
+          "too many opening parentheses"
+        when :too_many_closing_parentheses
+          "too many closing parentheses"
+        when :unexpected_zero_width_match
+          "unexpected zero-width match (:#{meta.fetch(:category)}) at '#{meta.fetch(:at)}'"
+        else
+          raise ::ArgumentError, "Unhandled #{reason}"
+        end
+
+      raise TokenizerError.for(reason, **meta), message
+    end
+  end
+end

data/lib/dentaku/version.rb

@@ -0,0 +1,3 @@
+module Dentaku
+  VERSION = "3.5.2"
+end

data/lib/dentaku/visitor/infix.rb

@@ -0,0 +1,82 @@
+# infix visitor
+#
+# use this visitor in a processor to get infix visiting order
+#
+#  visitor     node     deps
+#    accept ->   visit left ->
+#                process
+#                visit right ->
+module Dentaku
+  module Visitor
+    module Infix
+      def visit(ast)
+        ast.accept(self)
+      end
+
+      def process(_ast)
+        raise NotImplementedError
+      end
+
+      def visit_function(node)
+        node.args.each do |arg|
+          visit(arg)
+        end
+        process(node)
+      end
+
+      def visit_identifier(node)
+        process(node)
+      end
+
+      def visit_operation(node)
+        visit(node.left) if node.left
+        process(node)
+        visit(node.right) if node.right
+      end
+
+      def visit_operand(node)
+        process(node)
+      end
+
+      def visit_case(node)
+        process(node)
+      end
+
+      def visit_switch(node)
+        process(node)
+        end
+
+      def visit_case_conditional(node)
+        process(node)
+      end
+
+      def visit_when(node)
+        process(node)
+      end
+
+      def visit_then(node)
+        process(node)
+      end
+
+      def visit_else(node)
+        process(node)
+      end
+
+      def visit_negation(node)
+        process(node)
+      end
+
+      def visit_access(node)
+        process(node)
+      end
+
+      def visit_literal(node)
+        process(node)
+      end
+
+      def visit_nil(node)
+        process(node)
+      end
+    end
+  end
+end

data/lib/dentaku.rb

@@ -0,0 +1,69 @@
+require "bigdecimal"
+require "concurrent"
+require "dentaku/calculator"
+require "dentaku/version"
+
+module Dentaku
+  @enable_ast_caching = false
+  @enable_dependency_order_caching = false
+  @enable_identifier_caching = false
+  @aliases = {}
+
+  def self.evaluate(expression, data = {}, &block)
+    calculator.value.evaluate(expression, data, &block)
+  end
+
+  def self.evaluate!(expression, data = {}, &block)
+    calculator.value.evaluate!(expression, data, &block)
+  end
+
+  def self.enable_caching!
+    enable_ast_cache!
+    enable_dependency_order_cache!
+    enable_identifier_cache!
+  end
+
+  def self.enable_ast_cache!
+    @enable_ast_caching = true
+  end
+
+  def self.cache_ast?
+    @enable_ast_caching
+  end
+
+  def self.enable_dependency_order_cache!
+    @enable_dependency_order_caching = true
+  end
+
+  def self.cache_dependency_order?
+    @enable_dependency_order_caching
+  end
+
+  def self.enable_identifier_cache!
+    @enable_identifier_caching = true
+  end
+
+  def self.cache_identifier?
+    @enable_identifier_caching
+  end
+
+  def self.aliases
+    @aliases
+  end
+
+  def self.aliases=(hash)
+    @aliases = hash
+  end
+
+  def self.calculator
+    @calculator ||= Concurrent::ThreadLocalVar.new { Dentaku::Calculator.new }
+  end
+end
+
+def Dentaku(expression, data = {})
+  Dentaku.evaluate(expression, data)
+end
+
+def Dentaku!(expression, data = {})
+  Dentaku.evaluate!(expression, data)
+end

data/spec/ast/addition_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'dentaku/ast/arithmetic'
+
+require 'dentaku/token'
+
+describe Dentaku::AST::Addition do
+  let(:five) { Dentaku::AST::Numeric.new Dentaku::Token.new(:numeric, 5) }
+  let(:six)  { Dentaku::AST::Numeric.new Dentaku::Token.new(:numeric, 6) }
+
+  let(:t)    { Dentaku::AST::Numeric.new Dentaku::Token.new(:logical, true) }
+
+  it 'allows access to its sub-trees' do
+    node = described_class.new(five, six)
+    expect(node.left).to eq(five)
+    expect(node.right).to eq(six)
+  end
+
+  it 'performs addition' do
+    node = described_class.new(five, six)
+    expect(node.value).to eq(11)
+  end
+
+  it 'requires numeric operands' do
+    expect {
+      described_class.new(five, t)
+    }.to raise_error(Dentaku::NodeError, /requires numeric operands/)
+
+    expression = Dentaku::AST::Multiplication.new(five, five)
+    group = Dentaku::AST::Grouping.new(expression)
+
+    expect {
+      described_class.new(group, five)
+    }.not_to raise_error
+  end
+
+  it 'allows operands that respond to addition' do
+    # Sample struct that has a custom definition for addition
+
+    Operand = Struct.new(:value) do
+      def +(other)
+        case other
+        when Operand
+          value + other.value
+        when Numeric
+          value + other
+        end
+      end
+    end
+
+    operand_five = Dentaku::AST::Numeric.new Dentaku::Token.new(:numeric, Operand.new(5))
+    operand_six = Dentaku::AST::Numeric.new Dentaku::Token.new(:numeric, Operand.new(6))
+
+    expect {
+      described_class.new(operand_five, operand_six)
+    }.not_to raise_error
+
+    expect {
+      described_class.new(operand_five, six)
+    }.not_to raise_error
+
+  end
+end

data/spec/ast/all_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+require 'dentaku/ast/functions/all'
+require 'dentaku'
+
+describe Dentaku::AST::All do
+  let(:calculator) { Dentaku::Calculator.new }
+  it 'performs ALL operation' do
+    result = Dentaku('ALL(vals, val, val > 1)', vals: [1, 2, 3])
+    expect(result).to eq(false)
+  end
+
+  it 'works with a single value if needed for some reason' do
+    result = Dentaku('ALL(vals, val, val > 1)', vals: 1)
+    expect(result).to eq(false)
+
+    result = Dentaku('ALL(vals, val, val > 1)', vals: 2)
+    expect(result).to eq(true)
+  end
+
+  it 'raises argument error if a string is passed as identifier' do
+    expect { calculator.evaluate!('ALL({1, 2, 3}, "val", val % 2 == 0)') }.to raise_error(
+      Dentaku::ArgumentError, 'ALL() requires second argument to be an identifier'
+    )
+  end
+end