hayadentaku 3.5.7

data/lib/dentaku/token_matcher.rb

@@ -0,0 +1,138 @@
+require 'dentaku/token'
+
+module Dentaku
+  class TokenMatcher
+    attr_reader :children, :categories, :values
+
+    def initialize(categories = nil, values = nil, children = [])
+      # store categories and values as hash to optimize key lookup, h/t @jan-mangs
+      @categories = [categories].compact.flatten.each_with_object({}) { |c, h| h[c] = 1 }
+      @values     = [values].compact.flatten.each_with_object({}) { |v, h| h[v] = 1 }
+      @children   = children.compact
+      @invert     = false
+
+      @min = 1
+      @max = 1
+      @range = (@min..@max)
+    end
+
+    def |(other_matcher)
+      self.class.new(:nomatch, :nomatch, leaf_matchers + other_matcher.leaf_matchers)
+    end
+
+    def invert
+      @invert = ! @invert
+      self
+    end
+
+    def ==(token)
+      leaf_matcher? ? matches_token?(token) : any_child_matches_token?(token)
+    end
+
+    def match(token_stream, offset = 0)
+      matched_tokens = []
+      matched = false
+
+      while self == token_stream[matched_tokens.length + offset] && matched_tokens.length < @max
+        matched_tokens << token_stream[matched_tokens.length + offset]
+      end
+
+      if @range.cover?(matched_tokens.length)
+        matched = true
+      end
+
+      [matched, matched_tokens]
+    end
+
+    def caret
+      @caret = true
+      self
+    end
+
+    def caret?
+      @caret
+    end
+
+    def star
+      @min = 0
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def plus
+      @max = Float::INFINITY
+      @range = (@min..@max)
+      self
+    end
+
+    def leaf_matcher?
+      children.empty?
+    end
+
+    def leaf_matchers
+      leaf_matcher? ? [self] : children
+    end
+
+    private
+
+    def any_child_matches_token?(token)
+      children.any? { |child| child == token }
+    end
+
+    def matches_token?(token)
+      return false if token.nil?
+      (category_match(token.category) && value_match(token.value)) ^ @invert
+    end
+
+    def category_match(category)
+      @categories.empty? || @categories.key?(category)
+    end
+
+    def value_match(value)
+      @values.empty? || @values.key?(value)
+    end
+
+    def self.datetime;       new(:datetime);                       end
+    def self.numeric;        new(:numeric);                        end
+    def self.string;         new(:string);                         end
+    def self.logical;        new(:logical);                        end
+    def self.value
+      new(:datetime) | new(:numeric) | new(:string) | new(:logical)
+    end
+
+    def self.addsub;         new(:operator, [:add, :subtract]);    end
+    def self.subtract;       new(:operator, :subtract);            end
+    def self.anchored_minus; new(:operator, :subtract).caret;      end
+    def self.muldiv;         new(:operator, [:multiply, :divide]); end
+    def self.pow;            new(:operator, :pow);                 end
+    def self.mod;            new(:operator, :mod);                 end
+    def self.combinator;     new(:combinator);                     end
+
+    def self.comparator;     new(:comparator);                     end
+    def self.comp_gt;        new(:comparator, [:gt, :ge]);         end
+    def self.comp_lt;        new(:comparator, [:lt, :le]);         end
+
+    def self.open;           new(:grouping, :open);                end
+    def self.close;          new(:grouping, :close);               end
+    def self.comma;          new(:grouping, :comma);               end
+    def self.non_group;      new(:grouping).invert;                end
+    def self.non_group_star; new(:grouping).invert.star;           end
+    def self.non_close_plus; new(:grouping, :close).invert.plus;   end
+    def self.arguments;      (value | comma).plus;                 end
+
+    def self.if;             new(:function, :if);                  end
+    def self.round;          new(:function, :round);               end
+    def self.roundup;        new(:function, :roundup);             end
+    def self.rounddown;      new(:function, :rounddown);           end
+    def self.not;            new(:function, :not);                 end
+
+    def self.method_missing(name, *args, &block)
+      new(:function, name)
+    end
+
+    def self.respond_to_missing?(name, include_priv)
+      true
+    end
+  end
+end

data/lib/dentaku/token_matchers.rb

@@ -0,0 +1,29 @@
+module Dentaku
+  module TokenMatchers
+    def self.token_matchers(*symbols)
+      symbols.map { |s| matcher(s) }
+    end
+
+    def self.function_token_matchers(function_name, *symbols)
+      token_matchers(:open, *symbols, :close).unshift(
+        TokenMatcher.send(function_name)
+      )
+    end
+
+    def self.matcher(symbol)
+      @matchers ||= [
+        :datetime, :numeric, :string, :addsub, :subtract, :muldiv, :pow, :mod,
+        :comparator, :comp_gt, :comp_lt, :open, :close, :comma,
+        :non_close_plus, :non_group, :non_group_star, :arguments,
+        :logical, :combinator, :if, :round, :roundup, :rounddown, :not,
+        :anchored_minus, :math_neg_pow, :math_neg_mul
+      ].each_with_object({}) do |name, matchers|
+        matchers[name] = TokenMatcher.send(name)
+      end
+
+      @matchers.fetch(symbol) do
+        raise "Unknown token symbol #{ symbol }"
+      end
+    end
+  end
+end

data/lib/dentaku/token_scanner.rb

@@ -0,0 +1,240 @@
+require 'bigdecimal'
+require 'strscan'
+require 'time'
+require 'dentaku/string_casing'
+require 'dentaku/token'
+
+module Dentaku
+  class TokenScanner
+    extend StringCasing
+
+    DATE_TIME_REGEXP = /\d{2}\d{2}?-\d{1,2}-\d{1,2}([ |T]\d{1,2}:\d{1,2}:\d{1,2}(\.\d*)?)? ?(Z|((\+|\-)\d{2}\:?\d{2}))?(?!\d)/.freeze
+
+    def initialize(category, regexp, converter = nil, condition = nil)
+      @category  = category
+      @regexp    = %r{\A(#{ regexp })}i
+      # StringScanner anchors implicitly at the current position, so an
+      # unanchored copy of the same pattern is what its `scan` wants.
+      @ss_regexp = %r{(#{ regexp })}i
+      @converter = converter
+      @condition = condition
+    end
+
+    # Legacy entry point: takes a string, returns false or an Array of Tokens.
+    def scan(string, last_token = nil)
+      if (m = @regexp.match(string)) && (@condition.nil? || @condition.call(last_token))
+        value = raw = m.to_s
+        value = @converter.call(raw) if @converter
+
+        return Array(value).map do |v|
+          Token === v ? v : Token.new(@category, v, raw)
+        end
+      end
+
+      false
+    end
+
+    # Fast path used by the Tokenizer: scans directly against a StringScanner
+    # without slicing strings, and returns either a single Token, an Array of
+    # Tokens, or nil. Avoids the per-call `Array(value).map` allocation in the
+    # common (single-token) case.
+    def scan_at(strscanner, last_token = nil)
+      return nil if @condition && !@condition.call(last_token)
+
+      raw = strscanner.scan(@ss_regexp)
+      return nil unless raw
+
+      if @converter
+        value = @converter.call(raw)
+        if value.is_a?(Array)
+          value.map! { |v| Token === v ? v : Token.new(@category, v, raw) }
+        elsif Token === value
+          value
+        else
+          Token.new(@category, value, raw)
+        end
+      else
+        Token.new(@category, raw, raw)
+      end
+    end
+
+    class << self
+      attr_reader :case_sensitive
+
+      def available_scanners
+        [
+          :null,
+          :whitespace,
+          :datetime, # before numeric so it can pick up timestamps
+          :numeric,
+          :hexadecimal,
+          :double_quoted_string,
+          :single_quoted_string,
+          :negate,
+          :combinator,
+          :operator,
+          :grouping,
+          :array,
+          :access,
+          :case_statement,
+          :comparator,
+          :boolean,
+          :function,
+          :identifier,
+          :quoted_identifier
+        ]
+      end
+
+      def register_default_scanners
+        register_scanners(available_scanners)
+      end
+
+      def register_scanners(scanner_ids)
+        @scanners = scanner_ids.each_with_object({}) do |id, scanners|
+          scanners[id] = self.send(id)
+        end
+        reset_scanner_cache!
+      end
+
+      def register_scanner(id, scanner)
+        @scanners[id] = scanner
+        reset_scanner_cache!
+      end
+
+      def scanners=(scanner_ids)
+        @scanners.select! { |k, v| scanner_ids.include?(k) }
+        reset_scanner_cache!
+      end
+
+      def scanners(options = {})
+        @case_sensitive = options.fetch(:case_sensitive, false)
+        raw_date_literals = options.fetch(:raw_date_literals, true)
+
+        # Cache the two possible scanner lists so repeated tokenize calls don't
+        # rebuild the array each time (and the inner tokenize loop doesn't
+        # rebuild it on every iteration).
+        if raw_date_literals
+          @cached_full ||= @scanners.values
+        else
+          @cached_no_datetime ||= @scanners.reject { |k, _| k == :datetime }.values
+        end
+      end
+
+      # Invalidate cached scanner lists; called by register_scanner(s) so tests
+      # that swap scanners in and out keep working.
+      def reset_scanner_cache!
+        @cached_full = nil
+        @cached_no_datetime = nil
+      end
+
+      def whitespace
+        new(:whitespace, '\s+')
+      end
+
+      def null
+        new(:null, 'null\b')
+      end
+
+      # NOTE: Convert to DateTime as Array(Time) returns the parts of the time for some reason
+      def datetime
+        new(:datetime, DATE_TIME_REGEXP, lambda { |raw| Time.parse(raw).to_datetime })
+      end
+
+      def numeric
+        new(:numeric, '((?:\d+(\.\d+)?|\.\d+)(?:(e|E)(\+|-)?\d+)?)\b', lambda { |raw|
+          raw =~ /(\.|e|E)/ ? BigDecimal(raw) : raw.to_i
+        })
+      end
+
+      def hexadecimal
+        new(:numeric, '(0x[0-9a-f]+)\b', lambda { |raw| raw[2..-1].to_i(16) })
+      end
+
+      def double_quoted_string
+        new(:string, '"[^"]*"', lambda { |raw| raw.gsub(/^"|"$/, '') })
+      end
+
+      def single_quoted_string
+        new(:string, "'[^']*'", lambda { |raw| raw.gsub(/^'|'$/, '') })
+      end
+
+      def negate
+        new(:operator, '-', lambda { |raw| :negate }, lambda { |last_token|
+          last_token.nil?               ||
+          last_token.is?(:operator)     ||
+          last_token.is?(:comparator)   ||
+          last_token.is?(:combinator)   ||
+          last_token.value == :open     ||
+          last_token.value == :comma    ||
+          last_token.value == :lbracket ||
+          last_token.value == :array_start
+        })
+      end
+
+      def operator
+        names = {
+          pow: '^', add: '+', subtract: '-', multiply: '*', divide: '/', mod: '%', bitor: '|', bitand: '&', bitshiftleft: '<<', bitshiftright: '>>'
+        }.invert
+        new(:operator, '\^|\+|-|\*|\/|%|\||&|<<|>>', lambda { |raw| names[raw] })
+      end
+
+      def grouping
+        names = { open: '(', close: ')', comma: ',' }.invert
+        new(:grouping, '\(|\)|,', lambda { |raw| names[raw] })
+      end
+
+      def array
+        names = { array_start: '{', array_end: '}', }.invert
+        new(:array, '\{|\}|,', lambda { |raw| names[raw] })
+      end
+
+      def access
+        names = { lbracket: '[', rbracket: ']' }.invert
+        new(:access, '\[|\]', lambda { |raw| names[raw] })
+      end
+
+      def case_statement
+        names = { open: 'case', close: 'end', then: 'then', when: 'when', else: 'else' }.invert
+        new(:case, '(case|end|then|when|else)\b', lambda { |raw| names[raw.downcase] })
+      end
+
+      def comparator
+        names = { le: '<=', ge: '>=', ne: '!=', lt: '<', gt: '>', eq: '=' }.invert
+        alternate = { ne: '<>', eq: '==' }.invert
+        new(:comparator, '<=|>=|!=|<>|<|>|==|=', lambda { |raw| names[raw] || alternate[raw] })
+      end
+
+      def combinator
+        names = { and: '&&', or: '||' }.invert
+        new(:combinator, '(and|or|&&|\|\|)\s', lambda { |raw|
+          norm = raw.strip.downcase
+          names.fetch(norm) { norm.to_sym }
+        })
+      end
+
+      def boolean
+        new(:logical, '(true|false)\b', lambda { |raw| raw.strip.downcase == 'true' })
+      end
+
+      def function
+        new(:function, '\w+!?\s*\(', lambda do |raw|
+          function_name = raw.gsub('(', '')
+          [
+            Token.new(:function, function_name.strip.downcase.to_sym, function_name),
+            Token.new(:grouping, :open, '(')
+          ]
+        end)
+      end
+
+      def identifier
+        new(:identifier, '[[[:word:]]\.]+\b', lambda { |raw| standardize_case(raw.strip) })
+      end
+
+      def quoted_identifier
+        new(:identifier, '`[^`]*`', lambda { |raw| raw.gsub(/^`|`$/, '') })
+      end
+    end
+
+    register_default_scanners
+  end
+end

data/lib/dentaku/tokenizer.rb

@@ -0,0 +1,127 @@
+require 'strscan'
+require 'dentaku/token'
+require 'dentaku/token_matcher'
+require 'dentaku/token_scanner'
+
+module Dentaku
+  class Tokenizer
+    attr_reader :aliases
+
+    LPAREN = TokenMatcher.new(:grouping, :open)
+    RPAREN = TokenMatcher.new(:grouping, :close)
+
+    def tokenize(string, options = {})
+      @nesting = 0
+      @tokens  = []
+      @aliases = options.fetch(:aliases, global_aliases)
+      input    = strip_comments(string.to_s.dup)
+      input    = replace_aliases(input)
+
+      scanner_options = {
+        case_sensitive: options.fetch(:case_sensitive, false),
+        raw_date_literals: options.fetch(:raw_date_literals, true)
+      }
+
+      # Hoist the scanner list out of the per-position loop. The previous code
+      # rebuilt this Array (via Hash#select + .values) on every advance, which
+      # was a significant chunk of tokenize allocations.
+      scanners = TokenScanner.scanners(scanner_options)
+
+      ss = StringScanner.new(input)
+
+      until ss.eos?
+        last_token = @tokens.last
+        scanned = false
+
+        scanners.each do |scanner|
+          result = scanner.scan_at(ss, last_token)
+          next unless result
+
+          if result.is_a?(Array)
+            result.each { |t| handle_token(t, ss) }
+          else
+            handle_token(result, ss)
+          end
+
+          scanned = true
+          break
+        end
+
+        fail! :parse_error, at: ss.rest unless scanned
+      end
+
+      fail! :too_many_opening_parentheses if @nesting > 0
+
+      @tokens
+    end
+
+    def last_token
+      @tokens.last
+    end
+
+    private
+
+    def handle_token(token, ss)
+      if token.empty?
+        fail! :unexpected_zero_width_match,
+              token_category: token.category, at: ss.rest
+      end
+
+      @nesting += 1 if LPAREN == token
+      @nesting -= 1 if RPAREN == token
+      fail! :too_many_closing_parentheses if @nesting < 0
+
+      @tokens << token unless token.is?(:whitespace)
+    end
+
+    public
+
+    def strip_comments(input)
+      input.gsub(/\/\*[^*]*\*+(?:[^*\/][^*]*\*+)*\//, '')
+    end
+
+    def replace_aliases(string)
+      return string unless @aliases.any?
+
+      string.gsub!(alias_regex) do |match|
+        match_regex = /^#{Regexp.escape(match)}$/i
+
+        @aliases.detect do |(_key, aliases)|
+          !aliases.grep(match_regex).empty?
+        end.first
+      end
+
+      string
+    end
+
+    def alias_regex
+      values = @aliases.values.flatten.join('|')
+      /(?<=\p{Punct}|[[:space:]]|\A)(#{values})(?=\()/i
+    end
+
+    private
+
+    def global_aliases
+      return {} unless Dentaku.respond_to?(:aliases)
+      Dentaku.aliases
+    end
+
+    def fail!(reason, **meta)
+      message =
+        case reason
+        when :parse_error
+          "parse error at: '#{meta.fetch(:at)}'"
+        when :too_many_opening_parentheses
+          "too many opening parentheses"
+        when :too_many_closing_parentheses
+          "too many closing parentheses"
+        when :unexpected_zero_width_match
+          "unexpected zero-width match (:#{meta.fetch(:category)}) at '#{meta.fetch(:at)}'"
+        else
+          raise ::ArgumentError, "Unhandled #{reason}"
+        end
+
+      raise TokenizerError.for(reason, **meta), message
+    end
+  end
+end

data/lib/dentaku/version.rb

@@ -0,0 +1,3 @@
+module Dentaku
+  VERSION = "3.5.7"
+end

data/lib/dentaku/visitor/infix.rb

@@ -0,0 +1,86 @@
+# infix visitor
+#
+# use this visitor in a processor to get infix visiting order
+#
+#  visitor     node     deps
+#    accept ->   visit left ->
+#                process
+#                visit right ->
+module Dentaku
+  module Visitor
+    module Infix
+      def visit(ast)
+        ast.accept(self)
+      end
+
+      def process(_ast)
+        raise NotImplementedError
+      end
+
+      def visit_function(node)
+        node.args.each do |arg|
+          visit(arg)
+        end
+        process(node)
+      end
+
+      def visit_identifier(node)
+        process(node)
+      end
+
+      def visit_operation(node)
+        visit(node.left) if node.left
+        process(node)
+        visit(node.right) if node.right
+      end
+
+      def visit_operand(node)
+        process(node)
+      end
+
+      def visit_case(node)
+        process(node)
+      end
+
+      def visit_switch(node)
+        process(node)
+        end
+
+      def visit_case_conditional(node)
+        process(node)
+      end
+
+      def visit_when(node)
+        process(node)
+      end
+
+      def visit_then(node)
+        process(node)
+      end
+
+      def visit_else(node)
+        process(node)
+      end
+
+      def visit_negation(node)
+        process(node)
+      end
+
+      def visit_access(node)
+        process(node)
+      end
+
+      def visit_literal(node)
+        process(node)
+      end
+
+      def visit_nil(node)
+        process(node)
+      end
+
+      def visit_array(node)
+        process(node)
+      end
+    end
+  end
+end

data/lib/dentaku.rb

@@ -0,0 +1,69 @@
+require "bigdecimal"
+require "concurrent"
+require "dentaku/calculator"
+require "dentaku/version"
+
+module Dentaku
+  @enable_ast_caching = false
+  @enable_dependency_order_caching = false
+  @enable_identifier_caching = false
+  @aliases = {}
+
+  def self.evaluate(expression, data = {}, &block)
+    calculator.value.evaluate(expression, data, &block)
+  end
+
+  def self.evaluate!(expression, data = {}, &block)
+    calculator.value.evaluate!(expression, data, &block)
+  end
+
+  def self.enable_caching!
+    enable_ast_cache!
+    enable_dependency_order_cache!
+    enable_identifier_cache!
+  end
+
+  def self.enable_ast_cache!
+    @enable_ast_caching = true
+  end
+
+  def self.cache_ast?
+    @enable_ast_caching
+  end
+
+  def self.enable_dependency_order_cache!
+    @enable_dependency_order_caching = true
+  end
+
+  def self.cache_dependency_order?
+    @enable_dependency_order_caching
+  end
+
+  def self.enable_identifier_cache!
+    @enable_identifier_caching = true
+  end
+
+  def self.cache_identifier?
+    @enable_identifier_caching
+  end
+
+  def self.aliases
+    @aliases
+  end
+
+  def self.aliases=(hash)
+    @aliases = hash
+  end
+
+  def self.calculator
+    @calculator ||= Concurrent::ThreadLocalVar.new { Dentaku::Calculator.new }
+  end
+end
+
+def Dentaku(expression, data = {})
+  Dentaku.evaluate(expression, data)
+end
+
+def Dentaku!(expression, data = {})
+  Dentaku.evaluate!(expression, data)
+end