search-engine-for-typesense 1.0.0

data/lib/search_engine/compiler.rb

@@ -0,0 +1,383 @@
+# frozen_string_literal: true
+
+require 'search_engine/filters/sanitizer'
+require 'search_engine/ast'
+
+module SearchEngine
+  # Compiler for turning Predicate AST into Typesense `filter_by` strings.
+  #
+  # Pure and deterministic: no side effects, no I/O. Uses
+  # `SearchEngine::Filters::Sanitizer` for all quoting/escaping to ensure
+  # consistent rendering with the `where` DSL.
+  module Compiler
+    class Error < StandardError; end
+    class UnsupportedNode < Error; end
+
+    module_function
+
+    # Compile an AST node or an array of nodes (implicit AND) into a Typesense
+    # `filter_by` string.
+    #
+    # - Nil or empty input returns an empty String
+    # - Arrays at the top-level are treated as implicit AND
+    # - Group nodes are always parenthesized
+    # - Raw fragments are passed-through
+    #
+    # @param ast [SearchEngine::AST::Node, Array<SearchEngine::AST::Node>, nil]
+    # @param klass [Class] optional model class for context (used for observability)
+    # @return [String]
+    # @note Emits "search_engine.compile" via ActiveSupport::Notifications with
+    #   payload: { collection, klass, node_count, duration_ms, source: :ast }
+    def compile(ast, klass: nil)
+      root = coerce_root(ast)
+      return '' unless root
+
+      compiled = nil
+      if defined?(ActiveSupport::Notifications)
+        payload = {
+          collection: safe_collection_for_klass(klass),
+          klass: safe_klass_name(klass),
+          node_count: count_nodes(root),
+          source: :ast
+        }
+        SearchEngine::Instrumentation.instrument('search_engine.compile', payload) do |_ctx|
+          compiled = compile_node(root, parent_prec: 0, klass: klass)
+        end
+        compiled
+      else
+        compile_node(root, parent_prec: 0, klass: klass)
+      end
+    end
+
+    # --- Internals ---------------------------------------------------------
+
+    def coerce_root(ast)
+      return nil if ast.nil?
+
+      if ast.is_a?(Array)
+        nodes = ast.flatten.compact.select { |n| n.is_a?(SearchEngine::AST::Node) }
+        return nil if nodes.empty?
+        return nodes.first if nodes.length == 1
+
+        return SearchEngine::AST.and_(*nodes)
+      end
+
+      return ast if ast.is_a?(SearchEngine::AST::Node)
+
+      raise Error, "Compiler: unsupported root input #{ast.class}"
+    end
+    private_class_method :coerce_root
+
+    def compile_node(node, parent_prec:, klass: nil)
+      case node
+      when SearchEngine::AST::Eq
+        compile_binary(node.field, ':=', node.value, klass: klass)
+      when SearchEngine::AST::NotEq
+        compile_binary(node.field, ':!=', node.value, klass: klass)
+      when SearchEngine::AST::Gt
+        compile_binary(node.field, ':>', node.value, klass: klass)
+      when SearchEngine::AST::Gte
+        compile_binary(node.field, ':>=', node.value, klass: klass)
+      when SearchEngine::AST::Lt
+        compile_binary(node.field, ':<', node.value, klass: klass)
+      when SearchEngine::AST::Lte
+        compile_binary(node.field, ':<=', node.value, klass: klass)
+      when SearchEngine::AST::In
+        compile_binary(node.field, ':=', node.values, klass: klass)
+      when SearchEngine::AST::NotIn
+        compile_binary(node.field, ':!=', node.values, klass: klass)
+      when SearchEngine::AST::And
+        compile_boolean(node.children, ' && ', parent_prec: parent_prec, my_prec: precedence(:and), klass: klass)
+      when SearchEngine::AST::Or
+        compile_or(node, parent_prec, klass: klass)
+      when SearchEngine::AST::Group
+        "(#{compile_node(node.children.first, parent_prec: 0, klass: klass)})"
+      when SearchEngine::AST::Raw
+        node.fragment
+      when SearchEngine::AST::Matches
+        raise UnsupportedNode, 'Typesense filter_by does not support MATCHES; use AST::Raw if needed.'
+      when SearchEngine::AST::Prefix
+        raise UnsupportedNode, 'Typesense filter_by does not support PREFIX; use AST::Raw if needed.'
+      else
+        raise Error, "Compiler: unknown node #{node.class}"
+      end
+    end
+    private_class_method :compile_node
+
+    def compile_or(node, parent_prec, klass: nil)
+      compiled = compile_boolean(
+        node.children,
+        ' || ',
+        parent_prec: parent_prec,
+        my_prec: precedence(:or),
+        klass: klass
+      )
+      if node.children.length == 2 && node.children.last.is_a?(SearchEngine::AST::And)
+        left_str, right_str = compiled.split(' || ', 2)
+        compiled = "#{left_str} || (#{right_str})"
+      end
+      compiled
+    end
+    private_class_method :compile_or
+
+    def compile_binary(field, op, value, klass: nil)
+      fstr = field.to_s
+      if (m = fstr.match(/^\$(\w+)\.(.+)$/))
+        assoc = m[1]
+        inner = m[2]
+        # Render joined field as $assoc(inner OP value) per expected Typesense join filter syntax
+        target_klass = begin
+          cfg = klass.respond_to?(:join_for) ? klass.join_for(assoc.to_sym) : nil
+          coll = cfg ? cfg[:collection] : nil
+          coll ? SearchEngine.collection_for(coll) : nil
+        rescue StandardError
+          nil
+        end
+        rhs = quote(value, field: inner, klass: target_klass || klass)
+        "$#{assoc}(#{inner}#{op}#{rhs})"
+      else
+        rhs = quote(value, field: fstr, klass: klass)
+        binary(field, op, rhs)
+      end
+    end
+    private_class_method :compile_binary
+
+    def binary(field, op, rhs)
+      "#{field}#{op}#{rhs}"
+    end
+    private_class_method :binary
+
+    def compile_boolean(children, joiner, parent_prec:, my_prec:, klass: nil)
+      # For conjunctions only, merge multiple $assoc(inner ...) predicates targeting
+      # the same association into a single $assoc(inner && inner ...) expression
+      # to satisfy Typesense join filter rules.
+      if joiner == ' && '
+        merged = merge_join_predicates(children, parent_prec: parent_prec, my_prec: my_prec, klass: klass,
+joiner: joiner
+        )
+        return merged if merged
+      end
+
+      # Fallback/default behavior
+      parts = children.map do |child|
+        cstr = compile_node(child, parent_prec: my_prec, klass: klass)
+        if needs_parentheses?(child, parent_prec: my_prec)
+          "(#{cstr})"
+        else
+          cstr
+        end
+      end
+      inner = parts.join(joiner)
+
+      # Parent adds parens if its precedence is greater than ours
+      if parent_prec > my_prec
+        "(#{inner})"
+      else
+        inner
+      end
+    end
+    private_class_method :compile_boolean
+
+    # Merge multiple join predicates targeting the same association into consolidated expressions.
+    #
+    # For AND operations, collects all predicates for each association and merges them
+    # into a single $assoc(inner && inner ...) expression at the first position where
+    # that association appeared.
+    #
+    # @param children [Array<SearchEngine::AST::Node>] child nodes to process
+    # @param parent_prec [Integer] parent operator precedence
+    # @param my_prec [Integer] current operator precedence
+    # @param klass [Class, nil] model class for context
+    # @param joiner [String] operator joiner (' && ' or ' || ')
+    # @return [String, nil] merged expression string, or nil if no joins to merge
+    def merge_join_predicates(children, parent_prec:, my_prec:, klass:, joiner:)
+      items = [] # [{ pos:, str: }]
+      assoc_first_pos = {}
+      assoc_inner_map = {} # { assoc => [inner_str, ...] }
+
+      children.each_with_index do |child, idx|
+        inner = extract_join_inner_binary(child, klass: klass)
+        if inner
+          assoc, inner_expr = inner
+          assoc_first_pos[assoc] = idx unless assoc_first_pos.key?(assoc)
+          assoc_inner_map[assoc] ||= []
+          assoc_inner_map[assoc] << inner_expr
+          next
+        end
+
+        cstr = compile_node(child, parent_prec: my_prec, klass: klass)
+        cstr = "(#{cstr})" if needs_parentheses?(child, parent_prec: my_prec)
+        items << { pos: idx, str: cstr }
+      end
+
+      return nil if assoc_inner_map.empty?
+
+      # Emit one consolidated token per assoc at the first position it appeared
+      assoc_first_pos.sort_by { |_a, pos| pos }.each do |assoc, pos|
+        inners = Array(assoc_inner_map[assoc]).flatten.compact
+        next if inners.empty?
+
+        token = "$#{assoc}(#{inners.join(' && ')})"
+        items << { pos: pos, str: token }
+      end
+
+      inner = items.sort_by { |it| it[:pos] }.map { |it| it[:str] }.join(joiner)
+      parent_prec > my_prec ? "(#{inner})" : inner
+    end
+    private_class_method :merge_join_predicates
+
+    # Try to extract join association and compiled inner expression for a binary node
+    # with a joined field like "$assoc.field". Returns [assoc(String), inner(String)] or nil.
+    def extract_join_inner_binary(node, klass: nil)
+      case node
+      when SearchEngine::AST::Eq, SearchEngine::AST::NotEq,
+           SearchEngine::AST::Gt, SearchEngine::AST::Gte,
+           SearchEngine::AST::Lt, SearchEngine::AST::Lte,
+           SearchEngine::AST::In, SearchEngine::AST::NotIn
+        field = node.respond_to?(:field) ? node.field.to_s : nil
+        return nil unless field&.start_with?('$')
+
+        m = field.match(/^\$(\w+)\.(.+)$/)
+        return nil unless m
+
+        assoc = m[1]
+        inner_field = m[2]
+        op = op_for(node)
+        target_klass = begin
+          cfg = klass.respond_to?(:join_for) ? klass.join_for(assoc.to_sym) : nil
+          coll = cfg ? cfg[:collection] : nil
+          coll ? SearchEngine.collection_for(coll) : nil
+        rescue StandardError
+          nil
+        end
+        rhs = if node.respond_to?(:value)
+                quote(node.value, field: inner_field, klass: target_klass || klass)
+              elsif node.respond_to?(:values)
+                quote(node.values, field: inner_field, klass: target_klass || klass)
+              else
+                return nil
+              end
+
+        [assoc, "#{inner_field}#{op}#{rhs}"]
+      end
+    end
+    private_class_method :extract_join_inner_binary
+
+    def op_for(node)
+      case node
+      when SearchEngine::AST::Eq then ':='
+      when SearchEngine::AST::NotEq then ':!='
+      when SearchEngine::AST::Gt then ':>'
+      when SearchEngine::AST::Gte then ':>='
+      when SearchEngine::AST::Lt then ':<'
+      when SearchEngine::AST::Lte then ':<='
+      when SearchEngine::AST::In then ':='
+      when SearchEngine::AST::NotIn then ':!='
+      else
+        raise Error, "Unknown binary node for join extraction: #{node.class}"
+      end
+    end
+    private_class_method :op_for
+
+    def quote(value, field: nil, klass: nil)
+      # Typesense requires reference fields to use string values, even if the field type is int64
+      converted_value = if klass && field && field_reference?(klass, field)
+                          convert_for_reference_field(value)
+                        else
+                          value
+                        end
+
+      # Use conditional scalar quoting for scalars; preserve array element quoting rules
+      if converted_value.is_a?(Array)
+        SearchEngine::Filters::Sanitizer.quote(converted_value)
+      else
+        SearchEngine::Filters::Sanitizer.quote_scalar_for_filter(converted_value)
+      end
+    end
+    private_class_method :quote
+
+    # Check if a field has a reference attribute in the schema.
+    # Uses the same logic as Schema.build_references_by_local_key to ensure consistency.
+    def field_reference?(klass, field_name)
+      return false unless klass
+      return false unless klass.respond_to?(:joins_config)
+
+      configs = klass.joins_config || {}
+      configs.each_value do |cfg|
+        # Only belongs_to/belongs_to_many contribute references to schema
+        kind = (cfg[:kind] || :belongs_to).to_sym
+        next if %i[has_one has_many].include?(kind)
+
+        lk = cfg[:local_key]
+        next if lk.nil?
+
+        return true if lk.to_sym == field_name.to_sym
+      end
+
+      false
+    end
+    private_class_method :field_reference?
+
+    # Convert numeric values to strings for reference fields
+    def convert_for_reference_field(value)
+      if value.is_a?(Array)
+        value.map { |v| convert_for_reference_field(v) }
+      elsif value.is_a?(Numeric)
+        value.to_s
+      else
+        value
+      end
+    end
+    private_class_method :convert_for_reference_field
+
+    def needs_parentheses?(child, parent_prec:)
+      child_prec = precedence(child)
+      # Parenthesize when child binds looser than parent
+      child_prec < parent_prec
+    end
+    private_class_method :needs_parentheses?
+
+    def precedence(node)
+      case node
+      when Symbol
+        return 20 if node == :and
+        return 10 if node == :or
+
+        100
+      when SearchEngine::AST::And
+        20
+      when SearchEngine::AST::Or
+        10
+      when SearchEngine::AST::Group
+        # Group is a special case; caller always wraps it explicitly
+        100
+      else
+        # Leaves (Eq, In, etc.) bind tight
+        100
+      end
+    end
+    private_class_method :precedence
+
+    def count_nodes(node)
+      return 0 unless node.is_a?(SearchEngine::AST::Node)
+
+      1 + Array(node.children).sum { |child| count_nodes(child) }
+    end
+    private_class_method :count_nodes
+
+    def safe_klass_name(klass)
+      return nil unless klass
+
+      klass.respond_to?(:name) && klass.name ? klass.name : klass.to_s
+    end
+    private_class_method :safe_klass_name
+
+    def safe_collection_for_klass(klass)
+      return nil unless klass
+      return klass.collection if klass.respond_to?(:collection) && klass.collection
+
+      nil
+    end
+    private_class_method :safe_collection_for_klass
+  end
+end

data/lib/search_engine/config/observability.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Config
+    # Observability and structured logging configuration.
+    class Observability
+      # @return [Boolean] enable the compact logging subscriber automatically
+      attr_accessor :enabled
+      # @return [Symbol] :kv or :json
+      attr_accessor :log_format
+      # @return [Integer] maximum message length for error samples in logs
+      attr_accessor :max_message_length
+      # @return [Boolean] include short error messages in logs for batch/stale events
+      attr_accessor :include_error_messages
+      # @return [Boolean] also emit legacy event aliases where applicable
+      attr_accessor :emit_legacy_event_aliases
+
+      def initialize
+        @enabled = true
+        @log_format = :kv
+        @max_message_length = 200
+        @include_error_messages = false
+        @emit_legacy_event_aliases = true
+      end
+    end
+  end
+end

data/lib/search_engine/config/presets.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module SearchEngine
+  class Config
+    # Default presets resolution configuration.
+    # Controls namespacing and enablement.
+    class Presets
+      # @return [Boolean] when false, namespace is ignored but declared tokens remain usable
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets
+      attr_accessor :enabled
+      # @return [String, nil] optional namespace prepended to preset names when enabled
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets
+      attr_accessor :namespace
+
+      def initialize
+        @enabled = true
+        @namespace = nil
+        @locked_domains = %i[filter_by sort_by include_fields exclude_fields]
+        @locked_domains_set = nil
+      end
+
+      # Normalize a Boolean-like value.
+      # Accepts true/false, or common String forms ("true","false","1","0","yes","no","on","off").
+      # @param value [Object]
+      # @return [Boolean]
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#config-default-preset
+      def self.normalize_enabled(value)
+        return true  if value == true
+        return false if value == false
+
+        if value.is_a?(String)
+          v = value.strip.downcase
+          return true  if %w[1 true yes on].include?(v)
+          return false if %w[0 false no off].include?(v)
+        end
+
+        value
+      end
+
+      # Normalize namespace to a non-empty String or return original for validation.
+      # @param value [Object]
+      # @return [String, nil, Object]
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#config-default-preset
+      def self.normalize_namespace(value)
+        return nil if value.nil?
+
+        if value.is_a?(String)
+          ns = value.strip
+          return nil if ns.empty?
+
+          return ns
+        end
+
+        value
+      end
+
+      # Assign locked domains; accepts Array/Set or a single value. Values are
+      # normalized to Symbols. Internal membership checks use a frozen Set.
+      # @param value [Array<#to_sym>, Set<#to_sym>, #to_sym, nil]
+      # @return [void]
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#strategies-merge-only-lock
+      def locked_domains=(value)
+        list =
+          case value
+          when nil then []
+          when Set then value.to_a
+          when Array then value
+          else Array(value)
+          end
+        syms = list.compact.map { |k| k.respond_to?(:to_sym) ? k.to_sym : k }.grep(Symbol)
+        @locked_domains = syms
+        @locked_domains_set = syms.to_set.freeze
+      end
+
+      # Return the locked domains as an Array of Symbols.
+      # @return [Array<Symbol>]
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#strategies-merge-only-lock
+      def locked_domains
+        Array(@locked_domains).map(&:to_sym)
+      end
+
+      # Return a frozen Set of locked domains for fast membership checks.
+      # @return [Set<Symbol>]
+      # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/presets#strategies-merge-only-lock
+      def locked_domains_set
+        @locked_domains_set ||= locked_domains.to_set.freeze
+      end
+    end
+  end
+end

data/lib/search_engine/config/selection.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Config
+    # Selection/hydration configuration.
+    # Controls strictness of missing attributes during hydration.
+    class Selection
+      # @return [Boolean] when true, missing requested fields raise MissingField
+      attr_accessor :strict_missing
+
+      def initialize
+        @strict_missing = false
+      end
+    end
+  end
+end

data/lib/search_engine/config/typesense.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Config
+    # Typesense transport/client configuration.
+    # Holds connection details, timeouts, and retry knobs.
+    # Public fields mirror the facade to preserve API compatibility via forwarders.
+    #
+    # @example
+    #   ts = SearchEngine::Config::Typesense.new
+    #   ts.host = 'localhost'
+    #   ts.port = 8108
+    class Typesense
+      # @return [String, nil] secret Typesense API key (redacted separately)
+      attr_accessor :api_key
+      # @return [String] hostname of the Typesense server
+      attr_accessor :host
+      # @return [Integer] TCP port for the Typesense server
+      attr_accessor :port
+      # @return [String] one of "http" or "https"
+      attr_reader :protocol
+      # @return [Integer] request total timeout in milliseconds
+      attr_accessor :timeout_ms
+      # @return [Integer] connect/open timeout in milliseconds
+      attr_accessor :open_timeout_ms
+      # @return [Hash] retry policy with keys { attempts: Integer, backoff: Float or Range<Float> }
+      attr_accessor :retries
+
+      def initialize
+        @api_key = nil
+        @host = 'localhost'
+        @port = 8108
+        @protocol = 'http'
+        @timeout_ms = 3_600_000
+        @open_timeout_ms = 1_000
+        @retries = { attempts: 2, backoff: (10.0..60.0) }
+      end
+
+      # Normalize protocol assignment to default to 'http' when blank.
+      # @param value [String, nil]
+      # @return [void]
+      def protocol=(value)
+        s = value.to_s
+        @protocol = s.strip.present? ? s : 'http'
+      end
+    end
+  end
+end

data/lib/search_engine/config/validators.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Config
+    # Validation helpers for configuration.
+    # Keep messages identical to the previous inline implementations.
+    module Validators
+      module_function
+
+      def validate_protocol(protocol)
+        return [] if %w[http https].include?(protocol.to_s)
+
+        ['protocol must be "http" or "https"']
+      end
+
+      def validate_host(host)
+        return [] unless host.nil? || host.to_s.strip.empty?
+
+        ['host must be present']
+      end
+
+      def validate_port(port)
+        return [] if port.is_a?(Integer) && port.positive?
+
+        ['port must be a positive Integer']
+      end
+
+      def validate_timeouts(timeout_ms, open_timeout_ms)
+        errors = []
+        errors << 'timeout_ms must be a non-negative Integer' unless timeout_ms.is_a?(Integer) && !timeout_ms.negative?
+        unless open_timeout_ms.is_a?(Integer) && !open_timeout_ms.negative?
+          errors << 'open_timeout_ms must be a non-negative Integer'
+        end
+        errors
+      end
+
+      def validate_retries(retries)
+        return ['retries must be a Hash with keys :attempts and :backoff'] unless retries_valid_shape?(retries)
+
+        errors = []
+        attempts = retries[:attempts]
+        backoff = retries[:backoff]
+
+        unless attempts.is_a?(Integer) && !attempts.negative?
+          errors << 'retries[:attempts] must be a non-negative Integer'
+        end
+
+        if backoff.is_a?(Numeric)
+          if backoff.negative?
+            errors << 'retries[:backoff] must be a non-negative Float or Range of non-negative Floats'
+          end
+        elsif backoff.is_a?(Range)
+          b = backoff.begin
+          e = backoff.end
+          valid = b.is_a?(Numeric) && e.is_a?(Numeric) && b >= 0 && e >= 0 && b <= e
+          errors << 'retries[:backoff] must be a non-negative Float or Range of non-negative Floats' unless valid
+        else
+          errors << 'retries[:backoff] must be a non-negative Float or Range of non-negative Floats'
+        end
+
+        errors
+      end
+
+      def retries_valid_shape?(retries)
+        retries.is_a?(Hash) && retries.key?(:attempts) && retries.key?(:backoff)
+      end
+
+      def validate_cache(cache_ttl_s)
+        return [] if cache_ttl_s.is_a?(Integer) && !cache_ttl_s.negative?
+
+        ['cache_ttl_s must be a non-negative Integer']
+      end
+
+      def validate_multi_search_limit(limit)
+        return [] if limit.is_a?(Integer) && !limit.negative?
+
+        ['multi_search_limit must be a non-negative Integer']
+      end
+
+      def validate_presets(presets)
+        errors = []
+        en = presets.enabled
+        ns = presets.namespace
+        ld = Array(presets.locked_domains)
+
+        errors << 'presets.enabled must be a Boolean' unless [true, false].include?(en)
+        unless ns.nil? || (ns.is_a?(String) && !ns.strip.empty?)
+          errors << 'presets.namespace must be a non-empty String or nil'
+        end
+        unless ld.is_a?(Array) && ld.all? { |k| k.is_a?(Symbol) }
+          errors << 'presets.locked_domains must be an Array of Symbols'
+        end
+        errors
+      end
+    end
+  end
+end