search-engine-for-typesense 1.0.0

data/lib/search_engine/engine.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Rails engine for the SearchEngine gem.
+  # Configures autoloading and eager-loading paths.
+  class Engine < ::Rails::Engine
+    engine_name 'search_engine'
+    # isolate_namespace SearchEngine # enable later if controllers/routes appear
+
+    # Ensure Zeitwerk loads from lib/
+    config.autoload_paths << root.join('lib').to_s
+
+    # Ensure app/search_engine is eager-loadable in production
+    config.paths.add 'app/search_engine', eager_load: true
+
+    initializer 'search_engine.configuration' do
+      cfg = SearchEngine.config
+      # Hydrate only blank/unset fields from ENV to avoid clobbering
+      # host app overrides. ENV resolution is centralized in Config.
+      cfg.hydrate_from_env!(ENV, override_existing: false)
+      cfg.warn_if_incomplete!
+    end
+
+    # Ignore hyphenated compatibility shim and CLI entrypoint so Zeitwerk doesn't try to constantize them.
+    # These files are manually required (CLI is required from Rake tasks).
+    initializer 'search_engine.zeitwerk_ignores', before: :set_autoload_paths do
+      # Rails 6.1+ exposes a loader per engine via `loader`. Guard presence for safety.
+      loader = respond_to?(:loader) ? self.loader : nil
+      shim = root.join('lib', 'search-engine-for-typesense.rb').to_s
+      cli_file = root.join('lib', 'search_engine', 'cli.rb').to_s
+      loader&.ignore(shim)
+      loader&.ignore(cli_file)
+
+      # Also ensure Rails global autoloaders ignore these files, since the engine
+      # adds lib/ to autoload paths and the main/once loaders may scan it.
+      if defined?(Rails) && Rails.respond_to?(:autoloaders)
+        al = Rails.autoloaders
+        al.main.ignore(shim) if al.respond_to?(:main)
+        al.once.ignore(shim) if al.respond_to?(:once)
+        al.main.ignore(cli_file) if al.respond_to?(:main)
+        al.once.ignore(cli_file) if al.respond_to?(:once)
+      end
+    end
+
+    initializer 'search_engine.observability' do
+      cfg = SearchEngine.config
+
+      # Prefer new structured LoggingSubscriber when configured; otherwise
+      # fall back to legacy Notifications::CompactLogger gated by cfg.observability.
+      begin
+        require 'search_engine/logging_subscriber'
+      rescue LoadError
+        # no-op; allow running without ActiveSupport
+      end
+
+      if defined?(SearchEngine::LoggingSubscriber)
+        logging_cfg = cfg.respond_to?(:logging) ? cfg.logging : nil
+        # Opt-out when mode is nil or sample is explicitly 0.0
+        if logging_cfg.respond_to?(:mode) && !logging_cfg.mode.nil?
+          sample = logging_cfg.respond_to?(:sample) ? logging_cfg.sample : nil
+          if sample.nil? || sample.to_f > 0.0
+            SearchEngine::LoggingSubscriber.install!(logging_cfg)
+            next
+          end
+        end
+      end
+
+      next unless cfg.observability&.enabled
+
+      # Defer requiring subscriber to runtime to avoid eager load issues
+      begin
+        require 'search_engine/notifications/compact_logger'
+      rescue LoadError
+        # no-op; allow running without ActiveSupport
+      end
+
+      if defined?(SearchEngine::Notifications::CompactLogger)
+        # Subscribe once per boot; store handle in a class ivar in the subscriber
+        SearchEngine::Notifications::CompactLogger.subscribe(
+          logger: cfg.logger,
+          level: :info,
+          include_params: false
+        )
+      end
+    end
+
+    initializer 'search_engine.opentelemetry' do
+      SearchEngine.config
+      begin
+        require 'search_engine/otel'
+      rescue LoadError
+        # no-op; adapter is fully optional
+      end
+
+      if defined?(SearchEngine::Otel)
+        # Start adapter only when SDK is present and config enables it
+        SearchEngine::Otel.start!
+      end
+    end
+
+    initializer 'search_engine.console_helpers' do
+      if defined?(Rails::Console) || $PROGRAM_NAME&.end_with?('console')
+        begin
+          require 'search_engine/console_helpers'
+          SearchEngine::ConsoleHelpers.install!
+        rescue LoadError
+          # no-op; helpers are optional
+        end
+      end
+    end
+
+    # Manage a dedicated Zeitwerk loader for host app SearchEngine models.
+    # Loads after Rails so application models/constants are available.
+    initializer 'search_engine.models_loader' do
+      # Resolve configured path; allow disabling via nil/false/empty.
+      cfg = SearchEngine.config
+      models_path_value = cfg.respond_to?(:search_engine_models) ? cfg.search_engine_models : nil
+      next if models_path_value.nil? || models_path_value == false || models_path_value.to_s.strip.empty?
+
+      require 'pathname'
+      path = Pathname.new(models_path_value.to_s)
+      path = Rails.root.join(path) unless path.absolute?
+      path_s = path.to_s
+      next unless File.directory?(path_s)
+
+      # Ensure Rails' autoloaders do not also manage this directory.
+      if defined?(Rails) && Rails.respond_to?(:autoloaders)
+        al = Rails.autoloaders
+        %i[main once].each do |key|
+          al.public_send(key).ignore(path_s) if al.respond_to?(key)
+        end
+      end
+
+      # Create or reuse a dedicated loader under SearchEngine namespace.
+      loader = SearchEngine.instance_variable_get(:@_models_loader)
+      unless loader
+        loader = Zeitwerk::Loader.new
+        loader.tag = 'search_engine.models'
+        # Reuse Rails' inflector for consistent constantization rules.
+        if defined?(Rails) && Rails.respond_to?(:autoloaders) && Rails.autoloaders.respond_to?(:main)
+          loader.inflector = Rails.autoloaders.main.inflector
+        end
+        loader.push_dir(path_s, namespace: SearchEngine)
+        loader.enable_reloading if defined?(Rails) && Rails.env.development?
+        SearchEngine.instance_variable_set(:@_models_loader, loader)
+      end
+
+      # Setup immediately so host to_prepare callbacks that run early can rely on
+      # SearchEngine models being available.
+      unless SearchEngine.instance_variable_defined?(:@_models_loader_setup)
+        loader.setup
+        SearchEngine.instance_variable_set(:@_models_loader_setup, true)
+      end
+
+      # Set up and coordinate with Rails reloader lifecycle.
+      config.to_prepare do
+        l = SearchEngine.instance_variable_get(:@_models_loader)
+        next unless l
+
+        l.reload if defined?(Rails) && Rails.env.development?
+        # Always eager-load configured SearchEngine models so their `collection` calls
+        # register mappings at boot, regardless of Rails.eager_load setting.
+        l.eager_load
+      end
+    end
+  end
+end

data/lib/search_engine/errors.rb

@@ -0,0 +1,290 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Public error hierarchy for the SearchEngine client wrapper.
+  #
+  # These exceptions provide a stable contract to callers regardless of the
+  # underlying HTTP client or the Typesense gem's internal error types.
+  module Errors
+    # Base error for all SearchEngine failures.
+    # Carries optional structured metadata for enhanced DX.
+    #
+    # Keyword options are optional and backwards-compatible. Existing call sites
+    # that pass only a message remain valid.
+    #
+    # @!attribute [r] hint
+    #   @return [String, nil] short actionable suggestion (no secrets)
+    # @!attribute [r] doc
+    #   @return [String, nil] docs URL with optional anchor (e.g., "https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/query-dsl#operators")
+    # @!attribute [r] details
+    #   @return [Object, nil] machine-readable context (JSON-serializable)
+    # @!attribute [r] code
+    #   @return [Symbol, nil] stable symbolic code when defined by subclasses
+    # @abstract
+    class Error < StandardError
+      attr_reader :hint, :doc, :details, :code
+
+      # @param message [String]
+      # @param hint [String, nil]
+      # @param doc [String, nil]
+      # @param details [Object, nil]
+      # @param code [Symbol, nil]
+      def initialize(message = nil, hint: nil, doc: nil, details: nil, code: nil, **_ignore)
+        super(message)
+        @hint = presence_or_nil(hint)
+        @doc = presence_or_nil(doc)
+        @details = sanitize_details(details)
+        @code = code
+      end
+
+      # Return a stable, redaction-aware hash for logging/telemetry.
+      # Keys are predictable for downstream processing.
+      # @return [Hash]
+      def to_h
+        base = {
+          type: self.class.name,
+          message: to_base_message,
+          hint: @hint,
+          doc: @doc,
+          details: @details
+        }
+        base[:code] = @code if @code
+        prune_nils(base)
+      end
+
+      # Preserve historic message but append a concise suffix when hint/doc present.
+      # Single-line for log friendliness.
+      # @return [String]
+      def to_s
+        base = to_base_message
+        suffix = []
+        suffix << "Hint: #{@hint}" if @hint
+        suffix << "see #{@doc}" if @doc
+        return base if suffix.empty?
+
+        "#{base} — #{suffix.join(' ')}"
+      end
+
+      private
+
+      def to_base_message
+        # Call Exception#to_s directly to avoid our overridden to_s suffix
+        Exception.instance_method(:to_s).bind_call(self).to_s
+      end
+
+      def sanitize_details(obj)
+        return nil if obj.nil?
+
+        if defined?(SearchEngine::Observability)
+          begin
+            red = SearchEngine::Observability.redact(obj)
+            return jsonable(red)
+          rescue StandardError
+            return jsonable(obj)
+          end
+        end
+
+        jsonable(obj)
+      end
+
+      def jsonable(obj)
+        case obj
+        when Hash
+          obj.each_with_object({}) { |(k, v), h| h[k.to_sym] = jsonable(v) }
+        when Array
+          obj.map { |v| jsonable(v) }
+        when Numeric, TrueClass, FalseClass, NilClass, String
+          obj
+        else
+          obj.to_s
+        end
+      end
+
+      def prune_nils(h)
+        h.each_with_object({}) do |(k, v), acc|
+          acc[k] = v unless v.nil?
+        end
+      end
+
+      def presence_or_nil(v)
+        return nil if v.nil?
+
+        s = v.to_s
+        s.strip.empty? ? nil : s
+      end
+    end
+
+    # Raised when a request exceeds the configured timeout budget.
+    #
+    # Typical causes include connect/open timeouts or read timeouts surfaced by
+    # the underlying HTTP client used by the official Typesense gem.
+    class Timeout < Error; end
+
+    # Raised for network-level connectivity issues prior to receiving a response.
+    #
+    # Examples: DNS resolution failures, refused TCP connections, TLS handshake
+    # errors, or other socket-level errors.
+    class Connection < Error; end
+
+    # Raised when Typesense responds with a non-2xx HTTP status code.
+    #
+    # Carries the HTTP status and the parsed error body (when available) to aid
+    # in debugging and programmatic handling upstream.
+    class Api < Error
+      # @return [Integer] HTTP status code
+      attr_reader :status
+
+      # @return [Object, nil] Parsed error body (Hash/String), when available
+      attr_reader :body
+
+      # @param msg [String]
+      # @param status [Integer]
+      # @param body [Object, nil]
+      def initialize(msg, status:, body: nil, **kw)
+        super(msg, **kw)
+        @status = status
+        @body = body
+      end
+    end
+
+    # Raised when wrapper-level validation fails before making a request.
+    #
+    # Use this for actionable, developer-facing messages that indicate a caller
+    # constructed an invalid request (e.g., blank collection name).
+    class InvalidParams < Error; end
+
+    # Raised when a provided field name is unknown or disallowed for a model.
+    #
+    # Typical cause: a typo or using a field that is not declared via
+    # {SearchEngine::Base.attribute}.
+    class InvalidField < Error; end
+
+    # Raised when a base attribute referenced by the Field Selection DSL is not
+    # declared on the model.
+    #
+    # Prefer this over {InvalidField} for selection-time validation to provide
+    # developer-friendly guidance and suggestions.
+    class UnknownField < Error; end
+
+    # Raised when an operator or fragment token is not recognized by the SQL-ish
+    # grammar accepted by the Parser.
+    class InvalidOperator < Error; end
+
+    # Raised when a value cannot be coerced to the declared attribute type, or
+    # when its shape is incompatible (e.g., empty array for IN/NOT IN).
+    class InvalidType < Error; end
+
+    # Raised when a requested join association is not declared for a model.
+    #
+    # Typical cause: a typo or referencing an association that has not been
+    # registered via {SearchEngine::Base.join}.
+    class UnknownJoin < Error; end
+
+    # Raised when an association reference is invalid for the model and should
+    # be declared via {SearchEngine::Base.join}.
+    #
+    # Prefer this for high-level validation messaging with guidance and
+    # suggestions ("did you mean ..."), while keeping {UnknownJoin} for
+    # lower-level registry lookups.
+    class InvalidJoin < Error; end
+
+    # Raised when a query references a joined association field without applying
+    # the association on the relation via {SearchEngine::Relation#joins}.
+    #
+    # Example: calling `where(authors: { last_name: "Rowling" })` without
+    # `.joins(:authors)` on the relation first.
+    class JoinNotApplied < Error; end
+
+    # Raised when a nested attribute referenced by the Field Selection DSL is
+    # not declared on the joined association's target model.
+    #
+    # Typical cause: a typo in a nested field name or a stale attribute map.
+    class UnknownJoinField < Error; end
+
+    # Raised when selection inputs are malformed or ambiguous and cannot be
+    # deterministically normalized (e.g., invalid nested shapes or incompatible
+    # payload types).
+    class ConflictingSelection < Error; end
+
+    # Raised when grouping DSL is used with invalid inputs.
+    #
+    # Use for actionable messages like unknown field names, invalid limit values,
+    # or non-boolean missing_values.
+    #
+    # @example Unknown field with suggestion
+    #   raise SearchEngine::Errors::InvalidGroup, "InvalidGroup: unknown field :brand for grouping on SearchEngine::Product (did you mean :brand_id?)"
+    class InvalidGroup < Error; end
+
+    # Raised when grouping references unsupported constructs such as joined/path fields
+    # (e.g., "$assoc.field"). Only base fields are supported for grouping.
+    #
+    # @example
+    #   raise SearchEngine::Errors::UnsupportedGroupField, 'UnsupportedGroupField: grouping supports base fields only (got "$authors.last_name")'
+    class UnsupportedGroupField < Error; end
+
+    # Raised when strict selection is enabled and a requested field is absent
+    # in the hydrated document (e.g., excluded by API mapping).
+    #
+    # This error is actionable and guides remediation: adjust the relation's
+    # selection (select/exclude/reselect), relax strictness, or ensure the
+    # upstream Typesense include/exclude mapping includes the fields.
+    class MissingField < Error; end
+
+    # Raised when a materializer requests fields that are not permitted by the
+    # relation's effective selection (include − exclude, with exclude taking precedence).
+    #
+    # Used by selection-aware materializers like {SearchEngine::Relation#pluck},
+    # {SearchEngine::Relation#pick}, and {SearchEngine::Relation#ids} to fail fast
+    # before any network call.
+    class InvalidSelection < Error; end
+
+    # Raised when a curated ID does not match the configured pattern.
+    #
+    # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/curation
+    # @example
+    #   raise SearchEngine::Errors::InvalidCuratedId, 'InvalidCuratedId: "foo bar" is not a valid curated ID. Expected pattern: /\A[\w\-:\.]+\z/. Try removing illegal characters.'
+    class InvalidCuratedId < Error; end
+
+    # Raised when pinned/hidden lists exceed configured limits after normalization.
+    #
+    # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/curation
+    # @example
+    #   raise SearchEngine::Errors::CurationLimitExceeded, 'CurationLimitExceeded: pinned list exceeds max_pins=50 (attempted 51). Reduce inputs or raise the limit in SearchEngine.config.curation.'
+    class CurationLimitExceeded < Error; end
+
+    # Raised when an override tag is blank or invalid per allowed pattern.
+    #
+    # @see https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/curation
+    # @example
+    #   raise SearchEngine::Errors::InvalidOverrideTag, 'InvalidOverrideTag: "" is invalid. Use non-blank strings that match the allowed pattern.'
+    class InvalidOverrideTag < Error; end
+
+    # Raised when an option value is invalid or unsupported for a public API.
+    #
+    # Used by DSL methods to fail fast with actionable hints.
+    # Typical causes: invalid HTML tag tokens, negative integers for thresholds,
+    # or empty field lists where at least one field is required.
+    #
+    # @example
+    #   raise SearchEngine::Errors::InvalidOption.new(
+    #     'InvalidOption: tag must be a simple HTML-like token',
+    #     hint: 'Use a simple tag like <em> or <mark>',
+    #     doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/highlighting#options'
+    #   )
+    class InvalidOption < Error; end
+
+    # Raised when a query result exceeds the configured post-fetch hits ceiling.
+    #
+    # Use this for actionable messages when {Relation#validate_hits!} is set and
+    # the backend reports a total hits count above the allowed maximum.
+    #
+    # @example
+    #   raise SearchEngine::Errors::HitLimitExceeded.new(
+    #     'HitLimitExceeded: 12000 results exceed max=10000',
+    #     hint: 'Increase `validate_hits!(max:)` or narrow your filters.',
+    #     doc: 'https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/hit-limits#validation',
+    #     details: { total_hits: 12_000, max: 10_000, collection: 'products' }
+    #   )
+    class HitLimitExceeded < Error; end
+  end
+end

data/lib/search_engine/filters/sanitizer.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  module Filters
+    # Sanitizer utilities for Typesense-compatible filters.
+    #
+    # Provides quoting/escaping and helpers to build normalized filter strings
+    # from hashes and templates with placeholders.
+    module Sanitizer
+      module_function
+
+      # Quote a Ruby value into a Typesense filter literal.
+      #
+      # - NilClass => "null"
+      # - TrueClass/FalseClass => "true"/"false"
+      # - Numeric => as-is (stringified)
+      # - String => double-quoted with minimal escaping for \ and "
+      # - Time/DateTime/Date => ISO8601 string, then quoted as a string
+      # - Array => one-level flatten, each element quoted, wrapped with [ ... ]
+      #
+      # @param value [Object]
+      # @return [String]
+      def quote(value)
+        case value
+        when NilClass
+          'null'
+        when TrueClass
+          'true'
+        when FalseClass
+          'false'
+        when Numeric
+          value.to_s
+        when String
+          %("#{escape_string(value)}")
+        when Time
+          %("#{value.iso8601}")
+        when DateTime
+          %("#{value.iso8601}")
+        when Date
+          %("#{value.iso8601}")
+        when Array
+          elements = value.flatten(1).map { |el| quote(el) }
+          "[#{elements.join(', ')}]"
+        else
+          if value.respond_to?(:to_time)
+            %("#{value.to_time.iso8601}")
+          else
+            %("#{escape_string(value.to_s)}")
+          end
+        end
+      end
+
+      # Quote a scalar Ruby value for Typesense filters with conditional quoting for strings.
+      #
+      # Rules (based on Typesense filter_by syntax):
+      # - Strings that match a safe token pattern (e.g., Active, ACTIVE_1, foo-bar) are emitted bare
+      # - Reserved words true/false/null remain bare only when actually booleans/nil; string forms are quoted
+      # - Strings with any other characters are double-quoted with escaping
+      # - Arrays are delegated to +quote+ to preserve element quoting rules
+      #
+      # @param value [Object]
+      # @return [String]
+      def quote_scalar_for_filter(value)
+        return quote(value) if value.is_a?(Array)
+
+        case value
+        when NilClass
+          'null'
+        when TrueClass
+          'true'
+        when FalseClass
+          'false'
+        when Numeric
+          value.to_s
+        when String, Symbol
+          str = value.to_s
+          lc = str.strip.downcase
+          # Avoid ambiguity with special literals when user passes them as strings
+          return %("#{escape_string(str)}") if %w[true false null].include?(lc)
+
+          if safe_bare_string?(str)
+            str
+          else
+            %("#{escape_string(str)}")
+          end
+        when Time
+          %("#{value.iso8601}")
+        when DateTime
+          %("#{value.iso8601}")
+        when Date
+          %("#{value.iso8601}")
+        else
+          if value.respond_to?(:to_time)
+            %("#{value.to_time.iso8601}")
+          else
+            %("#{escape_string(value.to_s)}")
+          end
+        end
+      end
+
+      # Build normalized filter fragments from a Hash.
+      # Scalars become "field:=<quoted>", arrays become "field:=<quoted_list>".
+      #
+      # @param hash [Hash{#to_sym=>Object}]
+      # @param _attributes_map [Hash] (ignored here; validation should be done by caller)
+      # @return [Array<String>]
+      def build_from_hash(hash, _attributes_map = nil)
+        raise ArgumentError, 'filters hash must be a Hash' unless hash.is_a?(Hash)
+
+        hash.map do |key, raw|
+          field = key.to_sym.to_s
+          if array_like?(raw)
+            "#{field}:=#{quote(Array(raw))}"
+          else
+            "#{field}:=#{quote_scalar_for_filter(raw)}"
+          end
+        end
+      end
+
+      # Apply placeholder substitution for templates with '?' markers.
+      #
+      # Each unescaped '?' is replaced with a quoted argument from +args+ in order.
+      #
+      # @param template [String]
+      # @param args [Array<Object>]
+      # @return [String]
+      def apply_placeholders(template, args)
+        raise ArgumentError, 'template must be a String' unless template.is_a?(String)
+        raise ArgumentError, 'args must be an Array' unless args.is_a?(Array)
+
+        needed = count_placeholders(template)
+        provided = args.length
+        raise ArgumentError, "expected #{needed} args for #{needed} placeholders, got #{provided}" if needed != provided
+
+        idx = -1
+        template.gsub(/(?<!\\)\?/) do
+          idx += 1
+          val = args[idx]
+          val.is_a?(Array) ? quote(val) : quote_scalar_for_filter(val)
+        end
+      end
+
+      # Count unescaped '?' placeholders.
+      # @param template [String]
+      # @return [Integer]
+      def count_placeholders(template)
+        count = 0
+        escaped = false
+        template.each_char do |ch|
+          if escaped
+            escaped = false
+            next
+          end
+          if ch == '\\'
+            escaped = true
+          elsif ch == '?'
+            count += 1
+          end
+        end
+        count
+      end
+
+      # Escape a raw string for inclusion inside double quotes.
+      # @param str [String]
+      # @return [String]
+      def escape_string(str)
+        str.gsub('\\', '\\\\').gsub('"', '\\"')
+      end
+
+      # Determine whether a string can be emitted bare without quotes in filter_by.
+      # Safe if it matches: starts with a letter or underscore; then letters/digits/underscore/hyphen.
+      # This avoids ambiguity with numbers/booleans/null and special characters.
+      def safe_bare_string?(str)
+        return false if str.nil? || str.empty?
+
+        # Disallow surrounding/backtick/dquote characters quickly
+        return false if str.include?('"') || str.include?('`') || str.include?(',') || str.include?(' ')
+
+        # Must start with a letter or underscore; subsequent chars may include digits or hyphens/underscores
+        !!(str =~ /^[A-Za-z_][A-Za-z0-9_-]*$/)
+      end
+
+      # @api private
+      def array_like?(value)
+        value.is_a?(Array)
+      end
+    end
+  end
+end