search-engine-for-typesense 30.1.6.0 → 30.1.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52454481281edd904b9d0a3b68e191b6c8499426194b37f81e47f9fdc1a62435
-  data.tar.gz: 6adbd869d14f77faabf6ea10c42bc9f141d247a713ea016e97ab4188ce31564c
+  metadata.gz: ea3e7da2359ce1d9b4990147c6627e00a726cfa4e75cc340649ae561b32e42fe
+  data.tar.gz: 6b14b6b9e39cab91210e16ecced55f19948831d9f949cb2969f0be5731229cf1
 SHA512:
-  metadata.gz: '08c22e27bf1e8c039d453584c34af900c4e50b7b55efbad70a29854e8154dfa5353ccb24095f991aaa9ae38bb55cf9a799a5a183e07344a63f4b050c0cf5a23f'
-  data.tar.gz: b54156dde160147eb1463d1c108f38c15436a52fcf3958f7462f7acaa49b1b844525aa971f4c1bdd4de482962f21935463ff26c4cb332a6ecd1f90e2da43b105
+  metadata.gz: 6c2252a8d88ebe47b54bb7ac7ed2d65f2e0ed82b3e011ffeb3f5fb3756bff75de417c055a1de8c71fa513e51a6cfcb70b1025e6463164ff5de7018b1fcec1df8
+  data.tar.gz: 5519c44a440ed906df1673baf2682b1630171c0a2efb103feeb3300700d2d2653d521432fa332d2bc344c2cf717ba0ae7aa4d3feaaa6dcc7c82cfc0ba1486a4f

data/README.md

@@ -1,9 +1,6 @@
 # Search Engine for Typesense [![CI][ci-badge]][ci-url] [![Gem][gem-badge]][gem-url] [![Docs][docs-badge]][docs-url]
 [![Typesense](https://img.shields.io/badge/Typesense-Typesense-blue)](https://typesense.org) [![Typesense Ruby gem](https://img.shields.io/badge/Typesense%20Ruby%20gem-TypesenseRubyGem-blue)](https://github.com/typesense/typesense-ruby)
 
-> [!WARNING]
-> **⚠️ This project is under maintenance – work in progress. APIs and docs may change. ⚠️**
-
 Mountless Rails::Engine for [Typesense](https://typesense.org). Expressive Relation/DSL with JOINs, grouping, presets/curation — with strong DX and observability.
 
 > [!NOTE]

data/lib/generators/search_engine/install/templates/initializer.rb.tt

@@ -15,6 +15,7 @@
 # - CLI (Doctor): https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/cli
 # - DX helpers: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/dx
 # - Configuration: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/configuration
+# - Vector search: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/vector-search
 SearchEngine.configure do |c|
   c.host = ENV.fetch('TYPESENSE_HOST', 'localhost')
   c.port = Integer(ENV.fetch('TYPESENSE_PORT', 8108))
@@ -82,6 +83,26 @@ SearchEngine.configure do |c|
   # Override the logger used by the engine (defaults to Rails.logger). Example:
   # c.logger = Rails.logger
 
+  # --- Embedding / Vector search -------------------------------------------
+  # See: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/vector-search
+
+  # Default embedding model used by the `embedding` DSL macro when no
+  # per-field model: override is provided. Typesense ships built-in models
+  # (e.g. "ts/all-MiniLM-L12-v2"); remote providers like OpenAI are also
+  # supported — set api_key below or per-field.
+  # c.embedding.model = 'ts/all-MiniLM-L12-v2'
+
+  # API key for remote embedding providers (e.g. OpenAI). Keep in ENV.
+  # c.embedding.api_key = ENV['EMBEDDING_API_KEY']
+
+  # Extra model_config merged into every Typesense embed block.
+  # c.embedding.model_config = {}
+
+  # Tolerance for vector_search weights sum validation (weights must sum
+  # to ~1.0 within this tolerance). Raise when many small weights cause
+  # floating-point drift. Default: 0.01
+  # c.embedding.weights_sum_tolerance = 0.01
+
   # --- Presets -------------------------------------------------------------
   # See: https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/presets
 
@@ -125,6 +146,11 @@ SearchEngine.configure do |c|
   # c.observability.include_error_messages = false
   # c.observability.emit_legacy_event_aliases = true
 
+  # Redact raw float arrays inside vector_query strings in logs/telemetry.
+  # When true (default), vectors are replaced with [<N dims>].
+  # Set to false to see raw vectors for debugging.
+  # c.observability.redact_vectors = true
+
   # OpenTelemetry integration (only when the SDK is present). Defaults shown.
   # c.opentelemetry = { enabled: false, service_name: 'search_engine' }
 

data/lib/generators/search_engine/model/templates/model.rb.tt

@@ -4,9 +4,18 @@
 # Docs:
 # - https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/quickstart
 # - https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/field-selection#guardrails--errors
+# - https://nikita-shkoda.mintlify.app/projects/search-engine-for-typesense/v30.1/vector-search
 class SearchEngine::<%= class_name %> < SearchEngine::Base
   collection "<%= @collection_name %>"
 <% Array(@attributes).each do |(name, type)| -%>
   attribute :<%= name %>, :<%= type %>
 <% end -%>
+
+  # --- Vector search (optional) ---
+  # Auto-embedding: Typesense generates vectors from the listed text fields.
+  # Requires config.embedding.model or a per-field model: override.
+  # embedding from: %i[name description]
+  #
+  # External embedding: bring your own vectors (num_dim required).
+  # embedding :my_vector, num_dim: 384, suffix: false
 end

data/lib/search_engine/base/model_dsl.rb

@@ -96,7 +96,9 @@ module SearchEngine
         #
         # @param name [#to_sym]
         # @param type [Object] type descriptor (e.g., :string, :integer)
-        # @param index [Boolean, nil] when false, omit from compiled Typesense schema (still hydrated/displayed)
+        # @param index [Boolean, nil] when false, omit from compiled Typesense schema (still
+        #   hydrated/displayed). Fields referenced by an embedding's `from:` are kept in the
+        #   schema with Typesense-native `"index": false` instead of being omitted entirely.
         # @param locale [String, nil]
         # @param optional [Boolean, nil]
         # @param sort [Boolean, nil]

data/lib/search_engine/mapper.rb

@@ -566,7 +566,7 @@ module SearchEngine
         base_fields.reject! { |fname| fname.to_s.include?('.') }
         required = base_fields.to_set
         opts.each do |fname, o|
-          next unless o.is_a?(Hash) && o[:optional]
+          next unless o.is_a?(Hash) && (o[:optional] || o[:index] == false)
 
           required.delete(fname.to_sym)
         end

data/lib/search_engine/schema.rb

@@ -35,7 +35,7 @@ module SearchEngine
     }.freeze
 
     FIELD_COMPARE_KEYS = %i[
-      type reference async_reference locale sort optional infix facet
+      type reference async_reference locale sort optional index infix facet
       embed num_dim hnsw_params
     ].freeze
     PHYSICAL_SUFFIX_RE = /_\d{8}_\d{6}_\d{3}\z/
@@ -44,9 +44,14 @@ module SearchEngine
       # Build a Typesense-compatible schema hash from a model class DSL.
       #
       # The output includes only keys that are supported and declared via the DSL.
-      # Fields explicitly marked with `index: false` are intentionally omitted
-      # from the compiled schema (they can still be sent in documents and will
-      # be hydrated/displayed, but are not indexed in memory).
+      # Fields explicitly marked with `index: false` are generally omitted from
+      # the compiled schema (they can still be sent in documents and will be
+      # hydrated/displayed, but are not indexed in memory).
+      #
+      # **Exception:** fields with `index: false` that are referenced by an
+      # embedding's `from:` list are included with Typesense-native
+      # `"index": false` and `"optional": true` so the embedding model can
+      # read their values without keyword-indexing them.
       #
       # @param klass [Class] model class inheriting from {SearchEngine::Base}
       # @return [Hash] frozen schema hash with symbol keys
@@ -462,6 +467,7 @@ module SearchEngine
         attribute_options = klass.respond_to?(:attribute_options) ? (klass.attribute_options || {}) : {}
         references_by_local_key = build_references_by_local_key(klass, client: client)
         async_reference_by_local_key = build_async_reference_by_local_key(klass)
+        embed_source_fields = collect_embed_source_fields(attribute_options)
 
         fields_array = []
         needs_nested_fields = false
@@ -470,14 +476,16 @@ module SearchEngine
           validate_attribute_type!(attribute_name, type_descriptor)
 
           opts = attribute_options[attribute_name.to_sym] || {}
-          # Skip non-indexed attributes and any nested fields under a non-indexed base
+          # Skip non-indexed attributes and any nested fields under a non-indexed base,
+          # unless the field is referenced by an embedding's from: list.
           base_index_false = false
           if attribute_name.to_s.include?('.')
             base_sym = attribute_name.to_s.split('.', 2).first.to_sym
             base_opts = attribute_options[base_sym] || {}
             base_index_false = (base_opts[:index] == false)
           end
-          next if opts[:index] == false || base_index_false
+          effectively_unindexed = opts[:index] == false || base_index_false
+          next if effectively_unindexed && !embed_source_fields.include?(attribute_name.to_sym)
 
           ts_type = typesense_type_for(type_descriptor)
 
@@ -536,14 +544,34 @@ module SearchEngine
         %w[object object[]].include?(ts_type)
       end
 
+      # Collect field names referenced by any embedding's from: list.
+      # @param attribute_options [Hash] model attribute options
+      # @return [Set<Symbol>] field names used as embedding sources
+      def collect_embed_source_fields(attribute_options)
+        sources = Set.new
+        attribute_options.each_value do |opts|
+          next unless opts.is_a?(Hash)
+
+          embed = opts[:embed]
+          next unless embed.is_a?(Hash)
+
+          from = embed[:from] || embed['from']
+          Array(from).each { |f| sources << f.to_sym }
+        end
+        sources
+      end
+      private :collect_embed_source_fields
+
       def build_field_entry(attribute_name, ts_type, references_by_local_key, async_reference_by_local_key, opts)
+        unindexed = opts[:index] == false
         {
           name: attribute_name.to_s,
           type: ts_type,
           **{
+            index: unindexed ? false : nil,
             locale: opts[:locale],
             sort: opts[:sort],
-            optional: opts[:optional],
+            optional: unindexed ? true : opts[:optional],
             infix: opts[:infix],
             facet: opts[:facet],
             reference: references_by_local_key[attribute_name.to_sym],
@@ -621,7 +649,7 @@ module SearchEngine
         entry = { name: fname, type: normalize_type(ftype) }
         entry[:reference] = fref.to_s unless fref.nil? || fref.to_s.strip.empty?
 
-        %i[locale sort optional infix facet async_reference].each do |k|
+        %i[locale sort optional index infix facet async_reference].each do |k|
           val = field[k] || field[k.to_s]
           entry[k] = val unless val.nil?
         end

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.6.0'
+  VERSION = '30.1.6.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.6.0
+  version: 30.1.6.2
 platform: ruby
 authors:
 - Nikita Shkoda