sqlite_search 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e39fa70c6de966aae8026bb76778f58701d7e2ffaa20cf3ede7ed1b77df4488
-  data.tar.gz: 2664f27e51d316d7d9f99c507e9213c7ee564015a3345974a7ccc880641caba5
+  metadata.gz: bb082a5c1485cc56178a03f0b1421f5365f3e4b0ce225179119d1c3bee298ece
+  data.tar.gz: ca18eb64941711935abbbf55d0e64a88cb7f87a2837b80febabdcac7cb884f9e
 SHA512:
-  metadata.gz: 77745b678a9c28d335d4a440d3d90323bbecf775ba9fa4d747ca493f5e158eda6d4eea92117f4552c1ab47f9052de20b96ff0fc341cfb88b1bddcc12214cdacb
-  data.tar.gz: 433e12eee264c82ed27a0b9eb5ffaa90db564fc20a67ba8d7c9a2a0c09b175be1e5f2e079ddf63ab6cff8e978f6c25a1cda46db30cd8e8342d459dd6da6c6d6c
+  metadata.gz: ba5d6fa12b18ed351e8b3d5404ca5f9b4da36ab617d4672f8b9188772c111b216f98d89be590dfe45816d7127af6875e6d53d106140d7476d931f7552ee2fb2f
+  data.tar.gz: 7cc03facced61e28ad74fd1db2973657327ce87d949999a99b6d4f985685649ed7f7a4b6d36c8f275962c282abd65b7ac88c13aefc23af9fca0782ad155b567d

data/CHANGELOG.md

@@ -1,10 +1,8 @@
 # Changelog
 
-All notable changes to this project are documented here. The format is based on
-[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows
-[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+All notable changes to this project will be documented in this file.
 
-## [Unreleased]
+## [0.1.0] - 2026-10-03
 
 ### Added
 
@@ -22,5 +20,11 @@ All notable changes to this project are documented here. The format is based on
 - Exact pre-filtering by chaining: conditions placed before `.search`/`.semantic`
   (for example `Post.where(tenant_id: 5).search("...")`) push into both arms.
 - Rails generators for the FTS5 and vec migrations.
-
-[Unreleased]: https://github.com/radioactive-labs/sqlite_search/commits/main
+- Derived text: `source:` and `watch:` on `fts5_scope` and `vec_scope` index text
+  computed in Ruby and resync when the watched attributes change.
+- `vec_scope ..., sync: :manual` with `record.reembed` for apps that run their own
+  embedding pipeline.
+- Relevance thresholds: `order_by_rank(threshold:)`, and per-arm
+  `fts5_threshold:` / `vec_threshold:` on hybrid scopes.
+- A reranker can return `[record, score]` pairs, which set each result's
+  `<name>_score` to the reranker's score.

data/README.md

@@ -14,7 +14,20 @@ module with BM25 relevance ranking. Vector (semantic) search uses
 [`neighbor`](https://github.com/ankane/neighbor) gem, with your app supplying
 embeddings via a callback. Hybrid search fuses the two with Reciprocal Rank
 Fusion and an optional reranking step. There are no database triggers and no
-background service to run. Works on Rails 8.0+.
+search server to run. Works on Rails 8.0+.
+
+## Used in production
+
+<a href="https://universalchatbot.com">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/assets/universal-chatbot-dark.svg">
+    <img src="docs/assets/universal-chatbot-light.svg" alt="Universal Chatbot" height="33">
+  </picture>
+</a>
+
+sqlite_search runs the knowledge base search in
+[Universal Chatbot](https://universalchatbot.com), where it handles keyword,
+semantic, and hybrid retrieval over customer documents on SQLite in production.
 
 ## 30-second tour
 
@@ -30,7 +43,6 @@ end
 
 # app/models/post.rb
 class Post < ApplicationRecord
-  include SqliteSearch::Model
   fts5_scope :search, against: { title: 2.0, body: 1.0 }
 end
 
@@ -52,9 +64,11 @@ Add the gem:
 gem "sqlite_search"
 ```
 
-Then `bundle install`. In a Rails app the model concern and the migration
-helpers are wired in automatically. Vector and hybrid search need two more gems;
-see [Vector search](#vector-search).
+Then `bundle install`. In a Rails app the model DSL (`fts5_scope`, `vec_scope`,
+`hybrid_scope`) and the migration helpers are available in every model and
+migration automatically. Outside Rails, `include SqliteSearch::Model` in your
+models and `ActiveRecord::Migration.include(SqliteSearch::Migration)`. Vector
+and hybrid search need two more gems; see [Vector search](#vector-search).
 
 ## Full-text search
 
@@ -91,7 +105,6 @@ database.
 
 ```ruby
 class Post < ApplicationRecord
-  include SqliteSearch::Model
   fts5_scope :search, against: { title: 2.0, body: 1.0 }
 end
 ```
@@ -102,6 +115,30 @@ changes. They run inside the transaction, so the index write is atomic with the
 row: if it fails, both roll back. A model can declare more than one index
 (`fts5_scope :search_body, against: :body`) and each gets its own scope.
 
+### Index derived text
+
+When the text to index is not a plain column (assembled from JSON, cleaned up,
+or stemmed in Ruby), name a method with `source:`. It returns a hash of FTS
+column to text, and `against:` still names those columns and their weights.
+`watch:` lists the attributes whose change triggers a resync:
+
+```ruby
+class Article < ApplicationRecord
+  fts5_scope :keyword, against: { text: 1.0, tags: 2.0 },
+    source: :search_document, watch: [:body, :metadata]
+
+  def search_document
+    { text: body, tags: metadata["tags"].join(" ") }
+  end
+end
+```
+
+`watch:` is required with `source:` (without `source:` it defaults to the
+`against:` columns). `reindex` rebuilds a `source:` index record by record in
+Ruby, inside one transaction. `backfill: true` in the migration copies columns
+straight from the table, so it does not apply here (it raises if the table lacks
+the `against:` columns); run `reindex` after the migration instead.
+
 ### Query
 
 ```ruby
@@ -129,7 +166,10 @@ posts.first.search_rank   # higher is more relevant
 ```
 
 `order_by_rank` orders by SQLite's `bm25()`, inverted so higher means better, and
-honors the per-column weights from `against:`. Each ranked record carries a
+honors the per-column weights from `against:`. It replaces any `order` chained
+before it; chain an `order` after it to add a tiebreaker.
+`order_by_rank(threshold: 8.0)` keeps only rows whose rank is at least the
+threshold. Each ranked record carries a
 `<name>_rank` reader (`search_rank` for a scope named `:search`).
 
 ### Reindexing
@@ -157,6 +197,17 @@ gem "neighbor"
 gem "sqlite-vec"
 ```
 
+Then have `neighbor` load the sqlite-vec extension on every connection:
+
+```
+rails g neighbor:sqlite
+```
+
+That writes `config/initializers/neighbor.rb`, which calls
+`Neighbor::SQLite.initialize!`. Do this before running a `create_vec_index`
+migration: without it, migrations and `db:schema:load` (which do not load your
+models) fail with `no such module: vec0`.
+
 ### Register an embedder
 
 sqlite_search calls this block whenever it needs to turn text into a vector, both
@@ -201,6 +252,24 @@ vec_scope :semantic_search, against: [:title, :body], dimensions: 768, sync: :in
 
 Route the job to a specific queue with `SqliteSearch.config.job_queue = :embeddings`.
 
+Pass `sync: :manual` when your app already runs its own embedding pipeline (to
+track progress on the row, say). Saves then never embed, destroys still remove
+the vector, and your code embeds a record when it is ready:
+
+```ruby
+vec_scope :semantic_search, against: :body, dimensions: 768, sync: :manual
+
+post.reembed(:semantic_search)   # embed this record now
+```
+
+To embed text that is not a plain column, name a method with `source:` that
+returns the text, and list the attributes it depends on in `watch:` (not needed
+with `sync: :manual`, which never re-embeds on save):
+
+```ruby
+vec_scope :semantic_search, source: :embedding_text, watch: [:body, :metadata], dimensions: 768
+```
+
 ### Query
 
 ```ruby
@@ -214,6 +283,9 @@ is closer). `threshold:` filters by relevance: on a cosine scope it is a minimum
 similarity, on a euclidean or taxicab scope it is a maximum distance. A blank or
 nil query returns `.none`.
 
+Results come back nearest first, replacing any `order` chained before the
+search.
+
 Re-embed after a bulk write the same way you reindex FTS5:
 
 ```ruby
@@ -222,6 +294,9 @@ Post.reembed                     # every vec_scope on the model
 ```
 
 or `rake sqlite_search:reembed[Post,semantic_search]` from the command line.
+Re-embedding overwrites vectors row by row, so the index stays searchable while
+it runs and an embedder failure partway leaves the remaining rows' vectors in
+place. It then drops vectors whose row no longer exists.
 
 ## Hybrid search
 
@@ -232,8 +307,6 @@ declared:
 
 ```ruby
 class Post < ApplicationRecord
-  include SqliteSearch::Model
-
   fts5_scope :search_body, against: :body
   vec_scope :semantic_search, against: :body, dimensions: 768, sync: :inline
   hybrid_scope :search, fts5: :search_body, vec: :semantic_search
@@ -247,28 +320,53 @@ hybrid name collides with an arm's name). `k:` sets the RRF constant (default
 
 ```ruby
 posts = Post.search("coffee", limit: 20)
-posts.first.search_score   # fused score, higher is better
+posts.first.search_score   # fused (or reranker) score, higher is better
 ```
 
 Each arm produces a ranked candidate list, Reciprocal Rank Fusion combines them,
 an optional reranker reorders the result, and the top `limit` records come back
-as a relation ordered to match, each carrying a `<name>_score` reader. `limit:`
+as a relation in fused order (replacing any `order` chained before the search), each carrying a `<name>_score` reader. `limit:`
 defaults to 20. A blank query returns `.none`. Pass `rerank: false` to skip the
 reranker and return the plain fused order.
 
+Each arm can drop weak matches before fusion, so a poor match from one arm is
+not lifted into the results by the other. `fts5_threshold:` is a minimum
+keyword rank, and `vec_threshold:` is the vector scope's `threshold:` (minimum
+similarity on a cosine scope):
+
+```ruby
+Post.search("coffee", fts5_threshold: 8.0, vec_threshold: 0.3)
+```
+
 ### Reranking
 
 Register a reranker once and every hybrid scope uses it, unless a call opts out:
 
 ```ruby
 SqliteSearch.reranker do |query, documents, model:, scope:|
-  # documents are the fused candidate records, already loaded, in RRF order.
-  # Return them reordered; a subset is fine, unknown records are ignored.
-  MyRerankClient.rerank(query, documents)
+  # One relevance score per text, from your cross-encoder or rerank API.
+  scores = MyRerankClient.score(query, documents.map(&:body))
+  documents.zip(scores).sort_by { |_doc, score| -score }
 end
 ```
 
-`model:` is the model class the scope was declared on, and `scope:` is the hybrid
+`documents` is an `Array` of loaded records of the searched model, the same
+objects `Post.find` returns, in fused (RRF) order. It holds every fused
+candidate, before the `limit:` cut: each arm contributes up to
+`[limit * 3, 100].min`, so at most 200 records. You choose which attributes to
+send to your reranker (`body` above, or the text you embed).
+
+Return `[record, score]` pairs in the order you want, as above, and each
+record's `<name>_score` becomes your reranker's score. You can also return just
+the records, reordered, and `<name>_score` keeps the fused RRF score. Either way
+the order you return is the result order. Records are matched back by primary
+key: a candidate you leave out is dropped from the results, a record that was
+not a candidate is ignored, a repeat of a record is ignored, and an empty array
+gives `.none`. The top `limit` of your order comes back. Score all the records
+or none: mixing pairs and bare records would put two score scales in one
+column, so it counts as a reranker failure.
+
+`model:` is the class the search was called on, and `scope:` is the hybrid
 scope name (the same pair is passed to the embedder block). Reranking is
 best-effort: if the block raises, the failure is logged and the search falls back
 to the fused order, so a broken reranker never takes down a search.
@@ -305,6 +403,9 @@ already-fused set; raise `k:` or `limit:` if you need more rows to survive it.
 
 ## Limitations and notes
 
+**Ruby 3.3 or newer.** Vector and hybrid search run on `neighbor` 1.x, which
+requires Ruby 3.3.
+
 **ActiveRecord 8.0 or newer.** The migration helpers and their `schema.rb`
 round-trip rely on `create_virtual_table`, which arrived in Rails 8.0. The gem
 does not run on 7.1 or 7.2, and the gemspec enforces that.
@@ -357,11 +458,6 @@ lazy relation. `.search` does the most work, re-running both arms, the fusion,
 and any reranker (possibly a network call) on every invocation, so do not call
 either one inside a loop.
 
-**Hybrid's vector arm has no relevance threshold.** A bare `.semantic_search` call takes
-`threshold:`, but `hybrid_scope` does not, so the vector arm always feeds its
-nearest neighbors into the fusion even for a weak semantic match. Per-arm
-thresholds may come later.
-
 **The candidate pool is capped.** A hybrid scope pulls `[limit * 3, 100].min`
 candidates from each arm before fusing, so a very large `limit:` still fuses from
 at most 100 per arm.

data/lib/generators/sqlite_search/fts5_generator.rb

@@ -16,6 +16,7 @@ module SqliteSearch
         desc: "Index/scope name; the FTS table becomes <table>_<index>_fts (default: search)"
 
       def create_migration_file
+        raise Thor::Error, "Name at least one column to index, e.g. rails g sqlite_search:fts5 Post title body" if columns.empty?
         if options[:weights]
           weights = options[:weights].split(",")
           if weights.length != columns.length
@@ -24,18 +25,16 @@ module SqliteSearch
           end
         end
 
-        migration_template "create_fts5_index.rb.tt", "db/migrate/create_#{index_name}_fts5.rb"
+        migration_template "create_fts5_index.rb.tt", "db/migrate/create_#{table_name}_#{index_name}_fts5.rb"
       end
 
       private
 
-      def table_name = name.tableize
-
       def index_name
         options[:index] || "search"
       end
 
-      def migration_class_suffix = "#{index_name.camelize}Fts5"
+      def migration_class_suffix = "#{table_name.camelize}#{index_name.camelize}Fts5"
 
       def migration_version = "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"
 

data/lib/generators/sqlite_search/vec_generator.rb

@@ -14,15 +14,14 @@ module SqliteSearch
       class_option :dimensions, type: :numeric, required: true, desc: "Embedding dimensions (e.g. 768)"
 
       def create_migration_file
-        migration_template "create_vec_index.rb.tt", "db/migrate/create_#{index_name}_vec.rb"
+        migration_template "create_vec_index.rb.tt", "db/migrate/create_#{table_name}_#{index_name}_vec.rb"
       end
 
       private
 
-      def table_name = name.tableize
       def index_name = options[:index]
       def dimensions = options[:dimensions].to_i
-      def migration_class_suffix = "#{index_name.camelize}Vec"
+      def migration_class_suffix = "#{table_name.camelize}#{index_name.camelize}Vec"
       def migration_version = "#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"
     end
   end

data/lib/sqlite_search/embed_job.rb

@@ -19,7 +19,8 @@ module SqliteSearch
 
       def perform(model_name, id, scope_name)
         klass = model_name.constantize
-        record = klass.find_by(klass.primary_key => id)
+        # unscoped: a record hidden by a default_scope still needs its vector.
+        record = klass.unscoped.find_by(klass.primary_key => id)
         return unless record # deleted before the job ran
         definition = klass.sqlite_search_vec_definitions.fetch(scope_name.to_sym)
         SqliteSearch::Vec::Backend.new(definition).embed_and_store(record)

data/lib/sqlite_search/fts5/backend.rb

@@ -20,8 +20,7 @@ module SqliteSearch
           end
           conn.transaction do
             delete_row(conn, id)
-            values = @definition.columns.map { |c| record.public_send(c) }
-            insert_row(conn, id, values) unless values.all? { |v| v.nil? || v.to_s.empty? }
+            insert_record(conn, id, record)
           end
         end
       end
@@ -32,7 +31,11 @@ module SqliteSearch
         end
       end
 
-      def rebuild(model)
+      # Rebuilds from the class that declared the scope, not the one reindex was
+      # called on: an STI hierarchy shares one FTS table, so rebuilding from a
+      # subclass must still refill its siblings' rows.
+      def rebuild
+        model = @definition.model
         pk_type = model.columns_hash[model.primary_key.to_s]&.type
         unless pk_type == :integer
           raise SqliteSearch::Error,
@@ -41,15 +44,18 @@ module SqliteSearch
         end
 
         model.with_connection do |conn|
-          conn.execute("DELETE FROM #{quoted(conn)}")
-          col_list = @definition.columns.map { |c| conn.quote_column_name(c) }.join(", ")
-          non_blank = @definition.columns.map { |c| "COALESCE(#{conn.quote_column_name(c)}, '')" }.join(" || ")
-          conn.execute(<<~SQL.squish)
-            INSERT INTO #{quoted(conn)} (rowid, #{col_list})
-            SELECT #{conn.quote_column_name(model.primary_key)}, #{col_list}
-            FROM #{conn.quote_table_name(model.table_name)}
-            WHERE (#{non_blank}) <> ''
-          SQL
+          # One transaction, so readers never see an empty index and a failed
+          # insert leaves the old index in place.
+          conn.transaction do
+            conn.execute("DELETE FROM #{quoted(conn)}")
+            if @definition.source
+              # Derived text only exists in Ruby, so rebuild record by record.
+              model.unscoped.find_each { |record| insert_record(conn, record.public_send(model.primary_key), record) }
+            else
+              conn.execute(SqliteSearch::Fts5.copy_sql(conn, fts_table: @definition.table_name, source_table: model.table_name,
+                primary_key: model.primary_key, columns: @definition.columns))
+            end
+          end
         end
       end
 
@@ -61,6 +67,11 @@ module SqliteSearch
         conn.execute("DELETE FROM #{quoted(conn)} WHERE rowid = #{conn.quote(id)}")
       end
 
+      def insert_record(conn, id, record)
+        values = @definition.values_for(record)
+        insert_row(conn, id, values) unless values.all? { |v| v.nil? || v.to_s.empty? }
+      end
+
       def insert_row(conn, id, values)
         col_list = @definition.columns.map { |c| conn.quote_column_name(c) }.join(", ")
         vals = ([id] + values).map { |v| conn.quote(v) }.join(", ")

data/lib/sqlite_search/fts5/definition.rb

@@ -6,27 +6,50 @@ module SqliteSearch
   module Fts5
     # Immutable per-scope configuration.
     class Definition
-      attr_reader :model, :name, :columns, :weights, :tokenizer, :table_name
+      attr_reader :model, :name, :columns, :weights, :source, :table_name
 
-      def initialize(model:, name:, against:, tokenizer:)
+      # against: names the FTS columns (and their weights). By default each
+      # column's text is the record attribute of the same name. source: names
+      # a method returning {column => text} instead, for text derived in Ruby.
+      # watch: lists the attributes whose change triggers a resync (default:
+      # the against: columns).
+      def initialize(model:, name:, against:, source: nil, watch: nil)
         @model = model
         @name = name.to_sym
         @columns = SqliteSearch::Fts5.columns_for(against)
         @weights = SqliteSearch::Fts5.weights_for(against)
-        @tokenizer = tokenizer
+        @source = source
+        @watch = watch
         @table_name = "#{model.table_name}_#{@name}_fts"
       end
 
       def column_names = columns.map(&:to_s)
+      def watch_names = (@watch || columns).map(&:to_s)
+
+      # The text to index for a record, one value per column, in column order.
+      def values_for(record)
+        return columns.map { |c| record.public_send(c) } unless source
+        document = record.public_send(source)
+        columns.map { |c| document.fetch(c) }
+      end
 
       # SQLite bm25() returns lower (more negative) = better. We ORDER BY it
       # ascending, and expose -bm25 as the rank so higher = better.
       def bm25_expression(connection)
         args = [connection.quote_table_name(table_name)]
-        args.concat(weights.map { |w| format("%g", w) }) if weights
+        args.concat(table_weights(connection).map { |w| format("%g", w) }) if weights
         "bm25(#{args.join(", ")})"
       end
 
+      # bm25() takes weights by the FTS table's column position, which need not
+      # match the order of the against: hash, so line them up by column name.
+      def table_weights(connection)
+        @table_weights ||= begin
+          by_column = column_names.zip(weights).to_h
+          connection.columns(table_name).map { |c| by_column.fetch(c.name, 1.0) }
+        end
+      end
+
       def rank_column = "#{name}_rank"
     end
   end

data/lib/sqlite_search/fts5.rb

@@ -17,10 +17,23 @@ module SqliteSearch
       against.is_a?(Hash) ? against.values.map(&:to_f) : nil
     end
 
+    # INSERT ... SELECT that copies source-table columns into an FTS table,
+    # keyed by primary key and skipping rows with no text at all.
+    def copy_sql(conn, fts_table:, source_table:, primary_key:, columns:)
+      col_list = columns.map { |c| conn.quote_column_name(c) }.join(", ")
+      non_blank = columns.map { |c| "COALESCE(#{conn.quote_column_name(c)}, '')" }.join(" || ")
+      <<~SQL.squish
+        INSERT INTO #{conn.quote_table_name(fts_table)} (rowid, #{col_list})
+        SELECT #{conn.quote_column_name(primary_key)}, #{col_list}
+        FROM #{conn.quote_table_name(source_table)}
+        WHERE (#{non_blank}) <> ''
+      SQL
+    end
+
     # Extended onto the .none relation returned for blank queries so that
     # .order_by_rank chains safely (returns the same empty relation).
     module NullRank
-      def order_by_rank = self
+      def order_by_rank(threshold: nil) = self
     end
   end
 end

data/lib/sqlite_search/hybrid.rb

@@ -14,8 +14,11 @@ module SqliteSearch
     end
 
     # Best-effort rerank. `fused` is [[id, score], ...]; returns the same shape
-    # reordered by the reranker (RRF scores preserved), or the input unchanged
-    # on any reranker failure.
+    # in the reranker's order, or the input unchanged on any reranker failure.
+    # The reranker returns records, or [record, score] pairs to replace each
+    # record's RRF score with its own. Records outside `fused` and repeats are
+    # dropped. Scoring only some records would mix two score scales in one
+    # column, so it counts as a reranker failure.
     def rerank(query, fused, model:, scope_name:, reranker:)
       return fused unless reranker
       ids = fused.map(&:first)
@@ -23,15 +26,18 @@ module SqliteSearch
       records = ids.filter_map { |id| by_id[id] }
       scores = fused.to_h
       begin
-        reordered = reranker.call(query, records, model: model, scope: scope_name)
-        reordered.filter_map do |rec|
+        reordered = reranker.call(query, records, model: model, scope: scope_name).filter_map do |entry|
+          rec, rerank_score = entry.is_a?(Array) ? entry : [entry, nil]
           id = rec.public_send(model.primary_key)
-          [id, scores[id]] if scores.key?(id)
+          [id, rerank_score] if scores.key?(id)
+        end.uniq(&:first)
+        scored = reordered.count { |(_, rerank_score)| rerank_score }
+        unless scored.zero? || scored == reordered.size
+          raise SqliteSearch::Error, "reranker scored #{scored} of #{reordered.size} records; return all records or all [record, score] pairs"
         end
+        reordered.map { |id, rerank_score| [id, rerank_score || scores[id]] }
       rescue => e
-        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
-          Rails.logger.warn { "sqlite_search: rerank failed, using fused order (#{e.class}: #{e.message})" }
-        end
+        ActiveRecord::Base.logger&.warn { "sqlite_search: rerank failed, using fused order (#{e.class}: #{e.message})" }
         fused
       end
     end

data/lib/sqlite_search/migration.rb

@@ -16,7 +16,9 @@ module SqliteSearch
       options << "tokenize = '#{tokenizer}'"
       connection.create_virtual_table(fts_table, :fts5, options)
 
-      backfill_fts5_index(table, fts_table, columns, primary_key) if backfill
+      return unless backfill
+      # Seeding only applies going up; dropping the table undoes it.
+      reversible { |dir| dir.up { backfill_fts5_index(table, fts_table, columns, primary_key) } }
     end
 
     # vec0 indexes are keyed by an integer `id` column holding the source row's
@@ -33,14 +35,14 @@ module SqliteSearch
     private
 
     def backfill_fts5_index(table, fts_table, columns, primary_key)
-      col_list = columns.map { |c| connection.quote_column_name(c) }.join(", ")
-      non_blank = columns.map { |c| "COALESCE(#{connection.quote_column_name(c)}, '')" }.join(" || ")
-      connection.execute(<<~SQL.squish)
-        INSERT INTO #{connection.quote_table_name(fts_table)} (rowid, #{col_list})
-        SELECT #{connection.quote_column_name(primary_key)}, #{col_list}
-        FROM #{connection.quote_table_name(table)}
-        WHERE (#{non_blank}) <> ''
-      SQL
+      missing = columns.map(&:to_s) - connection.columns(table).map(&:name)
+      if missing.any?
+        raise SqliteSearch::Error,
+          "create_fts5_index backfill: copies columns straight from #{table}, which has no #{missing.join(", ")}. " \
+          "For an index built with source:, drop backfill: and run Model.reindex after migrating."
+      end
+      connection.execute(SqliteSearch::Fts5.copy_sql(connection, fts_table: fts_table, source_table: table,
+        primary_key: primary_key, columns: columns))
     end
   end
 end

data/lib/sqlite_search/model.rb

@@ -15,33 +15,40 @@ module SqliteSearch
         superclass.sqlite_search_fts5_definitions.merge(own)
       end
 
-      def fts5_scope(name, against:, tokenizer: "porter unicode61")
-        definition = Fts5::Definition.new(model: self, name: name, against: against, tokenizer: tokenizer)
+      def fts5_scope(name, against:, source: nil, watch: nil)
+        if source && watch.nil?
+          raise SqliteSearch::Error, "fts5_scope :#{name} uses source:, so it needs watch: (the attributes that change the text)."
+        end
+        definition = Fts5::Definition.new(model: self, name: name, against: against, source: source, watch: watch)
         (@sqlite_search_fts5_definitions ||= {})[definition.name] = definition
 
         scope name, ->(query = nil, prefix: false, raw: nil) do
           match = raw || SqliteSearch::Query.build(query, prefix: prefix)
           next none.extending(SqliteSearch::Fts5::NullRank) if match.nil? || match.to_s.empty?
 
-          fts = connection.quote_table_name(definition.table_name)
-          pk = "#{connection.quote_table_name(table_name)}.#{connection.quote_column_name(primary_key)}"
+          fts, pk = with_connection do |conn|
+            [conn.quote_table_name(definition.table_name), "#{quoted_table_name}.#{conn.quote_column_name(primary_key)}"]
+          end
 
           # bm25() is only valid in a query that MATCHes the fts table, so
-          # order_by_rank joins the fts table and re-applies MATCH here.
+          # order_by_rank joins the fts table and re-applies MATCH here. It
+          # reorders: relevance replaces any order chained before the search.
           rank_module = Module.new do
-            define_method(:order_by_rank) do
-              bm25 = definition.bm25_expression(connection)
-              joins("JOIN #{fts} ON #{fts}.rowid = #{pk}")
-                .where("#{fts} MATCH ?", match)
-                .select("#{connection.quote_table_name(table_name)}.*, -#{bm25} AS #{definition.rank_column}")
-                .order(Arel.sql(bm25))
+            define_method(:order_by_rank) do |threshold: nil|
+              bm25 = with_connection { |conn| definition.bm25_expression(conn) }
+              ranked = joins("JOIN #{fts} ON #{fts}.rowid = #{pk}").where("#{fts} MATCH ?", match)
+              ranked = ranked.where("-#{bm25} >= ?", threshold) if threshold
+              ranked
+                .select("#{quoted_table_name}.*, -#{bm25} AS #{definition.rank_column}")
+                .reorder(Arel.sql(bm25))
+                .extending(SqliteSearch::ScoredRelation)
             end
           end
 
           where("#{pk} IN (SELECT rowid FROM #{fts} WHERE #{fts} MATCH ?)", match).extending(rank_module)
         end
 
-        cols = definition.column_names
+        cols = definition.watch_names
         # FTS5 sync runs inside the transaction (after_save/after_destroy), not
         # after_commit: an FTS5 write is a cheap local write, so keeping it in the
         # transaction makes the index atomic with the row. A failed write rolls
@@ -58,7 +65,7 @@ module SqliteSearch
 
       def reindex(name = nil)
         definitions = name ? [sqlite_search_fts5_definitions.fetch(name.to_sym)] : sqlite_search_fts5_definitions.values
-        definitions.each { |definition| SqliteSearch::Fts5::Backend.new(definition).rebuild(self) }
+        definitions.each { |definition| SqliteSearch::Fts5::Backend.new(definition).rebuild }
       end
 
       def sqlite_search_vec_definitions
@@ -67,10 +74,19 @@ module SqliteSearch
         superclass.sqlite_search_vec_definitions.merge(own)
       end
 
-      def vec_scope(name, against:, dimensions:, distance: :cosine, embedder: nil, sync: :async)
+      # sync: :async (enqueue EmbedJob on save), :inline (embed in the save
+      # callback), or :manual (never on save; call record.reembed yourself).
+      def vec_scope(name, dimensions:, against: nil, source: nil, watch: nil, distance: :cosine, embedder: nil, sync: :async)
+        unless SqliteSearch::Vec::SYNC_MODES.include?(sync)
+          raise SqliteSearch::Error, "Unsupported sync #{sync.inspect}. Use one of: #{SqliteSearch::Vec::SYNC_MODES.join(", ")}."
+        end
+        if source && watch.nil? && sync != :manual
+          raise SqliteSearch::Error, "vec_scope :#{name} uses source:, so it needs watch: (the attributes that change the text)."
+        end
         SqliteSearch::Vec.load!
         definition = SqliteSearch::Vec::Definition.new(
-          model: self, name: name, against: against, dimensions: dimensions, distance: distance, embedder: embedder
+          model: self, name: name, against: against, source: source, watch: watch,
+          dimensions: dimensions, distance: distance, embedder: embedder
         )
         (@sqlite_search_vec_definitions ||= {})[definition.name] = definition
 
@@ -80,9 +96,10 @@ module SqliteSearch
           vector = definition.embed(query.to_s, record_model: klass)
 
           caller_conditions = all.only(:where, :joins)
-          src = connection.quote_table_name(table_name)
-          vec = connection.quote_table_name(definition.table_name)
-          pkc = connection.quote_column_name(primary_key)
+          src = quoted_table_name
+          vec, pkc = with_connection do |conn|
+            [conn.quote_table_name(definition.table_name), conn.quote_column_name(primary_key)]
+          end
 
           knn = definition.neighbor_model
             .joins("JOIN #{src} ON #{src}.#{pkc} = #{vec}.id")
@@ -101,27 +118,38 @@ module SqliteSearch
 
           ids = hits.map(&:first)
           distances = hits.to_h
-          pk_sql = "#{quoted_table_name}.#{pkc}"
+          pk_sql = "#{src}.#{pkc}"
           # Order and per-row scores are computed in Ruby (from the KNN), so carry
           # them as selected CASE columns: <name>_distance is then a real attribute.
-          order = Arel.sql(SqliteSearch::Sql.id_case(pk_sql, ids.each_with_index.to_h, connection))
-          cols = ["#{quoted_table_name}.*", "#{SqliteSearch::Sql.id_case(pk_sql, distances, connection)} AS #{definition.distance_method}"]
-          if definition.cosine?
-            cols << "#{SqliteSearch::Sql.id_case(pk_sql, distances.transform_values { |d| 1.0 - d }, connection)} AS #{definition.similarity_method}"
+          # Nearness replaces any order chained before the search.
+          order, cols = with_connection do |conn|
+            cols = ["#{src}.*", "#{SqliteSearch::Sql.id_case(pk_sql, distances, conn)} AS #{definition.distance_method}"]
+            if definition.cosine?
+              cols << "#{SqliteSearch::Sql.id_case(pk_sql, distances.transform_values { |d| 1.0 - d }, conn)} AS #{definition.similarity_method}"
+            end
+            [Arel.sql(SqliteSearch::Sql.id_case(pk_sql, ids.each_with_index.to_h, conn)), cols]
           end
-          where(primary_key => ids).order(order).select(cols.join(", "))
+          where(primary_key => ids).reorder(order).select(cols.join(", ")).extending(SqliteSearch::ScoredRelation)
         end
 
-        vec_cols = definition.column_names
+        vec_cols = definition.watch_names
         vec_sync = sync
-        SqliteSearch.ensure_embed_job! unless vec_sync == :inline
-        after_save_commit do
-          if (saved_changes.keys & vec_cols).any?
-            backend = SqliteSearch::Vec::Backend.new(definition)
-            if vec_sync == :inline
-              backend.embed_and_store(self)
-            else
-              SqliteSearch::EmbedJob.perform_later(self.class.name, public_send(self.class.primary_key), definition.name.to_s)
+        SqliteSearch.ensure_embed_job! if vec_sync == :async
+        unless vec_sync == :manual
+          # saved_changes in an after_commit callback only reflects the last save
+          # of the transaction, so note a watched change on each save and act on
+          # it at commit.
+          after_save do
+            (@sqlite_search_pending_embeds ||= Set.new) << definition.name if (saved_changes.keys & vec_cols).any?
+          end
+          after_rollback { @sqlite_search_pending_embeds = nil }
+          after_save_commit do
+            if @sqlite_search_pending_embeds&.delete?(definition.name)
+              if vec_sync == :inline
+                SqliteSearch::Vec::Backend.new(definition).embed_and_store(self)
+              else
+                SqliteSearch::EmbedJob.perform_later(self.class.name, public_send(self.class.primary_key), definition.name.to_s)
+              end
             end
           end
         end
@@ -157,13 +185,13 @@ module SqliteSearch
         rrf_k = k
         score_method = "#{name}_score"
 
-        scope name, ->(query = nil, limit: 20, rerank: true) do
+        scope name, ->(query = nil, limit: 20, rerank: true, fts5_threshold: nil, vec_threshold: nil) do
           next none if query.nil? || query.to_s.strip.empty?
           # candidate pool per arm before fusion; capped so a large limit: can't over-fetch
           pool = [limit * 3, 100].min
 
-          fts_ids = all.public_send(fts5_name, query).order_by_rank.limit(pool).pluck(primary_key)
-          vec_ids = all.public_send(vec_name, query, k: pool).pluck(primary_key)
+          fts_ids = all.public_send(fts5_name, query).order_by_rank(threshold: fts5_threshold).limit(pool).pluck(primary_key)
+          vec_ids = all.public_send(vec_name, query, k: pool, threshold: vec_threshold).pluck(primary_key)
 
           fused = SqliteSearch::Hybrid.rrf(fts_ids, vec_ids, k: rrf_k)
           next none if fused.empty?
@@ -174,14 +202,28 @@ module SqliteSearch
           fused = fused.first(limit)
           ids = fused.map(&:first)
           scores = fused.to_h
-          pk_sql = "#{quoted_table_name}.#{connection.quote_column_name(primary_key)}"
           # Fused rank and score come from Ruby, so carry them as SQL: the score
           # becomes a real <name>_score attribute (works with pluck, first, etc.).
-          order = Arel.sql(SqliteSearch::Sql.id_case(pk_sql, ids.each_with_index.to_h, connection))
-          score_col = "#{SqliteSearch::Sql.id_case(pk_sql, scores, connection)} AS #{score_method}"
-          where(primary_key => ids).order(order).select("#{quoted_table_name}.*, #{score_col}")
+          # The fused order replaces any order chained before the search.
+          order, score_col = with_connection do |conn|
+            pk_sql = "#{quoted_table_name}.#{conn.quote_column_name(primary_key)}"
+            [
+              Arel.sql(SqliteSearch::Sql.id_case(pk_sql, ids.each_with_index.to_h, conn)),
+              "#{SqliteSearch::Sql.id_case(pk_sql, scores, conn)} AS #{score_method}"
+            ]
+          end
+          where(primary_key => ids).reorder(order).select("#{quoted_table_name}.*, #{score_col}")
+            .extending(SqliteSearch::ScoredRelation)
         end
       end
     end
+
+    # Embed this record into one vec index (or every one) now. This is how a
+    # sync: :manual scope is kept current, typically from the app's own job.
+    def reembed(name = nil)
+      definitions = self.class.sqlite_search_vec_definitions
+      selected = name ? [definitions.fetch(name.to_sym)] : definitions.values
+      selected.each { |definition| SqliteSearch::Vec::Backend.new(definition).embed_and_store(self) }
+    end
   end
 end

data/lib/sqlite_search/query.rb

@@ -8,7 +8,9 @@ module SqliteSearch
   # operators, punctuation) is dropped, so untrusted input cannot inject
   # MATCH syntax. Terms are AND-joined. Returns nil when nothing usable.
   class Query
-    PHRASE = /"([^"]+)"/
+    # A quoted phrase, or a run of text between quotes. A quote with no partner
+    # matches neither and is skipped.
+    SEGMENT = /"([^"]*)"|([^"]+)/
     TERM_CHARS = /[^[:alnum:]_]+/
     RESERVED = /\A(?:AND|OR|NOT|NEAR)\z/
 
@@ -22,7 +24,7 @@ module SqliteSearch
     end
 
     def to_match
-      tokens = phrases + words
+      tokens = tokenize
       return nil if tokens.empty?
 
       tokens[-1] = "#{tokens[-1]}*" if @prefix && !quoted?(tokens[-1])
@@ -31,12 +33,16 @@ module SqliteSearch
 
     private
 
-    def phrases
-      @raw.scan(PHRASE).map { |(inner)| %("#{inner.strip}") }.reject { |p| p == '""' }
-    end
-
-    def words
-      @raw.gsub(PHRASE, " ").split(TERM_CHARS).reject { |w| w.empty? || w.match?(RESERVED) }
+    # Phrases and words in the order they were typed, so prefix: widens the
+    # last thing typed: never a finished word that came before a phrase.
+    def tokenize
+      @raw.scan(SEGMENT).flat_map do |phrase, text|
+        if phrase
+          phrase.strip.empty? ? [] : [%("#{phrase.strip}")]
+        else
+          text.split(TERM_CHARS).reject { |w| w.empty? || w.match?(RESERVED) }
+        end
+      end
     end
 
     def quoted?(token)

data/lib/sqlite_search/railtie.rb

@@ -15,6 +15,13 @@ module SqliteSearch
       end
     end
 
+    # Define EmbedJob as soon as ActiveJob loads, not when the first model with
+    # a vec_scope loads: a worker process deserializes the job by class name,
+    # possibly before any model has been loaded.
+    initializer "sqlite_search.active_job" do
+      ActiveSupport.on_load(:active_job) { SqliteSearch.ensure_embed_job! }
+    end
+
     rake_tasks do
       load File.expand_path("../tasks/sqlite_search.rake", __dir__)
     end

data/lib/sqlite_search/scored_relation.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SqliteSearch
+  # Extended onto relations that select a computed column (rank, distance,
+  # score) alongside table.*. A bare #count would wrap that whole select list
+  # in COUNT(...), which is invalid SQL, so count rows instead.
+  module ScoredRelation
+    def count(column_name = (:all unless block_given?), &) = super
+  end
+end

data/lib/sqlite_search/vec/backend.rb

@@ -16,8 +16,8 @@ module SqliteSearch
         end
 
         model = @definition.neighbor_model
-        text = SqliteSearch::Vec.text_for(record, @definition.columns)
-        if text.empty?
+        text = @definition.text_for(record)
+        if text.strip.empty?
           model.where(id: id).delete_all
           return
         end
@@ -34,9 +34,13 @@ module SqliteSearch
         @definition.neighbor_model.where(id: id).delete_all
       end
 
+      # Re-embeds in place, row by row, so the index stays searchable throughout
+      # and a failed embed leaves the remaining rows' vectors untouched. Then
+      # drops vectors whose source row is gone. Default scopes are ignored, as
+      # the FTS5 rebuild ignores them.
       def reembed(model)
-        @definition.neighbor_model.delete_all
-        model.find_each { |record| embed_and_store(record) }
+        model.unscoped.find_each { |record| embed_and_store(record) }
+        @definition.neighbor_model.where.not(id: model.base_class.unscoped.select(model.primary_key)).delete_all
       end
     end
   end

data/lib/sqlite_search/vec/definition.rb

@@ -5,12 +5,20 @@ require "sqlite_search/vec"
 module SqliteSearch
   module Vec
     class Definition
-      attr_reader :model, :name, :columns, :dimensions, :distance, :table_name
+      attr_reader :model, :name, :columns, :source, :dimensions, :distance, :table_name
 
-      def initialize(model:, name:, against:, dimensions:, distance: :cosine, embedder: nil)
+      # Text to embed comes from either against: (attributes joined by
+      # newlines) or source: (a method returning the text). watch: lists the
+      # attributes whose change triggers a re-embed; it defaults to against:.
+      def initialize(model:, name:, dimensions:, against: nil, source: nil, watch: nil, distance: :cosine, embedder: nil)
+        if against.nil? == source.nil?
+          raise SqliteSearch::Error, "vec_scope :#{name} needs exactly one of against: or source:."
+        end
         @model = model
         @name = name.to_sym
         @columns = SqliteSearch::Vec.columns_for(against)
+        @source = source
+        @watch = watch
         @dimensions = dimensions
         @distance = distance.to_sym
         @embedder = embedder
@@ -20,6 +28,15 @@ module SqliteSearch
       end
 
       def column_names = columns.map(&:to_s)
+      def watch_names = (@watch || columns).map(&:to_s)
+
+      # Text handed to the embedder: the source text, or the against
+      # attributes joined with blanks dropped.
+      def text_for(record)
+        return record.public_send(source).to_s if source
+        columns.map { |c| record.public_send(c) }.reject { |v| v.nil? || v.to_s.strip.empty? }.join("\n")
+      end
+
       def cosine? = distance == :cosine
       def distance_method = "#{name}_distance"
       def similarity_method = "#{name}_similarity"
@@ -38,7 +55,10 @@ module SqliteSearch
         @neighbor_model ||= begin
           tbl = table_name
           dims = dimensions
-          Class.new(ActiveRecord::Base) do
+          # Subclass the model's abstract parent (ApplicationRecord, or the
+          # abstract class of a secondary database) so the vec table is read
+          # and written on the model's own database.
+          Class.new(model.base_class.superclass) do
             self.table_name = tbl
             self.primary_key = "id"
             has_neighbors :embedding, dimensions: dims

data/lib/sqlite_search/vec.rb

@@ -8,6 +8,7 @@ module SqliteSearch
     # is kept in step to keep the schema honest. inner_product is queryable by
     # neighbor but is not a valid vec0 distance_metric, so it is not offered.
     DISTANCE_METRICS = {cosine: "cosine", euclidean: "l2", taxicab: "l1"}.freeze
+    SYNC_MODES = %i[async inline manual].freeze
 
     @loaded = false
 
@@ -38,14 +39,10 @@ module SqliteSearch
 
     def columns_for(against)
       case against
+      when nil then []
       when Array then against.map(&:to_sym)
       else [against.to_sym]
       end
     end
-
-    # Text handed to the embedder: the against columns joined, blanks dropped.
-    def text_for(record, columns)
-      columns.map { |c| record.public_send(c) }.reject { |v| v.nil? || v.to_s.strip.empty? }.join("\n")
-    end
   end
 end

data/lib/sqlite_search/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SqliteSearch
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/lib/sqlite_search.rb

@@ -4,6 +4,7 @@ require "active_record"
 require "sqlite_search/version"
 require "sqlite_search/errors"
 require "sqlite_search/sql"
+require "sqlite_search/scored_relation"
 require "sqlite_search/config"
 require "sqlite_search/vec"
 require "sqlite_search/vec/definition"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqlite_search
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Radioactive Labs
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-02 00:00:00.000000000 Z
+date: 2026-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -168,6 +168,7 @@ files:
 - lib/sqlite_search/model.rb
 - lib/sqlite_search/query.rb
 - lib/sqlite_search/railtie.rb
+- lib/sqlite_search/scored_relation.rb
 - lib/sqlite_search/sql.rb
 - lib/sqlite_search/vec.rb
 - lib/sqlite_search/vec/backend.rb
@@ -179,7 +180,6 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  homepage_uri: https://github.com/radioactive-labs/sqlite_search
   source_code_uri: https://github.com/radioactive-labs/sqlite_search
   changelog_uri: https://github.com/radioactive-labs/sqlite_search/blob/main/CHANGELOG.md
   bug_tracker_uri: https://github.com/radioactive-labs/sqlite_search/issues
@@ -193,7 +193,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="