truffler 0.1.0

data/lib/truffler/embeddings/fake_embedder.rb

@@ -0,0 +1,45 @@
+require "digest"
+
+module Truffler
+  module Embeddings
+    # Deterministic vectors for tests and benchmarks: each token adds a signed
+    # unit at a position chosen by its hash, and the sum is normalized, so
+    # texts that share words point the same way.
+    class FakeEmbedder < Embedder
+      attr_reader :calls
+
+      def initialize(model: nil)
+        @model = model
+        @error = nil
+        @calls = []
+      end
+
+      def fail_with(error)
+        @error = error
+        self
+      end
+
+      def recover!
+        @error = nil
+        self
+      end
+
+      def perform(texts, model:, dimensions:)
+        @calls << { texts: texts, model: model, dimensions: dimensions }
+        raise @error if @error
+
+        { vectors: texts.map { |text| vector_for(text, dimensions) }, model: @model || model }
+      end
+
+      def vector_for(text, dimensions)
+        vector = Array.new(dimensions, 0.0)
+        text.to_s.downcase.scan(/[[:alnum:]]+/).each do |token|
+          digest = Digest::SHA256.digest(token)
+          vector[digest.unpack1("N") % dimensions] += digest.getbyte(4).even? ? 1.0 : -1.0
+        end
+        norm = Math.sqrt(vector.sum { |value| value * value })
+        norm.zero? ? vector : vector.map { |value| value / norm }
+      end
+    end
+  end
+end

data/lib/truffler/embeddings/label_vector.rb

@@ -0,0 +1,81 @@
+module Truffler
+  module Embeddings
+    # A record's labels as a named-dimension vector (KTD20): one float per
+    # storage key in sorted order, choice options expanded, stored in
+    # truffler_embeddings beside the optional text vector and tagged with the
+    # vocabulary version that fixed the key order. Unlabeled keys are 0.0.
+    # Vectors are built from truffler_labels only, so a vocabulary change
+    # rebuilds them without asking Jev. Lens labels in the record's labeling
+    # scope (KTD21) extend the vector after the declared ones.
+    class LabelVector
+      attr_reader :model
+
+      def initialize(model)
+        @model = model
+      end
+
+      def keys(tenant_key = nil)
+        vocabulary.labels_for(tenant_key: tenant_key, all_users: true).values.flat_map { |label| label.storage_keys(tenant_key) }.sort
+      end
+
+      def write(record_ids, tenant_key:)
+        return 0 if record_ids.empty?
+
+        keys = keys(tenant_key)
+        version = version_for(tenant_key)
+        values = Records::Label.where(record_type: record_type, record_id: record_ids, label_key: keys)
+          .pluck(:record_id, :label_key, :value)
+          .each_with_object(Hash.new { |hash, id| hash[id] = {} }) { |(id, key, value), map| map[id.to_s][key] = value }
+        now = Time.current
+        rows = record_ids.uniq.map do |id|
+          vector = keys.map { |key| values[id.to_s].fetch(key, 0.0) }
+          { record_type: record_type, record_id: id, tenant_key: tenant_key, label_vector: Records::Embedding.pack(vector),
+            label_vocabulary_version: version, created_at: now, updated_at: now }
+        end
+        Records::Embedding.upsert_all(rows, unique_by: %i[record_type record_id],
+          update_only: %i[tenant_key label_vector label_vocabulary_version updated_at])
+        rows.size
+      end
+
+      # Rewrites every vector whose vocabulary version is not current, for
+      # one tenant or all of them. Returns the number of records rewritten.
+      def rebuild(tenant_key: :all, batch_size: 500)
+        tenants = tenant_key == :all ? Records::Label.where(record_type: record_type).distinct.pluck(:tenant_key) : [ tenant_key ]
+        tenants.sum do |tenant|
+          version = version_for(tenant)
+          current = Records::Embedding.for_model(model).where(tenant_key: tenant, label_vocabulary_version: version).select(:record_id)
+          ids = Records::Label.where(record_type: record_type, tenant_key: tenant).where.not(record_id: current).distinct.pluck(:record_id)
+          ids.each_slice(batch_size).sum { |slice| write(slice, tenant_key: tenant) }
+        end
+      end
+
+      # The stored vector keyed by label, or nil when it is missing or was
+      # written under another vocabulary version.
+      def read(record)
+        tenant_key = definition.tenant_key_for(record)
+        row = Records::Embedding.for_model(model).find_by(record_id: record.id)
+        return unless row&.label_vector && row.label_vocabulary_version == version_for(tenant_key)
+
+        keys(tenant_key).zip(row.label_values).to_h
+      end
+
+      private
+
+      def definition
+        model.truffler_definition
+      end
+
+      def vocabulary
+        definition.vocabulary
+      end
+
+      def version_for(tenant_key)
+        vocabulary.version(tenant_key: tenant_key, all_users: true)
+      end
+
+      def record_type
+        model.polymorphic_name
+      end
+    end
+  end
+end

data/lib/truffler/embeddings/neighbor_store.rb

@@ -0,0 +1,66 @@
+module Truffler
+  module Embeddings
+    # Cosine distance computed by the database: pgvector's `<=>` on a vector
+    # column, or sqlite-vec's `vec_distance_cosine` on float32 blobs (the
+    # host loads the extension). Because similarity is plain SQL, search can
+    # score text inline in its one query as an exact scan of the tenant's
+    # rows; an approximate (HNSW) index is an opt-in host migration.
+    class NeighborStore < VectorStore
+      TABLE = "truffler_embeddings".freeze
+
+      def self.available?(connection, table: TABLE, column: "embedding")
+        !dialect(connection, table: table, column: column).nil?
+      end
+
+      def self.dialect(connection, table: TABLE, column: "embedding")
+        case connection.adapter_name
+        when /postg/i
+          :postgres if connection.columns(table).find { |item| item.name == column.to_s }&.sql_type.to_s.start_with?("vector")
+        when /sqlite/i
+          :sqlite if sqlite_vec?(connection)
+        end
+      end
+
+      def self.sqlite_vec?(connection)
+        connection.select_value("SELECT vec_version()").present?
+      rescue ActiveRecord::StatementInvalid
+        false
+      end
+
+      def self.distance_sql(dialect, connection, column_sql, vector)
+        case dialect
+        when :sqlite then "vec_distance_cosine(#{column_sql}, X'#{Records::Embedding.pack(vector).unpack1('H*')}')"
+        when :postgres then "(#{column_sql} <=> #{connection.quote("[#{vector.map(&:to_f).join(',')}]")}::vector)"
+        else raise Error, "the neighbor vector store needs pgvector or the sqlite-vec extension"
+        end
+      end
+
+      def initialize(dialect: nil)
+        @dialect = dialect
+      end
+
+      def nearest(model, tenant_key:, vector:, k: DEFAULT_K)
+        distance = distance_sql(model, "#{TABLE}.embedding", vector)
+        embeddings(model, tenant_key, vector.size).order(Arel.sql(distance)).limit(k)
+          .pluck(:record_id, Arel.sql("1 - #{distance}")).map { |id, similarity| [ id, similarity.to_f ] }
+      end
+
+      def similarity_sql(model, tenant_key:, vector:, k: nil)
+        similarity = "1 - #{distance_sql(model, "#{TABLE}.embedding", vector)}"
+        subquery = embeddings(model, tenant_key, vector.size).where("#{TABLE}.record_id = #{primary_key_sql(model)}")
+        Arel.sql("COALESCE((#{subquery.select(Arel.sql(similarity)).to_sql}), 0.0)")
+      end
+
+      def inline_sql?(_model)
+        true
+      end
+
+      private
+
+      def distance_sql(model, column_sql, vector)
+        @dialect ||= self.class.dialect(model.connection)
+        self.class.distance_sql(@dialect, model.connection, column_sql, vector)
+      end
+    end
+  end
+end

data/lib/truffler/embeddings/ruby_llm_embedder.rb

@@ -0,0 +1,25 @@
+require "ruby_llm"
+
+module Truffler
+  module Embeddings
+    # The default embedder: `RubyLLM.embed`, which ruby_llm 1.x and 2 share.
+    # ruby_llm is not a truffler dependency; this file loads only when the
+    # default embedder is used.
+    class RubyLLMEmbedder < Embedder
+      def perform(texts, model:, dimensions:)
+        embedding = RubyLLM.embed(texts, model: model, dimensions: dimensions)
+        vectors = embedding.vectors
+        vectors = [ vectors ] if vectors.first.is_a?(Numeric)
+        { vectors: vectors, model: embedding.model, input_tokens: input_tokens(embedding) }
+      end
+
+      private
+
+      def input_tokens(embedding)
+        return embedding.input_tokens if embedding.respond_to?(:input_tokens)
+
+        embedding.tokens&.input if embedding.respond_to?(:tokens)
+      end
+    end
+  end
+end

data/lib/truffler/embeddings/ruby_store.rb

@@ -0,0 +1,12 @@
+module Truffler
+  module Embeddings
+    # Packed float blobs in truffler_embeddings with exact cosine computed in
+    # Ruby over the tenant's rows. For tests and small sets.
+    class RubyStore < VectorStore
+      def nearest(model, tenant_key:, vector:, k: DEFAULT_K)
+        rows = embeddings(model, tenant_key, vector.size).pluck(:record_id, :embedding)
+        rows.map { |id, blob| [ id, self.class.cosine(vector, Records::Embedding.unpack(blob)) ] }.max_by(k) { |_, similarity| similarity }
+      end
+    end
+  end
+end

data/lib/truffler/embeddings/vector_store.rb

@@ -0,0 +1,84 @@
+module Truffler
+  module Embeddings
+    # Nearest-neighbor search over one tenant's vectors. `nearest` returns
+    # `[record_id, cosine_similarity]` pairs, most similar first.
+    # `similarity_sql` returns a scalar SQL expression over the model's table
+    # that search can put in its SELECT or ORDER BY: stores that compute
+    # similarity in the database (`inline_sql?`) scan every row exactly, and
+    # the rest fall back to a CASE over the top-K neighbors.
+    class VectorStore
+      ADAPTERS = %i[auto ruby neighbor].freeze
+      DEFAULT_K = 200
+
+      def self.for(model, config: Truffler.config)
+        embeddings = model.truffler_definition.embeddings
+        return unless embeddings
+        return ColumnStore.new(embeddings[:column]) if embeddings.key?(:column)
+
+        case config.vector_store&.to_sym
+        when :ruby then RubyStore.new
+        when :neighbor then NeighborStore.new
+        when :auto then NeighborStore.available?(model.connection) ? NeighborStore.new : RubyStore.new
+        else raise Error, "config.vector_store must be one of #{ADAPTERS.join(', ')}"
+        end
+      end
+
+      def self.cosine(left, right)
+        dot = left_norm = right_norm = 0.0
+        left.each_with_index do |value, index|
+          other = right[index]
+          dot += value * other
+          left_norm += value * value
+          right_norm += other * other
+        end
+        left_norm.zero? || right_norm.zero? ? 0.0 : dot / Math.sqrt(left_norm * right_norm)
+      end
+
+      def nearest(model, tenant_key:, vector:, k: DEFAULT_K)
+        raise NotImplementedError, "#{self.class.name}#nearest"
+      end
+
+      def similarity_sql(model, tenant_key:, vector:, k: DEFAULT_K)
+        pairs = nearest(model, tenant_key: tenant_key, vector: vector, k: k)
+        return Arel.sql("0.0") if pairs.empty?
+
+        connection = model.connection
+        whens = pairs.map { |id, similarity| "WHEN #{connection.quote(id)} THEN #{Float(similarity)}" }
+        Arel.sql("(CASE #{primary_key_sql(model)} #{whens.join(' ')} ELSE 0.0 END)")
+      end
+
+      def inline_sql?(_model)
+        false
+      end
+
+      def write(model, record, vector, fingerprint:)
+        now = Time.current
+        Records::Embedding.upsert(
+          { record_type: model.polymorphic_name, record_id: record.id, tenant_key: model.truffler_definition.tenant_key_for(record),
+            fingerprint: fingerprint, embedding: Records::Embedding.encode(vector), dimensions: vector.size,
+            created_at: now, updated_at: now },
+          unique_by: %i[record_type record_id],
+          update_only: %i[tenant_key fingerprint embedding dimensions updated_at]
+        )
+      end
+
+      private
+
+      def embeddings(model, tenant_key, dimensions)
+        check_tenant!(model, tenant_key)
+        Records::Embedding.for_model(model).with_vector.where(tenant_key: tenant_key&.to_s, dimensions: dimensions)
+      end
+
+      def check_tenant!(model, tenant_key)
+        return unless model.truffler_definition.scoped? && tenant_key.nil?
+
+        raise MissingScope, "#{model.name}: vector search needs a tenant"
+      end
+
+      def primary_key_sql(model)
+        connection = model.connection
+        "#{connection.quote_table_name(model.table_name)}.#{connection.quote_column_name(model.primary_key)}"
+      end
+    end
+  end
+end

data/lib/truffler/embeddings.rb

@@ -0,0 +1,30 @@
+module Truffler
+  # Optional text embeddings (R10, R11) and the label vectors of KTD20. The
+  # embedding text is built from the declared fields inside the embed step
+  # only and is never stored.
+  module Embeddings
+    module_function
+
+    def embedder
+      Truffler.config.embedder || RubyLLMEmbedder.new
+    end
+
+    # Gem-managed embeddings, as opposed to a host column the gem only reads.
+    def managed?(definition)
+      definition.embeddings.present? && !definition.embeddings.key?(:column)
+    end
+
+    # Changes when the embedding model, the width, or the embedded fields
+    # change, which marks every stored vector stale.
+    def fingerprint(definition)
+      settings = definition.embeddings
+      Canonical.digest(model: settings[:model], dimensions: settings[:dimensions], fields: definition.fields)
+    end
+
+    def text_for(definition, record, max_chars: Truffler.config.max_field_chars)
+      definition.field_values(record).filter_map do |field, value|
+        "#{field}: #{value.to_s.truncate(max_chars, omission: '')}" if value.present?
+      end.join("\n")
+    end
+  end
+end

data/lib/truffler/errors.rb

@@ -0,0 +1,40 @@
+module Truffler
+  class Error < StandardError; end
+
+  class DefinitionError < Error; end
+  class SuppliedLabelFailed < Error; end
+  class MissingScope < Error; end
+  class LiveCallInTest < Error; end
+  class IncompleteAnswers < Error; end
+  class CassetteMiss < Error; end
+  class BudgetExhausted < Error; end
+  class TenantMismatch < Error; end
+  class NotAuthorized < Error; end
+  class InvalidLens < Error; end
+  class LensSpendCapExceeded < Error; end
+  class InvalidSuppliedAnswer < Error; end
+
+  # Carries the HTTP status and the original error's class name only, never a
+  # response body, which can echo record text back.
+  class ClientError < Error
+    attr_reader :status, :error_class
+
+    def self.from(error)
+      new(status: status_of(error), error_class: error.class.name)
+    end
+
+    def self.status_of(error)
+      status = error.status if error.respond_to?(:status)
+      response = error.response if error.respond_to?(:response)
+      status ||= response.status if response.respond_to?(:status)
+      status ||= response[:status] if response.is_a?(Hash)
+      status&.to_i
+    end
+
+    def initialize(status: nil, error_class: nil)
+      @status = status
+      @error_class = error_class
+      super([ "Jev request failed", status && "(status #{status})", error_class ].compact.join(" "))
+    end
+  end
+end

data/lib/truffler/instrumentation.rb

@@ -0,0 +1,17 @@
+module Truffler
+  module Instrumentation
+    module_function
+
+    def instrument(event, payload = {})
+      ActiveSupport::Notifications.instrument("truffler.#{event}", Redaction.safe(payload))
+    end
+
+    def monotonic_ms
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
+    end
+
+    def elapsed_ms(started)
+      (monotonic_ms - started).round(2)
+    end
+  end
+end

data/lib/truffler/jobs/backfill_job.rb

@@ -0,0 +1,45 @@
+module Truffler
+  module Jobs
+    # Backfills one model's stale, missing, failed, and demoted labels at
+    # backfill priority. Arguments are the record type, the id cursor, the
+    # spend so far, the cap, and the retry attempt, never record text. A
+    # budget denial or a page limit reschedules the job from its cursor; the
+    # spend cap ends it. A Jev error reschedules it with backoff and the spend
+    # already made, so retries cannot push past the cap; after MAX_ATTEMPTS
+    # the released rows wait for the ResumeJob sweep.
+    class BackfillJob < ActiveJob::Base
+      MAX_PAGES = 20
+      MAX_ATTEMPTS = 10
+      BUDGET_WAIT = 30.seconds
+
+      queue_as { Truffler.config.queue_name }
+
+      def perform(record_type, cursor: nil, spent: 0.0, spend_cap: Truffler.config.backfill_spend_cap, max_pages: MAX_PAGES,
+        attempt: 0)
+        model = record_type.safe_constantize
+        return unless model.respond_to?(:truffler_definition) && model.truffler_definition
+
+        result = Labeling::Backfill.new(model, cursor: cursor, spent: spent, spend_cap: spend_cap).run(max_pages: max_pages)
+        Instrumentation.instrument(:backfill, record_type: record_type, outcome: result.status,
+          labeled_count: result.labeled, request_count: result.requests, cost: result.cost)
+
+        follow_up = { cursor: result.cursor, spent: spent + result.cost, spend_cap: spend_cap, max_pages: max_pages }
+        case result.status
+        when :budget_denied then self.class.set(wait: BUDGET_WAIT).perform_later(record_type, **follow_up)
+        when :paused then self.class.perform_later(record_type, **follow_up)
+        when :client_error then retry_later(record_type, follow_up, attempt + 1)
+        when :complete, :spend_cap_reached then nil
+        else raise ArgumentError, "unknown backfill status #{result.status.inspect}"
+        end
+      end
+
+      private
+
+      def retry_later(record_type, follow_up, attempt)
+        return if attempt >= MAX_ATTEMPTS
+
+        self.class.set(wait: (attempt**4) + 2).perform_later(record_type, **follow_up, attempt: attempt)
+      end
+    end
+  end
+end

data/lib/truffler/jobs/embed_job.rb

@@ -0,0 +1,33 @@
+module Truffler
+  module Jobs
+    # Embeds one record's declared fields and stores the vector. Arguments are
+    # the record type and id only. On an embedder failure nothing is written,
+    # `embedded_at` stays as it was, and the job retries; labels are untouched.
+    class EmbedJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      retry_on ClientError, IncompleteAnswers, wait: :polynomially_longer, attempts: 10 do
+        nil
+      end
+
+      def perform(record_type, record_id)
+        model = record_type.safe_constantize
+        definition = model.try(:truffler_definition)
+        return unless definition && Embeddings.managed?(definition)
+
+        record = model.find_by(model.primary_key => record_id)
+        return Records::Embedding.where(record_type: record_type, record_id: record_id).delete_all unless record
+
+        settings = definition.embeddings
+        fingerprint = Embeddings.fingerprint(definition)
+        text = Embeddings.text_for(definition, record)
+        vector = Embeddings.embedder.embed([ text ], model: settings[:model], dimensions: settings[:dimensions]).vectors.first
+        Embeddings::VectorStore.for(model).write(model, record, vector, fingerprint: fingerprint)
+
+        now = Time.current
+        Records::RecordState.where(record_type: record_type, record_id: record.id)
+          .update_all(embedding_fingerprint: fingerprint, embedded_at: now, updated_at: now)
+      end
+    end
+  end
+end

data/lib/truffler/jobs/encode_query_job.rb

@@ -0,0 +1,17 @@
+module Truffler
+  module Jobs
+    # Encodes one pending query at encode priority. The only argument is the
+    # cache key; the query itself waits in the cache (encrypted on encrypted
+    # models). A Jev failure is dropped: the in-flight marker is released,
+    # so the next keystroke of that query prefetches again.
+    class EncodeQueryJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      discard_on ClientError, IncompleteAnswers
+
+      def perform(cache_key)
+        QueryEncoding::Encoder.new.encode(cache_key)
+      end
+    end
+  end
+end

data/lib/truffler/jobs/expire_lenses_job.rb

@@ -0,0 +1,15 @@
+module Truffler
+  module Jobs
+    # Expires lenses unused for `config.lenses.expire_after` (R43). Takes no
+    # arguments; hosts schedule it, for example daily.
+    class ExpireLensesJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      def perform
+        expired = Lenses::Lens.expire_unused!
+        Instrumentation.instrument("lens_expire", expired_count: expired)
+        expired
+      end
+    end
+  end
+end

data/lib/truffler/jobs/label_flush_job.rb

@@ -0,0 +1,39 @@
+module Truffler
+  module Jobs
+    # Labels one tenant's pending live records. Arguments are the record type
+    # and tenant key only. On a Jev or budget failure the claimed rows go back
+    # to pending (or failed after max_attempts) and the job retries. Rows over
+    # the tenant's live cap drop to backfill priority and a delayed backfill
+    # is scheduled for them.
+    class LabelFlushJob < ActiveJob::Base
+      RETRYABLE = [ ClientError, BudgetExhausted, IncompleteAnswers ].freeze
+
+      queue_as { Truffler.config.queue_name }
+
+      retry_on(*RETRYABLE, wait: :polynomially_longer, attempts: 10) { nil }
+
+      def perform(record_type, tenant_key)
+        model = record_type.safe_constantize
+        return unless model.respond_to?(:truffler_definition) && model.truffler_definition
+
+        queue = Labeling::Queue.new(model)
+        queue.clear_marker(tenant_key)
+        states = queue.claim(tenant_key, priority: :live, limit: Truffler.config.batch_size)
+        return if states.empty?
+
+        begin
+          result = Labeling::Labeler.new(model).label(states, priority: :live)
+        rescue *RETRYABLE => error
+          queue.release(states, error)
+          raise
+        end
+
+        if result.demoted
+          queue.demote(states)
+          queue.schedule_backfill
+        end
+        queue.schedule(tenant_key) if queue.pending?(tenant_key, priority: :live)
+      end
+    end
+  end
+end

data/lib/truffler/jobs/lens_backfill_job.rb

@@ -0,0 +1,33 @@
+module Truffler
+  module Jobs
+    # Backfills one lens's scope after it is activated or restored (R41,
+    # R45). Its only argument is the lens id. A budget denial or the batch
+    # limit reschedules it; the spend cap, expiry, or completion ends it.
+    class LensBackfillJob < ActiveJob::Base
+      RETRYABLE = [ ClientError, IncompleteAnswers ].freeze
+      MAX_BATCHES = 20
+      BUDGET_WAIT = 30.seconds
+
+      queue_as { Truffler.config.queue_name }
+
+      retry_on(*RETRYABLE, wait: :polynomially_longer, attempts: 10) { nil }
+
+      def perform(lens_id, max_batches: MAX_BATCHES)
+        lens = Lenses::Lens.find_by(id: lens_id)
+        return unless lens&.active?
+
+        result = Lenses::Backfill.new(lens, max_batches: max_batches).run
+        Instrumentation.instrument(:lens_backfill, record_type: lens.record_type, tenant_key: lens.tenant_key, lens_id: lens.id,
+          outcome: result.status, labeled_count: result.labeled, request_count: result.requests, cost: result.spent_usd)
+
+        case result.status
+        when :budget_denied then self.class.set(wait: BUDGET_WAIT).perform_later(lens_id, max_batches: max_batches)
+        when :paused then self.class.perform_later(lens_id, max_batches: max_batches)
+        when :complete, :spend_cap_reached, :inactive then nil
+        else raise ArgumentError, "unknown lens backfill status #{result.status.inspect}"
+        end
+        result
+      end
+    end
+  end
+end

data/lib/truffler/jobs/provider_search_job.rb

@@ -0,0 +1,18 @@
+module Truffler
+  module Jobs
+    # Runs the provider backup search for one Smart run (R19). Arguments are
+    # the run id plus the searching tenant and user keys; the query text is
+    # read from the run, never carried in the job. See Truffler::Providers for
+    # the run interface. An expired run is a no-op.
+    class ProviderSearchJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      def perform(run_id, tenant_key, user_key)
+        run = Providers.find_run(run_id)
+        return unless run
+
+        Providers::Runner.new(run, tenant_key: tenant_key, user_key: user_key).call
+      end
+    end
+  end
+end

data/lib/truffler/jobs/prune_query_misses_job.rb

@@ -0,0 +1,15 @@
+module Truffler
+  module Jobs
+    # Enforces the miss log retention window (R29). Takes no arguments; hosts
+    # schedule it, for example daily.
+    class PruneQueryMissesJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      def perform
+        deleted = Records::QueryMiss.expired.in_batches(of: 1_000).delete_all
+        Instrumentation.instrument("miss_prune", deleted_count: deleted)
+        deleted
+      end
+    end
+  end
+end

data/lib/truffler/jobs/rerank_chunk_job.rb

@@ -0,0 +1,14 @@
+module Truffler
+  module Jobs
+    # Reranks one chunk of a Smart run and pings the searcher. Arguments are
+    # the run id and the chunk index only. A Jev failure marks the chunk
+    # failed so its buckets still resolve; it is not retried.
+    class RerankChunkJob < ActiveJob::Base
+      queue_as { Truffler.config.queue_name }
+
+      def perform(run_id, index)
+        SmartSearch::Reranker.new.call(SmartSearch::Run.load(run_id), Integer(index))
+      end
+    end
+  end
+end