truffler 0.1.0

data/lib/truffler/providers/runner.rb

@@ -0,0 +1,67 @@
+module Truffler
+  module Providers
+    # Calls the declared provider for one run, scoped to the searching
+    # user's account, and stores exactly one terminal section state:
+    # :results, :empty, or :unavailable. A cancelled run is left alone.
+    class Runner
+      def initialize(run, tenant_key:, user_key:)
+        @run = run
+        @tenant_key = tenant_key
+        @user_key = user_key
+      end
+
+      def call
+        return if cancelled?
+
+        provider = Providers.declared(@run.model.truffler_definition)
+        return unless provider
+
+        started = Instrumentation.monotonic_ms
+        state = search(provider)
+        return if cancelled?
+
+        @run.update_section(SECTION, state)
+        instrument(state, started)
+        state
+      end
+
+      private
+
+      def search(provider)
+        return Providers.state(provider, :unavailable, reason: :scope_mismatch) unless scope_matches?
+
+        text = query_text
+        return Providers.state(provider, :unavailable, reason: :no_query) if text.blank?
+
+        results = Array.wrap(provider.search.call(text, tenant: @tenant_key, user: @user_key))
+        results.empty? ? Providers.state(provider, :empty) : Providers.state(provider, :results, results: results)
+      rescue StandardError => error
+        Providers.state(provider, :unavailable, error_class: error.class.name)
+      end
+
+      # The job's keys come from the run's creator; a run that knows its own
+      # keys must agree, so one account's provider never answers another's run.
+      def scope_matches?
+        %i[tenant_key user_key].all? do |key|
+          !@run.respond_to?(key) || @run.public_send(key)&.to_s == instance_variable_get("@#{key}")&.to_s
+        end
+      end
+
+      def query_text
+        query = @run.query
+        (query.respond_to?(:raw) ? query.raw : query.to_s).strip
+      end
+
+      def cancelled?
+        @run.respond_to?(:cancelled?) && @run.cancelled?
+      end
+
+      def instrument(state, started)
+        Instrumentation.instrument("provider_search", run_id: @run.id.to_s, record_type: @run.model.polymorphic_name,
+          tenant_key: @tenant_key, user_key: @user_key, section: SECTION.to_s, status: state[:status],
+          reason: state[:reason], result_count: Array(state[:results]).size, error_class: state[:error_class],
+          latency_ms: Instrumentation.elapsed_ms(started))
+      end
+    end
+  end
+end

data/lib/truffler/providers.rb

@@ -0,0 +1,57 @@
+module Truffler
+  # Provider backup search (R19): on the explicit action, a host-declared
+  # provider (Gmail for Cora) runs when the query asks for exact text or the
+  # local results are weak, and its state renders in its own section below
+  # every local section.
+  #
+  # The run passed to `start` and returned by `run_finder` is duck-typed. It
+  # must respond to:
+  #
+  #   id                          -> the run id, the only run reference jobs carry
+  #   model                       -> the searched ActiveRecord class
+  #   query                       -> the raw query text (a String or Search::Query),
+  #                                  decrypted by the run on encrypted models
+  #   update_section(:provider, state)
+  #                               -> stores the section state and pings the host
+  #                                  (`section: provider`); called once per transition
+  #   candidate_ids               -> the local candidate snapshot, used for the weak
+  #                                  check when `start` gets neither `local_result:`
+  #                                  nor a run that answers `local_weak?`
+  #
+  # and may respond to `local_weak?` (the keystroke's own weak verdict, which
+  # the weak check prefers over `candidate_ids`), `cancelled?`, `tenant_key`,
+  # and `user_key`, which the job honors when present.
+  module Providers
+    SECTION = :provider
+    STATUSES = %i[pending results empty unavailable].freeze
+
+    mattr_accessor :run_finder, default: ->(run_id) { SmartSearch::Run.load(run_id) }
+
+    module_function
+
+    # Returns :exact_text or :weak_local when the provider was enqueued, nil
+    # when it does not run. `local_result` is the keystroke Search::Result
+    # for the same query; its invite row carries the model's weak definition.
+    def start(run, query:, tenant_key:, user_key:, local_result: nil)
+      Backup.new(run, query: query, tenant_key: tenant_key, user_key: user_key, local_result: local_result).start
+    end
+
+    def find_run(run_id)
+      run_finder.call(run_id)
+    end
+
+    # The first declared provider, or nil.
+    def declared(definition)
+      name, options = definition.providers.first
+      Provider.new(name, options[:label], options[:search]) if name
+    end
+
+    def state(provider, status, **details)
+      raise ArgumentError, "unknown provider status #{status}" unless STATUSES.include?(status)
+
+      { status: status, name: provider.name, label: provider.label }.merge(details)
+    end
+
+    Provider = Struct.new(:name, :label, :search)
+  end
+end

data/lib/truffler/query_encoding/cache.rb

@@ -0,0 +1,71 @@
+module Truffler
+  module QueryEncoding
+    # Adds the in-flight side of query encoding to U8's encoding cache: a
+    # marker written with `unless_exist` so one job runs per query, and the
+    # pending query payload the job reads (encrypted on encrypted models).
+    # Both live only while the job is in flight; encodings and vectors are
+    # written through `Search::EncodingCache` and never hold query text.
+    class Cache
+      delegate :key, :vector_key, :read, :write, :read_vector, :write_vector, to: :@encodings
+
+      def initialize(store: Truffler.config.cache_store)
+        @store = store
+        @encodings = Search::EncodingCache.new(store: store)
+      end
+
+      def in_flight_key(cache_key)
+        "#{cache_key}/in_flight"
+      end
+
+      def payload_key(cache_key)
+        "#{cache_key}/payload"
+      end
+
+      # True when this caller took the marker, false when a job already holds it.
+      def claim(cache_key, expires_in: IN_FLIGHT_TTL)
+        @store.write(in_flight_key(cache_key), true, unless_exist: true, expires_in: expires_in).present?
+      end
+
+      def in_flight?(cache_key)
+        @store.exist?(in_flight_key(cache_key))
+      end
+
+      def release(cache_key)
+        @store.delete(payload_key(cache_key))
+        @store.delete(in_flight_key(cache_key))
+      end
+
+      def encoded?(cache_key)
+        @store.exist?(cache_key)
+      end
+
+      def read_encoding(cache_key, query)
+        Search::Encoding.load(@store.read(cache_key), query || Search::Query.new(""))
+      end
+
+      # Returns false when the query cannot be stored safely: an encrypted
+      # model without ActiveRecord encryption configured.
+      def write_payload(cache_key, model, query, tenant_key:, user_key:, expires_in: IN_FLIGHT_TTL)
+        encrypted = Misses.encrypted_model?(model)
+        return false if encrypted && !Misses.encryption_configured?
+
+        @store.write(payload_key(cache_key), { "record_type" => model.polymorphic_name, "tenant_key" => tenant_key,
+          "user_key" => user_key, "query" => Misses.seal(model, query.normalized), "encrypted" => encrypted }, expires_in: expires_in)
+        true
+      end
+
+      # {model:, query:, tenant_key:, user_key:} or nil once expired, released,
+      # or undecryptable.
+      def read_payload(cache_key)
+        payload = @store.read(payload_key(cache_key))
+        model = payload && payload["record_type"].safe_constantize
+        return unless model.try(:truffler_definition)
+
+        text = payload["encrypted"] ? ActiveRecord::Encryption.encryptor.decrypt(payload["query"]) : payload["query"]
+        { model: model, query: Search::Query.new(text), tenant_key: payload["tenant_key"], user_key: payload["user_key"] }
+      rescue ActiveRecord::Encryption::Errors::Base
+        nil
+      end
+    end
+  end
+end

data/lib/truffler/query_encoding/encoder.rb

@@ -0,0 +1,183 @@
+module Truffler
+  module QueryEncoding
+    # Encodes one pending query (KTD9). It asks a fixed question set: each
+    # label gets `filter | boost | ignore`, each choice label also gets its
+    # options plus `none`, and each of the first 12 word tokens gets
+    # `keyword | label_term | filler`. Exact-text tokens (digits, dates,
+    # quoted phrases, emails, identifiers) are keywords decided locally and
+    # never asked (R18). Query text travels only in `state` ("query" and
+    # "tokens"); a token question names its word by position, `tokens[n]`,
+    # so searcher text never lands in an instruction (R8).
+    #
+    # Answers become a `Search::Encoding` with the KTD20 intent vector: boost
+    # gives the declared boost, filter narrows and adds `filter_weight`
+    # (default 0), ignore gives nothing. The encoding is cached whenever it
+    # lands, late or not (AE6). With gem-managed embeddings the query vector
+    # is embedded and cached in the same pass.
+    class Encoder
+      INTENTS = {
+        "filter" => "The query asks only for records where this holds",
+        "boost" => "The query prefers records where this holds but does not require it",
+        "ignore" => "The query does not mention this"
+      }.freeze
+      TOKEN_ROLES = {
+        "keyword" => "A word to match in the record text",
+        "label_term" => "A word that names one of the labels above rather than text to match",
+        "filler" => "A word that carries no meaning for the search"
+      }.freeze
+      NO_OPTION = "none".freeze
+
+      Request = Data.define(:state, :questions, :token_ids, :exact_tokens, :unasked_tokens)
+
+      attr_reader :client, :budget, :cache
+
+      def initialize(client: Truffler.config.client, budget: Budget.new, cache: Cache.new,
+        clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, sleeper: ->(seconds) { sleep(seconds) })
+        @client = client
+        @budget = budget
+        @cache = cache
+        @clock = clock
+        @sleeper = sleeper
+      end
+
+      def request(model, query, tenant_key:, user_key: nil)
+        labels = labels(model, tenant_key, user_key)
+        questions = Questions.new
+        labels.each_value do |label|
+          questions.choice(:"intent__#{label.question_key}", instructions: intent_instructions(label), criteria: INTENTS)
+        end
+        labels.each_value do |label|
+          next unless label.type == :choice
+
+          options = label.options(tenant_key).merge(NO_OPTION => "The query names none of these")
+          questions.choice(:"option__#{label.question_key}", instructions: %(Which "#{label.key}" option does the search query ask about?),
+            criteria: options)
+        end
+
+        words = query.tokens.each_with_index.reject { |token, _| query.exact_tokens.include?(token) }
+        asked = words.first(MAX_TOKEN_QUESTIONS)
+        token_ids = asked.to_h do |_token, position|
+          id = :"token__#{position}"
+          questions.choice(id, instructions: %(In the search query, what is the word tokens[#{position}]?), criteria: TOKEN_ROLES)
+          [ position, id.to_s ]
+        end
+
+        Request.new(state: { "query" => query.normalized, "tokens" => query.tokens }, questions: questions.to_h, token_ids: token_ids,
+          exact_tokens: query.exact_tokens, unasked_tokens: words.drop(MAX_TOKEN_QUESTIONS).map(&:first))
+      end
+
+      # Encodes the query pending under `cache_key`. Returns the encoding, or
+      # nil when the payload expired, the vocabulary moved on, or the encode
+      # budget was denied (a silent skip). Always releases the in-flight marker.
+      def encode(cache_key)
+        pending = cache.read_payload(cache_key)
+        return unless pending
+
+        model, query, tenant_key, user_key = pending.values_at(:model, :query, :tenant_key, :user_key)
+        return unless cache.key(model, query, tenant_key: tenant_key, user_key: user_key) == cache_key
+
+        encoding = cache.encoded?(cache_key) ? cache.read_encoding(cache_key, query) : encode_labels(model, query, tenant_key, user_key)
+        embed_query(model, query, tenant_key)
+        encoding
+      ensure
+        cache.release(cache_key)
+      end
+
+      # Polls the cache until the encoding lands or the deadline (seconds
+      # from now) passes; nil at the deadline. Pass `query:` to recover the
+      # keyword and label-term tokens; otherwise the pending payload supplies it.
+      def await(cache_key, deadline:, query: nil, interval: 0.05)
+        stop = @clock.call + deadline.to_f
+        query ||= cache.read_payload(cache_key)&.dig(:query)
+        loop do
+          return cache.read_encoding(cache_key, query) if cache.encoded?(cache_key)
+
+          remaining = stop - @clock.call
+          return if remaining <= 0
+
+          @sleeper.call([ interval, remaining ].min)
+        end
+      end
+
+      def encoding_for(model, request, answers, tenant_key:, user_key: nil)
+        filters = {}
+        boosts = {}
+        intent = {}
+        labels(model, tenant_key, user_key).each_value do |label|
+          key = storage_key(label, answers, tenant_key)
+          next unless key
+
+          case answers.choice("intent__#{label.question_key}")
+          when "filter"
+            filters[key] = label.filter_at || DEFAULT_FILTER_AT
+            intent[key] = label.filter_weight
+          when "boost"
+            boosts[key] = intent[key] = label.boost || DEFAULT_BOOST
+          end
+        end
+
+        roles = request.token_ids.transform_values { |id| answers.choice(id) }
+        query = Search::Query.new(request.state["query"])
+        keyword_tokens = query.tokens.each_with_index.filter_map do |token, position|
+          token if roles.fetch(position, "keyword") == "keyword"
+        end
+        label_term_tokens = roles.filter_map { |position, role| query.tokens[position] if role == "label_term" }
+        Search::Encoding.new(filters: filters, boosts: boosts, intent_vector: intent, keyword_tokens: keyword_tokens,
+          label_term_tokens: label_term_tokens)
+      end
+
+      private
+
+      def encode_labels(model, query, tenant_key, user_key)
+        return if labels(model, tenant_key, user_key).empty?
+
+        started = Instrumentation.monotonic_ms
+        decision = budget.acquire(priority: :encode, user_key: user_key)
+        if decision.denied?
+          instrument(model, tenant_key, started, outcome: "skipped", reason: decision.reason)
+          return
+        end
+
+        request = request(model, query, tenant_key: tenant_key, user_key: user_key)
+        answers = client.ask(state: request.state, questions: request.questions, priority: decision.priority)
+        encoding = encoding_for(model, request, answers, tenant_key: tenant_key, user_key: user_key)
+        cache.write(model, query, encoding, tenant_key: tenant_key, user_key: user_key)
+        Misses.hook.call(model, tenant_key: tenant_key, user_key: user_key, query: query.normalized) if encoding.empty?
+        instrument(model, tenant_key, started, outcome: encoding.empty? ? "empty" : "encoded", question_count: request.questions.size,
+          filter_count: encoding.filters.size, boost_count: encoding.intent_vector.size)
+        encoding
+      end
+
+      def embed_query(model, query, tenant_key)
+        definition = model.truffler_definition
+        return unless Embeddings.managed?(definition) && cache.read_vector(model, query, tenant_key: tenant_key).nil?
+
+        settings = definition.embeddings
+        vector = Embeddings.embedder.embed([ query.normalized ], model: settings[:model], dimensions: settings[:dimensions]).vectors.first
+        cache.write_vector(model, query, vector, tenant_key: tenant_key)
+      end
+
+      def labels(model, tenant_key, user_key)
+        model.truffler_definition.vocabulary.labels_for(tenant_key: tenant_key, user_key: user_key)
+      end
+
+      # The label's storage key the query names, or nil for a choice label
+      # whose option answer is `none`.
+      def storage_key(label, answers, tenant_key)
+        return label.key unless label.type == :choice
+
+        option = answers.choice("option__#{label.question_key}")
+        "#{label.key}:#{option}" if option != NO_OPTION && label.options(tenant_key).key?(option)
+      end
+
+      def intent_instructions(label)
+        %(How does the search query use the label "#{label.key}" (#{label.description})?)
+      end
+
+      def instrument(model, tenant_key, started, **payload)
+        Instrumentation.instrument("encode", { record_type: model.polymorphic_name, tenant_key: tenant_key,
+          latency_ms: Instrumentation.elapsed_ms(started) }.merge(payload))
+      end
+    end
+  end
+end

data/lib/truffler/query_encoding/prefetch.rb

@@ -0,0 +1,34 @@
+module Truffler
+  module QueryEncoding
+    # The default `config.encoding_prefetch`. Called by keystroke search on a
+    # cache miss; enqueues one `EncodeQueryJob(cache_key)` per in-flight window
+    # and returns true while an encoding is in flight. It never calls Jev.
+    class Prefetch
+      def initialize(cache: nil)
+        @cache = cache
+      end
+
+      def call(model, query, cache_key:, tenant_key:, user_key:)
+        cache = @cache || Cache.new
+        return false unless work?(cache, model, query, cache_key, tenant_key)
+        return true unless cache.claim(cache_key)
+
+        unless cache.write_payload(cache_key, model, query, tenant_key: tenant_key, user_key: user_key)
+          cache.release(cache_key)
+          return false
+        end
+
+        Jobs::EncodeQueryJob.perform_later(cache_key)
+        true
+      end
+
+      private
+
+      def work?(cache, model, query, cache_key, tenant_key)
+        definition = model.truffler_definition
+        (definition.labels.any? && !cache.encoded?(cache_key)) ||
+          (Embeddings.managed?(definition) && cache.read_vector(model, query, tenant_key: tenant_key).nil?)
+      end
+    end
+  end
+end

data/lib/truffler/query_encoding.rb

@@ -0,0 +1,12 @@
+module Truffler
+  # Query encoding (KTD9, KTD10): Jev turns a query into label filters,
+  # boosts, a sparse intent vector, and keyword splits. Keystroke search only
+  # reads the cached result; a miss hands the query to `Prefetch`, which
+  # enqueues one `EncodeQueryJob` per in-flight window.
+  module QueryEncoding
+    IN_FLIGHT_TTL = 2.minutes
+    DEFAULT_FILTER_AT = 0.5
+    DEFAULT_BOOST = 1.0
+    MAX_TOKEN_QUESTIONS = 12
+  end
+end

data/lib/truffler/questions.rb

@@ -0,0 +1,78 @@
+module Truffler
+  # Builds TypeSafe questions as plain wire-shape hashes keyed by string id:
+  #
+  #   Truffler::Questions.build do |q|
+  #     q.noul :spam, instructions: "Is this spam?", criteria: { true => "...", false => "..." }
+  #     q.choice :tone, instructions: "Tone?", criteria: { calm: nil, angry: "Hostile words" }
+  #     q.score :urgency, instructions: "How urgent?", criteria: [ "Whenever", "Today" ]
+  #   end
+  class Questions
+    ID = /\A[a-z0-9_]+\z/
+    SEPARATOR = "__".freeze
+
+    def self.build
+      new.tap { |questions| yield questions }.to_h
+    end
+
+    def self.tag(prefix, index)
+      format("%s%03d", prefix, index)
+    end
+
+    def self.tagged_id(tag, key)
+      "#{tag}#{SEPARATOR}#{key}"
+    end
+
+    def self.split_id(id)
+      id.to_s.split(SEPARATOR, 2)
+    end
+
+    def self.valid_id?(id)
+      ID.match?(id.to_s)
+    end
+
+    def initialize
+      @questions = {}
+    end
+
+    def noul(id, instructions:, criteria: nil)
+      criteria = criteria&.to_h { |answer, description| [ answer.to_s, description ] }
+      if criteria && !(criteria.keys - %w[true false]).empty?
+        raise ArgumentError, "#{id}: noul criteria keys must be true and false"
+      end
+
+      add(id, "noul", instructions, criteria)
+    end
+
+    def choice(id, instructions:, criteria:)
+      criteria = criteria.to_h { |option| [ option, nil ] } if criteria.is_a?(Array)
+      raise ArgumentError, "#{id}: choice criteria must name at least one option" if criteria.blank?
+
+      add(id, "choice", instructions, criteria.to_h { |option, description| [ option.to_s, description ] })
+    end
+
+    def score(id, instructions:, criteria:)
+      levels = criteria.is_a?(Hash) ? criteria.sort_by { |level, _| level.to_i }.map(&:last) : Array(criteria)
+      raise ArgumentError, "#{id}: score criteria need at least two ordered levels" if levels.size < 2
+
+      add(id, "score", instructions, levels)
+    end
+
+    def to_h
+      @questions.deep_dup
+    end
+
+    private
+
+    def add(id, type, instructions, criteria)
+      key = id.to_s
+      raise ArgumentError, "question id #{id.inspect} must match #{ID.source}" unless self.class.valid_id?(key)
+      raise ArgumentError, "question id #{key} is already defined" if @questions.key?(key)
+      raise ArgumentError, "#{key}: instructions are required" if instructions.blank?
+
+      question = { "type" => type, "instructions" => Canonical.sort(instructions) }
+      question["criteria"] = criteria unless criteria.nil?
+      @questions[key] = question
+      self
+    end
+  end
+end

data/lib/truffler/railtie.rb

@@ -0,0 +1,11 @@
+module Truffler
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      Dir[File.expand_path("../tasks/**/*.rake", __dir__)].each { |file| load file }
+    end
+
+    generators do
+      require "generators/truffler/install/install_generator"
+    end
+  end
+end

data/lib/truffler/records/embedding.rb

@@ -0,0 +1,39 @@
+module Truffler
+  module Records
+    # One row per record holding its optional text vector and its label vector
+    # (KTD20). Neither column holds anything derived from text except floats.
+    # Blobs are packed little-endian float32, the layout sqlite-vec reads.
+    class Embedding < ActiveRecord::Base
+      self.table_name = "truffler_embeddings"
+
+      scope :for_model, ->(model) { where(record_type: model.polymorphic_name) }
+      scope :with_vector, -> { where.not(embedding: nil) }
+
+      def self.pack(vector)
+        vector.map(&:to_f).pack("e*")
+      end
+
+      def self.unpack(value)
+        case value
+        when nil then nil
+        when Array then value.map(&:to_f)
+        else value.encoding == ::Encoding::BINARY ? value.unpack("e*") : JSON.parse(value).map(&:to_f)
+        end
+      end
+
+      # The stored form for the `embedding` column: a blob, or pgvector's text
+      # literal when the install generator wrote a vector column.
+      def self.encode(vector)
+        columns_hash["embedding"].type == :binary ? pack(vector) : "[#{vector.map(&:to_f).join(',')}]"
+      end
+
+      def vector
+        self.class.unpack(embedding)
+      end
+
+      def label_values
+        self.class.unpack(label_vector)
+      end
+    end
+  end
+end

data/lib/truffler/records/label.rb

@@ -0,0 +1,11 @@
+module Truffler
+  module Records
+    # One numeric answer per record and label key. Choice labels store one row
+    # per option as "label:option".
+    class Label < ActiveRecord::Base
+      self.table_name = "truffler_labels"
+
+      scope :for_label, ->(key) { where(label_key: key).or(where("label_key LIKE ?", "#{sanitize_sql_like(key)}:%")) }
+    end
+  end
+end

data/lib/truffler/records/query_miss.rb

@@ -0,0 +1,20 @@
+module Truffler
+  module Records
+    # A query whose encoding matched no label. Digests are keyed HMACs;
+    # `query_text` is normalized plaintext on plaintext models, AR-encryption
+    # ciphertext on encrypted models when it is configured, and null otherwise.
+    class QueryMiss < ActiveRecord::Base
+      self.table_name = "truffler_query_misses"
+
+      scope :for_model, ->(model) { where(record_type: model.polymorphic_name) }
+      scope :retained, ->(now = Time.current) { where(created_at: (now - Truffler.config.miss_retention)..) }
+      scope :expired, ->(now = Time.current) { where(created_at: ...(now - Truffler.config.miss_retention)) }
+
+      # The normalized query, decrypted on encrypted models; nil when no text
+      # was stored or it can no longer be decrypted.
+      def query
+        Misses.unseal(record_type.safe_constantize, query_text)
+      end
+    end
+  end
+end

data/lib/truffler/records/record_state.rb

@@ -0,0 +1,30 @@
+module Truffler
+  module Records
+    # The durable labeling queue: one row per record, carrying its status,
+    # priority, attempts, and the vocabulary version it was labeled with.
+    class RecordState < ActiveRecord::Base
+      self.table_name = "truffler_record_states"
+
+      STATUSES = %w[pending labeling labeled failed].freeze
+      PRIORITIES = %w[live backfill].freeze
+
+      scope :for_model, ->(model) { where(record_type: model.polymorphic_name) }
+
+      # Only rows still claimed: a record edited mid-flight was reset to
+      # pending and must be labeled again.
+      def self.mark_supplied_failed(ids, error_class:)
+        now = Time.current
+        set = "attempts = attempts + 1, status = ?, priority = 'backfill', last_error_class = ?, claimed_at = NULL, updated_at = ?"
+        scope = where(id: ids)
+        scope.where("attempts + 1 >= ?", Truffler.config.max_attempts).update_all([ set, "failed", error_class, now ])
+        scope.where.not(status: "failed").update_all([ set, "pending", error_class, now ])
+      end
+
+      def self.mark_labeled(ids, version:)
+        now = Time.current
+        where(id: ids, status: "labeling").update_all(status: "labeled", vocabulary_version: version, labeled_at: now, attempts: 0,
+          last_error_class: nil, claimed_at: nil, updated_at: now)
+      end
+    end
+  end
+end

data/lib/truffler/redaction.rb

@@ -0,0 +1,24 @@
+module Truffler
+  # The privacy allowlist for anything the gem logs, instruments, or stores
+  # about a call: ids, counts, tokens, cost, model, latency, digests, and error
+  # class names. Every other key is dropped, so record and query text cannot
+  # leak through a payload by accident.
+  module Redaction
+    KEYS = %i[
+      priority model cost input_tokens tokens_estimated latency_ms error_class status outcome reason
+      record_type tenant_key user_key surface section sources vocabulary_version label_key
+    ].to_set.freeze
+    SUFFIXES = %w[_id _ids _count _digest _ms].freeze
+
+    module_function
+
+    def safe(payload)
+      payload.to_h.select { |key, _| allowed?(key) }
+    end
+
+    def allowed?(key)
+      key = key.to_sym
+      KEYS.include?(key) || SUFFIXES.any? { |suffix| key.end_with?(suffix) }
+    end
+  end
+end

data/lib/truffler/registry.rb

@@ -0,0 +1,16 @@
+module Truffler
+  # Declared models by class name, so code reloading never pins stale classes.
+  class Registry
+    def initialize
+      @names = Set.new
+    end
+
+    def register(model)
+      @names << model.name
+    end
+
+    def models
+      @names.filter_map(&:safe_constantize)
+    end
+  end
+end