truffler 0.1.4 → 0.1.6

data/lib/truffler/labeling/backfill.rb

@@ -2,20 +2,28 @@ module Truffler
   module Labeling
     # Relabels one model's records whose labels are missing, stale (labeled
     # under another vocabulary version), failed, or demoted to backfill
-    # priority. It walks newest-first below an id cursor, splits each page by
-    # tenant, packs batch_size records per request at backfill priority, and
-    # asks only the stale questions. Live pending rows belong to the flush job
-    # and are never touched.
+    # priority. It walks newest-first below an id cursor over the
+    # definition's index_scope (one tenant's records with `tenant_key:`),
+    # splits each page by tenant, skips disabled tenants, packs batch_size
+    # records per request at backfill priority, and asks only the stale
+    # questions. Live pending rows belong to the flush job and are never
+    # touched.
     #
     # Resumable: the cursor moves past a page only once the whole page is
     # done, and records already current are skipped, so a rerun never asks
     # Jev about them again. A spend cap stops the run before a request would
     # exceed it; host-supplied labels cost nothing, so they are still written
     # once the cap is reached. The cap counts everything spent under the
-    # model's current app-wide vocabulary version, kept in the
-    # truffler_backfill_spends ledger, so reruns and overlapping jobs share
-    # it; without that table it falls back to this run plus `spent:`. A Jev error releases the claimed rows and ends
-    # the run with `:client_error`, so the caller keeps the spend metered so far.
+    # current ledger version (the asked labels' fingerprints, so supplied
+    # labels never reset it) in the truffler_backfill_spends ledger, so
+    # reruns and overlapping jobs share it. Tenant-scoped models keep one
+    # ledger per tenant (backfill_spend_cap_scope :tenant, the default), so
+    # the cap applies to each tenant: a whole-model run skips a tenant at its
+    # cap, keeps labeling the others, and ends with `:spend_cap_reached`; a
+    # tenant run stops there. Unscoped models, and :app, keep one app-wide
+    # ledger. Without that table the cap falls back to this run plus
+    # `spent:`. A Jev error releases the claimed rows and ends the run with
+    # `:client_error`, so the caller keeps the spend metered so far.
     #
     # A budget denial ends the run with `:budget_denied`, unless the run
     # waits: then it backs off (see .backoff) and retries from the same
@@ -94,29 +102,46 @@ module Truffler
         end
       end
 
-      def self.status(model)
-        new(model).status
+      def self.status(model, tenant_key: nil)
+        new(model, tenant_key: tenant_key).status
       end
 
-      # The ledger row for the model's current vocabulary version, or nil
-      # when nothing was spent yet or the ledger table is missing.
-      def self.spend(model)
+      # The ledger row for the current ledger version (the tenant's, for a
+      # tenant ledger), or nil when nothing was spent yet or the ledger table
+      # is missing. A row still keyed by the pre-0.1.6 vocabulary version
+      # counts until a backfill takes it over.
+      def self.spend(model, tenant_key: nil)
         return unless Records::BackfillSpend.available?
 
-        Records::BackfillSpend.for_model(model).find_by(vocabulary_version: ledger_version(model))
+        ledgers = Records::BackfillSpend.for_ledger(model, tenant_key)
+        ledgers.find_by(vocabulary_version: ledger_version(model, tenant_key)) ||
+          ledgers.find_by(vocabulary_version: legacy_ledger_version(model, tenant_key))
       end
 
       # Zeroes the current vocabulary version's ledger in place, so a chain
       # still running keeps its row and continues against the fresh total.
-      def self.reset_spend!(model)
+      # `all_tenants: true` zeroes every tenant ledger of the model.
+      def self.reset_spend!(model, tenant_key: nil, all_tenants: false)
         return unless Records::BackfillSpend.available?
 
-        Records::BackfillSpend.for_model(model).where(vocabulary_version: ledger_version(model))
-          .update_all(spent_usd: 0.0, requests: 0, updated_at: Time.current)
+        ledgers = if all_tenants && Records::BackfillSpend.tenant_ledgers?
+          Records::BackfillSpend.for_model(model).where.not(tenant_key: nil)
+        else
+          Records::BackfillSpend.for_ledger(model, tenant_key)
+            .where(vocabulary_version: [ ledger_version(model, tenant_key), legacy_ledger_version(model, tenant_key) ])
+        end
+        ledgers.update_all(spent_usd: 0.0, requests: 0, updated_at: Time.current)
       end
 
-      def self.ledger_version(model)
-        model.truffler_definition.vocabulary.version(all_users: true)
+      # Spend ledgers key on the asked labels only (Vocabulary#ledger_version),
+      # so a supplied-label change never resets the cap.
+      def self.ledger_version(model, tenant_key = nil)
+        model.truffler_definition.vocabulary.ledger_version(tenant_key: tenant_key, all_users: true)
+      end
+
+      # What ledgers were keyed by before 0.1.6: the whole vocabulary version.
+      def self.legacy_ledger_version(model, tenant_key = nil)
+        model.truffler_definition.vocabulary.version(tenant_key: tenant_key, all_users: true)
       end
 
       # Seconds to wait after `denials` consecutive budget denials with no
@@ -128,9 +153,10 @@ module Truffler
 
       attr_reader :model, :batch_size, :page_size
 
-      def initialize(model, spend_cap: Truffler.config.backfill_spend_cap, batch_size: Truffler.config.batch_size,
+      def initialize(model, tenant_key: nil, spend_cap: Truffler.config.backfill_spend_cap, batch_size: Truffler.config.batch_size,
         page_size: nil, cursor: nil, spent: 0.0, client: Truffler.config.client, budget: Budget.new)
         @model = model
+        @tenant_key = tenant_key&.to_s if model.truffler_definition.scoped?
         model.truffler_definition.validate_columns!
         @batch_size = batch_size
         @page_size = page_size || batch_size * 5
@@ -140,6 +166,9 @@ module Truffler
         @client = client
         @budget = budget
         @versions = {}
+        @meters = {}
+        @enabled = {}
+        @capped = Set.new
       end
 
       # `progress` is called with the result so far and the delay before each
@@ -147,17 +176,17 @@ module Truffler
       def run(max_pages: nil, wait: false, max_duration: nil, sleeper: self.class.sleeper, clock: self.class.clock,
         progress: nil)
         @labeled = 0
-        @started_cost = meter.cost
+        @started_cost = spent_cost
         @pages = 0
         deadline = max_duration && clock.call + max_duration
         denials = 0
 
         loop do
-          before = [ @labeled, meter.requests ]
+          before = [ @labeled, requests ]
           status = sweep(max_pages, deadline, clock)
           return result(status) unless wait && status == :budget_denied
 
-          denials = 0 unless before == [ @labeled, meter.requests ]
+          denials = 0 unless before == [ @labeled, requests ]
           delay = self.class.backoff(denials, @retry_after)
           return result(:paused) if deadline && clock.call + delay > deadline
 
@@ -167,11 +196,16 @@ module Truffler
         end
       end
 
+      # Counts over what the backfill may touch: index_scope, enabled tenants,
+      # and the one tenant when `tenant_key:` is given.
       def status
-        counts = states.group(:status).count
-        labeled = states.where(status: "labeled").group(:tenant_key, :vocabulary_version).count
+        scope, tenants = status_scope
+        tracked = states.where(record_id: scope.select(model.arel_table[model.primary_key]))
+        tracked = tracked.where(tenant_key: tenants) if tenants
+        counts = tracked.group(:status).count
+        labeled = tracked.where(status: "labeled").group(:tenant_key, :vocabulary_version).count
         stale = labeled.sum { |(tenant_key, version), count| version == version_for(tenant_key) ? 0 : count }
-        { total: model.count, missing: model.joins(state_join).where("#{STATES}.id IS NULL").count,
+        { total: scope.count, missing: scope.joins(state_join).where("#{STATES}.id IS NULL").count,
           pending: counts["pending"].to_i, labeling: counts["labeling"].to_i, labeled: counts["labeled"].to_i,
           failed: counts["failed"].to_i, stale: stale, current: labeled.values.sum - stale }
       end
@@ -190,30 +224,67 @@ module Truffler
         Records::RecordState.for_model(model)
       end
 
+      # [relation, tenant keys or nil]: the records status counts, and the
+      # tenants it covers when it narrows to some.
+      def status_scope
+        scope = definition.index_relation(model.all)
+        return [ scope, nil ] unless definition.scoped?
+        return [ scope.where(definition.tenant_column => @tenant_key), [ @tenant_key ] ] if @tenant_key
+        return [ scope, nil ] unless Truffler.config.tenant_enabled
+
+        tenants = scope.reorder(nil).distinct.pluck(definition.tenant_column).map(&:to_s).select { |tenant_key| enabled?(tenant_key) }
+        [ scope.where(definition.tenant_column => tenants), tenants ]
+      end
+
       def queue
         @queue ||= Queue.new(model)
       end
 
-      # Spend carried in with `spent:` is ignored when the ledger holds it.
-      def meter
-        @meter ||= begin
-          ledger = Records::BackfillSpend.ledger(model, version_for(nil)) if Records::BackfillSpend.available?
+      # One meter per ledger: per tenant for tenant ledgers, else one for the
+      # run. Spend carried in with `spent:` is ignored when the ledger holds it.
+      def meter(tenant_key)
+        ledger_tenant = ledger_available? ? definition.ledger_tenant(tenant_key) : nil
+        @meters[ledger_tenant] ||= begin
+          if ledger_available?
+            ledger = Records::BackfillSpend.ledger(model, self.class.ledger_version(model, ledger_tenant), tenant_key: ledger_tenant,
+              legacy_version: version_for(ledger_tenant))
+          end
           SpendMeter.new(@client, cap: @spend_cap, spent: ledger ? 0.0 : @spent, ledger: ledger)
         end
       end
 
+      def ledger_available?
+        return @ledger_available if defined?(@ledger_available)
+
+        @ledger_available = Records::BackfillSpend.available?
+      end
+
+      def requests
+        @meters.each_value.sum(&:requests)
+      end
+
+      def spent_cost
+        @meters.each_value.sum(&:cost)
+      end
+
       def version_for(tenant_key)
         @versions[tenant_key] ||= definition.vocabulary.version(tenant_key: tenant_key, all_users: true)
       end
 
+      def enabled?(tenant_key)
+        @enabled.fetch(tenant_key) { @enabled[tenant_key] = definition.tenant_enabled?(tenant_key) }
+      end
+
       def result(status)
-        Result.new(status: status, labeled: @labeled, requests: meter.requests, cost: meter.cost - @started_cost,
+        Result.new(status: status, labeled: @labeled, requests: requests, cost: spent_cost - @started_cost,
           cursor: @cursor, retry_after: (@retry_after if status == :budget_denied))
       end
 
       # Walks pages below @cursor until done or stopped, returning the status.
       def sweep(max_pages, deadline, clock)
         @retry_after = nil
+        return complete if @tenant_key && !enabled?(@tenant_key)
+
         loop do
           scanned, rows = page(@cursor)
           return complete if scanned.empty?
@@ -221,6 +292,8 @@ module Truffler
 
           rows.group_by(&:last).each do |tenant_key, tenant_rows|
             tenant_rows.map(&:first).each_slice(batch_size) do |ids|
+              break if @capped.include?(tenant_key)
+
               stop = label(ids, tenant_key)
               return stop if stop
             end
@@ -234,7 +307,7 @@ module Truffler
 
       def complete
         @cursor = nil
-        :complete
+        @capped.any? ? :spend_cap_reached : :complete
       end
 
       # Returns the scanned ids (for the cursor) and the [id, tenant_key] rows
@@ -242,7 +315,8 @@ module Truffler
       # lenses) compare versions here because each tenant has its own.
       def page(cursor)
         pk = model.primary_key
-        scope = model.joins(state_join).where(needs_labeling_sql)
+        scope = definition.index_relation(model.joins(state_join).where(needs_labeling_sql))
+        scope = scope.where(definition.tenant_column => @tenant_key) if @tenant_key
         scope = scope.where(model.arel_table[pk].lt(cursor)) if cursor
         tenant = definition.scoped? ? model.arel_table[definition.tenant_column] : Arel.sql("NULL")
         plucked = scope.reorder(pk => :desc).limit(page_size)
@@ -250,7 +324,7 @@ module Truffler
 
         rows = plucked.filter_map do |id, tenant_key, status, version|
           tenant_key = tenant_key&.to_s
-          [ id, tenant_key ] unless status == "labeled" && version == version_for(tenant_key)
+          [ id, tenant_key ] unless (status == "labeled" && version == version_for(tenant_key)) || !enabled?(tenant_key)
         end
         [ plucked.map(&:first), rows ]
       end
@@ -277,16 +351,20 @@ module Truffler
 
       # Labels one tenant chunk. Returns nil when done, or the status that
       # stops the run; claimed rows that were not labeled go back to pending
-      # at backfill priority.
+      # at backfill priority. A tenant ledger at its cap only stops that
+      # tenant in a whole-model run.
       def label(ids, tenant_key)
         claimed = queue.claim_backfill(ids, tenant_key)
         return if claimed.empty?
 
         begin
-          Labeler.new(model, client: meter, budget: @budget).label(claimed, priority: :backfill)
+          Labeler.new(model, client: meter(tenant_key), budget: @budget).label(claimed, priority: :backfill)
         rescue SpendCapReached
           queue.demote(claimed)
-          return :spend_cap_reached
+          return :spend_cap_reached if @tenant_key || definition.ledger_tenant(tenant_key).nil? || !ledger_available?
+
+          @capped << tenant_key
+          return
         rescue BudgetExhausted => error
           queue.demote(claimed)
           @retry_after = error.retry_after

data/lib/truffler/labeling/labeler.rb

@@ -20,46 +20,48 @@ module Truffler
       end
 
       def label(states, priority:)
-        return Result.new(labeled: 0, requests: 0, cost: 0.0, demoted: false) if states.empty?
-
-        tenant_key = tenant_key_of(states)
-        records = load_records(states)
-        vocabulary = definition.vocabulary
-        @labels = vocabulary.labels_for(tenant_key: tenant_key, all_users: true)
-        fingerprints = vocabulary.fingerprints(tenant_key: tenant_key, all_users: true)
-        version = vocabulary.version(tenant_key: tenant_key, all_users: true)
-        states_by_id = states.index_by { |state| state.record_id.to_s }
-
-        stored = stored_fingerprints(records)
-        askable = askable_labels
-        stale = records.map { |record| [ record, stale_keys(stored[record.id.to_s].to_h, fingerprints, tenant_key, askable) ] }
-        supplied = askable.select(&:supplied?).map(&:key)
-        supplier = Supplied.new(model)
-        written = supplier.write(stale.map { |record, keys| [ record, keys & supplied ] }, tenant_key: tenant_key)
-        current, pending = stale.map { |record, keys| [ record, keys - supplied ] }.partition { |_, keys| keys.empty? }
-        settled = current.reject { |record, _| supplier.failed_ids.include?(record.id) }
-        Records::RecordState.mark_labeled(settled.map { |record, _| states_by_id[record.id.to_s].id }, version: version)
-        Embeddings::LabelVector.new(model).write(current.map { |record, _| record.id } - written, tenant_key: tenant_key)
-
-        requests = RequestBuilder.new(definition, tenant_key: tenant_key, labels: @labels).build(pending)
-        cost = 0.0
-        requests.each_with_index do |request, index|
-          decision = budget.acquire(priority: priority, tenant_key: (tenant_key if index.zero?),
-            records: index.zero? ? pending.size : 1)
-          if decision.demoted?
-            retry_failed_supplied(supplier.failed_ids, states_by_id)
-            return Result.new(labeled: current.size, requests: index, cost: cost, demoted: true)
+        Current.scope do
+          return Result.new(labeled: 0, requests: 0, cost: 0.0, demoted: false) if states.empty?
+
+          tenant_key = tenant_key_of(states)
+          records = load_records(states, tenant_key)
+          vocabulary = definition.vocabulary
+          @labels = vocabulary.labels_for(tenant_key: tenant_key, all_users: true)
+          fingerprints = vocabulary.fingerprints(tenant_key: tenant_key, all_users: true)
+          version = vocabulary.version(tenant_key: tenant_key, all_users: true)
+          states_by_id = states.index_by { |state| state.record_id.to_s }
+
+          stored = stored_fingerprints(records)
+          askable = askable_labels
+          stale = records.map { |record| [ record, stale_keys(stored[record.id.to_s].to_h, fingerprints, tenant_key, askable) ] }
+          supplied = askable.select(&:supplied?).map(&:key)
+          supplier = Supplied.new(model)
+          written = supplier.write(stale.map { |record, keys| [ record, keys & supplied ] }, tenant_key: tenant_key)
+          current, pending = stale.map { |record, keys| [ record, keys - supplied ] }.partition { |_, keys| keys.empty? }
+          settled = current.reject { |record, _| supplier.failed_ids.include?(record.id) }
+          Records::RecordState.mark_labeled(settled.map { |record, _| states_by_id[record.id.to_s].id }, version: version)
+          Embeddings::LabelVector.new(model).write(current.map { |record, _| record.id } - written, tenant_key: tenant_key)
+
+          requests = RequestBuilder.new(definition, tenant_key: tenant_key, labels: @labels).build(pending)
+          cost = 0.0
+          requests.each_with_index do |request, index|
+            decision = budget.acquire(priority: priority, tenant_key: (tenant_key if index.zero?),
+              records: index.zero? ? pending.size : 1)
+            if decision.demoted?
+              retry_failed_supplied(supplier.failed_ids, states_by_id)
+              return Result.new(labeled: current.size, requests: index, cost: cost, demoted: true)
+            end
+            raise BudgetExhausted.new("no Jev budget for #{priority} labeling", retry_after: decision.retry_after) if decision.denied?
+
+            answers = client.ask(state: request.state, questions: request.questions, priority: decision.priority)
+            cost += answers.usage&.cost.to_f
+            charge_lenses(request, answers.usage&.cost.to_f)
+            store(request, answers, fingerprints, tenant_key, version, states_by_id)
           end
-          raise BudgetExhausted.new("no Jev budget for #{priority} labeling", retry_after: decision.retry_after) if decision.denied?
 
-          answers = client.ask(state: request.state, questions: request.questions, priority: decision.priority)
-          cost += answers.usage&.cost.to_f
-          charge_lenses(request, answers.usage&.cost.to_f)
-          store(request, answers, fingerprints, tenant_key, version, states_by_id)
+          retry_failed_supplied(supplier.failed_ids, states_by_id)
+          Result.new(labeled: records.size, requests: requests.size, cost: cost, demoted: false)
         end
-
-        retry_failed_supplied(supplier.failed_ids, states_by_id)
-        Result.new(labeled: records.size, requests: requests.size, cost: cost, demoted: false)
       end
 
       private
@@ -87,17 +89,31 @@ module Truffler
         tenants.first
       end
 
-      def load_records(states)
-        records = model.where(model.primary_key => states.map(&:record_id)).to_a
+      # Records deleted or outside index_scope / index_if lose their state
+      # rows instead of being labeled. A disabled tenant keeps its state rows:
+      # they go back to pending at backfill priority, so re-enabling it
+      # relabels only what is stale instead of every record.
+      def load_records(states, tenant_key)
+        unless definition.tenant_enabled?(tenant_key)
+          Records::RecordState.where(id: states.map(&:id))
+            .update_all(status: "pending", priority: "backfill", claimed_at: nil, updated_at: Time.current)
+          return []
+        end
+
+        records = definition.index_relation(model.where(model.primary_key => states.map(&:record_id))).to_a
+          .select { |record| definition.index_if.nil? || definition.index_if.call(record) }
         found = records.map { |record| record.id.to_s }
         gone = states.reject { |state| found.include?(state.record_id.to_s) }
         Records::RecordState.where(id: gone.map(&:id)).delete_all if gone.any?
         records
       end
 
+      # A label is current when it has stored rows and all carry its current
+      # fingerprint; choice labels store only some options (see sparse_choice).
       def stale_keys(stored, fingerprints, tenant_key, askable)
         askable.reject do |label|
-          label.storage_keys(tenant_key).all? { |key| stored[key] == fingerprints[label.key] }
+          present = label.storage_keys(tenant_key).select { |key| stored.key?(key) }
+          present.any? && present.all? { |key| stored[key] == fingerprints[label.key] }
         end.map(&:key)
       end
 
@@ -154,7 +170,8 @@ module Truffler
         case label.type
         when :noul then [ [ label.key, answers.noul(id) ] ]
         when :score then [ [ label.key, answers.score(id) ] ]
-        when :choice then label.options(tenant_key).keys.map { |option| [ "#{label.key}:#{option}", answers.probability(id, option) ] }
+        when :choice
+          LabelDefinition.sparse_choice(label.options(tenant_key).keys.to_h { |option| [ "#{label.key}:#{option}", answers.probability(id, option) ] }).to_a
         end
       end
     end

data/lib/truffler/labeling/queue.rb

@@ -3,6 +3,8 @@ module Truffler
     # State transitions on truffler_record_states for one model, plus the
     # deduplicated scheduling of LabelFlushJob per tenant and of BackfillJob
     # for demoted rows. Jobs carry only the record type and tenant key.
+    # Records the definition does not index (a disabled tenant, `index_if`)
+    # are never enqueued.
     class Queue
       BACKFILL_WAIT = 1.minute
 
@@ -15,13 +17,15 @@ module Truffler
       end
 
       def enqueue(record, priority: :live)
+        return unless definition.indexable?(record)
+
         tenant_key = definition.tenant_key_for(record)
         now = Time.current
         Records::RecordState.upsert(
           { record_type: record_type, record_id: record.id, tenant_key: tenant_key, status: "pending",
             priority: priority.to_s, attempts: 0, created_at: now, updated_at: now },
           unique_by: %i[record_type record_id],
-          update_only: %i[tenant_key status priority attempts updated_at]
+          update_only: %i[tenant_key status priority attempts]
         )
         schedule(tenant_key)
       end
@@ -41,13 +45,14 @@ module Truffler
         job.perform_later(record_type, tenant_key)
       end
 
-      # Starts a backfill shortly after live rows are demoted over the tenant
-      # cap. One per model per marker lifetime; ResumeJob still catches rows a
-      # dropped job leaves behind.
-      def schedule_backfill
-        return unless config.cache_store.write(backfill_marker, true, unless_exist: true, expires_in: BACKFILL_WAIT + 300)
+      # Starts a backfill of the tenant shortly after its live rows are
+      # demoted over the tenant cap. One per tenant per marker lifetime;
+      # ResumeJob still catches rows a dropped job leaves behind.
+      def schedule_backfill(tenant_key = nil)
+        return unless definition.tenant_enabled?(tenant_key)
+        return unless config.cache_store.write(backfill_marker(tenant_key), true, unless_exist: true, expires_in: BACKFILL_WAIT + 300)
 
-        Jobs::BackfillJob.set(wait: BACKFILL_WAIT).perform_later(record_type)
+        Jobs::BackfillJob.set(wait: BACKFILL_WAIT).perform_later(record_type, **Jobs::BackfillJob.tenant_argument(model, tenant_key))
       end
 
       def clear_marker(tenant_key)
@@ -118,8 +123,8 @@ module Truffler
         "truffler/flush/#{record_type}/#{tenant_key}"
       end
 
-      def backfill_marker
-        "truffler/backfill/#{record_type}"
+      def backfill_marker(tenant_key)
+        "truffler/backfill/#{record_type}/#{tenant_key}"
       end
     end
   end

data/lib/truffler/labeling/supplied.rb

@@ -4,9 +4,13 @@ module Truffler
     # of one tenant into truffler_labels, under the same storage keys as asked
     # labels, and rewrites those records' label vectors. No Jev request, no
     # budget slot, no spend. A nil answer stores nothing, so the label reads
-    # as missing rather than 0. A `from` that raises or answers out of shape
-    # is instrumented as `supplied_label_failed`, skipped, and listed in
-    # `failed_ids` so the caller can retry it; its stored rows serve meanwhile.
+    # as missing rather than 0. Both failures are instrumented as
+    # `supplied_label_failed`. An answer out of shape (InvalidSuppliedAnswer)
+    # is permanent (`permanent: true`): it stores nothing for that label,
+    # like a nil answer, and the record settles, since retrying the same
+    # answer cannot succeed. A `from` that raises anything else is transient:
+    # the label is skipped and listed in `failed_ids` so the caller retries
+    # it (failed after max_attempts); its stored rows serve meanwhile.
     class Supplied
       attr_reader :model, :failed_ids
 
@@ -61,11 +65,18 @@ module Truffler
 
       def answer(label, record, tenant_key)
         label.supplied_values(record, tenant_key)
+      rescue InvalidSuppliedAnswer => error
+        instrument_failure(label, record, tenant_key, error, permanent: true)
+        nil
       rescue StandardError => error
-        Instrumentation.instrument(:supplied_label_failed, record_type: record_type, tenant_key: tenant_key, record_id: record.id,
-          label_key: label.key, error_class: error.class.name)
+        instrument_failure(label, record, tenant_key, error, permanent: false)
         :failed
       end
+
+      def instrument_failure(label, record, tenant_key, error, permanent:)
+        Instrumentation.instrument(:supplied_label_failed, record_type: record_type, tenant_key: tenant_key, record_id: record.id,
+          label_key: label.key, error_class: error.class.name, permanent: permanent)
+      end
     end
   end
 end

data/lib/truffler/lenses/backfill.rb

@@ -37,6 +37,8 @@ module Truffler
 
           attempted.concat(records.map(&:id))
           records.group_by { |record| definition.tenant_key_for(record) }.each do |tenant_key, slice|
+            next unless definition.tenant_enabled?(tenant_key)
+
             stop = label(slice, tenant_key)
             return result(stop) if stop
           end
@@ -70,22 +72,38 @@ module Truffler
       # lens row.
       def page(attempted)
         pk = model.primary_key
-        scope = model.all
-        scope = scope.where(definition.tenant_column => lens.tenant_key) if definition.scoped? && lens.tenant_key
+        scope = definition.index_relation(model.all)
+        if definition.scoped? && lens.tenant_key
+          return [] unless definition.tenant_enabled?(lens.tenant_key)
+
+          scope = scope.where(definition.tenant_column => lens.tenant_key)
+        elsif definition.scoped? && Truffler.config.tenant_enabled
+          scope = scope.where(definition.tenant_column => enabled_tenants)
+        end
         scope = scope.where.not(pk => attempted) if attempted.any?
         scope.where(Arel.sql(stale_sql)).reorder(definition.arrival_order).limit(batch_size).to_a
       end
 
+      # Tenants in index_scope that config.tenant_enabled allows, looked up
+      # once per run, so disabled tenants are never paged (and never refetched
+      # by the next LensBackfillJob).
+      def enabled_tenants
+        @enabled_tenants ||= definition.index_relation(model.all).reorder(nil).distinct.pluck(definition.tenant_column)
+          .select { |tenant_key| definition.tenant_enabled?(tenant_key.to_s) }
+      end
+
+      # Stale unless every lens label has a row under its current fingerprint;
+      # a choice label stores only some of its options.
       def stale_sql
-        expected = lens.labels.values.flat_map do |label|
-          print = Lenses.fingerprint(label.question)
-          label.storage_keys.map { |key| ActiveRecord::Base.sanitize_sql_array([ "(label_key = ? AND fingerprint = ?)", key, print ]) }
-        end
         pk = "#{model.quoted_table_name}.#{model.connection.quote_column_name(model.primary_key)}"
-        ActiveRecord::Base.sanitize_sql_array([
-          "(SELECT COUNT(*) FROM #{LABELS} WHERE #{LABELS}.record_type = ? AND #{LABELS}.record_id = #{pk} " \
-          "AND (#{expected.join(' OR ')})) < ?", model.polymorphic_name, expected.size
-        ])
+        current = lens.labels.values.map do |label|
+          ActiveRecord::Base.sanitize_sql_array([
+            "EXISTS (SELECT 1 FROM #{LABELS} WHERE #{LABELS}.record_type = ? AND #{LABELS}.record_id = #{pk} " \
+            "AND #{LABELS}.label_key IN (?) AND #{LABELS}.fingerprint = ?)",
+            model.polymorphic_name, label.storage_keys, Lenses.fingerprint(label.question)
+          ])
+        end
+        "NOT (#{current.join(' AND ')})"
       end
 
       # Labels one tenant's records through the labeler, which asks only

data/lib/truffler/model.rb

@@ -62,6 +62,8 @@ module Truffler
     # outside the watched columns, e.g. from the job that classified it.
     def truffler_refresh_labels!
       definition = self.class.truffler_definition
+      return self unless definition.indexable?(self)
+
       tenant_key = definition.tenant_key_for(self)
       keys = definition.supplied_labels.select { |label| label.available?(tenant_key) }.map(&:key)
       Labeling::Supplied.new(self.class).write([ [ self, keys ] ], tenant_key: tenant_key) if keys.any?
@@ -72,6 +74,8 @@ module Truffler
 
     def truffler_enqueue_labeling
       definition = self.class.truffler_definition
+      return unless definition.indexable?(self)
+
       if previously_new_record?
         Labeling::Queue.new(self.class).enqueue(self)
         return
@@ -105,6 +109,7 @@ module Truffler
       definition = self.class.truffler_definition
       return unless Embeddings.managed?(definition)
       return unless previously_new_record? || saved_changes.keys.intersect?(definition.relabel_columns)
+      return unless definition.indexable?(self)
 
       Jobs::EmbedJob.perform_later(self.class.polymorphic_name, id)
     end

data/lib/truffler/providers/backup.rb

@@ -39,7 +39,7 @@ module Truffler
       end
 
       def weak_local?
-        return @local_result.invite_row.present? if @local_result
+        return @local_result.local_weak? if @local_result
         return @run.local_weak? if @run.respond_to?(:local_weak?)
         unless @run.respond_to?(:candidate_ids)
           raise ArgumentError, "Providers.start needs local_result: or a run that responds to local_weak? or candidate_ids"