truffler 0.1.4 → 0.1.6

data/lib/truffler/query_encoding/encoder.rb

@@ -51,50 +51,55 @@ module Truffler
       end
 
       def request(model, query, tenant_key:, user_key: nil)
-        labels = labels(model, tenant_key, user_key)
-        questions = Questions.new
-        labels.each_value do |label|
-          questions.choice(:"intent__#{label.question_key}", instructions: intent_instructions(label), criteria: INTENTS)
-        end
-        labels.each_value do |label|
-          next unless label.type == :choice
+        Current.scope do
+          present = present_options(model, tenant_key, user_key)
+          labels = labels(model, tenant_key, user_key, present)
+          questions = Questions.new
+          labels.each_value do |label|
+            questions.choice(:"intent__#{label.question_key}", instructions: intent_instructions(label), criteria: INTENTS)
+          end
+          labels.each_value do |label|
+            next unless label.type == :choice
 
-          options = label.options(tenant_key).merge(NO_OPTION => "The query names none of these")
-          questions.choice(:"option__#{label.question_key}", instructions: %(Which "#{label.key}" option does the search query ask about?),
-            criteria: options)
-        end
+            options = PresentOptions.options(label, tenant_key, present).merge(NO_OPTION => "The query names none of these")
+            questions.choice(:"option__#{label.question_key}", instructions: %(Which "#{label.key}" option does the search query ask about?),
+              criteria: options)
+          end
 
-        words = query.tokens.each_with_index.reject do |token, position|
-          query.exact_tokens.include?(token) || query.time_position?(position)
-        end
-        asked = words.first(MAX_TOKEN_QUESTIONS)
-        token_ids = asked.to_h do |_token, position|
-          id = :"token__#{position}"
-          questions.choice(id, instructions: %(In the search query, what is the word tokens[#{position}]? The labels it may name, ) +
-            %(with their options, are in `labels`.), criteria: TOKEN_ROLES)
-          [ position, id.to_s ]
-        end
+          words = query.tokens.each_with_index.reject do |token, position|
+            query.exact_tokens.include?(token) || query.time_position?(position)
+          end
+          asked = words.first(MAX_TOKEN_QUESTIONS)
+          token_ids = asked.to_h do |_token, position|
+            id = :"token__#{position}"
+            questions.choice(id, instructions: %(In the search query, what is the word tokens[#{position}]? The labels it may name, ) +
+              %(with their options, are in `labels`.), criteria: TOKEN_ROLES)
+            [ position, id.to_s ]
+          end
 
-        state = { "query" => query.normalized, "tokens" => query.tokens, "labels" => vocabulary_state(labels, tenant_key) }
-        Request.new(state: state, questions: questions.to_h, token_ids: token_ids, exact_tokens: query.exact_tokens,
-          unasked_tokens: words.drop(MAX_TOKEN_QUESTIONS).map(&:first))
+          state = { "query" => query.normalized, "tokens" => query.tokens, "labels" => vocabulary_state(labels, tenant_key, present) }
+          Request.new(state: state, questions: questions.to_h, token_ids: token_ids, exact_tokens: query.exact_tokens,
+            unasked_tokens: words.drop(MAX_TOKEN_QUESTIONS).map(&:first))
+        end
       end
 
       # Encodes the query pending under `cache_key`. Returns the encoding, or
       # nil when the payload expired, the vocabulary moved on, or the encode
       # budget was denied (a silent skip). Always releases the in-flight marker.
       def encode(cache_key)
-        pending = cache.read_payload(cache_key)
-        return unless pending
-
-        model, query, tenant_key, user_key = pending.values_at(:model, :query, :tenant_key, :user_key)
-        return unless cache.key(model, query, tenant_key: tenant_key, user_key: user_key) == cache_key
-
-        encoding = cache.encoded?(cache_key) ? cache.read_encoding(cache_key, query) : encode_labels(model, query, tenant_key, user_key)
-        embed_query(model, query, tenant_key)
-        encoding
-      ensure
-        cache.release(cache_key)
+        Current.scope do
+          pending = cache.read_payload(cache_key)
+          return unless pending
+
+          model, query, tenant_key, user_key = pending.values_at(:model, :query, :tenant_key, :user_key)
+          return unless cache.key(model, query, tenant_key: tenant_key, user_key: user_key) == cache_key
+
+          encoding = cache.encoded?(cache_key) ? cache.read_encoding(cache_key, query) : encode_labels(model, query, tenant_key, user_key)
+          embed_query(model, query, tenant_key)
+          encoding
+        ensure
+          cache.release(cache_key)
+        end
       end
 
       # Polls the cache until the encoding lands or the deadline (seconds
@@ -118,8 +123,9 @@ module Truffler
         boosts = {}
         intent = {}
         names = {}
-        labels(model, tenant_key, user_key).each_value do |label|
-          key = storage_key(label, answers, tenant_key)
+        present = present_options(model, tenant_key, user_key)
+        labels(model, tenant_key, user_key, present).each_value do |label|
+          key = storage_key(label, answers, tenant_key, present)
           next unless key
 
           case answers.choice("intent__#{label.question_key}")
@@ -134,7 +140,8 @@ module Truffler
         end
 
         query = Search::Query.new(request.state["query"])
-        roles, sources, soft = reconcile(query, request.token_ids.transform_values { |id| answers.choice(id) }, names)
+        roles, sources, soft = reconcile(query, request.token_ids.transform_values { |id| answers.choice(id) }, names,
+          Search::Filler.label_words(model.truffler_definition, tenant_key))
         tokens = ->(positions) { positions.map { |position| query.tokens[position] } }
         Search::Encoding.new(filters: filters, boosts: boosts, intent_vector: intent,
           keyword_tokens: tokens.call(roles.keys.select { |position| roles[position] == "keyword" }),
@@ -174,25 +181,41 @@ module Truffler
         cache.write_vector(model, query, vector, tenant_key: tenant_key)
       end
 
-      def labels(model, tenant_key, user_key)
+      # The searcher's vocabulary; with skip_empty_options, minus choice
+      # labels the tenant has no option rows for (see PresentOptions).
+      def labels(model, tenant_key, user_key, present = present_options(model, tenant_key, user_key))
+        labels = vocabulary_labels(model, tenant_key, user_key)
+        return labels unless present
+
+        labels.select { |_, label| label.type != :choice || PresentOptions.options(label, tenant_key, present).any? }
+      end
+
+      def vocabulary_labels(model, tenant_key, user_key)
         model.truffler_definition.vocabulary.labels_for(tenant_key: tenant_key, user_key: user_key)
       end
 
+      def present_options(model, tenant_key, user_key)
+        return unless PresentOptions.enabled?
+
+        PresentOptions.new.keys(model, vocabulary_labels(model, tenant_key, user_key), tenant_key: tenant_key)
+      end
+
       # The label's storage key the query names, or nil for a choice label
       # whose option answer is NO_OPTION.
-      def storage_key(label, answers, tenant_key)
+      def storage_key(label, answers, tenant_key, present)
         return label.key unless label.type == :choice
 
         option = answers.choice("option__#{label.question_key}")
-        "#{label.key}:#{option}" if option != NO_OPTION && label.options(tenant_key).key?(option)
+        "#{label.key}:#{option}" if option != NO_OPTION && PresentOptions.options(label, tenant_key, present).key?(option)
       end
 
-      def vocabulary_state(labels, tenant_key)
+      def vocabulary_state(labels, tenant_key, present)
         labels.transform_values do |label|
           entry = { "description" => label.description }
           if label.type == :choice
-            entry["options"] = label.options(tenant_key).keys
-            names = label.option_names(tenant_key)
+            options = PresentOptions.options(label, tenant_key, present).keys
+            entry["options"] = options
+            names = label.option_names(tenant_key).slice(*options)
             entry["option_names"] = names if names.any?
           end
           entry
@@ -206,16 +229,19 @@ module Truffler
       # keyword naming an applied label becomes a label term; stopwords and
       # filler words become filler unless they are all that would be left of
       # an encoding that applies no label and no time range (Search::Filler).
-      # A word Jev called a label term that names no applied label locally
-      # is sourced to every applied label.
-      def reconcile(query, answered, names)
+      # A word naming any declared label (`keep`, see Filler.label_words) is
+      # never filler, even when Jev calls it that. A word Jev called a label
+      # term that names no applied label locally is sourced to every applied
+      # label.
+      def reconcile(query, answered, names, keep)
         roles = query.tokens.each_index.to_h do |position|
-          [ position, query.time_position?(position) ? "time" : answered.fetch(position, "keyword") ]
+          role = query.time_position?(position) ? "time" : answered.fetch(position, "keyword")
+          [ position, role == "filler" && keep.include?(query.tokens[position].singularize) ? "keyword" : role ]
         end
         matches = roles.keys.to_h { |position| [ position, roles[position] == "time" ? {} : label_matches(query.tokens[position], names) ] }
         words = roles.keys.select { |position| roles[position] == "keyword" && !query.exact_tokens.include?(query.tokens[position]) }
         words.each { |position| roles[position] = "label_term" if matches[position].any? }
-        filler = words.select { |position| roles[position] == "keyword" && Search::Filler.word?(query.tokens[position]) }
+        filler = words.select { |position| roles[position] == "keyword" && Search::Filler.word?(query.tokens[position], keep: keep) }
         Search::Filler.drop(filler, keyword_count: roles.values.count("keyword"), anchored: names.any? || !query.time_phrase.nil?,
           stopword: ->(position) { Search::Filler.stopword?(query.tokens[position]) })
           .each { |position| roles[position] = "filler" }

data/lib/truffler/query_encoding/present_options.rb

@@ -0,0 +1,54 @@
+module Truffler
+  module QueryEncoding
+    # With `config.skip_empty_options`, the choice options a tenant actually
+    # has: storage keys with at least one label row at or above
+    # `config.choice_min_probability` (above 0.0 when that is nil). Query
+    # encoding offers Jev only these, so it cannot pick "Source: email" in a
+    # tenant with no email sources.
+    #
+    # The set is digested into the encoding cache key, so encodings refresh
+    # when an option appears. It is read through the cache store for TTL, so
+    # a keystroke stays one SELECT; a new option reaches query encoding (and
+    # the cache key) within TTL.
+    class PresentOptions
+      TTL = 5.minutes
+
+      def initialize(store: Truffler.config.cache_store)
+        @store = store
+      end
+
+      def self.enabled?
+        Truffler.config.skip_empty_options == true
+      end
+
+      # The present storage keys among `labels`' choice options, or nil when
+      # skip_empty_options is off.
+      def keys(model, labels, tenant_key:)
+        return unless self.class.enabled?
+
+        candidates = labels.values.select { |label| label.type == :choice }.flat_map { |label| label.storage_keys(tenant_key) }.sort
+        return Set.new if candidates.empty?
+
+        min = Truffler.config.choice_min_probability
+        cache_key = "truffler/present_options/#{Canonical.digest(record_type: model.polymorphic_name, tenant_key: tenant_key&.to_s,
+          keys: candidates, min: min)}"
+        Set.new(@store.fetch(cache_key, expires_in: TTL) { present(model, candidates, tenant_key, min) })
+      end
+
+      # A label's options narrowed to `present` (all of them when nil).
+      def self.options(label, tenant_key, present)
+        options = label.options(tenant_key)
+        present ? options.select { |option, _| present.include?("#{label.key}:#{option}") } : options
+      end
+
+      private
+
+      def present(model, candidates, tenant_key, min)
+        rows = Records::Label.where(record_type: model.polymorphic_name, label_key: candidates)
+        rows = rows.where(tenant_key: tenant_key.to_s) if model.truffler_definition.scoped?
+        rows = min ? rows.where(value: min..) : rows.where(Records::Label.arel_table[:value].gt(0.0))
+        rows.distinct.pluck(:label_key).sort
+      end
+    end
+  end
+end

data/lib/truffler/records/backfill_spend.rb

@@ -1,11 +1,12 @@
 module Truffler
   module Records
-    # The backfill spend ledger: one row per model and app-wide vocabulary
-    # version, so a spend cap holds across runs, reruns, and overlapping
-    # BackfillJob chains. Spend is reserved and settled in SQL, never read,
-    # added to, and written back.
+    # The backfill spend ledger: one row per model, tenant (nil for the
+    # app-wide ledger), and vocabulary version, so a spend cap holds across
+    # runs, reruns, and overlapping BackfillJob chains. Spend is reserved and
+    # settled in SQL, never read, added to, and written back.
     class BackfillSpend < ActiveRecord::Base
       self.table_name = "truffler_backfill_spends"
+      TENANT_KEY_RECHECK = 1.minute
 
       scope :for_model, ->(model) { where(record_type: model.polymorphic_name) }
 
@@ -18,10 +19,63 @@ module Truffler
         false
       end
 
-      def self.ledger(model, version)
-        create_or_find_by!(record_type: model.polymorphic_name, vocabulary_version: version)
+      def self.for_ledger(model, tenant_key)
+        tenant_ledgers? ? for_model(model).where(tenant_key: tenant_key) : for_model(model)
       end
 
+      # Before `rails g truffler:upgrade` adds tenant_key, every tenant
+      # shares the app-wide row. Before 0.1.6 rows were keyed by the whole
+      # vocabulary version; pass it as `legacy_version:` and the first lookup
+      # takes that row over instead of starting from zero.
+      def self.ledger(model, version, tenant_key: nil, legacy_version: nil)
+        attributes = { record_type: model.polymorphic_name, vocabulary_version: version }
+        attributes[:tenant_key] = tenant_key if tenant_ledgers?
+        adopt(attributes, legacy_version) if legacy_version && legacy_version != version
+        create_or_find_by!(attributes)
+      end
+
+      def self.adopt(attributes, legacy_version)
+        return if exists?(attributes)
+
+        where(attributes.merge(vocabulary_version: legacy_version)).update_all(vocabulary_version: attributes[:vocabulary_version])
+      rescue ActiveRecord::RecordNotUnique
+        nil
+      end
+      private_class_method :adopt
+
+      # A worker booted before `db:migrate` added tenant_key has the old
+      # columns cached, so a miss reloads them at most once per
+      # TENANT_KEY_RECHECK and the worker moves to tenant ledgers without a
+      # restart.
+      def self.tenant_ledgers?
+        return true if column_names.include?("tenant_key")
+
+        if recheck_tenant_key?
+          reset_column_information
+          return true if column_names.include?("tenant_key")
+        end
+        warn_missing_tenant_key
+        false
+      end
+
+      def self.recheck_tenant_key?
+        now = Time.current
+        return false if @tenant_key_checked_at && now - @tenant_key_checked_at < TENANT_KEY_RECHECK
+
+        @tenant_key_checked_at = now
+        true
+      end
+      private_class_method :recheck_tenant_key?
+
+      def self.warn_missing_tenant_key
+        return if @missing_tenant_warned
+
+        @missing_tenant_warned = true
+        Truffler.config.logger.warn("[truffler] #{table_name}.tenant_key is missing, so backfill spend caps are app-wide. " \
+          "Run `bin/rails g truffler:upgrade && bin/rails db:migrate`.")
+      end
+      private_class_method :warn_missing_tenant_key
+
       def self.warn_missing
         return if @missing_warned
 

data/lib/truffler/redaction.rb

@@ -6,7 +6,7 @@ module Truffler
   module Redaction
     KEYS = %i[
       priority model cost input_tokens tokens_estimated latency_ms error_class status outcome reason
-      record_type tenant_key user_key surface section sources vocabulary_version label_key
+      record_type tenant_key user_key surface section sources vocabulary_version label_key permanent
     ].to_set.freeze
     SUFFIXES = %w[_id _ids _count _digest _ms].freeze
 

data/lib/truffler/search/encoding.rb

@@ -55,21 +55,40 @@ module Truffler
 
       # The encoding minus the chips the searcher removed (R20), matched by
       # storage key or by label key; "time" removes the time range. A label
-      # term whose every source label is gone becomes a keyword again.
-      def without(suppressed)
+      # term whose every source label is gone becomes a keyword again. `keep_words`
+      # (a set, or a callable returning one) holds the words that are never
+      # filler, as in `keywords`.
+      def without(suppressed, keep_words: nil)
         suppressed = Array(suppressed).map(&:to_s).to_set
         return self if suppressed.empty?
 
         keep = ->(key, _) { !suppressed.include?(key) && !suppressed.include?(self.class.split_key(key).first) }
         kept = { filters: filters.select(&keep), boosts: boosts.select(&keep), intent_vector: intent_vector.select(&keep) }
         applied = kept.values.flat_map(&:keys).to_set
-        freed = label_term_sources.select { |_, keys| keys.any? && keys.none? { |key| applied.include?(key) } }.keys
+        freed = freed_words(applied)
         kept_time = (time unless suppressed.include?(TimeRange.key))
         keywords = keyword_tokens && (keyword_tokens + freed).uniq
-        keywords = Filler.keywords(keywords + filler_tokens, anchored: false) if keywords && applied.empty? && kept_time.nil?
-        with(**kept, time: kept_time, label_term_tokens: label_term_tokens - freed,
-          label_term_sources: label_term_sources.except(*freed), soft_keyword_tokens: soft_keyword_tokens - freed,
-          keyword_tokens: keywords)
+        if keywords && applied.empty? && kept_time.nil?
+          keywords = Filler.keywords(keywords + filler_tokens, anchored: false, keep: keep_words.respond_to?(:call) ? keep_words.call : keep_words)
+        end
+        with(**kept, time: kept_time, **free(freed, keywords))
+      end
+
+      # The encoding with the filters `keys` demoted to soft boosts (zero-result
+      # relaxation, see Relaxation): they stay in the intent vector, at their
+      # intent weight or DEFAULT_BOOST when filtering gave them none, so records
+      # that match still rank first, but nothing requires them. A label term
+      # whose every applied source was relaxed becomes a keyword again, as in
+      # `without`.
+      def relax(keys)
+        relaxed = filters.slice(*Array(keys).map(&:to_s))
+        return self if relaxed.empty?
+
+        soft = relaxed.to_h { |key, _| [ key, intent_vector[key] || boosts[key] || QueryEncoding::DEFAULT_BOOST ] }
+        kept_filters = filters.except(*relaxed.keys)
+        freed = freed_words((kept_filters.keys + boosts.keys + intent_vector.keys).to_set - relaxed.keys)
+        with(filters: kept_filters, boosts: boosts.merge(soft), intent_vector: intent_vector.merge(soft),
+          **free(freed, keyword_tokens && (keyword_tokens + freed).uniq))
       end
 
       # Splits a storage key into its label key and choice option. Lens keys
@@ -85,10 +104,12 @@ module Truffler
       end
 
       # Without encoder decisions (a cold cache), every search token that is
-      # not a label term, minus filler words (see Filler).
-      def keywords(query)
+      # not a label term, minus filler words (see Filler). `keep` is called
+      # only then, for the words that are never filler.
+      def keywords(query, keep: nil)
         keyword_tokens ||
-          Filler.keywords(query.search_tokens - label_term_tokens, anchored: !empty? || !time.nil?, exact: query.exact_tokens)
+          Filler.keywords(query.search_tokens - label_term_tokens, anchored: !empty? || !time.nil?, exact: query.exact_tokens,
+            keep: keep&.call)
       end
 
       # The cache form: decisions plus token positions in the normalized
@@ -107,6 +128,16 @@ module Truffler
 
       private
 
+      # Label terms none of whose source keys is still applied.
+      def freed_words(applied)
+        label_term_sources.select { |_, keys| keys.any? && keys.none? { |key| applied.include?(key) } }.keys
+      end
+
+      def free(freed, keywords)
+        { label_term_tokens: label_term_tokens - freed, label_term_sources: label_term_sources.except(*freed),
+          soft_keyword_tokens: soft_keyword_tokens - freed, keyword_tokens: keywords }
+      end
+
       def weights(hash)
         hash.to_h.to_h { |key, weight| [ key.to_s, Float(weight) ] }
       end

data/lib/truffler/search/encoding_cache.rb

@@ -62,7 +62,18 @@ module Truffler
         definition = model.truffler_definition
         tenant = tenant_key&.to_s if definition.per_tenant_vocabulary?
         version = definition.vocabulary.encoding_version(tenant_key: tenant_key&.to_s, user_key: user_key)
-        Canonical.digest(record_type: model.polymorphic_name, query: query.normalized, vocabulary_version: version, tenant_key: tenant)
+        parts = { record_type: model.polymorphic_name, query: query.normalized, vocabulary_version: version, tenant_key: tenant }
+        present = present_options(model, tenant_key, user_key)
+        Canonical.digest(present ? parts.merge(present_options: Canonical.digest(present.to_a.sort)) : parts)
+      end
+
+      # With skip_empty_options, the tenant's present choice options (see
+      # QueryEncoding::PresentOptions), so an encoding refreshes when one appears.
+      def present_options(model, tenant_key, user_key)
+        return unless QueryEncoding::PresentOptions.enabled?
+
+        labels = model.truffler_definition.vocabulary.labels_for(tenant_key: tenant_key&.to_s, user_key: user_key)
+        QueryEncoding::PresentOptions.new.keys(model, labels, tenant_key: tenant_key&.to_s)
       end
     end
   end

data/lib/truffler/search/filler.rb

@@ -1,7 +1,7 @@
 module Truffler
   module Search
     # Words that carry no search meaning on their own: common stopwords and
-    # `config.filler_words` (generic nouns such as "customers" or "emails",
+    # `config.filler_words` (generic nouns such as "customers" or "items",
     # matched ignoring plurals). One rule serves the encoder's reconcile and
     # the cold-cache keywords: filler is dropped as a keyword unless dropping
     # it would leave the search with no keyword, no applied label, and no
@@ -12,7 +12,7 @@ module Truffler
         them there they this to up us was we what when where which who why with you your
       ].to_set.freeze
 
-      DEFAULT_WORDS = %w[customer customers people person user users message messages email emails item items stuff thing things].freeze
+      DEFAULT_WORDS = %w[customer customers people person user users message messages item items stuff thing things].freeze
 
       module_function
 
@@ -20,11 +20,34 @@ module Truffler
         STOPWORDS.include?(word.to_s.downcase)
       end
 
-      def word?(word, filler_words: Truffler.config.filler_words)
+      # `keep` holds singular words that are never filler (see label_words).
+      def word?(word, filler_words: Truffler.config.filler_words, keep: nil)
         word = word.to_s.downcase
+        return false if keep&.include?(word.singularize)
+
         stopword?(word) || Array(filler_words).any? { |filler| filler.to_s.downcase.singularize == word.singularize }
       end
 
+      # Singular words that name one of the model's declared labels for this
+      # tenant, applied or not: words of a label key, of a choice option key,
+      # and of an option's search text (not its description, which is prose).
+      # Such a word is never filler, so "text messages" still searches
+      # "messages" when an option's search text is "text message".
+      def label_words(definition, tenant_key = nil)
+        definition.labels.each_value.with_object(Set.new) do |label, words|
+          next unless label.available?(tenant_key)
+
+          names = [ label.key ]
+          if label.type == :choice
+            options = label.encoding_wording(tenant_key)[:options]
+            names.concat(options.keys, options.values.filter_map { |entry| entry[:search] })
+          end
+          names.each do |name|
+            name.to_s.downcase.split(/[^\p{Alnum}]+/).each { |word| words << word.singularize unless word.empty? || stopword?(word) }
+          end
+        end
+      end
+
       # The subset of `candidates` (droppable keywords) to drop, given how many
       # keywords there are and whether a label or time range is applied. When
       # nothing anchors the search and only droppable words are left, the
@@ -38,9 +61,11 @@ module Truffler
       end
 
       # `tokens` minus filler words, or only minus stopwords when nothing else
-      # would anchor the search. Exact tokens (a quoted "the") are never filler.
-      def keywords(tokens, anchored:, exact: [])
-        dropped = drop(tokens.select { |token| !exact.include?(token) && word?(token) }, keyword_count: tokens.size, anchored: anchored)
+      # would anchor the search. Exact tokens (a quoted "the") and `keep`
+      # words are never filler.
+      def keywords(tokens, anchored:, exact: [], keep: nil)
+        dropped = drop(tokens.select { |token| !exact.include?(token) && word?(token, keep: keep) }, keyword_count: tokens.size,
+          anchored: anchored)
         tokens - dropped
       end
     end

data/lib/truffler/search/keystroke.rb

@@ -36,26 +36,36 @@ module Truffler
       end
 
       def call
-        started = Instrumentation.monotonic_ms
-        watermark = Time.current
-        explicit_action = surface_action
-        cached = read_encoding
-        status = encoding_status(cached)
-        encoding = with_time(visible_lenses_only(cached&.without(suppressed), record_usage: true))
-        sql = sql(encoding)
-        records = sql.relation(scope, limit: limit).to_a
-        result = Result.new(records: records, query: query, encoding: encoding, encoding_status: status, watermark: watermark,
-          explicit_action: explicit_action, sources: sql.sources, invite_row: invite_row(records, cached),
-          weights: @weights, recount: ->(since) { count(since: since) })
-        instrument(result, started)
-        result
+        Current.scope do
+          started = Instrumentation.monotonic_ms
+          watermark = Time.current
+          explicit_action = surface_action
+          cached = read_encoding
+          status = encoding_status(cached)
+          encoding = visible_lenses_only(with_time(cached)&.without(suppressed, keep_words: label_words), record_usage: true)
+          sql = sql(encoding)
+          records = sql.relation(scope, limit: limit).to_a
+          relaxed = relax(encoding, records)
+          records, encoding, sql = relaxed.records, relaxed.encoding, sql(relaxed.encoding) if relaxed
+          relaxed_labels = relaxed&.relaxed_labels.to_a
+          result = Result.new(records: records, query: query, encoding: encoding, encoding_status: status, watermark: watermark,
+            explicit_action: explicit_action, sources: sql.sources, invite_row: invite_row(records, cached, status),
+            local_weak: local_weak?(records, cached), weights: @weights, relaxed_labels: relaxed_labels,
+            recount: ->(since) { count(since: since, relaxed: relaxed_labels) })
+          instrument(result, started)
+          result
+        end
       end
 
       # How many records the same search would return that arrived after
-      # `since` (R25). Reads the cache only and never prefetches.
-      def count(since:)
-        sql(with_time(visible_lenses_only(read_encoding&.without(suppressed)))).candidates(scope)
-          .where(model.arel_table[@definition.arrived_at_column].gt(since)).count
+      # `since` (R25), with the filters `relaxed` demoted as the search did.
+      # Reads the cache only and never prefetches.
+      def count(since:, relaxed: [])
+        Current.scope do
+          encoding = visible_lenses_only(with_time(read_encoding)&.without(suppressed, keep_words: label_words))
+          sql(relaxed.any? ? encoding&.relax(relaxed) : encoding).candidates(scope)
+            .where(model.arel_table[@definition.arrived_at_column].gt(since)).count
+        end
       end
 
       private
@@ -83,6 +93,10 @@ module Truffler
 
       # Drops lens keys this searcher cannot see (another user's personal
       # lens, an expired lens) and counts a use of the rest (R42, R43).
+      def label_words
+        -> { Filler.label_words(@definition, tenant_key) }
+      end
+
       def visible_lenses_only(encoding, record_usage: false)
         lens_keys = encoding ? (encoding.intent_vector.keys | encoding.filters.keys | encoding.boosts.keys).select { |key| lens_id(key) } : []
         return encoding if lens_keys.empty?
@@ -90,11 +104,12 @@ module Truffler
         visible = Lenses.labels(model, tenant_key: tenant_key, user_key: user_key).values.map(&:lens_id).uniq
         hidden, shown = lens_keys.partition { |key| !visible.include?(lens_id(key)) }
         Lenses.record_usage(shown.map { |key| lens_id(key) }.uniq) if record_usage
-        encoding.without(hidden)
+        encoding.without(hidden, keep_words: label_words)
       end
 
       # The query's time phrase, resolved on this search's clock, unless the
-      # searcher removed its chip.
+      # searcher removed its chip. Attached before `without`, which keeps
+      # filler dropped while a time range still anchors the search.
       def with_time(encoding)
         phrase = query.time_phrase
         return encoding if phrase.nil? || suppressed.include?(TimeRange.key)
@@ -133,23 +148,39 @@ module Truffler
         Sql.new(model, tenant_key: tenant_key, query: query, encoding: encoding, vector: read_vector, weights: @weights)
       end
 
-      # R21: the Smart search row. A model with no local text search whose
-      # encoding is not cached yet invites the action even when a blind index
-      # matched, because intent queries resolve on the action there (AE10).
-      def invite_row(records, cached)
+      # Only an empty result under filters pays for relaxation.
+      def relax(encoding, records)
+        return unless records.empty? && encoding&.filters&.any?
+
+        Relaxation.new(model, encoding, sql: method(:sql)).call(scope, limit: limit)
+      end
+
+      # R21: the Smart search row. A query whose encoding is not cached yet
+      # invites the action even when a blind index or keyword matched,
+      # because a first-time intent query resolves on the action (AE10): on a
+      # model with no local text search always, and with a `keyword` source
+      # while the encoding is in flight unless `invite_on_pending_encoding false`.
+      def invite_row(records, cached, status)
         return if query.blank?
 
-        reason =
-          if @definition.keyword.blank? && cached.nil? then :encoding_pending
-          elsif records.empty? then :empty
-          elsif records.size < @definition.weak_below then :weak
-          end
+        pending = cached.nil? && (@definition.keyword.blank? || (@definition.invite_on_pending_encoding && status == :pending))
+        reason = pending ? :encoding_pending : weak_reason(records)
         { query: query.raw.strip, reason: reason } if reason
       end
 
+      def weak_reason(records)
+        if records.empty? then :empty
+        elsif records.size < @definition.weak_below then :weak
+        end
+      end
+
+      def local_weak?(records, cached)
+        !query.blank? && (weak_reason(records).present? || (@definition.keyword.blank? && cached.nil?))
+      end
+
       def instrument(result, started)
         payload = { record_type: model.polymorphic_name, tenant_key: tenant_key, surface: surface, outcome: result.encoding_status,
-          result_count: result.records.size, filter_count: result.encoding&.filters&.size.to_i,
+          result_count: result.records.size, filter_count: result.encoding&.filters&.size.to_i, relaxed_count: result.relaxed_labels.size,
           boost_count: result.encoding&.intent_vector&.size.to_i, sources: result.sources.map(&:to_s),
           reason: result.invite_row&.dig(:reason), latency_ms: Instrumentation.elapsed_ms(started) }
         payload[:query_digest] = Misses.digest(:query, query.normalized) if Misses.encrypted_model?(model)