truffler 0.1.0 → 0.1.2

data/lib/truffler/labeling/backfill.rb

@@ -11,39 +11,84 @@ module Truffler
     # done, and records already current are skipped, so a rerun never asks
     # Jev about them again. A spend cap stops the run before a request would
     # exceed it; host-supplied labels cost nothing, so they are still written
-    # once the cap is reached. A Jev error releases the claimed rows and ends
+    # once the cap is reached. The cap counts everything spent under the
+    # model's current app-wide vocabulary version, kept in the
+    # truffler_backfill_spends ledger, so reruns and overlapping jobs share
+    # it; without that table it falls back to this run plus `spent:`. A Jev error releases the claimed rows and ends
     # the run with `:client_error`, so the caller keeps the spend metered so far.
+    #
+    # A budget denial ends the run with `:budget_denied`, unless the run
+    # waits: then it backs off (see .backoff) and retries from the same
+    # cursor until it completes, reaches the spend cap, or runs out of
+    # `max_duration` seconds, which pauses it with the cursor to resume from.
     class Backfill
-      Result = Data.define(:status, :labeled, :requests, :cost, :cursor)
+      Result = Data.define(:status, :labeled, :requests, :cost, :cursor, :retry_after)
+
+      INITIAL_BACKOFF = 1.0
+      MAX_BACKOFF = 30.0
+
+      class_attribute :sleeper, default: ->(seconds) { sleep(seconds) }
+      class_attribute :clock, default: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
 
       STATES = Records::RecordState.table_name
 
       class SpendCapReached < StandardError; end
 
       # Wraps the client to meter spend per request and refuse a request whose
-      # estimated cost would push spend past the cap.
+      # estimated cost would push spend past the cap. With a ledger the
+      # estimate is reserved in SQL before the request and settled to the
+      # reported cost after it, so concurrent meters on one ledger never
+      # both take the last of the cap.
       class SpendMeter
-        attr_reader :spent, :requests
+        attr_reader :requests, :cost
 
-        def initialize(client, cap:, spent:, config: Truffler.config)
+        def initialize(client, cap:, spent:, ledger: nil, config: Truffler.config)
           @client = client
           @cap = cap
           @spent = spent.to_f
+          @ledger = ledger
           @requests = 0
+          @cost = 0.0
           @config = config
         end
 
-        def ask(state:, questions:, **options)
-          raise SpendCapReached if @cap && @spent + estimate(state, questions) > @cap
+        def spent
+          @ledger ? @ledger.total : @spent
+        end
 
-          answers = @client.ask(state: state, questions: questions, **options)
-          @requests += 1
-          @spent += answers.usage&.cost.to_f
+        def ask(state:, questions:, **options)
+          estimate = estimate(state, questions)
+          reserve(estimate)
+          answers = nil
+          begin
+            answers = @client.ask(state: state, questions: questions, **options)
+          ensure
+            @ledger&.settle(-estimate, requests: 0) unless answers
+          end
+          record(answers.usage&.cost.to_f, estimate)
           answers
         end
 
         private
 
+        def record(cost, estimate)
+          @requests += 1
+          @cost += cost
+          if @ledger
+            @ledger.settle(cost - estimate)
+          else
+            @spent += cost
+          end
+        end
+
+        def reserve(estimate)
+          if @ledger
+            raise SpendCapReached unless @ledger.reserve(estimate, @cap)
+          elsif @cap && @spent + estimate > @cap
+            raise SpendCapReached
+          end
+        end
+
         def estimate(state, questions)
           @config.cost_for(Tokens.estimate({ state: state, questions: questions }))
         end
@@ -53,40 +98,72 @@ module Truffler
         new(model).status
       end
 
+      # The ledger row for the model's current vocabulary version, or nil
+      # when nothing was spent yet or the ledger table is missing.
+      def self.spend(model)
+        return unless Records::BackfillSpend.available?
+
+        Records::BackfillSpend.for_model(model).find_by(vocabulary_version: ledger_version(model))
+      end
+
+      # Zeroes the current vocabulary version's ledger in place, so a chain
+      # still running keeps its row and continues against the fresh total.
+      def self.reset_spend!(model)
+        return unless Records::BackfillSpend.available?
+
+        Records::BackfillSpend.for_model(model).where(vocabulary_version: ledger_version(model))
+          .update_all(spent_usd: 0.0, requests: 0, updated_at: Time.current)
+      end
+
+      def self.ledger_version(model)
+        model.truffler_definition.vocabulary.version(all_users: true)
+      end
+
+      # Seconds to wait after `denials` consecutive budget denials with no
+      # work in between: 1, 2, 4, ... capped at MAX_BACKOFF, or the budget's
+      # retry hint when that is longer.
+      def self.backoff(denials, retry_after = nil)
+        [ [ INITIAL_BACKOFF * (2**denials), MAX_BACKOFF ].min, retry_after.to_f ].max
+      end
+
       attr_reader :model, :batch_size, :page_size
 
       def initialize(model, spend_cap: Truffler.config.backfill_spend_cap, batch_size: Truffler.config.batch_size,
         page_size: nil, cursor: nil, spent: 0.0, client: Truffler.config.client, budget: Budget.new)
         @model = model
+        model.truffler_definition.validate_columns!
         @batch_size = batch_size
         @page_size = page_size || batch_size * 5
         @cursor = cursor
-        @meter = SpendMeter.new(client, cap: spend_cap, spent: spent)
+        @spend_cap = spend_cap
+        @spent = spent
+        @client = client
         @budget = budget
         @versions = {}
       end
 
-      def run(max_pages: nil)
+      # `progress` is called with the result so far and the delay before each
+      # wait; it carries counts, cost, and the cursor, never record text.
+      def run(max_pages: nil, wait: false, max_duration: nil, sleeper: self.class.sleeper, clock: self.class.clock,
+        progress: nil)
         @labeled = 0
-        @started_spent = @meter.spent
-        cursor = @cursor
-        pages = 0
+        @started_cost = meter.cost
+        @pages = 0
+        deadline = max_duration && clock.call + max_duration
+        denials = 0
 
         loop do
-          scanned, rows = page(cursor)
-          return result(:complete, nil) if scanned.empty?
-          return result(:paused, cursor) if max_pages && pages >= max_pages
+          before = [ @labeled, meter.requests ]
+          status = sweep(max_pages, deadline, clock)
+          return result(status) unless wait && status == :budget_denied
 
-          rows.group_by(&:last).each do |tenant_key, tenant_rows|
-            tenant_rows.map(&:first).each_slice(batch_size) do |ids|
-              stop = label(ids, tenant_key)
-              return result(stop, cursor) if stop
-            end
-          end
+          denials = 0 unless before == [ @labeled, meter.requests ]
+          delay = self.class.backoff(denials, @retry_after)
+          return result(:paused) if deadline && clock.call + delay > deadline
 
-          cursor = scanned.last
-          pages += 1
-          return result(:complete, nil) if scanned.size < page_size
+          progress&.call(result(status), delay)
+          sleeper.call(delay)
+          denials += 1
         end
       end
 
@@ -117,13 +194,47 @@ module Truffler
         @queue ||= Queue.new(model)
       end
 
+      # Spend carried in with `spent:` is ignored when the ledger holds it.
+      def meter
+        @meter ||= begin
+          ledger = Records::BackfillSpend.ledger(model, version_for(nil)) if Records::BackfillSpend.available?
+          SpendMeter.new(@client, cap: @spend_cap, spent: ledger ? 0.0 : @spent, ledger: ledger)
+        end
+      end
+
       def version_for(tenant_key)
         @versions[tenant_key] ||= definition.vocabulary.version(tenant_key: tenant_key, all_users: true)
       end
 
-      def result(status, cursor)
-        Result.new(status: status, labeled: @labeled, requests: @meter.requests, cost: @meter.spent - @started_spent,
-          cursor: cursor)
+      def result(status)
+        Result.new(status: status, labeled: @labeled, requests: meter.requests, cost: meter.cost - @started_cost,
+          cursor: @cursor, retry_after: (@retry_after if status == :budget_denied))
+      end
+
+      # Walks pages below @cursor until done or stopped, returning the status.
+      def sweep(max_pages, deadline, clock)
+        @retry_after = nil
+        loop do
+          scanned, rows = page(@cursor)
+          return complete if scanned.empty?
+          return :paused if (max_pages && @pages >= max_pages) || (deadline && clock.call >= deadline)
+
+          rows.group_by(&:last).each do |tenant_key, tenant_rows|
+            tenant_rows.map(&:first).each_slice(batch_size) do |ids|
+              stop = label(ids, tenant_key)
+              return stop if stop
+            end
+          end
+
+          @cursor = scanned.last
+          @pages += 1
+          return complete if scanned.size < page_size
+        end
+      end
+
+      def complete
+        @cursor = nil
+        :complete
       end
 
       # Returns the scanned ids (for the cursor) and the [id, tenant_key] rows
@@ -172,10 +283,14 @@ module Truffler
         return if claimed.empty?
 
         begin
-          Labeler.new(model, client: @meter, budget: @budget).label(claimed, priority: :backfill)
-        rescue BudgetExhausted, SpendCapReached => error
+          Labeler.new(model, client: meter, budget: @budget).label(claimed, priority: :backfill)
+        rescue SpendCapReached
+          queue.demote(claimed)
+          return :spend_cap_reached
+        rescue BudgetExhausted => error
           queue.demote(claimed)
-          return error.is_a?(SpendCapReached) ? :spend_cap_reached : :budget_denied
+          @retry_after = error.retry_after
+          return :budget_denied
         rescue ClientError, IncompleteAnswers => error
           queue.release(claimed, error)
           return :client_error

data/lib/truffler/labeling/labeler.rb

@@ -14,6 +14,7 @@ module Truffler
 
       def initialize(model, client: Truffler.config.client, budget: Budget.new)
         @model = model
+        model.truffler_definition.validate_columns!
         @client = client
         @budget = budget
       end
@@ -49,7 +50,7 @@ module Truffler
             retry_failed_supplied(supplier.failed_ids, states_by_id)
             return Result.new(labeled: current.size, requests: index, cost: cost, demoted: true)
           end
-          raise BudgetExhausted, "no Jev budget for #{priority} labeling" if decision.denied?
+          raise BudgetExhausted.new("no Jev budget for #{priority} labeling", retry_after: decision.retry_after) if decision.denied?
 
           answers = client.ask(state: request.state, questions: request.questions, priority: decision.priority)
           cost += answers.usage&.cost.to_f

data/lib/truffler/labeling/queue.rb

@@ -10,6 +10,7 @@ module Truffler
 
       def initialize(model, config: Truffler.config)
         @model = model
+        model.truffler_definition.validate_columns!
         @config = config
       end
 

data/lib/truffler/labeling/supplied.rb

@@ -12,6 +12,7 @@ module Truffler
 
       def initialize(model)
         @model = model
+        model.truffler_definition.validate_columns!
         @failed_ids = Set.new
       end
 

data/lib/truffler/lenses/drafter.rb

@@ -84,7 +84,8 @@ module Truffler
       # regenerated.
       def vocabulary_for(model, scope, lens)
         definition = model.truffler_definition
-        declared = definition.labels.to_h { |key, label| [ key, label.question(scope.tenant_key) ] }
+        declared = definition.labels.select { |_, label| label.available?(scope.tenant_key) }
+          .to_h { |key, label| [ key, label.question(scope.tenant_key) ] }
         visible = Lenses.visible_lenses(model, tenant_key: scope.tenant_key, user_digest: scope.user_digest)
         visible = visible.reject { |other| other.id == lens&.id }
         declared.merge(visible.each_with_object({}) { |other, all| all.merge!(other.storage_questions) })

data/lib/truffler/model.rb

@@ -62,8 +62,9 @@ module Truffler
     # outside the watched columns, e.g. from the job that classified it.
     def truffler_refresh_labels!
       definition = self.class.truffler_definition
-      keys = definition.supplied_labels.map(&:key)
-      Labeling::Supplied.new(self.class).write([ [ self, keys ] ], tenant_key: definition.tenant_key_for(self)) if keys.any?
+      tenant_key = definition.tenant_key_for(self)
+      keys = definition.supplied_labels.select { |label| label.available?(tenant_key) }.map(&:key)
+      Labeling::Supplied.new(self.class).write([ [ self, keys ] ], tenant_key: tenant_key) if keys.any?
       self
     end
 
@@ -77,10 +78,10 @@ module Truffler
       end
 
       changed = saved_changes.keys
-      if changed.intersect?([ *definition.fields, definition.tenant_column ].compact)
+      if changed.intersect?(definition.relabel_columns)
         truffler_expire_labels
       else
-        watched = definition.supplied_labels.select { |label| changed.intersect?(label.watch) }
+        watched = definition.labels.values.select { |label| changed.intersect?(label.watch) }
         return if watched.empty?
 
         truffler_expire_labels(watched.map(&:key))
@@ -103,7 +104,7 @@ module Truffler
     def truffler_enqueue_embedding
       definition = self.class.truffler_definition
       return unless Embeddings.managed?(definition)
-      return unless previously_new_record? || saved_changes.keys.intersect?([ *definition.fields, definition.tenant_column ].compact)
+      return unless previously_new_record? || saved_changes.keys.intersect?(definition.relabel_columns)
 
       Jobs::EmbedJob.perform_later(self.class.polymorphic_name, id)
     end

data/lib/truffler/query_encoding/encoder.rb

@@ -2,12 +2,21 @@ module Truffler
   module QueryEncoding
     # Encodes one pending query (KTD9). It asks a fixed question set: each
     # label gets `filter | boost | ignore`, each choice label also gets its
-    # options plus `none`, and each of the first 12 word tokens gets
+    # options plus `Truffler::NO_OPTION`, and each of the first 12 word tokens gets
     # `keyword | label_term | filler`. Exact-text tokens (digits, dates,
     # quoted phrases, emails, identifiers) are keywords decided locally and
     # never asked (R18). Query text travels only in `state` ("query" and
     # "tokens"); a token question names its word by position, `tokens[n]`,
-    # so searcher text never lands in an instruction (R8).
+    # so searcher text never lands in an instruction (R8). The label
+    # vocabulary, with choice option display names, rides along in
+    # `state["labels"]` so a token question can tell a word that names a
+    # label from text to match.
+    #
+    # Jev's word roles are then reconciled locally: a keyword that names a
+    # label the query applies (its key, a word of its key, the chosen option,
+    # or a word of that option's display name, ignoring case and plurals, or
+    # sharing its first three letters) becomes a label term, and a common
+    # stopword becomes filler.
     #
     # Answers become a `Search::Encoding` with the KTD20 intent vector: boost
     # gives the declared boost, filter narrows and adds `filter_weight`
@@ -22,10 +31,14 @@ module Truffler
       }.freeze
       TOKEN_ROLES = {
         "keyword" => "A word to match in the record text",
-        "label_term" => "A word that names one of the labels above rather than text to match",
+        "label_term" => "A word that names one of the labels in `labels`, or one of its options, rather than text to match",
         "filler" => "A word that carries no meaning for the search"
       }.freeze
-      NO_OPTION = "none".freeze
+      NO_OPTION = Truffler::NO_OPTION
+      STOPWORDS = %w[
+        a about all an and any are at be by for from have i in is it me my now of on or our please so some that the their
+        them there they this to up us was we what when where which who why with you your
+      ].to_set.freeze
 
       Request = Data.define(:state, :questions, :token_ids, :exact_tokens, :unasked_tokens)
 
@@ -54,16 +67,20 @@ module Truffler
             criteria: options)
         end
 
-        words = query.tokens.each_with_index.reject { |token, _| query.exact_tokens.include?(token) }
+        words = query.tokens.each_with_index.reject do |token, position|
+          query.exact_tokens.include?(token) || query.time_position?(position)
+        end
         asked = words.first(MAX_TOKEN_QUESTIONS)
         token_ids = asked.to_h do |_token, position|
           id = :"token__#{position}"
-          questions.choice(id, instructions: %(In the search query, what is the word tokens[#{position}]?), criteria: TOKEN_ROLES)
+          questions.choice(id, instructions: %(In the search query, what is the word tokens[#{position}]? The labels it may name, ) +
+            %(with their options, are in `labels`.), criteria: TOKEN_ROLES)
           [ position, id.to_s ]
         end
 
-        Request.new(state: { "query" => query.normalized, "tokens" => query.tokens }, questions: questions.to_h, token_ids: token_ids,
-          exact_tokens: query.exact_tokens, unasked_tokens: words.drop(MAX_TOKEN_QUESTIONS).map(&:first))
+        state = { "query" => query.normalized, "tokens" => query.tokens, "labels" => vocabulary_state(labels, tenant_key) }
+        Request.new(state: state, questions: questions.to_h, token_ids: token_ids, exact_tokens: query.exact_tokens,
+          unasked_tokens: words.drop(MAX_TOKEN_QUESTIONS).map(&:first))
       end
 
       # Encodes the query pending under `cache_key`. Returns the encoding, or
@@ -103,6 +120,7 @@ module Truffler
         filters = {}
         boosts = {}
         intent = {}
+        terms = []
         labels(model, tenant_key, user_key).each_value do |label|
           key = storage_key(label, answers, tenant_key)
           next unless key
@@ -113,15 +131,15 @@ module Truffler
             intent[key] = label.filter_weight
           when "boost"
             boosts[key] = intent[key] = label.boost || DEFAULT_BOOST
+          else next
           end
+          terms.concat(label_terms(key, option_name(label, key, tenant_key)))
         end
 
-        roles = request.token_ids.transform_values { |id| answers.choice(id) }
         query = Search::Query.new(request.state["query"])
-        keyword_tokens = query.tokens.each_with_index.filter_map do |token, position|
-          token if roles.fetch(position, "keyword") == "keyword"
-        end
-        label_term_tokens = roles.filter_map { |position, role| query.tokens[position] if role == "label_term" }
+        roles = reconcile(query, request.token_ids.transform_values { |id| answers.choice(id) }, terms.uniq)
+        keyword_tokens = query.tokens.each_index.filter_map { |position| query.tokens[position] if roles[position] == "keyword" }
+        label_term_tokens = query.tokens.each_index.filter_map { |position| query.tokens[position] if roles[position] == "label_term" }
         Search::Encoding.new(filters: filters, boosts: boosts, intent_vector: intent, keyword_tokens: keyword_tokens,
           label_term_tokens: label_term_tokens)
       end
@@ -162,7 +180,7 @@ module Truffler
       end
 
       # The label's storage key the query names, or nil for a choice label
-      # whose option answer is `none`.
+      # whose option answer is NO_OPTION.
       def storage_key(label, answers, tenant_key)
         return label.key unless label.type == :choice
 
@@ -170,6 +188,63 @@ module Truffler
         "#{label.key}:#{option}" if option != NO_OPTION && label.options(tenant_key).key?(option)
       end
 
+      def vocabulary_state(labels, tenant_key)
+        labels.transform_values do |label|
+          entry = { "description" => label.description }
+          if label.type == :choice
+            options = label.options(tenant_key)
+            entry["options"] = options.keys
+            names = options.compact
+            entry["option_names"] = names if names.any?
+          end
+          entry
+        end
+      end
+
+      # {position => role} for every token. Time phrase words are "time";
+      # exact tokens and unasked words are keywords; a keyword naming an
+      # applied label becomes a label term; stopwords become filler unless
+      # they are all that would be left of an encoding that applies nothing.
+      def reconcile(query, answered, terms)
+        roles = query.tokens.each_index.to_h do |position|
+          [ position, query.time_position?(position) ? "time" : answered.fetch(position, "keyword") ]
+        end
+        words = roles.keys.select { |position| roles[position] == "keyword" && !query.exact_tokens.include?(query.tokens[position]) }
+        words.each { |position| roles[position] = "label_term" if names_label?(query.tokens[position], terms) }
+        stopwords = words.select { |position| roles[position] == "keyword" && STOPWORDS.include?(query.tokens[position]) }
+        return roles if terms.empty? && roles.values.count("keyword") == stopwords.size
+
+        stopwords.each { |position| roles[position] = "filler" }
+        roles
+      end
+
+      # "category:billing" names "category" and "billing"; "needs_action"
+      # names "needs_action", "needs", and "action". An option's display
+      # name adds its words minus stopwords: "p_17" shown as "Spiral writing
+      # tool" also names "spiral", "writing", and "tool".
+      STEM = 3
+
+      def label_terms(storage_key, option_name = nil)
+        label, option = Search::Encoding.split_key(storage_key)
+        keys = [ label.split(":").last, option ].compact.flat_map { |name| [ name.downcase, *name.downcase.split(/[^\p{Alnum}]+/) ] }
+        words = option_name.to_s.downcase.split(/[^\p{Alnum}]+/).reject { |word| STOPWORDS.include?(word) }
+        (keys + words).reject(&:empty?).map(&:singularize)
+      end
+
+      def option_name(label, storage_key, tenant_key)
+        return unless label.type == :choice
+
+        label.options(tenant_key)[Search::Encoding.split_key(storage_key).last]
+      end
+
+      # "angry" names "anger": the same word ignoring plurals, or two words of
+      # four letters or more that share their first three letters. Only the
+      # applied labels' terms are compared, so the loose match stays safe.
+      def names_label?(word, terms)
+        word = word.singularize
+        terms.any? { |term| term == word || (word.length > STEM && term.length > STEM && word[0, STEM] == term[0, STEM]) }
+      end
+
       def intent_instructions(label)
         %(How does the search query use the label "#{label.key}" (#{label.description})?)
       end

data/lib/truffler/records/backfill_spend.rb

@@ -0,0 +1,51 @@
+module Truffler
+  module Records
+    # The backfill spend ledger: one row per model and app-wide vocabulary
+    # version, so a spend cap holds across runs, reruns, and overlapping
+    # BackfillJob chains. Spend is reserved and settled in SQL, never read,
+    # added to, and written back.
+    class BackfillSpend < ActiveRecord::Base
+      self.table_name = "truffler_backfill_spends"
+
+      scope :for_model, ->(model) { where(record_type: model.polymorphic_name) }
+
+      # False on apps that upgraded the gem without running
+      # `rails g truffler:upgrade`; warns once per process.
+      def self.available?
+        return true if connection.data_source_exists?(table_name)
+
+        warn_missing
+        false
+      end
+
+      def self.ledger(model, version)
+        create_or_find_by!(record_type: model.polymorphic_name, vocabulary_version: version)
+      end
+
+      def self.warn_missing
+        return if @missing_warned
+
+        @missing_warned = true
+        Truffler.config.logger.warn("[truffler] #{table_name} is missing, so backfill spend caps apply per run only. " \
+          "Run `bin/rails g truffler:upgrade && bin/rails db:migrate`.")
+      end
+      private_class_method :warn_missing
+
+      # Adds `amount` unless that would pass `cap`; true when reserved.
+      def reserve(amount, cap)
+        scope = self.class.where(id: id)
+        scope = scope.where("spent_usd + ? <= ?", amount, cap) if cap
+        scope.update_all([ "spent_usd = spent_usd + ?, updated_at = ?", amount, Time.current ]) == 1
+      end
+
+      def settle(amount, requests: 1)
+        self.class.where(id: id)
+          .update_all([ "spent_usd = spent_usd + ?, requests = requests + ?, updated_at = ?", amount, requests, Time.current ])
+      end
+
+      def total
+        self.class.where(id: id).pick(:spent_usd).to_f
+      end
+    end
+  end
+end

data/lib/truffler/search/encoding.rb

@@ -12,7 +12,9 @@ module Truffler
     # - `keyword_tokens`: query tokens for the keyword source; nil means
     #   every token that is not a label term.
     # - `label_term_tokens`: tokens that named a label rather than a keyword.
-    Encoding = Data.define(:filters, :boosts, :intent_vector, :keyword_tokens, :label_term_tokens) do
+    # - `time`: the query's `TimeRange`, resolved at search time and never
+    #   cached, since "today" moves.
+    Encoding = Data.define(:filters, :boosts, :intent_vector, :keyword_tokens, :label_term_tokens, :time) do
       def self.load(value, query)
         return value if value.is_a?(self)
         return if value.nil?
@@ -23,12 +25,12 @@ module Truffler
           label_term_tokens: Array(value["label_term_positions"]).filter_map { |position| tokens[position] })
       end
 
-      def initialize(filters: {}, boosts: {}, intent_vector: nil, keyword_tokens: nil, label_term_tokens: [])
+      def initialize(filters: {}, boosts: {}, intent_vector: nil, keyword_tokens: nil, label_term_tokens: [], time: nil)
         filters = weights(filters)
         boosts = weights(boosts)
         intent_vector = weights(intent_vector || boosts).reject { |_, weight| weight.zero? }
         super(filters: filters.freeze, boosts: boosts.freeze, intent_vector: intent_vector.freeze,
-          keyword_tokens: keyword_tokens&.map(&:to_s)&.freeze, label_term_tokens: Array(label_term_tokens).map(&:to_s).freeze)
+          keyword_tokens: keyword_tokens&.map(&:to_s)&.freeze, label_term_tokens: Array(label_term_tokens).map(&:to_s).freeze, time: time)
       end
 
       def empty?
@@ -36,13 +38,14 @@ module Truffler
       end
 
       # The encoding minus the chips the searcher removed (R20), matched by
-      # storage key or by label key.
+      # storage key or by label key; "time" removes the time range.
       def without(suppressed)
         suppressed = Array(suppressed).map(&:to_s).to_set
         return self if suppressed.empty?
 
         keep = ->(key, _) { !suppressed.include?(key) && !suppressed.include?(self.class.split_key(key).first) }
-        with(filters: filters.select(&keep), boosts: boosts.select(&keep), intent_vector: intent_vector.select(&keep))
+        with(filters: filters.select(&keep), boosts: boosts.select(&keep), intent_vector: intent_vector.select(&keep),
+          time: (time unless suppressed.include?(TimeRange.key)))
       end
 
       # Splits a storage key into its label key and choice option. Lens keys
@@ -58,7 +61,7 @@ module Truffler
       end
 
       def keywords(query)
-        keyword_tokens || (query.tokens - label_term_tokens)
+        keyword_tokens || (query.search_tokens - label_term_tokens)
       end
 
       # The cache form: decisions plus token positions in the normalized

data/lib/truffler/search/keystroke.rb

@@ -10,9 +10,10 @@ module Truffler
       attr_reader :model, :query, :tenant_key, :scope, :user_key, :suppressed, :surface, :limit
 
       def initialize(model, query, tenant:, scope:, user: nil, suppressed: [], surface: nil, limit: DEFAULT_LIMIT, weights: {},
-        cache: EncodingCache.new)
+        cache: EncodingCache.new, clock: -> { Time.current })
         @model = model
         @definition = model.try(:truffler_definition) || raise(DefinitionError, "#{model.name} has no truffler declaration")
+        @definition.validate_columns!
         @query = Query.wrap(query)
         @tenant_key = tenant&.to_s
         @scope = scope.nil? && !@definition.scoped? ? model.all : scope
@@ -22,6 +23,7 @@ module Truffler
         @limit = limit
         @weights = @definition.ranking.merge(weights.to_h { |key, weight| [ key.to_sym, Float(weight) ] })
         @cache = cache
+        @clock = clock
         check_scope!
       end
 
@@ -39,7 +41,7 @@ module Truffler
         explicit_action = surface_action
         cached = read_encoding
         status = encoding_status(cached)
-        encoding = visible_lenses_only(cached&.without(suppressed), record_usage: true)
+        encoding = with_time(visible_lenses_only(cached&.without(suppressed), record_usage: true))
         sql = sql(encoding)
         records = sql.relation(scope, limit: limit).to_a
         result = Result.new(records: records, query: query, encoding: encoding, encoding_status: status, watermark: watermark,
@@ -52,7 +54,7 @@ module Truffler
       # How many records the same search would return that arrived after
       # `since` (R25). Reads the cache only and never prefetches.
       def count(since:)
-        sql(visible_lenses_only(read_encoding&.without(suppressed))).candidates(scope)
+        sql(with_time(visible_lenses_only(read_encoding&.without(suppressed)))).candidates(scope)
           .where(model.arel_table[@definition.arrived_at_column].gt(since)).count
       end
 
@@ -91,6 +93,15 @@ module Truffler
         encoding.without(hidden)
       end
 
+      # The query's time phrase, resolved on this search's clock, unless the
+      # searcher removed its chip.
+      def with_time(encoding)
+        phrase = query.time_phrase
+        return encoding if phrase.nil? || suppressed.include?(TimeRange.key)
+
+        (encoding || Encoding.new).with(time: phrase.range(@clock.call))
+      end
+
       def lens_id(key)
         prefix, id = key.split(":", 3)
         Integer(id, exception: false) if prefix == Lenses::KEY_PREFIX