truffler 0.1.3 → 0.1.5

data/lib/truffler/search/sql.rb

@@ -3,14 +3,26 @@ module Truffler
     # Builds the one keystroke query of KTD8, scored per KTD20:
     #
     #   w_label * SUM(weight * value) over the intent's nonzero label keys
-    #   + w_text * text similarity (inline SQL, or the store's top-K CASE)
+    #   + w_text * text similarity (a top-K join, inline SQL, or the store's top-K CASE)
     #   + w_keyword * keyword hit + w_exact * exact-source hit
+    #   + SOFT_KEYWORD * w_keyword * soft keyword hit
     #
     # A weighted dot product, not cosine: cosine would divide out magnitude
     # and let a record high on unrelated labels outrank the one the query
     # asked for. Hard filters are EXISTS subqueries that run before scoring.
+    #
+    # At scale: when every record past the filters is a candidate (label-only
+    # and filtered searches), label scores come from one grouped aggregate
+    # LEFT JOINed on record_id instead of a subquery per row. A store with
+    # `neighbors_sql` (NeighborStore on Postgres) is LEFT JOINed the same
+    # way, so text similarity is read from the tenant's top-K. Relation
+    # sources run once, as `id = ANY(ARRAY(subquery))` on Postgres.
     class Sql
       LABELS = "truffler_labels".freeze
+      LABEL_SCORES = "truffler_label_scores".freeze
+      NEIGHBORS = "truffler_neighbors".freeze
+      # Share of the keyword weight a soft keyword hit adds (see Encoding).
+      SOFT_KEYWORD = 0.25
 
       attr_reader :model, :tenant_key, :query, :encoding, :vector, :weights
 
@@ -35,7 +47,8 @@ module Truffler
       # filter the filter decides membership and text matches only rank.
       def candidates(scope)
         base = base(scope)
-        return base if label_only? || encoding.filters.any?
+        base = base.joins(Arel.sql(neighbors_join_sql)) if neighbors_sql
+        return base if every_base_record?
 
         conditions = [ keyword_sql, exact_sql, (text_candidate_sql if text_score_sql) ].compact
         base.where(Arel.sql(conditions.any? ? conditions.map { |condition| "(#{condition})" }.join(" OR ") : "1 = 0"))
@@ -43,19 +56,21 @@ module Truffler
 
       def relation(scope, limit: nil)
         relation = candidates(scope)
+        relation = relation.joins(Arel.sql(label_scores_join_sql)) if grouped_label_scores?
         relation = relation.select(Arel.sql("#{table}.*")) if relation.select_values.empty?
         relation = relation.select(*score_columns.map { |name, sql| Arel.sql("(#{sql}) AS #{name}") })
         relation = relation.reorder(*ordering)
         limit ? relation.limit(limit) : relation
       end
 
+      # The label term: a per-candidate subquery when sources narrow the
+      # candidates, else the grouped join's score.
       def label_score_sql
-        intent = encoding.intent_vector
-        return if intent.empty?
+        return if encoding.intent_vector.empty?
+        return "COALESCE(#{connection.quote_table_name(LABEL_SCORES)}.score, 0.0)" if grouped_label_scores?
 
-        cases = intent.map { |key, weight| "WHEN #{quote(key)} THEN #{Float(weight)} * #{label_column('value')}" }.join(" ")
-        "COALESCE((SELECT SUM(CASE #{label_column('label_key')} #{cases} ELSE 0.0 END) FROM #{quoted_labels} " \
-          "WHERE #{label_scope_sql} AND #{label_column('label_key')} IN (#{intent.keys.map { |key| quote(key) }.join(', ')})), 0.0)"
+        "COALESCE((SELECT SUM(#{label_case_sql}) FROM #{quoted_labels} " \
+          "WHERE #{label_scope_sql} AND #{label_keys_sql}), 0.0)"
       end
 
       def sources
@@ -64,7 +79,7 @@ module Truffler
       end
 
       def keywords
-        encoding.keywords(query)
+        @keywords ||= encoding.keywords(query, keep: -> { Filler.label_words(definition, tenant_key) })
       end
 
       private
@@ -112,6 +127,46 @@ module Truffler
         range.to ? scope.where(arrived_at.lt(range.to)) : scope
       end
 
+      def label_case_sql
+        cases = encoding.intent_vector.map { |key, weight| "WHEN #{quote(key)} THEN #{Float(weight)} * #{label_column('value')}" }
+        "CASE #{label_column('label_key')} #{cases.join(' ')} ELSE 0.0 END"
+      end
+
+      def label_keys_sql
+        "#{label_column('label_key')} IN (#{encoding.intent_vector.keys.map { |key| quote(key) }.join(', ')})"
+      end
+
+      # Every record past the tenant, time, and filters is a candidate.
+      def every_base_record?
+        label_only? || encoding.filters.any?
+      end
+
+      def grouped_label_scores?
+        encoding.intent_vector.any? && every_base_record?
+      end
+
+      # One aggregate over the tenant's rows for the intent's keys, which
+      # `index_truffler_labels_for_search` (with INCLUDE (record_id)) serves
+      # as an index-only scan.
+      def label_scores_join_sql
+        tenant = definition.scoped? ? " AND #{label_column('tenant_key')} = #{quote(tenant_key)}" : ""
+        scores = connection.quote_table_name(LABEL_SCORES)
+        "LEFT JOIN (SELECT #{label_column('record_id')} AS record_id, SUM(#{label_case_sql}) AS score FROM #{quoted_labels} " \
+          "WHERE #{label_column('record_type')} = #{quote(model.polymorphic_name)}#{tenant} AND #{label_keys_sql} " \
+          "GROUP BY #{label_column('record_id')}) #{scores} ON #{scores}.record_id = #{primary_key}"
+      end
+
+      def neighbors_sql
+        return @neighbors_sql if defined?(@neighbors_sql)
+
+        @neighbors_sql = (store.neighbors_sql(model, tenant_key: tenant_key, vector: vector) if vector && store.respond_to?(:neighbors_sql))
+      end
+
+      def neighbors_join_sql
+        neighbors = connection.quote_table_name(NEIGHBORS)
+        "LEFT JOIN (#{neighbors_sql}) #{neighbors} ON #{neighbors}.record_id = #{primary_key}"
+      end
+
       def label_filter_sql(key, threshold)
         "EXISTS (SELECT 1 FROM #{quoted_labels} WHERE #{label_scope_sql} AND #{label_column('label_key')} = #{quote(key)} " \
           "AND #{label_column('value')} >= #{Float(threshold)})"
@@ -130,20 +185,28 @@ module Truffler
       def keyword_sql
         return @keyword_sql if defined?(@keyword_sql)
 
-        @keyword_sql = keywords.empty? ? nil : keyword_condition(definition.keyword)
+        @keyword_sql = keywords.empty? ? nil : keyword_condition(definition.keyword, keywords)
       end
 
-      def keyword_condition(source)
+      # Any soft keyword hit; soft keywords rank and never narrow.
+      def soft_keyword_sql
+        return @soft_keyword_sql if defined?(@soft_keyword_sql)
+
+        conditions = (encoding.soft_keyword_tokens - keywords).uniq.filter_map { |token| keyword_condition(definition.keyword, [ token ]) }
+        @soft_keyword_sql = conditions.empty? ? nil : conditions.map { |condition| "(#{condition})" }.join(" OR ")
+      end
+
+      def keyword_condition(source, tokens)
         case source
         when Array
           return if source.empty?
 
-          keywords.map do |token|
+          tokens.map do |token|
             pattern = quote("%#{model.sanitize_sql_like(token)}%")
             "(#{source.map { |name| "LOWER(#{column(name)}) LIKE #{pattern} ESCAPE '\\'" }.join(' OR ')})"
           end.join(" AND ")
         when nil then nil
-        else membership_sql(source.call(tenant_scope, keywords))
+        else membership_sql(source.call(tenant_scope, tokens))
         end
       end
 
@@ -160,7 +223,10 @@ module Truffler
       def text_score_sql
         return @text_score_sql if defined?(@text_score_sql)
 
-        @text_score_sql = (store.similarity_sql(model, tenant_key: tenant_key, vector: vector).to_s if vector && store)
+        @text_score_sql =
+          if neighbors_sql then "COALESCE(#{connection.quote_table_name(NEIGHBORS)}.similarity, 0.0)"
+          elsif vector && store then store.similarity_sql(model, tenant_key: tenant_key, vector: vector).to_s
+          end
       end
 
       def text_candidate_sql
@@ -175,9 +241,14 @@ module Truffler
         definition.scoped? ? model.where(definition.tenant_column => tenant_key) : model.all
       end
 
+      # An id list is the fast path. A relation runs once as an array on
+      # Postgres, where `IN (subquery)` becomes a hashed filter over every
+      # tenant row.
       def membership_sql(result)
         case result
-        when ActiveRecord::Relation then "#{primary_key} IN (#{result.reselect(result.klass.arel_table[result.klass.primary_key]).to_sql})"
+        when ActiveRecord::Relation
+          subquery = result.reselect(result.klass.arel_table[result.klass.primary_key]).to_sql
+          connection.adapter_name.match?(/postg/i) ? "#{primary_key} = ANY(ARRAY(#{subquery}))" : "#{primary_key} IN (#{subquery})"
         when nil then nil
         else
           ids = Array(result)
@@ -189,12 +260,19 @@ module Truffler
         terms = {
           truffler_label_score: (label_score_sql && "#{Float(weights[:label])} * #{label_score_sql}"),
           truffler_text_score: (text_score_sql && "#{Float(weights[:text])} * #{text_score_sql}"),
-          truffler_keyword_score: (keyword_sql && "#{Float(weights[:keyword])} * (CASE WHEN #{keyword_sql} THEN 1.0 ELSE 0.0 END)"),
+          truffler_keyword_score: keyword_score_sql,
           truffler_exact_score: (exact_sql && "#{Float(weights[:exact])} * (CASE WHEN #{exact_sql} THEN 1.0 ELSE 0.0 END)")
         }.compact
         { truffler_score: terms.any? ? terms.values.map { |term| "(#{term})" }.join(" + ") : "0.0", **terms }
       end
 
+      def keyword_score_sql
+        weight = Float(weights[:keyword])
+        terms = [ (keyword_sql && "#{weight} * (CASE WHEN #{keyword_sql} THEN 1.0 ELSE 0.0 END)"),
+          (soft_keyword_sql && "#{weight * SOFT_KEYWORD} * (CASE WHEN #{soft_keyword_sql} THEN 1.0 ELSE 0.0 END)") ].compact
+        terms.map { |term| "(#{term})" }.join(" + ") if terms.any?
+      end
+
       def ordering
         order_column, direction = definition.order
         [ Arel.sql("truffler_score DESC"), (Arel.sql("#{column(order_column)} #{direction == :asc ? 'ASC' : 'DESC'}") if order_column),

data/lib/truffler/smart_search/dispatcher.rb

@@ -64,7 +64,7 @@ module Truffler
           if @encodings.encoded?(key) then @encodings.read_encoding(key, query)
           elsif @encodings.in_flight?(key) then @encoder.await(key, deadline: @deadline, query: query)
           end
-        encoding&.without(run.suppressed)
+        encoding&.without(run.suppressed, keep_words: -> { Search::Filler.label_words(model.truffler_definition, run.tenant_key) })
       end
 
       # The snapshot narrowed to what the encoding allows, in the tenant, in

data/lib/truffler/smart_search/starter.rb

@@ -30,7 +30,7 @@ module Truffler
         local_ids = result.ids
         run = Run.create(@model, query: @query.raw.strip, tenant_key: keystroke.tenant_key, user_key: keystroke.user_key,
           surface: keystroke.surface, suppressed: @suppressed, pool_ids: pool(result, local_ids), local_ids: local_ids,
-          local_weak: result.invite_row.present?, explicit_action: result.explicit_action, store: @store)
+          local_weak: result.local_weak?, explicit_action: result.explicit_action, store: @store)
         previous = @store.supersede(run.record_type, run.tenant_key, run.user_key, run.surface, run.id)
         Run.load(previous, store: @store).cancel! if previous
         Instrumentation.instrument(:smart_search, run_id: run.id, record_type: run.record_type, tenant_key: run.tenant_key,

data/lib/truffler/version.rb

@@ -1,3 +1,3 @@
 module Truffler
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/lib/truffler/vocabulary.rb

@@ -1,9 +1,9 @@
 module Truffler
   # A label's fingerprint is the SHA-256 of its canonical question plus the
   # pinned Jev model; a host-supplied label's is its supplied fingerprint
-  # (type, options, version), which no Jev model change touches. The
-  # vocabulary version digests every fingerprint. A stored label is stale
-  # when its fingerprint differs from the current one.
+  # (type, option keys, version), which no Jev model change or rewording
+  # touches. The vocabulary version digests every fingerprint. A stored
+  # label is stale when its fingerprint differs from the current one.
   #
   # A scope's vocabulary also holds its active lens labels (KTD21), so a lens
   # changes the version only where it applies. user_key is the searcher key
@@ -39,6 +39,16 @@ module Truffler
       Canonical.digest(fingerprints(tenant_key: tenant_key, user_key: user_key, all_users: all_users))
     end
 
+    # The version query encodings are cached under: the labeling version plus
+    # a digest of the wording only query encoding reads (label descriptions,
+    # option descriptions and search texts), which labeling never sees for
+    # supplied labels and never sees for search texts.
+    def encoding_version(tenant_key: nil, user_key: nil)
+      labels = labels_for(tenant_key: tenant_key, user_key: user_key)
+      Canonical.digest(version: Canonical.digest(labels.transform_values { |label| fingerprint_of(label, tenant_key) }),
+        wording: labels.transform_values { |label| label.encoding_wording(tenant_key) }.compact)
+    end
+
     private
 
     def fingerprint_of(label, tenant_key)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: truffler
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Kieran Klaassen
@@ -102,12 +102,14 @@ files:
 - bench/cassettes/0858c5006c0bfac3e970ddaff3c139f6c110dbe3d8539fd95263706c637cdc8e.json
 - bench/cassettes/0c0eb7c27d76bd0b9263690a63cd92d6b68c305826318555fd5b084a09970b78.json
 - bench/cassettes/0fada0c0a103882e2895f7f455d73d2cbe2b4a81bf743bb19b2b4f6e0130126b.json
+- bench/cassettes/1037e0b63ba790c02d14d0ddfc18b68151c11d03ce88d764658044b6bab2e55b.json
 - bench/cassettes/1117d79ac7c43b21beaf75e510461f91cd4ecc3cdbbe461f1794644af4aaf8f3.json
 - bench/cassettes/11b7444a074cca4d0ef1535db0cf0edefe4c34eac4be3c593c9c2f33d3b038b8.json
 - bench/cassettes/11d00371d1b993675be00463159164536f1f8ed90a48652b0935eb9c36f8b684.json
 - bench/cassettes/1456ebe12616ce961d68a14eada62778a7271ebd3d1b4cb6eefd6857651c7afc.json
 - bench/cassettes/14c3693f64b0e9ea7e470277fccae6ab501d404185eec5bb9074ef4b75c572c0.json
 - bench/cassettes/14fe9f6d8055d41f5d970dc32ab640c213ea8782d365c82d32b9884ba4f28c67.json
+- bench/cassettes/19bba849a846d30a69dc385a337d67704fbe5194506fd3c914d3270d86495fde.json
 - bench/cassettes/1a1d8557e4698426c2467aa573a240357715d3d9e3fa77bf066672a8b6bf9a86.json
 - bench/cassettes/1b5a8aaaf98305560588246cd51f53c6c185a951f8018c26bf588954d9de1cea.json
 - bench/cassettes/1fe596be747e33e9c7e4d7c50de32613957ac66bee43679b4bed96c551b1fd0a.json
@@ -116,6 +118,7 @@ files:
 - bench/cassettes/2f6576325dfab696cec8d8b35fa89c29960bee15d10dab31c91cef507f4842be.json
 - bench/cassettes/33ec27d22c0f63dbbe93008f50af07fa8667a5c806a9fbfc7897683915412b40.json
 - bench/cassettes/351f745b0c47e0cf3272b0fea7fc4941b6aa733f013c55a9a19707b542543200.json
+- bench/cassettes/3768847dc57df8bf2328771e7763c0b8bd013a167244e7849967dac144420601.json
 - bench/cassettes/37d974fdc0c1876413111c244d363c8d2306e3229fec668c158b5c924ffdb983.json
 - bench/cassettes/39524467463dd612cbf03e314b9cfd8cbe1c5f06969da2289359cf7cf83de5da.json
 - bench/cassettes/3f2c758c111ab7f1163ba579f2329718afe15620507d356d3ebe2a053df237ea.json
@@ -199,6 +202,8 @@ files:
 - lib/generators/truffler/install/templates/initializer.rb.tt
 - lib/generators/truffler/install/templates/migration.rb.tt
 - lib/generators/truffler/upgrade/templates/backfill_spends_migration.rb.tt
+- lib/generators/truffler/upgrade/templates/backfill_spends_tenant_key_migration.rb.tt
+- lib/generators/truffler/upgrade/templates/labels_search_covering_migration.rb.tt
 - lib/generators/truffler/upgrade/upgrade_generator.rb
 - lib/tasks/truffler.rake
 - lib/tasks/truffler/bench.rake
@@ -221,6 +226,7 @@ files:
 - lib/truffler/clients/base.rb
 - lib/truffler/clients/callable.rb
 - lib/truffler/clients/cassette.rb
+- lib/truffler/clients/evaluator.rb
 - lib/truffler/clients/fake.rb
 - lib/truffler/clients/ruby_llm_typesafe.rb
 - lib/truffler/configuration.rb
@@ -294,6 +300,7 @@ files:
 - lib/truffler/registry.rb
 - lib/truffler/search/encoding.rb
 - lib/truffler/search/encoding_cache.rb
+- lib/truffler/search/filler.rb
 - lib/truffler/search/keystroke.rb
 - lib/truffler/search/query.rb
 - lib/truffler/search/result.rb