truffler 0.1.0

data/lib/truffler/search/encoding.rb

@@ -0,0 +1,84 @@
+module Truffler
+  module Search
+    # What query encoding (U9) decided about one query, as keystroke search
+    # reads it from the cache. Keys are label storage keys (`needs_action`,
+    # `category:billing`, `lens:<id>:<label>`).
+    #
+    # - `filters`: hard filters, key => minimum value, applied as EXISTS.
+    # - `boosts`: key => weight, shown as boost chips.
+    # - `intent_vector`: the sparse query vector of KTD20, key => weight, used
+    #   for the label term SUM(weight * value). Defaults to `boosts`; the
+    #   encoder decides whether filtered labels also carry weight here.
+    # - `keyword_tokens`: query tokens for the keyword source; nil means
+    #   every token that is not a label term.
+    # - `label_term_tokens`: tokens that named a label rather than a keyword.
+    Encoding = Data.define(:filters, :boosts, :intent_vector, :keyword_tokens, :label_term_tokens) do
+      def self.load(value, query)
+        return value if value.is_a?(self)
+        return if value.nil?
+
+        tokens = query.tokens
+        new(filters: value["filters"], boosts: value["boosts"], intent_vector: value["intent_vector"],
+          keyword_tokens: value["keyword_positions"]&.map { |position| tokens[position] }&.compact,
+          label_term_tokens: Array(value["label_term_positions"]).filter_map { |position| tokens[position] })
+      end
+
+      def initialize(filters: {}, boosts: {}, intent_vector: nil, keyword_tokens: nil, label_term_tokens: [])
+        filters = weights(filters)
+        boosts = weights(boosts)
+        intent_vector = weights(intent_vector || boosts).reject { |_, weight| weight.zero? }
+        super(filters: filters.freeze, boosts: boosts.freeze, intent_vector: intent_vector.freeze,
+          keyword_tokens: keyword_tokens&.map(&:to_s)&.freeze, label_term_tokens: Array(label_term_tokens).map(&:to_s).freeze)
+      end
+
+      def empty?
+        filters.empty? && boosts.empty? && intent_vector.empty?
+      end
+
+      # The encoding minus the chips the searcher removed (R20), matched by
+      # storage key or by label key.
+      def without(suppressed)
+        suppressed = Array(suppressed).map(&:to_s).to_set
+        return self if suppressed.empty?
+
+        keep = ->(key, _) { !suppressed.include?(key) && !suppressed.include?(self.class.split_key(key).first) }
+        with(filters: filters.select(&keep), boosts: boosts.select(&keep), intent_vector: intent_vector.select(&keep))
+      end
+
+      # Splits a storage key into its label key and choice option. Lens keys
+      # ("lens:<id>:<label>[:<option>]") carry two extra colons.
+      def self.split_key(key)
+        key = key.to_s
+        if key.start_with?("#{Lenses::KEY_PREFIX}:")
+          prefix, id, label, option = key.split(":", 4)
+          [ "#{prefix}:#{id}:#{label}", option ]
+        else
+          key.split(":", 2)
+        end
+      end
+
+      def keywords(query)
+        keyword_tokens || (query.tokens - label_term_tokens)
+      end
+
+      # The cache form: decisions plus token positions in the normalized
+      # query, so no query text is stored (the cache key already fixes the
+      # query).
+      def dump(query)
+        { "filters" => filters, "boosts" => boosts, "intent_vector" => intent_vector,
+          "keyword_positions" => keyword_tokens && positions(query, keyword_tokens),
+          "label_term_positions" => positions(query, label_term_tokens) }
+      end
+
+      private
+
+      def weights(hash)
+        hash.to_h.to_h { |key, weight| [ key.to_s, Float(weight) ] }
+      end
+
+      def positions(query, words)
+        query.tokens.each_index.select { |index| words.include?(query.tokens[index]) }
+      end
+    end
+  end
+end

data/lib/truffler/search/encoding_cache.rb

@@ -0,0 +1,68 @@
+module Truffler
+  module Search
+    # Where keystroke search finds query encodings and query vectors (R12,
+    # R15). Keys digest the model, the normalized query, and the vocabulary
+    # version, plus the tenant when the vocabulary has per-tenant choices.
+    # Values hold decisions and floats, never query text. Encoding keys use
+    # the searcher's vocabulary version, which holds the lenses that searcher
+    # can see (KTD21), so a personal lens's encoding is never shared; query
+    # vectors do not depend on lenses and stay keyed per tenant.
+    #
+    # On a miss, search calls `prefetch`, which hands off to
+    # `config.encoding_prefetch` (the query encoder, U9). The hook is called
+    # as `call(model, query, cache_key:, tenant_key:, user_key:)` and returns
+    # truthy when an encoding is now in flight. Without a hook nothing is
+    # encoded and the encoding status reads `:none`.
+    class EncodingCache
+      TTL = 7.days
+
+      def initialize(store: Truffler.config.cache_store)
+        @store = store
+      end
+
+      def key(model, query, tenant_key:, user_key: nil)
+        "truffler/enc/#{digest(model, query, tenant_key, user_key)}"
+      end
+
+      def vector_key(model, query, tenant_key:)
+        "truffler/vec/#{digest(model, query, tenant_key, nil)}"
+      end
+
+      def read(model, query, tenant_key:, user_key: nil)
+        query = Query.wrap(query)
+        Encoding.load(@store.read(key(model, query, tenant_key: tenant_key, user_key: user_key)), query)
+      end
+
+      def write(model, query, encoding, tenant_key:, user_key: nil, expires_in: TTL)
+        query = Query.wrap(query)
+        @store.write(key(model, query, tenant_key: tenant_key, user_key: user_key), encoding.dump(query), expires_in: expires_in)
+      end
+
+      def read_vector(model, query, tenant_key:)
+        @store.read(vector_key(model, Query.wrap(query), tenant_key: tenant_key))&.map(&:to_f)
+      end
+
+      def write_vector(model, query, vector, tenant_key:, expires_in: TTL)
+        @store.write(vector_key(model, Query.wrap(query), tenant_key: tenant_key), vector.map(&:to_f), expires_in: expires_in)
+      end
+
+      def prefetch(model, query, tenant_key:, user_key:)
+        hook = Truffler.config.encoding_prefetch
+        return false unless hook
+
+        query = Query.wrap(query)
+        hook.call(model, query, cache_key: key(model, query, tenant_key: tenant_key, user_key: user_key), tenant_key: tenant_key,
+          user_key: user_key).present?
+      end
+
+      private
+
+      def digest(model, query, tenant_key, user_key)
+        definition = model.truffler_definition
+        tenant = tenant_key&.to_s if definition.per_tenant_vocabulary?
+        version = definition.vocabulary.version(tenant_key: tenant_key&.to_s, user_key: user_key)
+        Canonical.digest(record_type: model.polymorphic_name, query: query.normalized, vocabulary_version: version, tenant_key: tenant)
+      end
+    end
+  end
+end

data/lib/truffler/search/keystroke.rb

@@ -0,0 +1,149 @@
+module Truffler
+  module Search
+    # A tenant-scoped keystroke search (KTD8). It reads the query encoding
+    # and query vector from the cache only, so no network call sits on the
+    # keystroke (R12); a miss hands the query to the prefetch hook and the
+    # results stand without it. Everything else is one SQL query.
+    class Keystroke
+      DEFAULT_LIMIT = 50
+
+      attr_reader :model, :query, :tenant_key, :scope, :user_key, :suppressed, :surface, :limit
+
+      def initialize(model, query, tenant:, scope:, user: nil, suppressed: [], surface: nil, limit: DEFAULT_LIMIT, weights: {},
+        cache: EncodingCache.new)
+        @model = model
+        @definition = model.try(:truffler_definition) || raise(DefinitionError, "#{model.name} has no truffler declaration")
+        @query = Query.wrap(query)
+        @tenant_key = tenant&.to_s
+        @scope = scope.nil? && !@definition.scoped? ? model.all : scope
+        @user_key = self.class.user_key(user)
+        @suppressed = Array(suppressed).map(&:to_s)
+        @surface = surface&.to_s
+        @limit = limit
+        @weights = @definition.ranking.merge(weights.to_h { |key, weight| [ key.to_sym, Float(weight) ] })
+        @cache = cache
+        check_scope!
+      end
+
+      def self.user_key(user)
+        case user
+        when nil then nil
+        when ActiveRecord::Base then "#{user.class.polymorphic_name}:#{user.id}"
+        else user.to_s
+        end
+      end
+
+      def call
+        started = Instrumentation.monotonic_ms
+        watermark = Time.current
+        explicit_action = surface_action
+        cached = read_encoding
+        status = encoding_status(cached)
+        encoding = visible_lenses_only(cached&.without(suppressed), record_usage: true)
+        sql = sql(encoding)
+        records = sql.relation(scope, limit: limit).to_a
+        result = Result.new(records: records, query: query, encoding: encoding, encoding_status: status, watermark: watermark,
+          explicit_action: explicit_action, sources: sql.sources, invite_row: invite_row(records, cached),
+          weights: @weights, recount: ->(since) { count(since: since) })
+        instrument(result, started)
+        result
+      end
+
+      # How many records the same search would return that arrived after
+      # `since` (R25). Reads the cache only and never prefetches.
+      def count(since:)
+        sql(visible_lenses_only(read_encoding&.without(suppressed))).candidates(scope)
+          .where(model.arel_table[@definition.arrived_at_column].gt(since)).count
+      end
+
+      private
+
+      def check_scope!
+        if @definition.scoped? && tenant_key.nil?
+          raise MissingScope, "#{model.name}.truffler needs tenant: (declared tenant #{@definition.tenant_column})"
+        end
+        return if scope.is_a?(ActiveRecord::Relation) && scope.klass <= model
+
+        raise MissingScope, "#{model.name}.truffler needs scope: to be a relation of #{model.name}"
+      end
+
+      def surface_action
+        return unless surface
+
+        @definition.surfaces.fetch(surface) { raise DefinitionError, "#{model.name} declares no surface #{surface}" }[:explicit_action]
+      end
+
+      def read_encoding
+        return if query.blank? || @definition.labels.empty?
+
+        @cache.read(model, query, tenant_key: tenant_key, user_key: user_key)
+      end
+
+      # Drops lens keys this searcher cannot see (another user's personal
+      # lens, an expired lens) and counts a use of the rest (R42, R43).
+      def visible_lenses_only(encoding, record_usage: false)
+        lens_keys = encoding ? (encoding.intent_vector.keys | encoding.filters.keys | encoding.boosts.keys).select { |key| lens_id(key) } : []
+        return encoding if lens_keys.empty?
+
+        visible = Lenses.labels(model, tenant_key: tenant_key, user_key: user_key).values.map(&:lens_id).uniq
+        hidden, shown = lens_keys.partition { |key| !visible.include?(lens_id(key)) }
+        Lenses.record_usage(shown.map { |key| lens_id(key) }.uniq) if record_usage
+        encoding.without(hidden)
+      end
+
+      def lens_id(key)
+        prefix, id = key.split(":", 3)
+        Integer(id, exception: false) if prefix == Lenses::KEY_PREFIX
+      end
+
+      def read_vector
+        return if query.blank? || !@definition.embeddings
+
+        @vector ||= @cache.read_vector(model, query, tenant_key: tenant_key)
+      end
+
+      # Prefetches on a missing encoding or query vector; the status reports
+      # the encoding only.
+      def encoding_status(cached)
+        return :none if query.blank?
+
+        in_flight = prefetch?(cached) && @cache.prefetch(model, query, tenant_key: tenant_key, user_key: user_key)
+        return :cached if cached
+        return :none if @definition.labels.empty?
+
+        in_flight ? :pending : :none
+      end
+
+      def prefetch?(cached)
+        (cached.nil? && @definition.labels.any?) || (@definition.embeddings.present? && read_vector.nil?)
+      end
+
+      def sql(encoding)
+        Sql.new(model, tenant_key: tenant_key, query: query, encoding: encoding, vector: read_vector, weights: @weights)
+      end
+
+      # R21: the Smart search row. A model with no local text search whose
+      # encoding is not cached yet invites the action even when a blind index
+      # matched, because intent queries resolve on the action there (AE10).
+      def invite_row(records, cached)
+        return if query.blank?
+
+        reason =
+          if @definition.keyword.blank? && cached.nil? then :encoding_pending
+          elsif records.empty? then :empty
+          elsif records.size < @definition.weak_below then :weak
+          end
+        { query: query.raw.strip, reason: reason } if reason
+      end
+
+      def instrument(result, started)
+        payload = { record_type: model.polymorphic_name, tenant_key: tenant_key, surface: surface, outcome: result.encoding_status,
+          result_count: result.records.size, filter_count: result.encoding&.filters&.size.to_i,
+          boost_count: result.encoding&.intent_vector&.size.to_i, sources: result.sources.map(&:to_s),
+          reason: result.invite_row&.dig(:reason), latency_ms: Instrumentation.elapsed_ms(started) }
+        payload[:query_digest] = Misses.digest(:query, query.normalized) if Misses.encrypted_model?(model)
+        Instrumentation.instrument("search", payload)
+      end
+    end
+  end
+end

data/lib/truffler/search/query.rb

@@ -0,0 +1,67 @@
+module Truffler
+  module Search
+    # A normalized, tokenized search query. Quoted phrases stay one token.
+    # Exact-text signals (quoted phrases, digit-bearing tokens, emails, and
+    # identifier shapes such as `INV-4471` or `order_id`) are detected
+    # locally and never asked of Jev (R18).
+    class Query
+      QUOTED = /"([^"]*)"/
+      EMAIL = /\A[^@\s]+@[^@\s]+\.[a-z]{2,}\z/
+      IDENTIFIER = /\A[a-z0-9]+(?:[-_.][a-z0-9]+)+\z/
+      EDGE_PUNCTUATION = /\A[^\p{Alnum}@]+|[^\p{Alnum}]+\z/
+
+      attr_reader :raw, :normalized, :tokens, :exact_tokens
+
+      def self.wrap(query)
+        query.is_a?(self) ? query : new(query)
+      end
+
+      def self.normalize(text)
+        text.to_s.unicode_normalize(:nfkc).downcase.squish
+      end
+
+      def initialize(raw)
+        @raw = raw.to_s
+        @normalized = self.class.normalize(raw)
+        @tokens, @exact_tokens = tokenize(normalized)
+      end
+
+      def blank?
+        tokens.empty?
+      end
+
+      def exact_text?
+        exact_tokens.any?
+      end
+
+      private
+
+      def tokenize(text)
+        tokens = []
+        exact = []
+        text.split(QUOTED, -1).each_with_index do |part, index|
+          if index.odd?
+            phrase = part.squish
+            next if phrase.empty?
+
+            tokens << phrase
+            exact << phrase
+          else
+            part.split.each do |word|
+              word = word.gsub(EDGE_PUNCTUATION, "")
+              next if word.empty?
+
+              tokens << word
+              exact << word if exact?(word)
+            end
+          end
+        end
+        [ tokens.freeze, exact.freeze ]
+      end
+
+      def exact?(word)
+        word.match?(/\d/) || word.match?(EMAIL) || word.match?(IDENTIFIER)
+      end
+    end
+  end
+end

data/lib/truffler/search/result.rb

@@ -0,0 +1,100 @@
+module Truffler
+  module Search
+    # One keystroke search, carrying every host UI contract: records, chips
+    # (R20), the Smart search invite row (R21), the encoding status, the
+    # watermark for "N new matches" (R25), the surface's explicit action
+    # (R23), and per-record score breakdowns for debugging.
+    class Result
+      ENCODING_STATUSES = %i[cached pending none].freeze
+      INVITE_REASONS = %i[weak empty encoding_pending].freeze
+      SCORE_COLUMNS = { label: "truffler_label_score", text: "truffler_text_score", keyword: "truffler_keyword_score",
+        exact: "truffler_exact_score" }.freeze
+
+      attr_reader :records, :query, :encoding, :encoding_status, :watermark, :explicit_action, :sources, :invite_row
+
+      def initialize(records:, query:, encoding:, encoding_status:, watermark:, explicit_action:, sources:, invite_row:, weights:, recount:)
+        @records = records
+        @query = query
+        @encoding = encoding
+        @encoding_status = encoding_status
+        @watermark = watermark
+        @explicit_action = explicit_action
+        @sources = sources
+        @invite_row = invite_row
+        @weights = weights
+        @recount = recount
+      end
+
+      def ids
+        records.map(&:id)
+      end
+
+      # Applied filters, then boosts, as `{key:, label:, kind:, name:}`.
+      def chips
+        return [] unless encoding
+
+        filters = encoding.filters.keys.map { |key| chip(key, :filter) }
+        filters + (encoding.boosts.keys - encoding.filters.keys).map { |key| chip(key, :boost) }
+      end
+
+      def score(record)
+        find(record)&.attributes&.fetch("truffler_score", nil)&.to_f
+      end
+
+      # The weighted terms that summed to the record's score.
+      def breakdown(record)
+        found = find(record)
+        return {} unless found
+
+        SCORE_COLUMNS.to_h { |term, name| [ term, found.attributes.fetch(name, 0.0).to_f ] }.merge(total: score(found))
+      end
+
+      # Each intent label's share of the label term, `key => w_label * weight
+      # * value`. Loaded on first use, so the keystroke itself stays one query.
+      def contributions(record)
+        id = record.respond_to?(:id) ? record.id : record
+        (@contributions ||= load_contributions).fetch(id.to_s, {})
+      end
+
+      def new_matches_count
+        @recount.call(watermark)
+      end
+
+      def promoted_ids(run)
+        return [] unless run
+
+        ids & Array(run.promoted_ids)
+      end
+
+      def smart_ranking_paused?(run = nil)
+        run&.status&.to_sym == :paused
+      end
+
+      private
+
+      def find(record)
+        id = record.respond_to?(:id) ? record.id : record
+        records.find { |item| item.id == id }
+      end
+
+      def chip(key, kind)
+        label, option = Encoding.split_key(key)
+        title = label.split(":").last.humanize
+        name = option ? "#{title}: #{option}" : title
+        { key: key, label: label, kind: kind, name: name }
+      end
+
+      def load_contributions
+        intent = encoding&.intent_vector.to_h
+        return {} if intent.empty? || records.empty?
+
+        model = records.first.class
+        rows = Records::Label.where(record_type: model.polymorphic_name, record_id: ids, label_key: intent.keys)
+          .pluck(:record_id, :label_key, :value)
+        rows.each_with_object(Hash.new { |hash, id| hash[id] = {} }) do |(id, key, value), map|
+          map[id.to_s][key] = @weights[:label] * intent[key] * value
+        end
+      end
+    end
+  end
+end

data/lib/truffler/search/sql.rb

@@ -0,0 +1,197 @@
+module Truffler
+  module Search
+    # Builds the one keystroke query of KTD8, scored per KTD20:
+    #
+    #   w_label * SUM(weight * value) over the intent's nonzero label keys
+    #   + w_text * text similarity (inline SQL, or the store's top-K CASE)
+    #   + w_keyword * keyword hit + w_exact * exact-source hit
+    #
+    # A weighted dot product, not cosine: cosine would divide out magnitude
+    # and let a record high on unrelated labels outrank the one the query
+    # asked for. Hard filters are EXISTS subqueries that run before scoring.
+    class Sql
+      LABELS = "truffler_labels".freeze
+
+      attr_reader :model, :tenant_key, :query, :encoding, :vector, :weights
+
+      def initialize(model, tenant_key:, query:, encoding: nil, vector: nil, weights: nil, store: nil)
+        @model = model
+        @tenant_key = tenant_key&.to_s
+        @query = query
+        @encoding = encoding || Encoding.new
+        @vector = vector
+        @weights = weights || definition.ranking
+        @store = store
+      end
+
+      # The caller's relation, ANDed with the tenant and the hard filters.
+      def base(scope)
+        scope = scope.where(definition.tenant_column => tenant_key) if definition.scoped?
+        encoding.filters.reduce(scope) { |relation, (key, threshold)| relation.where(Arel.sql(label_filter_sql(key, threshold))) }
+      end
+
+      # Every record the query can return, before ranking.
+      def candidates(scope)
+        base = base(scope)
+        return base if label_only?
+
+        conditions = [ keyword_sql, exact_sql, (text_candidate_sql if text_score_sql) ].compact
+        base.where(Arel.sql(conditions.any? ? conditions.map { |condition| "(#{condition})" }.join(" OR ") : "1 = 0"))
+      end
+
+      def relation(scope, limit: nil)
+        relation = candidates(scope)
+        relation = relation.select(Arel.sql("#{table}.*")) if relation.select_values.empty?
+        relation = relation.select(*score_columns.map { |name, sql| Arel.sql("(#{sql}) AS #{name}") })
+        relation = relation.reorder(*ordering)
+        limit ? relation.limit(limit) : relation
+      end
+
+      def label_score_sql
+        intent = encoding.intent_vector
+        return if intent.empty?
+
+        cases = intent.map { |key, weight| "WHEN #{quote(key)} THEN #{Float(weight)} * #{label_column('value')}" }.join(" ")
+        "COALESCE((SELECT SUM(CASE #{label_column('label_key')} #{cases} ELSE 0.0 END) FROM #{quoted_labels} " \
+          "WHERE #{label_scope_sql} AND #{label_column('label_key')} IN (#{intent.keys.map { |key| quote(key) }.join(', ')})), 0.0)"
+      end
+
+      def sources
+        [ (:keyword if keyword_sql), (:exact if exact_sql), (:vector if text_score_sql),
+          (:labels unless encoding.empty?) ].compact
+      end
+
+      def keywords
+        encoding.keywords(query)
+      end
+
+      private
+
+      def definition
+        model.truffler_definition
+      end
+
+      def connection
+        model.connection
+      end
+
+      def quote(value)
+        connection.quote(value)
+      end
+
+      def table
+        connection.quote_table_name(model.table_name)
+      end
+
+      def column(name)
+        "#{table}.#{connection.quote_column_name(name)}"
+      end
+
+      def primary_key
+        column(model.primary_key)
+      end
+
+      def quoted_labels
+        connection.quote_table_name(LABELS)
+      end
+
+      def label_column(name)
+        "#{quoted_labels}.#{connection.quote_column_name(name)}"
+      end
+
+      def label_scope_sql
+        tenant = definition.scoped? ? " AND #{label_column('tenant_key')} = #{quote(tenant_key)}" : ""
+        "#{label_column('record_type')} = #{quote(model.polymorphic_name)}#{tenant} AND #{label_column('record_id')} = #{primary_key}"
+      end
+
+      def label_filter_sql(key, threshold)
+        "EXISTS (SELECT 1 FROM #{quoted_labels} WHERE #{label_scope_sql} AND #{label_column('label_key')} = #{quote(key)} " \
+          "AND #{label_column('value')} >= #{Float(threshold)})"
+      end
+
+      # All label terms (or a blank query) means label-only matches: every
+      # record past the hard filters is a candidate, ranked by the label
+      # term. So is an applied encoding on a model with nothing local to
+      # search its keywords with.
+      def label_only?
+        return true if keywords.empty?
+
+        !encoding.empty? && definition.keyword.blank? && !text_score_sql && definition.exact_sources.empty?
+      end
+
+      def keyword_sql
+        return @keyword_sql if defined?(@keyword_sql)
+
+        @keyword_sql = keywords.empty? ? nil : keyword_condition(definition.keyword)
+      end
+
+      def keyword_condition(source)
+        case source
+        when Array
+          return if source.empty?
+
+          keywords.map do |token|
+            pattern = quote("%#{model.sanitize_sql_like(token)}%")
+            "(#{source.map { |name| "LOWER(#{column(name)}) LIKE #{pattern} ESCAPE '\\'" }.join(' OR ')})"
+          end.join(" AND ")
+        when nil then nil
+        else membership_sql(source.call(tenant_scope, keywords))
+        end
+      end
+
+      def exact_sql
+        return @exact_sql if defined?(@exact_sql)
+
+        values = query.blank? ? [] : (query.exact_tokens + [ query.normalized ]).uniq
+        conditions = definition.exact_sources.values.flat_map do |callable|
+          values.filter_map { |value| membership_sql(callable.call(tenant_scope, value)) }
+        end
+        @exact_sql = conditions.empty? ? nil : conditions.join(" OR ")
+      end
+
+      def text_score_sql
+        return @text_score_sql if defined?(@text_score_sql)
+
+        @text_score_sql = (store.similarity_sql(model, tenant_key: tenant_key, vector: vector).to_s if vector && store)
+      end
+
+      def text_candidate_sql
+        "#{text_score_sql} > #{Float(weights[:min_similarity])}"
+      end
+
+      def store
+        @store ||= Embeddings::VectorStore.for(model) if definition.embeddings
+      end
+
+      def tenant_scope
+        definition.scoped? ? model.where(definition.tenant_column => tenant_key) : model.all
+      end
+
+      def membership_sql(result)
+        case result
+        when ActiveRecord::Relation then "#{primary_key} IN (#{result.reselect(result.klass.arel_table[result.klass.primary_key]).to_sql})"
+        when nil then nil
+        else
+          ids = Array(result)
+          "#{primary_key} IN (#{ids.map { |id| quote(id) }.join(', ')})" if ids.any?
+        end
+      end
+
+      def score_columns
+        terms = {
+          truffler_label_score: (label_score_sql && "#{Float(weights[:label])} * #{label_score_sql}"),
+          truffler_text_score: (text_score_sql && "#{Float(weights[:text])} * #{text_score_sql}"),
+          truffler_keyword_score: (keyword_sql && "#{Float(weights[:keyword])} * (CASE WHEN #{keyword_sql} THEN 1.0 ELSE 0.0 END)"),
+          truffler_exact_score: (exact_sql && "#{Float(weights[:exact])} * (CASE WHEN #{exact_sql} THEN 1.0 ELSE 0.0 END)")
+        }.compact
+        { truffler_score: terms.any? ? terms.values.map { |term| "(#{term})" }.join(" + ") : "0.0", **terms }
+      end
+
+      def ordering
+        order_column, direction = definition.order
+        [ Arel.sql("truffler_score DESC"), (Arel.sql("#{column(order_column)} #{direction == :asc ? 'ASC' : 'DESC'}") if order_column),
+          Arel.sql("#{primary_key} DESC") ].compact
+      end
+    end
+  end
+end