@pithy-sh/support 0.5.0 → 0.6.0

package/src/store/hybrid.ts

@@ -0,0 +1,314 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { pithyCommandOf } from "@pithy-sh/core/src/cli/command";
+import { chunkByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
+import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
+import type { Logger } from "@pithy-sh/core/src/logger/logger";
+import type { VectorIndexConfig } from "@pithy-sh/vector/src/config/config";
+import type { VectorAi } from "@pithy-sh/vector/src/embed/embed";
+import type { VectorStore } from "@pithy-sh/vector/src/index/index";
+import { SUPPORT_SEARCH_TABLE, SUPPORT_THREADS_TABLE, type SupportDatabase } from "../data/tables";
+import { semanticHits } from "../semantic/query";
+import { DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT } from "./paging";
+import { type KeywordBackend, keywordHits, type MessageHit } from "./search";
+import { applyThreadFilters, type ListedThread, listedThreadOf, type ThreadFilters, viewerThreads } from "./threads";
+
+/**
+ * Hybrid search — keyword and semantic, run together and ranked as one.
+ *
+ * The two fail in opposite directions. Keyword finds the exact string and misses the paraphrase; semantic
+ * finds the paraphrase and misses the identifier, because it cannot retrieve an order number it has never
+ * seen. A support inbox needs both.
+ *
+ * **Search is not listing.** A query goes in, ranked threads come out, up to a limit, and there is no cursor:
+ * a ranked list has a top, not pages.
+ *
+ * ## How a result is ranked
+ *
+ * 1. Each backend returns **messages**, best first — FTS5 by `bm25`, the `LIKE` scan by recency, Vectorize by
+ *    similarity.
+ * 2. Each list rolls up to **threads**: a thread takes the rank of its best message, and appears once.
+ * 3. Hybrid fuses the two thread lists by **reciprocal rank fusion** over rank position — `1 / (60 + rank)`,
+ *    summed. No weights to tune, and a thread both backends found rises above one only one found. The raw
+ *    scores are never normalized into one number: each result carries its message's `bm25` and similarity as
+ *    they were, absent where that backend did not run or did not match.
+ * 4. A tie goes to the keyword hit, the better keyword rank first, and then to `lastMessageAt`, newest first.
+ *    Semantic search cannot retrieve an identifier, so for a query like `ORD-40912` the exact match is keyword's
+ *    rank 1 while semantic's rank 1 is some other thread: equal fused scores, and the exact match wins.
+ *
+ * **Filters apply after retrieval, in D1**, never as Vectorize metadata, which would go stale the moment a
+ * thread is archived, reassigned or reclassified. A narrow filter can therefore return fewer than the limit,
+ * and the result's count says how many it did.
+ *
+ * **Degrade, never fail.** A backend that is on and cannot answer — FTS5 configured and its table not
+ * provisioned, the semantic index or its binding absent, a model that does not answer — is skipped and named,
+ * and the rest of the search still runs.
+ */
+
+/** The search modes a caller may ask for. */
+export type SearchMode = "keyword" | "semantic" | "hybrid";
+
+/** A backend that can answer part of a search. */
+export type SearchBackend = KeywordBackend | "semantic";
+
+/** Why a backend that is on did not answer. */
+export type SkipReason = "unprovisioned" | "failed";
+
+/** The reciprocal-rank-fusion constant — the standard 60, which damps the head without flattening it. */
+export const RRF_K = 60;
+
+/** How many messages each backend retrieves before the roll-up. Vectorize's payload-free ceiling. */
+export const SEARCH_HITS = 100;
+
+/** One thread a backend ranked, with the message that ranked it. */
+export interface RankedThread {
+  /** The thread. */
+  threadId: string;
+  /** Its best-matching message. */
+  messageId: string;
+  /** Its rank in this backend's list, 1 first. */
+  rank: number;
+}
+
+/**
+ * Roll a backend's message hits up to threads: a thread takes the rank of its best message, and appears once.
+ * The input is best first, so the first message seen for a thread is its best.
+ */
+export function rollUp(hits: readonly MessageHit[]): RankedThread[] {
+  const seen = new Set<string>();
+  const ranked: RankedThread[] = [];
+  for (const hit of hits) {
+    if (seen.has(hit.threadId)) continue;
+    seen.add(hit.threadId);
+    ranked.push({ threadId: hit.threadId, messageId: hit.messageId, rank: ranked.length + 1 });
+  }
+  return ranked;
+}
+
+/** One thread after fusion: its fused score and the message the result names. */
+export interface FusedThread {
+  /** The thread. */
+  threadId: string;
+  /** The message the result names — the one from the backend that ranked this thread higher. */
+  messageId: string;
+  /** The reciprocal-rank-fusion score. Internal: it orders results and is never reported. */
+  fused: number;
+}
+
+/**
+ * Fuse ranked thread lists by reciprocal rank fusion. A thread found by several lists sums its terms.
+ *
+ * The message a result names comes from the list that ranked the thread highest, the first list winning a
+ * tie — the keyword list is passed first, so an exact match names the message that holds it.
+ */
+export function fuse(lists: readonly (readonly RankedThread[])[]): FusedThread[] {
+  const fused = new Map<string, FusedThread & { best: number }>();
+  for (const list of lists) {
+    for (const entry of list) {
+      const term = 1 / (RRF_K + entry.rank);
+      const known = fused.get(entry.threadId);
+      if (!known) {
+        fused.set(entry.threadId, {
+          threadId: entry.threadId,
+          messageId: entry.messageId,
+          fused: term,
+          best: entry.rank,
+        });
+        continue;
+      }
+      known.fused += term;
+      if (entry.rank < known.best) {
+        known.best = entry.rank;
+        known.messageId = entry.messageId;
+      }
+    }
+  }
+  return [...fused.values()].map(({ threadId, messageId, fused: score }) => ({ threadId, messageId, fused: score }));
+}
+
+/** What a search asks. */
+export interface SearchThreadsQuery extends ThreadFilters {
+  /** The words. */
+  q: string;
+  /** Which backends to run. Absent runs every backend that is on. */
+  mode?: SearchMode;
+  /** How many threads to return. Defaults to {@link DEFAULT_SEARCH_LIMIT}, capped at {@link MAX_SEARCH_LIMIT}. */
+  limit?: number;
+}
+
+/** What a search needs. The semantic seams are absent exactly when that backend cannot run here. */
+export interface SearchDeps {
+  /** The support tables. */
+  db: SupportDatabase;
+  /** Whether `search.fts` is on. */
+  fts: boolean;
+  /** The message index, when `search.semantic` is on. */
+  index?: VectorIndexConfig;
+  /** The Workers AI binding, when bound. */
+  ai?: VectorAi;
+  /** The message index's Vectorize binding, when bound. */
+  store?: VectorStore;
+  /** The calling viewer — whose read and snooze state each result carries. */
+  viewer: string;
+  /** Where a skipped backend is reported to the operator. */
+  log?: Logger;
+}
+
+/** One search result: a thread, and the message that matched best with its raw scores. */
+export interface SearchResult {
+  /** The thread, as this viewer sees it. */
+  thread: ListedThread;
+  /** The best-matching message. */
+  messageId: string;
+  /** That message's FTS5 `bm25`, raw — lower is better. Absent when FTS5 did not run or did not match it. */
+  bm25?: number;
+  /** That message's similarity to the query, raw. Absent when semantic search did not run or did not match it. */
+  similarity?: number;
+}
+
+/** What a search did, and what it found. */
+export interface SearchOutcome {
+  /** The mode that ran — the one asked for, or `hybrid` when none was. */
+  mode: SearchMode;
+  /** The backends that answered. */
+  ran: SearchBackend[];
+  /** The backends that were on and could not answer, and why. */
+  skipped: { backend: SearchBackend; reason: SkipReason }[];
+  /** The threads, best first. */
+  results: SearchResult[];
+}
+
+/** Whether an error is SQLite saying the full-text index does not exist. */
+function isMissingSearchTable(error: unknown): boolean {
+  const message = error instanceof Error ? error.message : String(error);
+  return message.includes("no such table") && message.includes(SUPPORT_SEARCH_TABLE);
+}
+
+/** A refusal for a mode this configuration cannot run. */
+function semanticOff(mode: SearchMode): ValidationError {
+  return new ValidationError({
+    message: `Search mode \`${mode}\` needs semantic search, and \`search.semantic\` is off.`,
+    action: `Turn on \`search.semantic\` in support()'s config and run \`${pithyCommandOf("support provision", {}, { runtime: true })}\`, or search with \`mode=keyword\`.`,
+    detail: `mode ${mode} requested with search.semantic off`,
+  });
+}
+
+/** Run a search. */
+export async function searchThreads(deps: SearchDeps, query: SearchThreadsQuery): Promise<SearchOutcome> {
+  const semanticOn = deps.index !== undefined;
+  if ((query.mode === "semantic" || query.mode === "hybrid") && !semanticOn) throw semanticOff(query.mode);
+  const mode: SearchMode = query.mode ?? "hybrid";
+  const limit = Math.min(Math.max(query.limit ?? DEFAULT_SEARCH_LIMIT, 1), MAX_SEARCH_LIMIT);
+
+  const ran: SearchBackend[] = [];
+  const skipped: SearchOutcome["skipped"] = [];
+  const lists: RankedThread[][] = [];
+  const keywordRank = new Map<string, number>();
+  const bm25 = new Map<string, number>();
+  const similarity = new Map<string, number>();
+
+  if (mode !== "semantic") {
+    let backend: KeywordBackend = deps.fts ? "fts" : "like";
+    let hits: MessageHit[];
+    try {
+      hits = await keywordHits(deps.db, query.q, { backend, limit: SEARCH_HITS });
+    } catch (error) {
+      // **The index being configured on is not the same as the index existing.** A project that turned
+      // `search.fts` on and has not re-provisioned gets the `LIKE` scan — slower, and still an answer —
+      // and the result says FTS5 was skipped. Narrow on purpose: a genuine query bug still surfaces.
+      if (backend !== "fts" || !isMissingSearchTable(error)) throw error;
+      deps.log?.warn("support FTS index is configured but absent — searching with a LIKE scan", {
+        action: `Run \`${pithyCommandOf("support provision", {}, { runtime: true })}\` to create and backfill it, or set \`search.fts: false\`.`,
+      });
+      skipped.push({ backend: "fts", reason: "unprovisioned" });
+      backend = "like";
+      hits = await keywordHits(deps.db, query.q, { backend, limit: SEARCH_HITS });
+    }
+    ran.push(backend);
+    if (backend === "fts") for (const hit of hits) if (hit.score !== undefined) bm25.set(hit.messageId, hit.score);
+    const keywordList = rollUp(hits);
+    for (const entry of keywordList) keywordRank.set(entry.threadId, entry.rank);
+    lists.push(keywordList);
+  }
+
+  if (mode !== "keyword" && deps.index) {
+    if (!deps.ai || !deps.store) {
+      // The index has not been provisioned for this Worker yet: its binding, or the model's, is absent.
+      deps.log?.warn("support semantic search is on but its index is not bound — skipping it", {
+        action: `Run \`${pithyCommandOf("support provision", {}, { runtime: true })}\` to create the index and bind it.`,
+      });
+      skipped.push({ backend: "semantic", reason: "unprovisioned" });
+    } else {
+      try {
+        const hits = await semanticHits({ db: deps.db, ai: deps.ai, store: deps.store, index: deps.index }, query.q);
+        for (const hit of hits) if (hit.score !== undefined) similarity.set(hit.messageId, hit.score);
+        lists.push(rollUp(hits));
+        ran.push("semantic");
+      } catch (error) {
+        deps.log?.warn("support semantic search could not answer — skipping it", {
+          detail: error instanceof Error ? error.message : String(error),
+        });
+        skipped.push({ backend: "semantic", reason: "failed" });
+      }
+    }
+  }
+
+  const fused = fuse(lists);
+  const threads = await hydrate(
+    deps.db,
+    deps.viewer,
+    fused.map((entry) => entry.threadId),
+    query,
+  );
+  const results = fused
+    .flatMap((entry) => {
+      const thread = threads.get(entry.threadId);
+      return thread ? [{ entry, thread }] : [];
+    })
+    .sort(
+      (a, b) =>
+        b.entry.fused - a.entry.fused ||
+        (keywordRank.get(a.entry.threadId) ?? Number.POSITIVE_INFINITY) -
+          (keywordRank.get(b.entry.threadId) ?? Number.POSITIVE_INFINITY) ||
+        b.thread.lastMessageAt.getTime() - a.thread.lastMessageAt.getTime() ||
+        (a.thread.id < b.thread.id ? 1 : a.thread.id > b.thread.id ? -1 : 0),
+    )
+    .slice(0, limit)
+    .map(({ entry, thread }): SearchResult => {
+      const keyword = bm25.get(entry.messageId);
+      const near = similarity.get(entry.messageId);
+      return {
+        thread,
+        messageId: entry.messageId,
+        ...(keyword !== undefined ? { bm25: keyword } : {}),
+        ...(near !== undefined ? { similarity: near } : {}),
+      };
+    });
+
+  return { mode, ran, skipped, results };
+}
+
+/** The filter values bind before the id list; this is the most a filtered read can bind besides it. */
+const HYDRATE_FIXED_PARAMETERS = 12;
+
+/** Read the candidate threads that pass the caller's filters, as this viewer sees them. */
+async function hydrate(
+  db: SupportDatabase,
+  viewer: string,
+  threadIds: readonly string[],
+  filters: ThreadFilters,
+): Promise<Map<string, ListedThread>> {
+  const threads = new Map<string, ListedThread>();
+  if (threadIds.length === 0) return threads;
+  for (const group of chunkByBoundParameters(threadIds, HYDRATE_FIXED_PARAMETERS)) {
+    const rows = await applyThreadFilters(viewerThreads(db, viewer), filters)
+      .where(`${SUPPORT_THREADS_TABLE}.id`, "in", group)
+      .execute();
+    for (const row of rows) {
+      const thread = listedThreadOf(row);
+      threads.set(thread.id, thread);
+    }
+  }
+  return threads;
+}

package/src/store/paging.ts

@@ -20,3 +20,17 @@ export const DEFAULT_PAGE_SIZE = 25;
 
 /** The most a caller may ask for in one page. */
 export const MAX_PAGE_SIZE = 100;
+
+/**
+ * The search route's bounds. Here rather than in `store/hybrid.ts` for the reason the page bounds are:
+ * `http/schemas.ts` is a client's business and reaches no Kysely.
+ */
+
+/** How many threads a search returns by default. */
+export const DEFAULT_SEARCH_LIMIT = 20;
+
+/** The most a search returns. A ranked list's tail is noise; fifty is past where anyone reads. */
+export const MAX_SEARCH_LIMIT = 50;
+
+/** The longest search term accepted — it becomes a scan predicate and a model input. */
+export const MAX_SEARCH_TERM = 200;

package/src/store/provisionIndex.ts

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import type { D1Database } from "@cloudflare/workers-types";
+import { supportDatabase } from "../data/tables";
+import { reindexAll } from "./search";
+import {
+  createSearchIndex,
+  dropSearchIndex,
+  SEARCH_OBJECTS,
+  type SearchIndexAction,
+  searchIndexAction,
+  searchIndexState,
+} from "./searchIndex";
+
+/**
+ * **Bring one database's full-text index into line with `search.fts`, and say what that took.**
+ *
+ * The body of `pithy support provision`'s search step, and of `pithy restore`'s rebuild of the index a
+ * backup leaves out (#848). One function for both, because the two have to agree: a restore that rebuilt
+ * the index some other way would be a second definition of a provisioned index, and the first time the
+ * triggers changed it would restore the old ones.
+ *
+ * It reads `sqlite_master` for the table and its triggers, decides with {@link searchIndexAction}, and
+ * applies the decision. A create and a repair backfill from `pithy_support_messages` straight away: an
+ * index that exists and is empty answers "no matches" for a word plainly in a message, because the `LIKE`
+ * fallback only fires while the table is absent. A drop takes the triggers first, then the table.
+ *
+ * Every statement is idempotent, so a re-run is a no-op that answers `none`.
+ */
+export async function provisionSearchIndex(database: D1Database, wanted: boolean): Promise<SearchIndexAction> {
+  const listed = await database
+    .prepare(`SELECT name FROM sqlite_master WHERE name IN (${SEARCH_OBJECTS.map(() => "?").join(", ")})`)
+    .bind(...SEARCH_OBJECTS)
+    .all<{ name: string }>();
+  const action = searchIndexAction(searchIndexState((listed.results ?? []).map((row) => row.name)), wanted);
+  if (action === "none") return action;
+  const db = supportDatabase(database);
+  if (action === "drop") {
+    await dropSearchIndex(db);
+    return action;
+  }
+  await createSearchIndex(db);
+  await reindexAll(db);
+  return action;
+}

package/src/store/search.ts

@@ -1,28 +1,23 @@
 // SPDX-FileCopyrightText: 2026 Pithy
 // SPDX-License-Identifier: MIT
 
-import { type Expression, type SqlBool, sql } from "kysely";
+import { type SqlBool, sql } from "kysely";
 import { SUPPORT_MESSAGES_TABLE, type SupportDatabase } from "../data/tables";
 
 /**
- * Text search over the inbox — two backends behind one predicate.
+ * Keyword retrieval over the inbox — two backends behind one call.
  *
- * **`LIKE` is the default and FTS5 is the opt-in**, which is the opposite of what it looks like it
- * should be. The reason is in `store/searchIndex.ts`: an FTS5 virtual table anywhere in a D1
- * database makes `wrangler d1 export` refuse to dump *the whole database*, so turning it on by
- * default would quietly cost every adopter their backup story for tables that have nothing to do
- * with support. `LIKE` is an unindexed scan and is completely adequate for an inbox measured in
- * thousands of messages, which is what this capability is for.
+ * **FTS5 is the default and `LIKE` is the fallback.** The index lives in a virtual table that
+ * `pithy support provision` creates beside its triggers, so a project that has not provisioned it — or
+ * that set `search.fts: false` — searches with an unindexed `LIKE` scan, which is adequate for an inbox
+ * measured in thousands of messages. `pithy backup` leaves the index out and `pithy restore` rebuilds it.
  *
- * Both paths return a **predicate over `pithy_support_threads`**, not a list of ids, so search
- * composes with the category/archived filters and — crucially — with cursor pagination. Returning
- * ids would mean either capping the match set (silent truncation, which reads as "that is all there
- * is") or paginating twice over.
+ * **Both return messages, ranked, not threads.** A thread is scored by its best-matching message, and the
+ * keyword half is fused with the semantic half by rank position in `store/hybrid.ts`. FTS5 ranks by `bm25`,
+ * which is reported raw. `LIKE` has no score, so its matches rank by recency, newest first.
  *
- * Either way this is keyword search, deliberately. People search a support inbox for a word they
- * remember — an order number, an error string, a surname — not for a concept. `@pithy-sh/vector` is
- * there for anyone who wants meaning-matching, and it is the nicer demo and the wrong default: it
- * cannot find `ORD-40912`.
+ * Keyword search is first-class beside semantic search: people search a support inbox for a word they
+ * remember — an order number, an error string, a surname — and semantic search cannot find `ORD-40912`.
  */
 
 /** The longest search term accepted. Bounded because it becomes a scan predicate. */
@@ -53,35 +48,61 @@ export function ftsQuery(term: string): string {
   return tokens.map((token) => `"${token.replace(/"/g, '""')}"`).join(" AND ");
 }
 
-/** Whether a term has anything searchable left after tokenizing. */
-export function isSearchable(term: string, fts: boolean): boolean {
-  return fts ? ftsQuery(term).length > 0 : term.trim().length > 0;
+/** One message a backend matched, in rank order. */
+export interface MessageHit {
+  /** The message. */
+  messageId: string;
+  /** Its thread. */
+  threadId: string;
+  /** The backend's own score for it, raw — `bm25` for FTS5, similarity for Vectorize, none for `LIKE`. */
+  score?: number;
 }
 
+/** Which keyword backend answered. */
+export type KeywordBackend = "fts" | "like";
+
 /**
- * A predicate matching threads whose messages contain `term`.
+ * The messages matching `term`, best first, at most `limit` of them.
  *
- * Correlated on the thread id so it drops into a `where` beside every other filter. The subquery is
- * `exists`, so it stops at the first matching message rather than counting them.
+ * A term that tokenizes to nothing — `???`, `@@@` — asked a question with no answer, and the honest answer
+ * is none: never the whole inbox rendered as search results.
  */
-export function searchPredicate(term: string, options: { fts: boolean }): Expression<SqlBool> {
-  if (options.fts) {
+export async function keywordHits(
+  db: SupportDatabase,
+  term: string,
+  options: { backend: KeywordBackend; limit: number },
+): Promise<MessageHit[]> {
+  if (options.backend === "fts") {
+    const match = ftsQuery(term);
+    if (match.length === 0) return [];
     // `<table> MATCH ?` naming the table in full is the documented FTS5 form, and the only one that
-    // works: the hidden match column keeps the table's own name, so aliasing the table to `s` and
-    // writing `s match ?` is `no such column: s`. `thread_id` is UNINDEXED, so the correlation is a
-    // filter over the match set rather than part of the match.
-    return sql<SqlBool>`exists (
-      select 1 from pithy_support_search
-      where pithy_support_search.thread_id = pithy_support_threads.id
-        and pithy_support_search match ${ftsQuery(term)}
-    )`;
+    // works: the hidden match column keeps the table's own name. `bm25()` is lower-is-better, so the
+    // ascending order is best first.
+    const { rows } = await sql<{ messageId: string; threadId: string; score: number }>`
+      select message_id as "messageId", thread_id as "threadId", bm25(pithy_support_search) as "score"
+      from pithy_support_search
+      where pithy_support_search match ${match}
+      order by "score" asc
+      limit ${options.limit}
+    `.execute(db);
+    return rows.map((row) => ({ messageId: row.messageId, threadId: row.threadId, score: Number(row.score) }));
   }
+  if (term.trim().length === 0) return [];
   const pattern = `%${escapeLike(term.slice(0, MAX_TERM))}%`;
-  return sql<SqlBool>`exists (
-    select 1 from pithy_support_messages m
-    where m.thread_id = pithy_support_threads.id
-      and (m.subject like ${pattern} escape '\\' or m.text_body like ${pattern} escape '\\')
-  )`;
+  const rows = await db
+    .selectFrom(SUPPORT_MESSAGES_TABLE)
+    .select(["id", "threadId"])
+    .where((eb) =>
+      eb.or([
+        sql<SqlBool>`${eb.ref("subject")} like ${pattern} escape '\\'`,
+        sql<SqlBool>`${eb.ref("textBody")} like ${pattern} escape '\\'`,
+      ]),
+    )
+    .orderBy("receivedAt", "desc")
+    .orderBy("id", "desc")
+    .limit(options.limit)
+    .execute();
+  return rows.map((row) => ({ messageId: row.id, threadId: row.threadId }));
 }
 
 /**

package/src/store/searchIndex.ts

@@ -154,10 +154,10 @@ export function searchIndexAction(state: SearchIndexState, wanted: boolean): Sea
 /**
  * The `CREATE TRIGGER` statements, one per {@link SEARCH_TRIGGERS} entry and in that order.
  *
- * **`BEGIN` and `END` are uppercase, and that is load-bearing.** Miniflare accepts a lowercase
- * compound-statement body; remote D1 rejects it as `incomplete input [code: 7500]`, so a lowercased
- * body passes every local test in this repository and fails on deploy. `searchIndex.test.ts` fails on
- * the lowercase form, because no Workers test can.
+ * **`BEGIN` and `END` are uppercase, by convention.** Remote D1 and Miniflare both accept a lowercase
+ * compound-statement body today, so nothing enforces the casing at run time. Older D1 rejected the
+ * lowercase form as `incomplete input [code: 7500]`, and uppercase is accepted everywhere, so the bodies
+ * keep it and `searchIndex.test.ts` pins it.
  *
  * **Every column is the physical snake_case name.** `CamelCasePlugin` snake-cases the identifiers the
  * query builder emits and never touches a string inside a `sql` template, so this is the one place in