@pithy-sh/support 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/README.md +8 -0
  2. package/dist/audit/actions.d.ts +9 -0
  3. package/dist/audit/actions.d.ts.map +1 -1
  4. package/dist/audit/actions.js +9 -0
  5. package/dist/capability.d.ts +2 -1
  6. package/dist/capability.d.ts.map +1 -1
  7. package/dist/capability.js +30 -6
  8. package/dist/config/config.d.ts +63 -2
  9. package/dist/config/config.d.ts.map +1 -1
  10. package/dist/config/config.js +63 -5
  11. package/dist/data/assignment.d.ts +43 -0
  12. package/dist/data/assignment.d.ts.map +1 -0
  13. package/dist/data/assignment.js +38 -0
  14. package/dist/data/embedding.d.ts +21 -0
  15. package/dist/data/embedding.d.ts.map +1 -0
  16. package/dist/data/embedding.js +23 -0
  17. package/dist/data/tables.d.ts +28 -0
  18. package/dist/data/tables.d.ts.map +1 -1
  19. package/dist/data/tables.js +10 -2
  20. package/dist/data/thread.d.ts +9 -4
  21. package/dist/data/thread.d.ts.map +1 -1
  22. package/dist/data/thread.js +9 -4
  23. package/dist/http/guards.d.ts +32 -14
  24. package/dist/http/guards.d.ts.map +1 -1
  25. package/dist/http/guards.js +44 -16
  26. package/dist/http/handlers.d.ts +35 -2
  27. package/dist/http/handlers.d.ts.map +1 -1
  28. package/dist/http/handlers.js +81 -8
  29. package/dist/http/resolve.d.ts +8 -0
  30. package/dist/http/resolve.d.ts.map +1 -1
  31. package/dist/http/resolve.js +20 -1
  32. package/dist/http/responses.d.ts +355 -0
  33. package/dist/http/responses.d.ts.map +1 -1
  34. package/dist/http/responses.js +64 -2
  35. package/dist/http/routes.d.ts +25 -11
  36. package/dist/http/routes.d.ts.map +1 -1
  37. package/dist/http/routes.js +34 -5
  38. package/dist/http/schemas.d.ts +67 -3
  39. package/dist/http/schemas.d.ts.map +1 -1
  40. package/dist/http/schemas.js +44 -5
  41. package/dist/http/scopes.d.ts +10 -5
  42. package/dist/http/scopes.d.ts.map +1 -1
  43. package/dist/http/scopes.js +42 -6
  44. package/dist/http/views.d.ts +9 -1
  45. package/dist/http/views.d.ts.map +1 -1
  46. package/dist/http/views.js +40 -1
  47. package/dist/inbound/ingest.d.ts +1 -1
  48. package/dist/inbound/ingest.d.ts.map +1 -1
  49. package/dist/inbound/ingest.js +3 -1
  50. package/dist/index.d.ts +5 -3
  51. package/dist/index.d.ts.map +1 -1
  52. package/dist/index.js +6 -4
  53. package/dist/migrations/0002_assignment.d.ts +20 -0
  54. package/dist/migrations/0002_assignment.d.ts.map +1 -0
  55. package/dist/migrations/0002_assignment.js +45 -0
  56. package/dist/migrations/0003_embeddings.d.ts +15 -0
  57. package/dist/migrations/0003_embeddings.d.ts.map +1 -0
  58. package/dist/migrations/0003_embeddings.js +23 -0
  59. package/dist/migrations/set.d.ts +13 -0
  60. package/dist/migrations/set.d.ts.map +1 -0
  61. package/dist/migrations/set.js +21 -0
  62. package/dist/presence/room.d.ts +35 -0
  63. package/dist/presence/room.d.ts.map +1 -0
  64. package/dist/presence/room.js +66 -0
  65. package/dist/provision/provisionSupport.d.ts +32 -0
  66. package/dist/provision/provisionSupport.d.ts.map +1 -1
  67. package/dist/provision/provisionSupport.js +10 -2
  68. package/dist/provision/resolveSupportConfig.d.ts.map +1 -1
  69. package/dist/provision/resolveSupportConfig.js +13 -2
  70. package/dist/purge/contribution.d.ts +49 -0
  71. package/dist/purge/contribution.d.ts.map +1 -0
  72. package/dist/purge/contribution.js +164 -0
  73. package/dist/secret/registry.d.ts +26 -2
  74. package/dist/secret/registry.d.ts.map +1 -1
  75. package/dist/secret/registry.js +15 -3
  76. package/dist/semantic/documents.d.ts +21 -0
  77. package/dist/semantic/documents.d.ts.map +1 -0
  78. package/dist/semantic/documents.js +128 -0
  79. package/dist/semantic/embed.d.ts +59 -0
  80. package/dist/semantic/embed.d.ts.map +1 -0
  81. package/dist/semantic/embed.js +59 -0
  82. package/dist/semantic/messageIndex.d.ts +46 -0
  83. package/dist/semantic/messageIndex.d.ts.map +1 -0
  84. package/dist/semantic/messageIndex.js +67 -0
  85. package/dist/semantic/query.d.ts +37 -0
  86. package/dist/semantic/query.d.ts.map +1 -0
  87. package/dist/semantic/query.js +46 -0
  88. package/dist/staff/staff.d.ts +22 -0
  89. package/dist/staff/staff.d.ts.map +1 -0
  90. package/dist/staff/staff.js +64 -0
  91. package/dist/store/assignment.d.ts +63 -0
  92. package/dist/store/assignment.d.ts.map +1 -0
  93. package/dist/store/assignment.js +86 -0
  94. package/dist/store/hybrid.d.ts +134 -0
  95. package/dist/store/hybrid.d.ts.map +1 -0
  96. package/dist/store/hybrid.js +185 -0
  97. package/dist/store/paging.d.ts +10 -0
  98. package/dist/store/paging.d.ts.map +1 -1
  99. package/dist/store/paging.js +11 -1
  100. package/dist/store/provisionIndex.d.ts +21 -0
  101. package/dist/store/provisionIndex.d.ts.map +1 -0
  102. package/dist/store/provisionIndex.js +36 -0
  103. package/dist/store/search.d.ts +18 -9
  104. package/dist/store/search.d.ts.map +1 -1
  105. package/dist/store/search.js +36 -34
  106. package/dist/store/searchIndex.d.ts +4 -4
  107. package/dist/store/searchIndex.js +4 -4
  108. package/dist/store/threads.d.ts +174 -8
  109. package/dist/store/threads.d.ts.map +1 -1
  110. package/dist/store/threads.js +33 -40
  111. package/dist/submission/submit.d.ts +1 -1
  112. package/dist/submission/submit.d.ts.map +1 -1
  113. package/dist/submission/submit.js +2 -1
  114. package/dist/test-utils/seed.d.ts +33 -0
  115. package/dist/test-utils/seed.d.ts.map +1 -0
  116. package/dist/test-utils/seed.js +102 -0
  117. package/dist/test-utils/semantic.d.ts +21 -0
  118. package/dist/test-utils/semantic.d.ts.map +1 -0
  119. package/dist/test-utils/semantic.js +107 -0
  120. package/dist/version.generated.d.ts +1 -1
  121. package/dist/version.generated.js +1 -1
  122. package/dist/workflows/specs.d.ts +23 -0
  123. package/dist/workflows/specs.d.ts.map +1 -1
  124. package/dist/workflows/specs.js +27 -7
  125. package/dist/workflows/worker.d.ts +15 -2
  126. package/dist/workflows/worker.d.ts.map +1 -1
  127. package/dist/workflows/worker.js +48 -4
  128. package/package.json +9 -7
  129. package/pithy.manifest.json +21 -3
  130. package/src/audit/actions.ts +9 -0
  131. package/src/capability.ts +43 -4
  132. package/src/config/config.ts +104 -5
  133. package/src/data/assignment.ts +55 -0
  134. package/src/data/embedding.ts +27 -0
  135. package/src/data/tables.ts +8 -0
  136. package/src/data/thread.ts +26 -4
  137. package/src/http/guards.ts +52 -15
  138. package/src/http/handlers.ts +125 -4
  139. package/src/http/resolve.ts +19 -0
  140. package/src/http/responses.ts +154 -0
  141. package/src/http/routes.ts +98 -12
  142. package/src/http/schemas.ts +117 -28
  143. package/src/http/scopes.ts +43 -6
  144. package/src/http/views.ts +44 -0
  145. package/src/inbound/ingest.ts +5 -2
  146. package/src/index.ts +5 -0
  147. package/src/migrations/0002_assignment.ts +65 -0
  148. package/src/migrations/0003_embeddings.ts +29 -0
  149. package/src/migrations/set.ts +21 -0
  150. package/src/presence/room.ts +94 -0
  151. package/src/provision/provisionSupport.ts +33 -1
  152. package/src/provision/resolveSupportConfig.ts +24 -13
  153. package/src/purge/contribution.ts +294 -0
  154. package/src/secret/registry.ts +17 -2
  155. package/src/semantic/documents.ts +178 -0
  156. package/src/semantic/embed.ts +99 -0
  157. package/src/semantic/messageIndex.ts +74 -0
  158. package/src/semantic/query.ts +66 -0
  159. package/src/staff/staff.ts +128 -0
  160. package/src/store/assignment.ts +179 -0
  161. package/src/store/hybrid.ts +314 -0
  162. package/src/store/paging.ts +14 -0
  163. package/src/store/provisionIndex.ts +46 -0
  164. package/src/store/search.ts +58 -37
  165. package/src/store/searchIndex.ts +4 -4
  166. package/src/store/threads.ts +91 -91
  167. package/src/submission/submit.ts +3 -2
  168. package/src/test-utils/seed.ts +141 -0
  169. package/src/test-utils/semantic.ts +110 -0
  170. package/src/version.generated.ts +1 -1
  171. package/src/workflows/specs.ts +34 -0
  172. package/src/workflows/worker.ts +84 -8
  173. package/src/workflows/wrangler.jsonc +8 -3
@@ -0,0 +1,185 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import { SUPPORT_THREADS_TABLE } from "../data/tables.js";
4
+ import { semanticHits } from "../semantic/query.js";
5
+ import "./paging.js";
6
+ import { keywordHits } from "./search.js";
7
+ import { applyThreadFilters, listedThreadOf, viewerThreads } from "./threads.js";
8
+ import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
9
+ import { pithyCommandOf } from "@pithy-sh/core/src/cli/command";
10
+ import { chunkByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
11
+ //#region src/store/hybrid.ts
12
+ /** The reciprocal-rank-fusion constant — the standard 60, which damps the head without flattening it. */
13
+ const RRF_K = 60;
14
+ /** How many messages each backend retrieves before the roll-up. Vectorize's payload-free ceiling. */
15
+ const SEARCH_HITS = 100;
16
+ /**
17
+ * Roll a backend's message hits up to threads: a thread takes the rank of its best message, and appears once.
18
+ * The input is best first, so the first message seen for a thread is its best.
19
+ */
20
+ function rollUp(hits) {
21
+ const seen = /* @__PURE__ */ new Set();
22
+ const ranked = [];
23
+ for (const hit of hits) {
24
+ if (seen.has(hit.threadId)) continue;
25
+ seen.add(hit.threadId);
26
+ ranked.push({
27
+ threadId: hit.threadId,
28
+ messageId: hit.messageId,
29
+ rank: ranked.length + 1
30
+ });
31
+ }
32
+ return ranked;
33
+ }
34
+ /**
35
+ * Fuse ranked thread lists by reciprocal rank fusion. A thread found by several lists sums its terms.
36
+ *
37
+ * The message a result names comes from the list that ranked the thread highest, the first list winning a
38
+ * tie — the keyword list is passed first, so an exact match names the message that holds it.
39
+ */
40
+ function fuse(lists) {
41
+ const fused = /* @__PURE__ */ new Map();
42
+ for (const list of lists) for (const entry of list) {
43
+ const term = 1 / (60 + entry.rank);
44
+ const known = fused.get(entry.threadId);
45
+ if (!known) {
46
+ fused.set(entry.threadId, {
47
+ threadId: entry.threadId,
48
+ messageId: entry.messageId,
49
+ fused: term,
50
+ best: entry.rank
51
+ });
52
+ continue;
53
+ }
54
+ known.fused += term;
55
+ if (entry.rank < known.best) {
56
+ known.best = entry.rank;
57
+ known.messageId = entry.messageId;
58
+ }
59
+ }
60
+ return [...fused.values()].map(({ threadId, messageId, fused: score }) => ({
61
+ threadId,
62
+ messageId,
63
+ fused: score
64
+ }));
65
+ }
66
+ /** Whether an error is SQLite saying the full-text index does not exist. */
67
+ function isMissingSearchTable(error) {
68
+ const message = error instanceof Error ? error.message : String(error);
69
+ return message.includes("no such table") && message.includes("pithy_support_search");
70
+ }
71
+ /** A refusal for a mode this configuration cannot run. */
72
+ function semanticOff(mode) {
73
+ return new ValidationError({
74
+ message: `Search mode \`${mode}\` needs semantic search, and \`search.semantic\` is off.`,
75
+ action: `Turn on \`search.semantic\` in support()'s config and run \`${pithyCommandOf("support provision", {}, { runtime: true })}\`, or search with \`mode=keyword\`.`,
76
+ detail: `mode ${mode} requested with search.semantic off`
77
+ });
78
+ }
79
+ /** Run a search. */
80
+ async function searchThreads(deps, query) {
81
+ const semanticOn = deps.index !== void 0;
82
+ if ((query.mode === "semantic" || query.mode === "hybrid") && !semanticOn) throw semanticOff(query.mode);
83
+ const mode = query.mode ?? "hybrid";
84
+ const limit = Math.min(Math.max(query.limit ?? 20, 1), 50);
85
+ const ran = [];
86
+ const skipped = [];
87
+ const lists = [];
88
+ const keywordRank = /* @__PURE__ */ new Map();
89
+ const bm25 = /* @__PURE__ */ new Map();
90
+ const similarity = /* @__PURE__ */ new Map();
91
+ if (mode !== "semantic") {
92
+ let backend = deps.fts ? "fts" : "like";
93
+ let hits;
94
+ try {
95
+ hits = await keywordHits(deps.db, query.q, {
96
+ backend,
97
+ limit: 100
98
+ });
99
+ } catch (error) {
100
+ if (backend !== "fts" || !isMissingSearchTable(error)) throw error;
101
+ deps.log?.warn("support FTS index is configured but absent — searching with a LIKE scan", { action: `Run \`${pithyCommandOf("support provision", {}, { runtime: true })}\` to create and backfill it, or set \`search.fts: false\`.` });
102
+ skipped.push({
103
+ backend: "fts",
104
+ reason: "unprovisioned"
105
+ });
106
+ backend = "like";
107
+ hits = await keywordHits(deps.db, query.q, {
108
+ backend,
109
+ limit: 100
110
+ });
111
+ }
112
+ ran.push(backend);
113
+ if (backend === "fts") {
114
+ for (const hit of hits) if (hit.score !== void 0) bm25.set(hit.messageId, hit.score);
115
+ }
116
+ const keywordList = rollUp(hits);
117
+ for (const entry of keywordList) keywordRank.set(entry.threadId, entry.rank);
118
+ lists.push(keywordList);
119
+ }
120
+ if (mode !== "keyword" && deps.index) {
121
+ if (!deps.ai || !deps.store) {
122
+ deps.log?.warn("support semantic search is on but its index is not bound — skipping it", { action: `Run \`${pithyCommandOf("support provision", {}, { runtime: true })}\` to create the index and bind it.` });
123
+ skipped.push({
124
+ backend: "semantic",
125
+ reason: "unprovisioned"
126
+ });
127
+ } else try {
128
+ const hits = await semanticHits({
129
+ db: deps.db,
130
+ ai: deps.ai,
131
+ store: deps.store,
132
+ index: deps.index
133
+ }, query.q);
134
+ for (const hit of hits) if (hit.score !== void 0) similarity.set(hit.messageId, hit.score);
135
+ lists.push(rollUp(hits));
136
+ ran.push("semantic");
137
+ } catch (error) {
138
+ deps.log?.warn("support semantic search could not answer — skipping it", { detail: error instanceof Error ? error.message : String(error) });
139
+ skipped.push({
140
+ backend: "semantic",
141
+ reason: "failed"
142
+ });
143
+ }
144
+ }
145
+ const fused = fuse(lists);
146
+ const threads = await hydrate(deps.db, deps.viewer, fused.map((entry) => entry.threadId), query);
147
+ return {
148
+ mode,
149
+ ran,
150
+ skipped,
151
+ results: fused.flatMap((entry) => {
152
+ const thread = threads.get(entry.threadId);
153
+ return thread ? [{
154
+ entry,
155
+ thread
156
+ }] : [];
157
+ }).sort((a, b) => b.entry.fused - a.entry.fused || (keywordRank.get(a.entry.threadId) ?? Number.POSITIVE_INFINITY) - (keywordRank.get(b.entry.threadId) ?? Number.POSITIVE_INFINITY) || b.thread.lastMessageAt.getTime() - a.thread.lastMessageAt.getTime() || (a.thread.id < b.thread.id ? 1 : a.thread.id > b.thread.id ? -1 : 0)).slice(0, limit).map(({ entry, thread }) => {
158
+ const keyword = bm25.get(entry.messageId);
159
+ const near = similarity.get(entry.messageId);
160
+ return {
161
+ thread,
162
+ messageId: entry.messageId,
163
+ ...keyword !== void 0 ? { bm25: keyword } : {},
164
+ ...near !== void 0 ? { similarity: near } : {}
165
+ };
166
+ })
167
+ };
168
+ }
169
+ /** The filter values bind before the id list; this is the most a filtered read can bind besides it. */
170
+ const HYDRATE_FIXED_PARAMETERS = 12;
171
+ /** Read the candidate threads that pass the caller's filters, as this viewer sees them. */
172
+ async function hydrate(db, viewer, threadIds, filters) {
173
+ const threads = /* @__PURE__ */ new Map();
174
+ if (threadIds.length === 0) return threads;
175
+ for (const group of chunkByBoundParameters(threadIds, HYDRATE_FIXED_PARAMETERS)) {
176
+ const rows = await applyThreadFilters(viewerThreads(db, viewer), filters).where(`${SUPPORT_THREADS_TABLE}.id`, "in", group).execute();
177
+ for (const row of rows) {
178
+ const thread = listedThreadOf(row);
179
+ threads.set(thread.id, thread);
180
+ }
181
+ }
182
+ return threads;
183
+ }
184
+ //#endregion
185
+ export { RRF_K, SEARCH_HITS, fuse, rollUp, searchThreads };
@@ -17,4 +17,14 @@
17
17
  export declare const DEFAULT_PAGE_SIZE = 25;
18
18
  /** The most a caller may ask for in one page. */
19
19
  export declare const MAX_PAGE_SIZE = 100;
20
+ /**
21
+ * The search route's bounds. Here rather than in `store/hybrid.ts` for the reason the page bounds are:
22
+ * `http/schemas.ts` is a client's business and reaches no Kysely.
23
+ */
24
+ /** How many threads a search returns by default. */
25
+ export declare const DEFAULT_SEARCH_LIMIT = 20;
26
+ /** The most a search returns. A ranked list's tail is noise; fifty is past where anyone reads. */
27
+ export declare const MAX_SEARCH_LIMIT = 50;
28
+ /** The longest search term accepted — it becomes a scan predicate and a model input. */
29
+ export declare const MAX_SEARCH_TERM = 200;
20
30
  //# sourceMappingURL=paging.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"paging.d.ts","sourceRoot":"","sources":["../../src/store/paging.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AAEH,gDAAgD;AAChD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,iDAAiD;AACjD,eAAO,MAAM,aAAa,MAAM,CAAC"}
1
+ {"version":3,"file":"paging.d.ts","sourceRoot":"","sources":["../../src/store/paging.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;;GAYG;AAEH,gDAAgD;AAChD,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAEpC,iDAAiD;AACjD,eAAO,MAAM,aAAa,MAAM,CAAC;AAEjC;;;GAGG;AAEH,oDAAoD;AACpD,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,kGAAkG;AAClG,eAAO,MAAM,gBAAgB,KAAK,CAAC;AAEnC,wFAAwF;AACxF,eAAO,MAAM,eAAe,MAAM,CAAC"}
@@ -18,5 +18,15 @@
18
18
  const DEFAULT_PAGE_SIZE = 25;
19
19
  /** The most a caller may ask for in one page. */
20
20
  const MAX_PAGE_SIZE = 100;
21
+ /**
22
+ * The search route's bounds. Here rather than in `store/hybrid.ts` for the reason the page bounds are:
23
+ * `http/schemas.ts` is a client's business and reaches no Kysely.
24
+ */
25
+ /** How many threads a search returns by default. */
26
+ const DEFAULT_SEARCH_LIMIT = 20;
27
+ /** The most a search returns. A ranked list's tail is noise; fifty is past where anyone reads. */
28
+ const MAX_SEARCH_LIMIT = 50;
29
+ /** The longest search term accepted — it becomes a scan predicate and a model input. */
30
+ const MAX_SEARCH_TERM = 200;
21
31
  //#endregion
22
- export { DEFAULT_PAGE_SIZE, MAX_PAGE_SIZE };
32
+ export { DEFAULT_PAGE_SIZE, DEFAULT_SEARCH_LIMIT, MAX_PAGE_SIZE, MAX_SEARCH_LIMIT, MAX_SEARCH_TERM };
@@ -0,0 +1,21 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import type { D1Database } from "@cloudflare/workers-types";
4
+ import { type SearchIndexAction } from "./searchIndex";
5
+ /**
6
+ * **Bring one database's full-text index into line with `search.fts`, and say what that took.**
7
+ *
8
+ * The body of `pithy support provision`'s search step, and of `pithy restore`'s rebuild of the index a
9
+ * backup leaves out (#848). One function for both, because the two have to agree: a restore that rebuilt
10
+ * the index some other way would be a second definition of a provisioned index, and the first time the
11
+ * triggers changed it would restore the old ones.
12
+ *
13
+ * It reads `sqlite_master` for the table and its triggers, decides with {@link searchIndexAction}, and
14
+ * applies the decision. A create and a repair backfill from `pithy_support_messages` straight away: an
15
+ * index that exists and is empty answers "no matches" for a word plainly in a message, because the `LIKE`
16
+ * fallback only fires while the table is absent. A drop takes the triggers first, then the table.
17
+ *
18
+ * Every statement is idempotent, so a re-run is a no-op that answers `none`.
19
+ */
20
+ export declare function provisionSearchIndex(database: D1Database, wanted: boolean): Promise<SearchIndexAction>;
21
+ //# sourceMappingURL=provisionIndex.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"provisionIndex.d.ts","sourceRoot":"","sources":["../../src/store/provisionIndex.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAG5D,OAAO,EAIL,KAAK,iBAAiB,EAGvB,MAAM,eAAe,CAAC;AAEvB;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,oBAAoB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAe5G"}
@@ -0,0 +1,36 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import { supportDatabase } from "../data/tables.js";
4
+ import { reindexAll } from "./search.js";
5
+ import { SEARCH_OBJECTS, createSearchIndex, dropSearchIndex, searchIndexAction, searchIndexState } from "./searchIndex.js";
6
+ //#region src/store/provisionIndex.ts
7
+ /**
8
+ * **Bring one database's full-text index into line with `search.fts`, and say what that took.**
9
+ *
10
+ * The body of `pithy support provision`'s search step, and of `pithy restore`'s rebuild of the index a
11
+ * backup leaves out (#848). One function for both, because the two have to agree: a restore that rebuilt
12
+ * the index some other way would be a second definition of a provisioned index, and the first time the
13
+ * triggers changed it would restore the old ones.
14
+ *
15
+ * It reads `sqlite_master` for the table and its triggers, decides with {@link searchIndexAction}, and
16
+ * applies the decision. A create and a repair backfill from `pithy_support_messages` straight away: an
17
+ * index that exists and is empty answers "no matches" for a word plainly in a message, because the `LIKE`
18
+ * fallback only fires while the table is absent. A drop takes the triggers first, then the table.
19
+ *
20
+ * Every statement is idempotent, so a re-run is a no-op that answers `none`.
21
+ */
22
+ async function provisionSearchIndex(database, wanted) {
23
+ const listed = await database.prepare(`SELECT name FROM sqlite_master WHERE name IN (${SEARCH_OBJECTS.map(() => "?").join(", ")})`).bind(...SEARCH_OBJECTS).all();
24
+ const action = searchIndexAction(searchIndexState((listed.results ?? []).map((row) => row.name)), wanted);
25
+ if (action === "none") return action;
26
+ const db = supportDatabase(database);
27
+ if (action === "drop") {
28
+ await dropSearchIndex(db);
29
+ return action;
30
+ }
31
+ await createSearchIndex(db);
32
+ await reindexAll(db);
33
+ return action;
34
+ }
35
+ //#endregion
36
+ export { provisionSearchIndex };
@@ -1,6 +1,5 @@
1
1
  // SPDX-FileCopyrightText: 2026 Pithy
2
2
  // SPDX-License-Identifier: MIT
3
- import { type Expression, type SqlBool } from "kysely";
4
3
  import { type SupportDatabase } from "../data/tables";
5
4
  /**
6
5
  * Turn a user's words into an FTS5 MATCH expression.
@@ -12,17 +11,27 @@ import { type SupportDatabase } from "../data/tables";
12
11
  * of the package applies to model output and to HTML.
13
12
  */
14
13
  export declare function ftsQuery(term: string): string;
15
- /** Whether a term has anything searchable left after tokenizing. */
16
- export declare function isSearchable(term: string, fts: boolean): boolean;
14
+ /** One message a backend matched, in rank order. */
15
+ export interface MessageHit {
16
+ /** The message. */
17
+ messageId: string;
18
+ /** Its thread. */
19
+ threadId: string;
20
+ /** The backend's own score for it, raw — `bm25` for FTS5, similarity for Vectorize, none for `LIKE`. */
21
+ score?: number;
22
+ }
23
+ /** Which keyword backend answered. */
24
+ export type KeywordBackend = "fts" | "like";
17
25
  /**
18
- * A predicate matching threads whose messages contain `term`.
26
+ * The messages matching `term`, best first, at most `limit` of them.
19
27
  *
20
- * Correlated on the thread id so it drops into a `where` beside every other filter. The subquery is
21
- * `exists`, so it stops at the first matching message rather than counting them.
28
+ * A term that tokenizes to nothing — `???`, `@@@` — asked a question with no answer, and the honest answer
29
+ * is none: never the whole inbox rendered as search results.
22
30
  */
23
- export declare function searchPredicate(term: string, options: {
24
- fts: boolean;
25
- }): Expression<SqlBool>;
31
+ export declare function keywordHits(db: SupportDatabase, term: string, options: {
32
+ backend: KeywordBackend;
33
+ limit: number;
34
+ }): Promise<MessageHit[]>;
26
35
  /**
27
36
  * Add a message to the full-text index, replacing any row already there for it.
28
37
  *
@@ -1 +1 @@
1
- {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/store/search.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,OAAO,EAAO,MAAM,QAAQ,CAAC;AAC5D,OAAO,EAA0B,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AA+B9E;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED,oEAAoE;AACpE,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAEhE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;IAAE,GAAG,EAAE,OAAO,CAAA;CAAE,GAAG,UAAU,CAAC,OAAO,CAAC,CAkB5F;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,eAAe,EACnB,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5E,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkB1F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,UAAU,CAC9B,EAAE,EAAE,eAAe,EACnB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GAC1E,OAAO,CAAC,MAAM,CAAC,CA4CjB"}
1
+ {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/store/search.ts"],"names":[],"mappings":"AAIA,OAAO,EAA0B,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AA0B9E;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAS7C;AAED,oDAAoD;AACpD,MAAM,WAAW,UAAU;IACzB,mBAAmB;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,wGAAwG;IACxG,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,sCAAsC;AACtC,MAAM,MAAM,cAAc,GAAG,KAAK,GAAG,MAAM,CAAC;AAE5C;;;;;GAKG;AACH,wBAAsB,WAAW,CAC/B,EAAE,EAAE,eAAe,EACnB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE;IAAE,OAAO,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAClD,OAAO,CAAC,UAAU,EAAE,CAAC,CAgCvB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,eAAe,EACnB,KAAK,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5E,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;;;GAKG;AACH,wBAAsB,aAAa,CAAC,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkB1F;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,UAAU,CAC9B,EAAE,EAAE,eAAe,EACnB,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GAC1E,OAAO,CAAC,MAAM,CAAC,CA4CjB"}
@@ -4,24 +4,19 @@ import { SUPPORT_MESSAGES_TABLE } from "../data/tables.js";
4
4
  import { sql } from "kysely";
5
5
  //#region src/store/search.ts
6
6
  /**
7
- * Text search over the inbox — two backends behind one predicate.
7
+ * Keyword retrieval over the inbox — two backends behind one call.
8
8
  *
9
- * **`LIKE` is the default and FTS5 is the opt-in**, which is the opposite of what it looks like it
10
- * should be. The reason is in `store/searchIndex.ts`: an FTS5 virtual table anywhere in a D1
11
- * database makes `wrangler d1 export` refuse to dump *the whole database*, so turning it on by
12
- * default would quietly cost every adopter their backup story for tables that have nothing to do
13
- * with support. `LIKE` is an unindexed scan and is completely adequate for an inbox measured in
14
- * thousands of messages, which is what this capability is for.
9
+ * **FTS5 is the default and `LIKE` is the fallback.** The index lives in a virtual table that
10
+ * `pithy support provision` creates beside its triggers, so a project that has not provisioned it — or
11
+ * that set `search.fts: false` — searches with an unindexed `LIKE` scan, which is adequate for an inbox
12
+ * measured in thousands of messages. `pithy backup` leaves the index out and `pithy restore` rebuilds it.
15
13
  *
16
- * Both paths return a **predicate over `pithy_support_threads`**, not a list of ids, so search
17
- * composes with the category/archived filters and — crucially — with cursor pagination. Returning
18
- * ids would mean either capping the match set (silent truncation, which reads as "that is all there
19
- * is") or paginating twice over.
14
+ * **Both return messages, ranked, not threads.** A thread is scored by its best-matching message, and the
15
+ * keyword half is fused with the semantic half by rank position in `store/hybrid.ts`. FTS5 ranks by `bm25`,
16
+ * which is reported raw. `LIKE` has no score, so its matches rank by recency, newest first.
20
17
  *
21
- * Either way this is keyword search, deliberately. People search a support inbox for a word they
22
- * remember — an order number, an error string, a surname — not for a concept. `@pithy-sh/vector` is
23
- * there for anyone who wants meaning-matching, and it is the nicer demo and the wrong default: it
24
- * cannot find `ORD-40912`.
18
+ * Keyword search is first-class beside semantic search: people search a support inbox for a word they
19
+ * remember — an order number, an error string, a surname — and semantic search cannot find `ORD-40912`.
25
20
  */
26
21
  /** The longest search term accepted. Bounded because it becomes a scan predicate. */
27
22
  const MAX_TERM = 200;
@@ -41,28 +36,35 @@ function escapeLike(term) {
41
36
  function ftsQuery(term) {
42
37
  return term.slice(0, MAX_TERM).split(/[^\p{L}\p{N}_-]+/u).filter((token) => token.length > 0).slice(0, 16).map((token) => `"${token.replace(/"/g, "\"\"")}"`).join(" AND ");
43
38
  }
44
- /** Whether a term has anything searchable left after tokenizing. */
45
- function isSearchable(term, fts) {
46
- return fts ? ftsQuery(term).length > 0 : term.trim().length > 0;
47
- }
48
39
  /**
49
- * A predicate matching threads whose messages contain `term`.
40
+ * The messages matching `term`, best first, at most `limit` of them.
50
41
  *
51
- * Correlated on the thread id so it drops into a `where` beside every other filter. The subquery is
52
- * `exists`, so it stops at the first matching message rather than counting them.
42
+ * A term that tokenizes to nothing — `???`, `@@@` — asked a question with no answer, and the honest answer
43
+ * is none: never the whole inbox rendered as search results.
53
44
  */
54
- function searchPredicate(term, options) {
55
- if (options.fts) return sql`exists (
56
- select 1 from pithy_support_search
57
- where pithy_support_search.thread_id = pithy_support_threads.id
58
- and pithy_support_search match ${ftsQuery(term)}
59
- )`;
45
+ async function keywordHits(db, term, options) {
46
+ if (options.backend === "fts") {
47
+ const match = ftsQuery(term);
48
+ if (match.length === 0) return [];
49
+ const { rows } = await sql`
50
+ select message_id as "messageId", thread_id as "threadId", bm25(pithy_support_search) as "score"
51
+ from pithy_support_search
52
+ where pithy_support_search match ${match}
53
+ order by "score" asc
54
+ limit ${options.limit}
55
+ `.execute(db);
56
+ return rows.map((row) => ({
57
+ messageId: row.messageId,
58
+ threadId: row.threadId,
59
+ score: Number(row.score)
60
+ }));
61
+ }
62
+ if (term.trim().length === 0) return [];
60
63
  const pattern = `%${escapeLike(term.slice(0, MAX_TERM))}%`;
61
- return sql`exists (
62
- select 1 from pithy_support_messages m
63
- where m.thread_id = pithy_support_threads.id
64
- and (m.subject like ${pattern} escape '\\' or m.text_body like ${pattern} escape '\\')
65
- )`;
64
+ return (await db.selectFrom(SUPPORT_MESSAGES_TABLE).select(["id", "threadId"]).where((eb) => eb.or([sql`${eb.ref("subject")} like ${pattern} escape '\\'`, sql`${eb.ref("textBody")} like ${pattern} escape '\\'`])).orderBy("receivedAt", "desc").orderBy("id", "desc").limit(options.limit).execute()).map((row) => ({
65
+ messageId: row.id,
66
+ threadId: row.threadId
67
+ }));
66
68
  }
67
69
  /**
68
70
  * Add a message to the full-text index, replacing any row already there for it.
@@ -166,4 +168,4 @@ async function reindexAll(db, options = {}) {
166
168
  return indexed;
167
169
  }
168
170
  //#endregion
169
- export { ftsQuery, indexMessage, isSearchable, reindexAll, reindexThread, searchPredicate };
171
+ export { ftsQuery, indexMessage, keywordHits, reindexAll, reindexThread };
@@ -129,10 +129,10 @@ export declare function searchIndexAction(state: SearchIndexState, wanted: boole
129
129
  /**
130
130
  * The `CREATE TRIGGER` statements, one per {@link SEARCH_TRIGGERS} entry and in that order.
131
131
  *
132
- * **`BEGIN` and `END` are uppercase, and that is load-bearing.** Miniflare accepts a lowercase
133
- * compound-statement body; remote D1 rejects it as `incomplete input [code: 7500]`, so a lowercased
134
- * body passes every local test in this repository and fails on deploy. `searchIndex.test.ts` fails on
135
- * the lowercase form, because no Workers test can.
132
+ * **`BEGIN` and `END` are uppercase, by convention.** Remote D1 and Miniflare both accept a lowercase
133
+ * compound-statement body today, so nothing enforces the casing at run time. Older D1 rejected the
134
+ * lowercase form as `incomplete input [code: 7500]`, and uppercase is accepted everywhere, so the bodies
135
+ * keep it and `searchIndex.test.ts` pins it.
136
136
  *
137
137
  * **Every column is the physical snake_case name.** `CamelCasePlugin` snake-cases the identifiers the
138
138
  * query builder emits and never touches a string inside a `sql` template, so this is the one place in
@@ -127,10 +127,10 @@ function searchIndexAction(state, wanted) {
127
127
  /**
128
128
  * The `CREATE TRIGGER` statements, one per {@link SEARCH_TRIGGERS} entry and in that order.
129
129
  *
130
- * **`BEGIN` and `END` are uppercase, and that is load-bearing.** Miniflare accepts a lowercase
131
- * compound-statement body; remote D1 rejects it as `incomplete input [code: 7500]`, so a lowercased
132
- * body passes every local test in this repository and fails on deploy. `searchIndex.test.ts` fails on
133
- * the lowercase form, because no Workers test can.
130
+ * **`BEGIN` and `END` are uppercase, by convention.** Remote D1 and Miniflare both accept a lowercase
131
+ * compound-statement body today, so nothing enforces the casing at run time. Older D1 rejected the
132
+ * lowercase form as `incomplete input [code: 7500]`, and uppercase is accepted everywhere, so the bodies
133
+ * keep it and `searchIndex.test.ts` pins it.
134
134
  *
135
135
  * **Every column is the physical snake_case name.** `CamelCasePlugin` snake-cases the identifiers the
136
136
  * query builder emits and never touches a string inside a `sql` template, so this is the one place in