@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,2480 @@
1
+ import { createHash } from "node:crypto";
2
+ import { constants, type BigIntStats } from "node:fs";
3
+ import {
4
+ chmod,
5
+ lstat,
6
+ mkdir,
7
+ mkdtemp,
8
+ open,
9
+ realpath,
10
+ rm,
11
+ stat,
12
+ } from "node:fs/promises";
13
+ import { homedir, tmpdir } from "node:os";
14
+ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
15
+
16
+ import type { Backlink, MetadataObject, Note, NoteConnections } from "./graph.js";
17
+ import {
18
+ describeSemanticProjection,
19
+ prepareSemanticProjection,
20
+ resolveSemanticDatabase,
21
+ withSemanticGenerationWriterLease,
22
+ type SemanticIndexIdentity,
23
+ type SemanticProjection,
24
+ type SemanticProjectionDescription,
25
+ type SemanticWriterLeaseOptions,
26
+ } from "./semantic-runtime.js";
27
+ import { fuseRankedCandidates, validateSearchQuery } from "./search.js";
28
+ import { scanVault, type VaultSnapshot } from "./vault.js";
29
+
30
+ /** QMD's small local default at an immutable Hugging Face repository revision. */
31
+ export const recommendedEmbeddingModel =
32
+ "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf#0f741b5a6585bd53aeb15cd1372c56f2a0f65e12";
33
+ /** SHA-256 of the immutable recommended EmbeddingGemma GGUF artifact. */
34
+ export const recommendedEmbeddingModelSha256 =
35
+ "b5ce9d77a3fc4b3b39ccb5643c36777911cc4eb46a66962eadfa3f5f60490d63";
36
+ export const MAX_EMBEDDING_MODEL_BYTES = 2 * 1_024 * 1_024 * 1_024;
37
+ export const MAX_SEMANTIC_DATABASE_IDENTITY_BYTES = 16 * 1_024;
38
+ const MAX_SEMANTIC_READ_SNAPSHOT_BYTES = 16 * 1_024 * 1_024 * 1_024;
39
+ const SHA256 = /^[0-9a-f]{64}$/u;
40
+
41
+ const semanticIndexSchema = 1;
42
+ export const qmdIndexerVersion =
43
+ "2.5.3+hraness.aa993dceb3ef8cfb71d470554ca437570f5a2b3c";
44
+ const collectionName = "kb";
45
+ const markdownPattern = "**/*.md";
46
+ const ignoredPatterns = ["index.md", "**/AGENTS.md"] as const;
47
+ const embeddingChunkStrategy = "regex";
48
+ const globalContext =
49
+ "A Markdown knowledge base. Source records preserve evidence; maintained notes contain current synthesis; explicit wikilinks define structural relationships.";
50
+ const collectionContext = {
51
+ "/": "Knowledge-base notes, clipped sources, plans, reports, and explicit contextual links.",
52
+ "/articles": "Captured source records and their acquisition provenance.",
53
+ "/notes": "Maintained concepts, comparisons, and current synthesis.",
54
+ "/plans": "Decisions, constraints, execution state, and verification evidence.",
55
+ "/riffs": "Voice-preserving first-person source thought.",
56
+ } as const;
57
+ const recommendedEmbeddingModelIdentity =
58
+ `${recommendedEmbeddingModel}@sha256:${recommendedEmbeddingModelSha256}`;
59
+ const semanticIndexIdentity: SemanticIndexIdentity = {
60
+ producer: { package: "@hraness/kb", schema: semanticIndexSchema },
61
+ indexer: { package: "@tobilu/qmd", version: qmdIndexerVersion },
62
+ collection: {
63
+ name: collectionName,
64
+ pattern: markdownPattern,
65
+ ignore: ignoredPatterns,
66
+ globalContext,
67
+ pathContexts: Object.entries(collectionContext).map(([path, context]) => ({ path, context })),
68
+ },
69
+ embedding: {
70
+ model: recommendedEmbeddingModelIdentity,
71
+ chunkStrategy: embeddingChunkStrategy,
72
+ },
73
+ };
74
+ // Keep this widened: a literal dynamic import makes TypeScript load QMD's public declarations.
75
+ const qmdModuleSpecifier: string = "@tobilu/qmd";
76
+
77
+ export type SemanticSearchMode = "hybrid" | "keyword" | "semantic";
78
+
79
+ export type SemanticIndexOptions = {
80
+ readonly root: string;
81
+ readonly database?: string;
82
+ /** Verified local bytes for the pinned model. Public results retain the stable model URI. */
83
+ readonly embeddingModelFile?: string;
84
+ readonly force?: boolean;
85
+ };
86
+
87
+ /**
88
+ * Opaque ownership handle for one privately copied, verified local embedding
89
+ * model. The machine-local source path is deliberately not part of this API.
90
+ */
91
+ export type VerifiedEmbeddingModelLease = {
92
+ readonly model: typeof recommendedEmbeddingModel;
93
+ /** Stop accepting new readers; the final retained reader removes the private copy. */
94
+ readonly close: () => Promise<void>;
95
+ };
96
+
97
+ export type SemanticSearchOptions = {
98
+ readonly root: string;
99
+ readonly query: string;
100
+ readonly database?: string;
101
+ readonly mode?: SemanticSearchMode;
102
+ readonly limit?: number;
103
+ readonly candidateLimit?: number;
104
+ readonly minScore?: number;
105
+ };
106
+
107
+ export type SemanticCollectionConfig = {
108
+ readonly global_context?: string;
109
+ readonly collections: Readonly<Record<string, {
110
+ readonly path: string;
111
+ readonly pattern: string;
112
+ readonly ignore?: readonly string[];
113
+ readonly context?: Readonly<Record<string, string>>;
114
+ }>>;
115
+ readonly models?: {
116
+ readonly embed?: string;
117
+ };
118
+ };
119
+
120
+ export type SemanticStoreOptions = {
121
+ readonly dbPath: string;
122
+ readonly config: SemanticCollectionConfig;
123
+ };
124
+
125
+ export type SemanticUpdateResult = {
126
+ readonly collections: number;
127
+ readonly indexed: number;
128
+ readonly updated: number;
129
+ readonly unchanged: number;
130
+ readonly removed: number;
131
+ readonly needsEmbedding: number;
132
+ };
133
+
134
+ export type SemanticEmbeddingFailure = {
135
+ readonly path: string;
136
+ readonly hash: string;
137
+ readonly seq: number;
138
+ readonly attempts: number;
139
+ readonly reason: string;
140
+ };
141
+
142
+ export type SemanticEmbeddingResult = {
143
+ readonly docsProcessed: number;
144
+ readonly chunksEmbedded: number;
145
+ readonly errors: number;
146
+ readonly failures?: readonly SemanticEmbeddingFailure[];
147
+ readonly durationMs: number;
148
+ };
149
+
150
+ export type SemanticIndexResult = {
151
+ readonly root: string;
152
+ readonly database: string;
153
+ readonly model: string;
154
+ readonly update: SemanticUpdateResult;
155
+ readonly embedding: SemanticEmbeddingResult | null;
156
+ };
157
+
158
+ export type SemanticSearchHit = {
159
+ readonly path: string;
160
+ readonly title: string;
161
+ readonly score: number;
162
+ readonly source: "fts" | "hybrid" | "vec";
163
+ readonly docid: string;
164
+ readonly modifiedAt?: string;
165
+ readonly line?: number;
166
+ readonly snippet: string;
167
+ readonly signals?: {
168
+ readonly keyword: boolean;
169
+ readonly semantic: boolean;
170
+ };
171
+ readonly tags: readonly string[];
172
+ readonly metadata: MetadataObject;
173
+ readonly inboundContextualCount: number;
174
+ readonly outboundContextualCount: number;
175
+ readonly backlinks: readonly Backlink[];
176
+ };
177
+
178
+ export type SemanticSearchResult = {
179
+ readonly root: string;
180
+ readonly database: string;
181
+ readonly model: string;
182
+ readonly mode: SemanticSearchMode;
183
+ readonly query: string;
184
+ readonly update: SemanticUpdateResult;
185
+ readonly embedding: SemanticEmbeddingResult | null;
186
+ /** Exact query-vector work, or null when the optional public QMD fallback hid it. */
187
+ readonly queryEmbedding?: SemanticQueryEmbeddingAccounting | null;
188
+ /** Raw backend-window evidence for post-filter completeness diagnostics. */
189
+ readonly rawWindow?: {
190
+ readonly requested: number;
191
+ readonly returned: number;
192
+ /** Rows rejected because they could not reconcile to this live snapshot. */
193
+ readonly discarded: number;
194
+ /** Rows intentionally excluded by the caller's minimum score. */
195
+ readonly thresholdRejected: number;
196
+ /** True when QMD returned fewer raw rows than the requested backend window. */
197
+ readonly exhausted: boolean;
198
+ };
199
+ readonly results: readonly SemanticSearchHit[];
200
+ };
201
+
202
+ export type SemanticSessionOptions = {
203
+ readonly root: string;
204
+ readonly database?: string;
205
+ /** Verified local bytes for the pinned model. Public results retain the stable model URI. */
206
+ readonly embeddingModelFile?: string;
207
+ /** Shared opaque lease for verified local bytes. Mutually exclusive with embeddingModelFile. */
208
+ readonly embeddingModelLease?: VerifiedEmbeddingModelLease;
209
+ /** Reject QMD versions that cannot expose store-local query-vector inference. */
210
+ readonly requireStoreLocalVectorBoundary?: boolean;
211
+ };
212
+
213
+ export type SemanticWarmCacheAttestationOptions = {
214
+ readonly root: string;
215
+ readonly database?: string;
216
+ /** A branded lease proves the pinned local model bytes remain available. */
217
+ readonly embeddingModelLease: VerifiedEmbeddingModelLease;
218
+ /** Exact SQLite main-file bytes and checkpointed sidecar absence sealed before readers open. */
219
+ readonly databaseSnapshotSeal: SemanticDatabaseSnapshotSeal;
220
+ };
221
+
222
+ export type SemanticDatabaseFileSeal = Readonly<{
223
+ readonly bytes: number;
224
+ readonly sha256: string;
225
+ }>;
226
+
227
+ /**
228
+ * The complete SQLite state a strict warm reader may consume. The canonical
229
+ * database is sealed by bytes while WAL/SHM/rollback-journal sidecars are
230
+ * sealed as absent, so SQLite can create any runtime sidecars only beside the
231
+ * disposable private copy.
232
+ */
233
+ export type SemanticDatabaseSnapshotSeal = Readonly<{
234
+ readonly database: SemanticDatabaseFileSeal;
235
+ /** A strict warm snapshot is checkpointed; canonical sidecars are forbidden. */
236
+ readonly wal: null;
237
+ readonly shm: null;
238
+ readonly journal: null;
239
+ }>;
240
+
241
+ /**
242
+ * The already-indexed, immutable projection required by a measured warm
243
+ * reader. Unlike a normal semantic session, opening this session cannot
244
+ * prepare, repair, update, or embed the document corpus.
245
+ */
246
+ export type SemanticWarmSearchSessionOptions = SemanticWarmCacheAttestationOptions;
247
+
248
+ export type SemanticWarmCacheReadiness = {
249
+ readonly model: typeof recommendedEmbeddingModel;
250
+ /** Canonical database path identity. No model or projection path is exposed. */
251
+ readonly database: string;
252
+ readonly pendingEmbeddings: 0;
253
+ };
254
+
255
+ export type SemanticWarmCacheCheckpointResult = Readonly<{
256
+ readonly database: string;
257
+ readonly wal: null;
258
+ readonly shm: null;
259
+ readonly journal: null;
260
+ }>;
261
+
262
+ export type SemanticAttestationStoreOptions = {
263
+ readonly dbPath: string;
264
+ };
265
+
266
+ export type SemanticWarmStoreOptions = SemanticAttestationStoreOptions & Readonly<{
267
+ /** Private verified bytes used only to construct this store's local LLM session. */
268
+ readonly embeddingModelSource: string;
269
+ }>;
270
+
271
+ export type SemanticQueryEmbeddingAccounting = {
272
+ readonly calls: number;
273
+ readonly inputTokens: number;
274
+ /** Wall duration of the actual embed call only; token counting is excluded. */
275
+ readonly durationMs: number;
276
+ };
277
+
278
+ export type SemanticSessionSearchOptions = Omit<
279
+ SemanticSearchOptions,
280
+ "root" | "database"
281
+ >;
282
+
283
+ export type SemanticSearchSession = {
284
+ readonly root: string;
285
+ readonly database: string;
286
+ readonly model: string;
287
+ readonly update: SemanticUpdateResult;
288
+ /** Searches share one live vault snapshot and one serialized QMD store. */
289
+ readonly search: (
290
+ options: SemanticSessionSearchOptions,
291
+ ) => Promise<SemanticSearchResult>;
292
+ /** Idempotently close the owned QMD store after queued searches settle. */
293
+ readonly close: () => Promise<void>;
294
+ };
295
+
296
+ type SemanticSearchDocument = {
297
+ readonly filepath: string;
298
+ readonly title: string;
299
+ readonly hash: string;
300
+ readonly docid: string;
301
+ readonly modifiedAt: string;
302
+ readonly score: number;
303
+ readonly source: "fts" | "vec";
304
+ readonly chunkPos?: number;
305
+ };
306
+
307
+ type SearchStore = {
308
+ readonly close: () => Promise<void>;
309
+ readonly update: (options: { readonly collections: readonly string[] }) => Promise<SemanticUpdateResult>;
310
+ readonly embed: (options: {
311
+ readonly collection: string;
312
+ readonly force: boolean;
313
+ readonly model: string;
314
+ readonly chunkStrategy: "regex";
315
+ }) => Promise<SemanticEmbeddingResult>;
316
+ readonly searchLex: (
317
+ query: string,
318
+ options: { readonly collection: string; readonly limit: number },
319
+ ) => Promise<readonly SemanticSearchDocument[]>;
320
+ readonly searchVector: (
321
+ query: string,
322
+ options: { readonly collection: string; readonly limit: number },
323
+ ) => Promise<{
324
+ readonly results: readonly SemanticSearchDocument[];
325
+ readonly accounting: SemanticQueryEmbeddingAccounting | null;
326
+ }>;
327
+ };
328
+
329
+ type SemanticQueryStore = Pick<SearchStore, "close" | "searchLex" | "searchVector">;
330
+
331
+ type WarmSearchStore = SemanticQueryStore & Readonly<{
332
+ readonly pendingEmbeddingCount: () => Promise<number>;
333
+ }>;
334
+
335
+ export type SemanticDependencies = {
336
+ readonly createStore?: (options: SemanticStoreOptions) => Promise<unknown>;
337
+ /** QMD opener over a disposable stable snapshot used only by strict warm readers. */
338
+ readonly createWarmSearchStore?: (options: SemanticWarmStoreOptions) => Promise<unknown>;
339
+ /** Config-free QMD opener over a disposable stable snapshot used only by attestation. */
340
+ readonly createAttestationStore?: (
341
+ options: SemanticAttestationStoreOptions,
342
+ ) => Promise<unknown>;
343
+ /** SQLite writer used only to checkpoint a finished private index before sealing. */
344
+ readonly openCheckpointDatabase?: (database: string) => Promise<unknown>;
345
+ readonly digestEmbeddingModelFile?: (path: string) => Promise<string>;
346
+ readonly cacheHome?: string;
347
+ readonly scanVault?: (root: string) => Promise<VaultSnapshot>;
348
+ readonly writerLease?: SemanticWriterLeaseOptions;
349
+ /** Monotonic clock injection used only to measure the actual query embed call. */
350
+ readonly now?: () => number;
351
+ };
352
+
353
+ /** Hash one bounded, regular, non-symlink model file through a stable descriptor. */
354
+ export async function sha256EmbeddingModelFile(path: string): Promise<string> {
355
+ const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
356
+ try {
357
+ const metadata = await handle.stat();
358
+ if (!metadata.isFile()) throw new TypeError("The embedding model must be a regular file.");
359
+ if (metadata.size > MAX_EMBEDDING_MODEL_BYTES) {
360
+ throw new RangeError(
361
+ `The embedding model exceeds ${MAX_EMBEDDING_MODEL_BYTES.toLocaleString("en-US")} bytes.`,
362
+ );
363
+ }
364
+ const hash = createHash("sha256");
365
+ const buffer = new Uint8Array(1_024 * 1_024);
366
+ let observed = 0;
367
+ while (true) {
368
+ const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
369
+ if (bytesRead === 0) break;
370
+ observed += bytesRead;
371
+ if (observed > MAX_EMBEDDING_MODEL_BYTES) {
372
+ throw new RangeError(
373
+ `The embedding model exceeds ${MAX_EMBEDDING_MODEL_BYTES.toLocaleString("en-US")} bytes.`,
374
+ );
375
+ }
376
+ hash.update(buffer.subarray(0, bytesRead));
377
+ }
378
+ const after = await handle.stat();
379
+ if (after.size !== metadata.size || observed !== metadata.size) {
380
+ throw new Error("The embedding model changed while its digest was computed; retry.");
381
+ }
382
+ return hash.digest("hex");
383
+ } finally {
384
+ await handle.close();
385
+ }
386
+ }
387
+
388
+ type EmbeddingModelSourceLease = Readonly<{
389
+ readonly source: string;
390
+ readonly release: () => Promise<void>;
391
+ }>;
392
+
393
+ type VerifiedEmbeddingModelLeaseState = {
394
+ readonly source: string;
395
+ readonly cleanup: () => Promise<void>;
396
+ references: number;
397
+ acceptingReaders: boolean;
398
+ cleanupPromise?: Promise<void>;
399
+ };
400
+
401
+ const verifiedEmbeddingModelLeaseStates = new WeakMap<
402
+ VerifiedEmbeddingModelLease,
403
+ VerifiedEmbeddingModelLeaseState
404
+ >();
405
+
406
+ async function writeEmbeddingModelBytes(
407
+ destination: Awaited<ReturnType<typeof open>>,
408
+ bytes: Uint8Array,
409
+ ): Promise<void> {
410
+ let offset = 0;
411
+ while (offset < bytes.byteLength) {
412
+ const written = await destination.write(bytes, offset, bytes.byteLength - offset, null);
413
+ if (written.bytesWritten < 1) throw new Error("The embedding model snapshot write stalled.");
414
+ offset += written.bytesWritten;
415
+ }
416
+ }
417
+
418
+ /** Copy verified bytes behind an unguessable private path before QMD can open them. */
419
+ async function verifiedIndexEmbeddingModelSource(
420
+ path: string | undefined,
421
+ dependencies: SemanticDependencies,
422
+ ): Promise<EmbeddingModelSourceLease> {
423
+ if (path === undefined) {
424
+ return Object.freeze({ source: recommendedEmbeddingModel, release: () => Promise.resolve() });
425
+ }
426
+ const sourcePath = resolve(path);
427
+ const directory = await mkdtemp(join(tmpdir(), "hraness-kb-embedding-model-"));
428
+ const destinationPath = join(directory, "pinned-model.gguf");
429
+ let source: Awaited<ReturnType<typeof open>> | undefined;
430
+ let destination: Awaited<ReturnType<typeof open>> | undefined;
431
+ try {
432
+ source = await open(sourcePath, constants.O_RDONLY | constants.O_NOFOLLOW);
433
+ destination = await open(
434
+ destinationPath,
435
+ constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,
436
+ 0o400,
437
+ );
438
+ const before = await source.stat();
439
+ if (!before.isFile()) throw new TypeError("The embedding model must be a regular file.");
440
+ if (before.size > MAX_EMBEDDING_MODEL_BYTES) {
441
+ throw new RangeError(
442
+ `The embedding model exceeds ${MAX_EMBEDDING_MODEL_BYTES.toLocaleString("en-US")} bytes.`,
443
+ );
444
+ }
445
+ const hash = createHash("sha256");
446
+ const buffer = new Uint8Array(1_024 * 1_024);
447
+ let observed = 0;
448
+ for (;;) {
449
+ const { bytesRead } = await source.read(buffer, 0, buffer.byteLength, null);
450
+ if (bytesRead === 0) break;
451
+ observed += bytesRead;
452
+ if (observed > MAX_EMBEDDING_MODEL_BYTES) {
453
+ throw new RangeError(
454
+ `The embedding model exceeds ${MAX_EMBEDDING_MODEL_BYTES.toLocaleString("en-US")} bytes.`,
455
+ );
456
+ }
457
+ const bytes = buffer.subarray(0, bytesRead);
458
+ hash.update(bytes);
459
+ await writeEmbeddingModelBytes(destination, bytes);
460
+ }
461
+ const after = await source.stat();
462
+ const copied = await destination.stat();
463
+ if (
464
+ after.dev !== before.dev
465
+ || after.ino !== before.ino
466
+ || after.size !== before.size
467
+ || observed !== before.size
468
+ || copied.size !== observed
469
+ ) {
470
+ throw new Error("The embedding model changed while its private snapshot was created; retry.");
471
+ }
472
+ await destination.sync();
473
+ await destination.close();
474
+ destination = undefined;
475
+ await source.close();
476
+ source = undefined;
477
+ const digest = dependencies.digestEmbeddingModelFile === undefined
478
+ ? hash.digest("hex")
479
+ : await dependencies.digestEmbeddingModelFile(destinationPath);
480
+ if (digest !== recommendedEmbeddingModelSha256) {
481
+ throw new Error(
482
+ "The local embedding model does not match the pinned recommended model SHA-256.",
483
+ );
484
+ }
485
+ await chmod(destinationPath, 0o400);
486
+ let released = false;
487
+ return Object.freeze({
488
+ source: destinationPath,
489
+ release: async () => {
490
+ if (released) return;
491
+ released = true;
492
+ await rm(directory, { recursive: true, force: true });
493
+ },
494
+ });
495
+ } catch (error: unknown) {
496
+ await destination?.close().catch(() => undefined);
497
+ await source?.close().catch(() => undefined);
498
+ await rm(directory, { recursive: true, force: true });
499
+ throw error;
500
+ }
501
+ }
502
+
503
+ function releaseEmbeddingModelReference(
504
+ state: VerifiedEmbeddingModelLeaseState,
505
+ ): Promise<void> {
506
+ if (state.references < 1) {
507
+ throw new Error("Verified embedding-model lease reference accounting underflowed.");
508
+ }
509
+ state.references -= 1;
510
+ if (state.references !== 0) return Promise.resolve();
511
+ state.cleanupPromise ??= state.cleanup();
512
+ return state.cleanupPromise;
513
+ }
514
+
515
+ function retainVerifiedEmbeddingModelLease(
516
+ lease: VerifiedEmbeddingModelLease,
517
+ ): EmbeddingModelSourceLease {
518
+ const state = verifiedEmbeddingModelLeaseStates.get(lease);
519
+ if (state === undefined) {
520
+ throw new TypeError("embeddingModelLease must be created by createVerifiedEmbeddingModelLease.");
521
+ }
522
+ if (!state.acceptingReaders) {
523
+ throw new Error("The verified embedding-model lease is closed.");
524
+ }
525
+ state.references += 1;
526
+ let released = false;
527
+ return Object.freeze({
528
+ source: state.source,
529
+ release: () => {
530
+ if (released) return Promise.resolve();
531
+ released = true;
532
+ return releaseEmbeddingModelReference(state);
533
+ },
534
+ });
535
+ }
536
+
537
+ /**
538
+ * Copy and verify one local model exactly once. Sessions retain the private
539
+ * copy without learning its path; closing the owner stops new retention while
540
+ * the last retained session removes the copy through reference counting.
541
+ */
542
+ export async function createVerifiedEmbeddingModelLease(
543
+ embeddingModelFile: string,
544
+ dependencies: SemanticDependencies = {},
545
+ ): Promise<VerifiedEmbeddingModelLease> {
546
+ if (typeof embeddingModelFile !== "string" || embeddingModelFile.trim() === "") {
547
+ throw new TypeError("embeddingModelFile must be a non-empty path string.");
548
+ }
549
+ const copied = await verifiedIndexEmbeddingModelSource(embeddingModelFile, dependencies);
550
+ const state: VerifiedEmbeddingModelLeaseState = {
551
+ source: copied.source,
552
+ cleanup: copied.release,
553
+ references: 1,
554
+ acceptingReaders: true,
555
+ };
556
+ let ownerClosePromise: Promise<void> | undefined;
557
+ const lease: VerifiedEmbeddingModelLease = Object.freeze({
558
+ model: recommendedEmbeddingModel,
559
+ close: () => {
560
+ if (ownerClosePromise !== undefined) return ownerClosePromise;
561
+ state.acceptingReaders = false;
562
+ ownerClosePromise = releaseEmbeddingModelReference(state);
563
+ return ownerClosePromise;
564
+ },
565
+ });
566
+ verifiedEmbeddingModelLeaseStates.set(lease, state);
567
+ return lease;
568
+ }
569
+
570
+ function requiredStoreLocalVectorBoundary(value: unknown): boolean {
571
+ if (value === undefined) return false;
572
+ if (typeof value !== "boolean") {
573
+ throw new TypeError("requireStoreLocalVectorBoundary must be a boolean.");
574
+ }
575
+ return value;
576
+ }
577
+
578
+ async function sessionEmbeddingModelSource(
579
+ options: Pick<SemanticSessionOptions, "embeddingModelFile" | "embeddingModelLease">,
580
+ dependencies: SemanticDependencies,
581
+ ): Promise<EmbeddingModelSourceLease> {
582
+ if (options.embeddingModelFile !== undefined && options.embeddingModelLease !== undefined) {
583
+ throw new TypeError(
584
+ "embeddingModelFile and embeddingModelLease are mutually exclusive.",
585
+ );
586
+ }
587
+ if (options.embeddingModelLease !== undefined) {
588
+ return retainVerifiedEmbeddingModelLease(options.embeddingModelLease);
589
+ }
590
+ if (options.embeddingModelFile === undefined) {
591
+ return Object.freeze({
592
+ source: recommendedEmbeddingModel,
593
+ release: () => Promise.resolve(),
594
+ });
595
+ }
596
+ const owner = await createVerifiedEmbeddingModelLease(
597
+ options.embeddingModelFile,
598
+ dependencies,
599
+ );
600
+ try {
601
+ const retained = retainVerifiedEmbeddingModelLease(owner);
602
+ await owner.close();
603
+ return retained;
604
+ } catch (error: unknown) {
605
+ await owner.close().catch(() => undefined);
606
+ throw error;
607
+ }
608
+ }
609
+
610
+ function cacheHome(dependencies: SemanticDependencies): string {
611
+ const configured = dependencies.cacheHome ?? process.env.XDG_CACHE_HOME;
612
+ if (configured !== undefined && configured.trim() !== "") {
613
+ return isAbsolute(configured) ? configured : resolve(configured);
614
+ }
615
+ return join(homedir(), ".cache");
616
+ }
617
+
618
+ export function semanticDatabasePath(root: string, dependencies: SemanticDependencies = {}): string {
619
+ const identity = createHash("sha256").update(resolve(root)).digest("hex").slice(0, 20);
620
+ return join(cacheHome(dependencies), "hraness-kb", "indexes", `${identity}.sqlite`);
621
+ }
622
+
623
+ async function resolvedDirectory(path: string): Promise<string> {
624
+ const root = await realpath(resolve(path));
625
+ if (!(await stat(root)).isDirectory()) throw new Error("Knowledge-base root must be a directory.");
626
+ return root;
627
+ }
628
+
629
+ function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
630
+ return typeof value === "object" && value !== null && !Array.isArray(value);
631
+ }
632
+
633
+ function boundaryRecord(value: unknown, label: string): Readonly<Record<string, unknown>> {
634
+ if (!isRecord(value)) throw new Error(`${label} must be an object.`);
635
+ return value;
636
+ }
637
+
638
+ function boundaryString(value: unknown, label: string): string {
639
+ if (typeof value !== "string") throw new Error(`${label} must be a string.`);
640
+ return value;
641
+ }
642
+
643
+ function boundaryNumber(value: unknown, label: string): number {
644
+ if (typeof value !== "number" || !Number.isFinite(value)) {
645
+ throw new Error(`${label} must be a finite number.`);
646
+ }
647
+ return value;
648
+ }
649
+
650
+ function boundaryCount(value: unknown, label: string): number {
651
+ const number = boundaryNumber(value, label);
652
+ if (!Number.isSafeInteger(number) || number < 0) {
653
+ throw new Error(`${label} must be a non-negative safe integer.`);
654
+ }
655
+ return number;
656
+ }
657
+
658
+ function boundaryArray(value: unknown, label: string): readonly unknown[] {
659
+ if (!Array.isArray(value)) throw new Error(`${label} must be an array.`);
660
+ return value;
661
+ }
662
+
663
+ function parseUpdateResult(value: unknown): SemanticUpdateResult {
664
+ const result = boundaryRecord(value, "QMD update result");
665
+ return {
666
+ collections: boundaryCount(result.collections, "QMD update result.collections"),
667
+ indexed: boundaryCount(result.indexed, "QMD update result.indexed"),
668
+ updated: boundaryCount(result.updated, "QMD update result.updated"),
669
+ unchanged: boundaryCount(result.unchanged, "QMD update result.unchanged"),
670
+ removed: boundaryCount(result.removed, "QMD update result.removed"),
671
+ needsEmbedding: boundaryCount(
672
+ result.needsEmbedding,
673
+ "QMD update result.needsEmbedding",
674
+ ),
675
+ };
676
+ }
677
+
678
+ function parseEmbeddingFailure(value: unknown, index: number): SemanticEmbeddingFailure {
679
+ const label = `QMD embedding result.failures[${index}]`;
680
+ const failure = boundaryRecord(value, label);
681
+ return {
682
+ path: boundaryString(failure.path, `${label}.path`),
683
+ hash: boundaryString(failure.hash, `${label}.hash`),
684
+ seq: boundaryCount(failure.seq, `${label}.seq`),
685
+ attempts: boundaryCount(failure.attempts, `${label}.attempts`),
686
+ reason: boundaryString(failure.reason, `${label}.reason`),
687
+ };
688
+ }
689
+
690
+ function parseEmbeddingResult(value: unknown): SemanticEmbeddingResult {
691
+ const result = boundaryRecord(value, "QMD embedding result");
692
+ const failures = result.failures === undefined
693
+ ? undefined
694
+ : boundaryArray(result.failures, "QMD embedding result.failures")
695
+ .map((failure, index) => parseEmbeddingFailure(failure, index));
696
+ return {
697
+ docsProcessed: boundaryCount(result.docsProcessed, "QMD embedding result.docsProcessed"),
698
+ chunksEmbedded: boundaryCount(result.chunksEmbedded, "QMD embedding result.chunksEmbedded"),
699
+ errors: boundaryCount(result.errors, "QMD embedding result.errors"),
700
+ ...(failures === undefined ? {} : { failures }),
701
+ durationMs: boundaryNumber(result.durationMs, "QMD embedding result.durationMs"),
702
+ };
703
+ }
704
+
705
+ function parseSearchDocument(value: unknown, index: number): SemanticSearchDocument {
706
+ const label = `QMD search result[${index}]`;
707
+ const result = boundaryRecord(value, label);
708
+ const source = result.source;
709
+ if (source !== "fts" && source !== "vec") {
710
+ throw new Error(`${label}.source must be "fts" or "vec".`);
711
+ }
712
+ const chunkPos = result.chunkPos === undefined
713
+ ? undefined
714
+ : boundaryCount(result.chunkPos, `${label}.chunkPos`);
715
+ return {
716
+ filepath: boundaryString(result.filepath, `${label}.filepath`),
717
+ title: boundaryString(result.title, `${label}.title`),
718
+ hash: boundaryString(result.hash, `${label}.hash`),
719
+ docid: boundaryString(result.docid, `${label}.docid`),
720
+ modifiedAt: boundaryString(result.modifiedAt, `${label}.modifiedAt`),
721
+ score: boundaryNumber(result.score, `${label}.score`),
722
+ source,
723
+ ...(chunkPos === undefined ? {} : { chunkPos }),
724
+ };
725
+ }
726
+
727
+ function boundedResultArray(
728
+ value: unknown,
729
+ label: string,
730
+ maximum: number,
731
+ ): readonly unknown[] {
732
+ const results = boundaryArray(value, label);
733
+ if (results.length > maximum) {
734
+ throw new Error(`${label} returned more than the requested ${maximum} results.`);
735
+ }
736
+ return results;
737
+ }
738
+
739
+ function parseSearchResults(
740
+ value: unknown,
741
+ maximum: number,
742
+ ): readonly SemanticSearchDocument[] {
743
+ return boundedResultArray(value, "QMD search results", maximum)
744
+ .map((result, index) => parseSearchDocument(result, index));
745
+ }
746
+
747
+ type UnknownMethod = (...arguments_: unknown[]) => Promise<unknown>;
748
+
749
+ function boundUnknownMethod(
750
+ owner: Readonly<Record<string, unknown>>,
751
+ name: string,
752
+ label: string,
753
+ ): UnknownMethod {
754
+ const method = owner[name];
755
+ if (typeof method !== "function") throw new Error(`${label}.${name} must be a function.`);
756
+ return async (...arguments_) => {
757
+ const returned: unknown = Reflect.apply(method, owner, arguments_);
758
+ return await returned;
759
+ };
760
+ }
761
+
762
+ type QmdInternalVectorBoundary = {
763
+ readonly pendingEmbeddingCount: () => Promise<number>;
764
+ readonly searchVector: (
765
+ query: string,
766
+ options: { readonly collection: string; readonly limit: number },
767
+ ) => Promise<{
768
+ readonly results: readonly SemanticSearchDocument[];
769
+ readonly accounting: SemanticQueryEmbeddingAccounting;
770
+ }>;
771
+ };
772
+
773
+ function measuredDuration(startedAt: number, finishedAt: number): number {
774
+ if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt) || finishedAt < startedAt) {
775
+ throw new Error("The query-embedding monotonic clock returned an invalid interval.");
776
+ }
777
+ const duration = finishedAt - startedAt;
778
+ return duration;
779
+ }
780
+
781
+ function internalVectorBoundary(
782
+ store: Readonly<Record<string, unknown>>,
783
+ modelIdentity: string,
784
+ now: () => number,
785
+ ): QmdInternalVectorBoundary | null {
786
+ // QMD 2.5.3's public searchVector omits its per-store LLM session and falls
787
+ // back to a process-global model. Its documented advanced internal boundary
788
+ // lets KB keep query inference on the same store-local, verified bytes while
789
+ // retaining a path-independent identity for derived vector rows.
790
+ if (store.internal === undefined) return null;
791
+ const internal = boundaryRecord(store.internal, "QMD store.internal");
792
+ const llm = boundaryRecord(internal.llm, "QMD store.internal.llm");
793
+ const getHashesNeedingEmbedding = boundUnknownMethod(
794
+ internal,
795
+ "getHashesNeedingEmbedding",
796
+ "QMD store.internal",
797
+ );
798
+ const searchVec = boundUnknownMethod(internal, "searchVec", "QMD store.internal");
799
+ const embed = boundUnknownMethod(llm, "embed", "QMD store.internal.llm");
800
+ const countTokens = boundUnknownMethod(
801
+ llm,
802
+ "countTokens",
803
+ "QMD store.internal.llm",
804
+ );
805
+ return {
806
+ pendingEmbeddingCount: async () => boundaryCount(
807
+ await getHashesNeedingEmbedding(modelIdentity),
808
+ "QMD store.internal.getHashesNeedingEmbedding result",
809
+ ),
810
+ searchVector: async (query, options) => {
811
+ let calls = 0;
812
+ let inputTokens = 0;
813
+ let durationMs = 0;
814
+ const session = Object.freeze({
815
+ countTokens: async (text: unknown) => boundaryCount(
816
+ await countTokens(
817
+ boundaryString(text, "QMD query embedding text"),
818
+ ),
819
+ "QMD store.internal.llm.countTokens result",
820
+ ),
821
+ embed: async (text: unknown, embedOptions?: unknown) => {
822
+ const exactText = boundaryString(text, "QMD query embedding text");
823
+ if (calls !== 0) {
824
+ throw new Error("QMD query-vector search must perform exactly one query embedding.");
825
+ }
826
+ inputTokens = boundaryCount(
827
+ await countTokens(exactText),
828
+ "QMD store.internal.llm.countTokens result",
829
+ );
830
+ calls = 1;
831
+ const startedAt = now();
832
+ try {
833
+ return await embed(exactText, embedOptions);
834
+ } finally {
835
+ durationMs = measuredDuration(startedAt, now());
836
+ }
837
+ },
838
+ });
839
+ const results = parseSearchResults(await searchVec(
840
+ query,
841
+ modelIdentity,
842
+ options.limit,
843
+ options.collection,
844
+ session,
845
+ ), options.limit);
846
+ if (calls !== 1) {
847
+ throw new Error(
848
+ `QMD query-vector search must perform exactly one query embedding; observed ${calls}.`,
849
+ );
850
+ }
851
+ return {
852
+ results,
853
+ accounting: Object.freeze({ calls, inputTokens, durationMs }),
854
+ };
855
+ },
856
+ };
857
+ }
858
+
859
+ function parseSearchStore(
860
+ value: unknown,
861
+ modelIdentity: string,
862
+ options: {
863
+ readonly requireStoreLocalVectorBoundary: boolean;
864
+ readonly now: () => number;
865
+ },
866
+ ): SearchStore {
867
+ const store = boundaryRecord(value, "QMD store");
868
+ const close = boundUnknownMethod(store, "close", "QMD store");
869
+ const embed = boundUnknownMethod(store, "embed", "QMD store");
870
+ const searchLex = boundUnknownMethod(store, "searchLex", "QMD store");
871
+ const searchVector = boundUnknownMethod(store, "searchVector", "QMD store");
872
+ const update = boundUnknownMethod(store, "update", "QMD store");
873
+ let internalVector: QmdInternalVectorBoundary | null = null;
874
+ try {
875
+ internalVector = internalVectorBoundary(store, modelIdentity, options.now);
876
+ } catch (error: unknown) {
877
+ if (options.requireStoreLocalVectorBoundary) throw error;
878
+ }
879
+ if (options.requireStoreLocalVectorBoundary && internalVector === null) {
880
+ throw new Error(
881
+ "QMD store-local vector search is required, but store.internal.searchVec with its LLM boundary is unavailable.",
882
+ );
883
+ }
884
+ return {
885
+ close: async () => {
886
+ await close();
887
+ },
888
+ embed: async (options) => parseEmbeddingResult(await embed(options)),
889
+ searchLex: async (query, options) =>
890
+ parseSearchResults(await searchLex(query, options), options.limit),
891
+ searchVector: internalVector?.searchVector
892
+ ?? (async (query, vectorOptions) => ({
893
+ results: parseSearchResults(
894
+ await searchVector(query, vectorOptions),
895
+ vectorOptions.limit,
896
+ ),
897
+ accounting: null,
898
+ })),
899
+ update: async (options) => {
900
+ const result = parseUpdateResult(await update(options));
901
+ if (internalVector === null) return result;
902
+ return {
903
+ ...result,
904
+ needsEmbedding: await internalVector.pendingEmbeddingCount(),
905
+ };
906
+ },
907
+ };
908
+ }
909
+
910
+ function parseWarmSearchStore(
911
+ value: unknown,
912
+ now: () => number,
913
+ ): WarmSearchStore {
914
+ const store = boundaryRecord(value, "QMD warm search store");
915
+ const close = boundUnknownMethod(store, "close", "QMD warm search store");
916
+ const searchLex = boundUnknownMethod(store, "searchLex", "QMD warm search store");
917
+ const internalVector = internalVectorBoundary(store, recommendedEmbeddingModel, now);
918
+ if (internalVector === null) {
919
+ throw new Error(
920
+ "QMD warm search requires store.internal.searchVec with its store-local LLM boundary.",
921
+ );
922
+ }
923
+ return Object.freeze({
924
+ close: async () => {
925
+ await close();
926
+ },
927
+ pendingEmbeddingCount: internalVector.pendingEmbeddingCount,
928
+ searchLex: async (query, options) =>
929
+ parseSearchResults(await searchLex(query, options), options.limit),
930
+ searchVector: internalVector.searchVector,
931
+ });
932
+ }
933
+
934
+ async function closeMalformedStore(value: unknown): Promise<void> {
935
+ if (!isRecord(value)) return;
936
+ const close = value.close;
937
+ if (typeof close !== "function") return;
938
+ try {
939
+ const returned: unknown = Reflect.apply(close, value, []);
940
+ await returned;
941
+ } catch {
942
+ // Preserve the boundary error that explains why the store was rejected.
943
+ }
944
+ }
945
+
946
+ async function openedSearchStore(
947
+ value: unknown,
948
+ modelIdentity: string,
949
+ options: {
950
+ readonly requireStoreLocalVectorBoundary: boolean;
951
+ readonly now: () => number;
952
+ },
953
+ ): Promise<SearchStore> {
954
+ try {
955
+ return parseSearchStore(value, modelIdentity, options);
956
+ } catch (error: unknown) {
957
+ await closeMalformedStore(value);
958
+ throw error;
959
+ }
960
+ }
961
+
962
+ async function openedWarmSearchStore(
963
+ value: unknown,
964
+ now: () => number,
965
+ ): Promise<WarmSearchStore> {
966
+ try {
967
+ return parseWarmSearchStore(value, now);
968
+ } catch (error: unknown) {
969
+ await closeMalformedStore(value);
970
+ throw error;
971
+ }
972
+ }
973
+
974
+ type IsolatedQmdDatabaseSnapshot = Readonly<{
975
+ readonly database: string;
976
+ readonly cleanup: () => Promise<void>;
977
+ }>;
978
+
979
+ function sameStableFileMetadata(left: BigIntStats, right: BigIntStats): boolean {
980
+ return left.dev === right.dev
981
+ && left.ino === right.ino
982
+ && left.mode === right.mode
983
+ && left.nlink === right.nlink
984
+ && left.size === right.size
985
+ && left.mtimeNs === right.mtimeNs
986
+ && left.ctimeNs === right.ctimeNs;
987
+ }
988
+
989
+ function missingFile(error: unknown): boolean {
990
+ return isRecord(error) && error.code === "ENOENT";
991
+ }
992
+
993
+ async function writeAll(
994
+ handle: Awaited<ReturnType<typeof open>>,
995
+ bytes: Uint8Array,
996
+ length: number,
997
+ position: number,
998
+ ): Promise<void> {
999
+ let written = 0;
1000
+ while (written < length) {
1001
+ const result = await handle.write(
1002
+ bytes,
1003
+ written,
1004
+ length - written,
1005
+ position + written,
1006
+ );
1007
+ if (result.bytesWritten === 0) {
1008
+ throw new Error("The isolated QMD snapshot stopped accepting bytes.");
1009
+ }
1010
+ written += result.bytesWritten;
1011
+ }
1012
+ }
1013
+
1014
+ async function copyStableSnapshotFile(
1015
+ source: string,
1016
+ destination: string,
1017
+ label: string,
1018
+ maximumBytes: number,
1019
+ expected: SemanticDatabaseFileSeal,
1020
+ ): Promise<SemanticDatabaseFileSeal> {
1021
+ if (
1022
+ !Number.isSafeInteger(expected.bytes)
1023
+ || expected.bytes < 0
1024
+ || expected.bytes > maximumBytes
1025
+ || !SHA256.test(expected.sha256)
1026
+ ) throw new TypeError(`${label} seal is invalid.`);
1027
+ const pathBefore = await lstat(source, { bigint: true });
1028
+ if (
1029
+ pathBefore.isSymbolicLink()
1030
+ || !pathBefore.isFile()
1031
+ || pathBefore.nlink !== 1n
1032
+ || pathBefore.size > BigInt(maximumBytes)
1033
+ || pathBefore.size !== BigInt(expected.bytes)
1034
+ ) {
1035
+ throw new Error(`${label} must be one bounded, singly linked regular file.`);
1036
+ }
1037
+ const sourceHandle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
1038
+ let destinationHandle: Awaited<ReturnType<typeof open>> | undefined;
1039
+ try {
1040
+ const before = await sourceHandle.stat({ bigint: true });
1041
+ if (!sameStableFileMetadata(pathBefore, before)) {
1042
+ throw new Error(`${label} changed before its isolated read snapshot was opened.`);
1043
+ }
1044
+ destinationHandle = await open(
1045
+ destination,
1046
+ constants.O_WRONLY
1047
+ | constants.O_CREAT
1048
+ | constants.O_EXCL
1049
+ | constants.O_NOFOLLOW,
1050
+ 0o600,
1051
+ );
1052
+ const buffer = new Uint8Array(1_024 * 1_024);
1053
+ const hash = createHash("sha256");
1054
+ let copied = 0;
1055
+ while (copied < Number(before.size)) {
1056
+ const requested = Math.min(buffer.byteLength, Number(before.size) - copied);
1057
+ const { bytesRead } = await sourceHandle.read(buffer, 0, requested, copied);
1058
+ if (bytesRead === 0) {
1059
+ throw new Error(`${label} ended before its declared size.`);
1060
+ }
1061
+ await writeAll(destinationHandle, buffer, bytesRead, copied);
1062
+ hash.update(buffer.subarray(0, bytesRead));
1063
+ copied += bytesRead;
1064
+ }
1065
+ await destinationHandle.sync();
1066
+ const [after, pathAfter] = await Promise.all([
1067
+ sourceHandle.stat({ bigint: true }),
1068
+ lstat(source, { bigint: true }),
1069
+ ]);
1070
+ if (
1071
+ copied !== Number(before.size)
1072
+ || !sameStableFileMetadata(before, after)
1073
+ || !sameStableFileMetadata(after, pathAfter)
1074
+ ) {
1075
+ throw new Error(`${label} changed while its isolated read snapshot was copied.`);
1076
+ }
1077
+ const observed = Object.freeze({ bytes: copied, sha256: hash.digest("hex") });
1078
+ if (observed.bytes !== expected.bytes || observed.sha256 !== expected.sha256) {
1079
+ throw new Error(`${label} does not match its sealed byte commitment.`);
1080
+ }
1081
+ return observed;
1082
+ } finally {
1083
+ await Promise.allSettled([
1084
+ sourceHandle.close(),
1085
+ destinationHandle?.close() ?? Promise.resolve(),
1086
+ ]);
1087
+ }
1088
+ }
1089
+
1090
+ async function assertAbsentSnapshotSidecar(path: string, label: string): Promise<void> {
1091
+ try {
1092
+ await lstat(path);
1093
+ } catch (error: unknown) {
1094
+ if (missingFile(error)) return;
1095
+ throw error;
1096
+ }
1097
+ throw new Error(`${label} appeared while the isolated read snapshot was copied.`);
1098
+ }
1099
+
1100
+ async function createIsolatedQmdDatabaseSnapshot(
1101
+ database: string,
1102
+ seal: SemanticDatabaseSnapshotSeal,
1103
+ ): Promise<IsolatedQmdDatabaseSnapshot> {
1104
+ const directory = await mkdtemp(join(tmpdir(), "hraness-kb-qmd-reader."));
1105
+ await chmod(directory, 0o700);
1106
+ let cleaned = false;
1107
+ const cleanup = async (): Promise<void> => {
1108
+ if (cleaned) return;
1109
+ cleaned = true;
1110
+ await rm(directory, { recursive: true, force: true });
1111
+ };
1112
+ try {
1113
+ const isolatedDatabase = join(directory, "snapshot.sqlite");
1114
+ if (seal.wal !== null || seal.shm !== null || seal.journal !== null) {
1115
+ throw new TypeError("Strict warm semantic snapshots require checkpointed SQLite state.");
1116
+ }
1117
+ const sourceWal = `${database}-wal`;
1118
+ const sourceShm = `${database}-shm`;
1119
+ const sourceJournal = `${database}-journal`;
1120
+ await Promise.all([
1121
+ assertAbsentSnapshotSidecar(sourceWal, "Semantic database WAL"),
1122
+ assertAbsentSnapshotSidecar(sourceShm, "Semantic database SHM"),
1123
+ assertAbsentSnapshotSidecar(sourceJournal, "Semantic database rollback journal"),
1124
+ ]);
1125
+ const databaseSeal = await copyStableSnapshotFile(
1126
+ database,
1127
+ isolatedDatabase,
1128
+ "Semantic database",
1129
+ MAX_SEMANTIC_READ_SNAPSHOT_BYTES,
1130
+ seal.database,
1131
+ );
1132
+ await Promise.all([
1133
+ assertAbsentSnapshotSidecar(sourceWal, "Semantic database WAL"),
1134
+ assertAbsentSnapshotSidecar(sourceShm, "Semantic database SHM"),
1135
+ assertAbsentSnapshotSidecar(sourceJournal, "Semantic database rollback journal"),
1136
+ ]);
1137
+ if (databaseSeal.bytes > MAX_SEMANTIC_READ_SNAPSHOT_BYTES) {
1138
+ throw new RangeError("Semantic database exceeds the read-snapshot byte bound.");
1139
+ }
1140
+ return Object.freeze({ database: isolatedDatabase, cleanup });
1141
+ } catch (error: unknown) {
1142
+ await cleanup();
1143
+ throw error;
1144
+ }
1145
+ }
1146
+
1147
+ function aggregateCloseFailures(
1148
+ settlements: readonly PromiseSettledResult<unknown>[],
1149
+ label: string,
1150
+ ): void {
1151
+ const failures = settlements.filter(
1152
+ (result): result is PromiseRejectedResult => result.status === "rejected",
1153
+ );
1154
+ if (failures.length > 0) {
1155
+ throw new AggregateError(
1156
+ failures.map(({ reason }): unknown => reason),
1157
+ `${label} did not close cleanly.`,
1158
+ );
1159
+ }
1160
+ }
1161
+
1162
+ async function closeIsolatedStore(
1163
+ close: () => Promise<void>,
1164
+ snapshot: IsolatedQmdDatabaseSnapshot,
1165
+ label: string,
1166
+ ): Promise<void> {
1167
+ const settlements: PromiseSettledResult<unknown>[] = [];
1168
+ try {
1169
+ await close();
1170
+ settlements.push({ status: "fulfilled", value: undefined });
1171
+ } catch (reason: unknown) {
1172
+ settlements.push({ status: "rejected", reason });
1173
+ }
1174
+ try {
1175
+ await snapshot.cleanup();
1176
+ settlements.push({ status: "fulfilled", value: undefined });
1177
+ } catch (reason: unknown) {
1178
+ settlements.push({ status: "rejected", reason });
1179
+ }
1180
+ aggregateCloseFailures(settlements, label);
1181
+ }
1182
+
1183
+ function isolatedWarmSearchStore(
1184
+ store: WarmSearchStore,
1185
+ snapshot: IsolatedQmdDatabaseSnapshot,
1186
+ ): WarmSearchStore {
1187
+ let closePromise: Promise<void> | undefined;
1188
+ return Object.freeze({
1189
+ ...store,
1190
+ close: () => {
1191
+ closePromise ??= closeIsolatedStore(
1192
+ store.close,
1193
+ snapshot,
1194
+ "Isolated QMD warm store",
1195
+ );
1196
+ return closePromise;
1197
+ },
1198
+ });
1199
+ }
1200
+
1201
+ function isolatedAttestationStore(
1202
+ store: SemanticAttestationStore,
1203
+ snapshot: IsolatedQmdDatabaseSnapshot,
1204
+ ): SemanticAttestationStore {
1205
+ let closePromise: Promise<void> | undefined;
1206
+ return Object.freeze({
1207
+ ...store,
1208
+ close: () => {
1209
+ closePromise ??= closeIsolatedStore(
1210
+ store.close,
1211
+ snapshot,
1212
+ "Isolated QMD attestation store",
1213
+ );
1214
+ return closePromise;
1215
+ },
1216
+ });
1217
+ }
1218
+
1219
+ function storeConfig(root: string, embeddingModelSource: string): SemanticCollectionConfig {
1220
+ return {
1221
+ global_context: globalContext,
1222
+ collections: {
1223
+ [collectionName]: {
1224
+ path: root,
1225
+ pattern: markdownPattern,
1226
+ ignore: ignoredPatterns,
1227
+ context: collectionContext,
1228
+ },
1229
+ },
1230
+ models: { embed: embeddingModelSource },
1231
+ };
1232
+ }
1233
+
1234
+ async function defaultCreateStore(options: SemanticStoreOptions): Promise<unknown> {
1235
+ const loaded: unknown = await import(qmdModuleSpecifier);
1236
+ const module = boundaryRecord(loaded, "QMD module");
1237
+ const createStore = boundUnknownMethod(module, "createStore", "QMD module");
1238
+ return await createStore(options);
1239
+ }
1240
+
1241
+ async function defaultCreateWarmSearchStore(
1242
+ options: SemanticWarmStoreOptions,
1243
+ ): Promise<unknown> {
1244
+ const loaded: unknown = await import(qmdModuleSpecifier);
1245
+ const module = boundaryRecord(loaded, "QMD module");
1246
+ const createStore = boundUnknownMethod(module, "createStore", "QMD module");
1247
+ // The caller passes an isolated stable snapshot, never the attested cache.
1248
+ // QMD has no read-only store mode and initializes schema/WAL state on every
1249
+ // open. Config-free mode keeps those unavoidable writes confined to the
1250
+ // disposable snapshot. The pinned fork's exact internal LLM boundary is
1251
+ // installed before any query because DB-only mode cannot select a model.
1252
+ const created = await createStore({ dbPath: options.dbPath });
1253
+ let localLlm: Readonly<Record<string, unknown>> | undefined;
1254
+ try {
1255
+ const store = boundaryRecord(created, "QMD warm search store");
1256
+ const internal = boundaryRecord(store.internal, "QMD warm search store.internal");
1257
+ const previousLlm = boundaryRecord(internal.llm, "QMD warm search store.internal.llm");
1258
+ const previousDispose = boundUnknownMethod(
1259
+ previousLlm,
1260
+ "dispose",
1261
+ "QMD warm search store.internal.llm",
1262
+ );
1263
+ const llmSpecifier = new URL("./llm.js", import.meta.resolve(qmdModuleSpecifier)).href;
1264
+ const loadedLlm: unknown = await import(llmSpecifier);
1265
+ const llmModule = boundaryRecord(loadedLlm, "QMD LLM module");
1266
+ if (typeof llmModule.LlamaCpp !== "function") {
1267
+ throw new Error("QMD LLM module.LlamaCpp must be a constructor.");
1268
+ }
1269
+ const constructed: unknown = Reflect.construct(llmModule.LlamaCpp, [{
1270
+ embedModel: options.embeddingModelSource,
1271
+ inactivityTimeoutMs: 5 * 60_000,
1272
+ disposeModelsOnInactivity: true,
1273
+ }]);
1274
+ localLlm = boundaryRecord(constructed, "QMD warm local LLM");
1275
+ if (localLlm.embedModelName !== options.embeddingModelSource) {
1276
+ throw new Error("QMD warm local LLM did not retain the verified model source.");
1277
+ }
1278
+ const localDispose = boundUnknownMethod(localLlm, "dispose", "QMD warm local LLM");
1279
+ if (!Reflect.set(internal, "llm", localLlm)) {
1280
+ throw new Error("QMD warm store rejected its verified local LLM boundary.");
1281
+ }
1282
+ await previousDispose();
1283
+ const close = boundUnknownMethod(store, "close", "QMD warm search store");
1284
+ let closed = false;
1285
+ return Object.freeze({
1286
+ ...store,
1287
+ internal,
1288
+ close: async () => {
1289
+ if (closed) return;
1290
+ closed = true;
1291
+ const settlements = await Promise.allSettled([localDispose(), close()]);
1292
+ const failures = settlements.filter(
1293
+ (result): result is PromiseRejectedResult => result.status === "rejected",
1294
+ );
1295
+ if (failures.length > 0) {
1296
+ throw new AggregateError(
1297
+ failures.map(({ reason }): unknown => reason),
1298
+ "QMD warm store did not close cleanly.",
1299
+ );
1300
+ }
1301
+ },
1302
+ });
1303
+ } catch (error: unknown) {
1304
+ if (localLlm !== undefined) {
1305
+ const dispose = localLlm.dispose;
1306
+ if (typeof dispose === "function") {
1307
+ await Promise.resolve(Reflect.apply(dispose, localLlm, [])).catch(() => undefined);
1308
+ }
1309
+ }
1310
+ await closeMalformedStore(created);
1311
+ throw error;
1312
+ }
1313
+ }
1314
+
1315
+ async function defaultCreateAttestationStore(
1316
+ options: SemanticAttestationStoreOptions,
1317
+ ): Promise<unknown> {
1318
+ const loaded: unknown = await import(qmdModuleSpecifier);
1319
+ const module = boundaryRecord(loaded, "QMD module");
1320
+ const createStore = boundUnknownMethod(module, "createStore", "QMD module");
1321
+ // This path is an isolated stable snapshot. QMD's config-free opener still
1322
+ // initializes SQLite, but it cannot mutate or lock the attested cache.
1323
+ return await createStore(options);
1324
+ }
1325
+
1326
+ type SemanticCheckpointDatabase = Readonly<{
1327
+ readonly checkpoint: () => Promise<unknown>;
1328
+ readonly close: () => Promise<void>;
1329
+ }>;
1330
+
1331
+ async function defaultOpenCheckpointDatabase(database: string): Promise<unknown> {
1332
+ const moduleSpecifier: string = "bun:sqlite";
1333
+ const loaded: unknown = await import(moduleSpecifier);
1334
+ const module = boundaryRecord(loaded, "Bun SQLite module");
1335
+ if (typeof module.Database !== "function") {
1336
+ throw new TypeError("Bun SQLite module.Database must be a constructor.");
1337
+ }
1338
+ const opened = boundaryRecord(
1339
+ Reflect.construct(module.Database, [database, { create: false, strict: true }]),
1340
+ "Bun SQLite checkpoint database",
1341
+ );
1342
+ const query = boundUnknownMethod(opened, "query", "Bun SQLite checkpoint database");
1343
+ const close = boundUnknownMethod(opened, "close", "Bun SQLite checkpoint database");
1344
+ return Object.freeze({
1345
+ checkpoint: async () => {
1346
+ const walStatement = boundaryRecord(
1347
+ await query("PRAGMA wal_checkpoint(TRUNCATE)"),
1348
+ "Bun SQLite WAL checkpoint statement",
1349
+ );
1350
+ const walGet = boundUnknownMethod(walStatement, "get", "Bun SQLite WAL checkpoint statement");
1351
+ const modeStatement = boundaryRecord(
1352
+ await query("PRAGMA journal_mode = DELETE"),
1353
+ "Bun SQLite journal-mode statement",
1354
+ );
1355
+ const modeGet = boundUnknownMethod(
1356
+ modeStatement,
1357
+ "get",
1358
+ "Bun SQLite journal-mode statement",
1359
+ );
1360
+ return Object.freeze({ wal: await walGet(), mode: await modeGet() });
1361
+ },
1362
+ close: async () => {
1363
+ await close();
1364
+ },
1365
+ });
1366
+ }
1367
+
1368
+ async function openedCheckpointDatabase(value: unknown): Promise<SemanticCheckpointDatabase> {
1369
+ try {
1370
+ const database = boundaryRecord(value, "Semantic checkpoint database");
1371
+ const checkpoint = boundUnknownMethod(database, "checkpoint", "Semantic checkpoint database");
1372
+ const close = boundUnknownMethod(database, "close", "Semantic checkpoint database");
1373
+ return Object.freeze({
1374
+ checkpoint: async () => {
1375
+ return await checkpoint();
1376
+ },
1377
+ close: async () => {
1378
+ await close();
1379
+ },
1380
+ });
1381
+ } catch (error: unknown) {
1382
+ await closeMalformedStore(value);
1383
+ throw error;
1384
+ }
1385
+ }
1386
+
1387
+ type SemanticAttestationStore = {
1388
+ readonly close: () => Promise<void>;
1389
+ readonly pendingEmbeddingCount: () => Promise<number>;
1390
+ };
1391
+
1392
+ function parseSemanticAttestationStore(value: unknown): SemanticAttestationStore {
1393
+ const store = boundaryRecord(value, "QMD attestation store");
1394
+ const close = boundUnknownMethod(store, "close", "QMD attestation store");
1395
+ const internal = boundaryRecord(store.internal, "QMD attestation store.internal");
1396
+ const llm = boundaryRecord(internal.llm, "QMD attestation store.internal.llm");
1397
+ const getHashesNeedingEmbedding = boundUnknownMethod(
1398
+ internal,
1399
+ "getHashesNeedingEmbedding",
1400
+ "QMD attestation store.internal",
1401
+ );
1402
+ // Attestation never invokes these methods. Requiring all three proves that
1403
+ // a later strict query can stay on the same store-local vector/LLM boundary.
1404
+ boundUnknownMethod(internal, "searchVec", "QMD attestation store.internal");
1405
+ boundUnknownMethod(llm, "countTokens", "QMD attestation store.internal.llm");
1406
+ boundUnknownMethod(llm, "embed", "QMD attestation store.internal.llm");
1407
+ return Object.freeze({
1408
+ close: async () => {
1409
+ await close();
1410
+ },
1411
+ pendingEmbeddingCount: async () => boundaryCount(
1412
+ await getHashesNeedingEmbedding(recommendedEmbeddingModel),
1413
+ "QMD attestation store.internal.getHashesNeedingEmbedding result",
1414
+ ),
1415
+ });
1416
+ }
1417
+
1418
+ async function openedSemanticAttestationStore(
1419
+ value: unknown,
1420
+ ): Promise<SemanticAttestationStore> {
1421
+ try {
1422
+ return parseSemanticAttestationStore(value);
1423
+ } catch (error: unknown) {
1424
+ await closeMalformedStore(value);
1425
+ throw error;
1426
+ }
1427
+ }
1428
+
1429
+ async function openStore(
1430
+ root: string,
1431
+ database: string,
1432
+ embeddingModelSource: string,
1433
+ dependencies: SemanticDependencies,
1434
+ requireStoreLocalVectorBoundary = false,
1435
+ ): Promise<SearchStore> {
1436
+ await mkdir(dirname(database), { recursive: true });
1437
+ const created = await (dependencies.createStore ?? defaultCreateStore)({
1438
+ dbPath: database,
1439
+ config: storeConfig(root, embeddingModelSource),
1440
+ });
1441
+ return await openedSearchStore(created, recommendedEmbeddingModel, {
1442
+ requireStoreLocalVectorBoundary,
1443
+ now: dependencies.now ?? performance.now.bind(performance),
1444
+ });
1445
+ }
1446
+
1447
+ async function assertExactWarmProjectionFile(
1448
+ path: string,
1449
+ expected: Uint8Array,
1450
+ label: string,
1451
+ ): Promise<void> {
1452
+ const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
1453
+ try {
1454
+ const before = await handle.stat();
1455
+ if (!before.isFile() || before.nlink !== 1 || before.size !== expected.byteLength) {
1456
+ throw new Error(`${label} does not match the immutable warm projection.`);
1457
+ }
1458
+ const observed = await handle.readFile();
1459
+ const after = await handle.stat();
1460
+ if (
1461
+ after.dev !== before.dev
1462
+ || after.ino !== before.ino
1463
+ || after.size !== before.size
1464
+ || !observed.equals(expected)
1465
+ ) {
1466
+ throw new Error(`${label} does not match the immutable warm projection.`);
1467
+ }
1468
+ } finally {
1469
+ await handle.close();
1470
+ }
1471
+ }
1472
+
1473
+ async function assertExistingWarmProjection(
1474
+ description: SemanticProjectionDescription,
1475
+ notes: readonly Note[],
1476
+ ): Promise<void> {
1477
+ let canonicalGeneration: string;
1478
+ try {
1479
+ canonicalGeneration = await realpath(description.generationPath);
1480
+ } catch (error: unknown) {
1481
+ throw new Error("The immutable warm semantic projection is absent.", { cause: error });
1482
+ }
1483
+ if (canonicalGeneration !== description.generationPath) {
1484
+ throw new Error("The immutable warm semantic projection changed identity.");
1485
+ }
1486
+ await assertExactWarmProjectionFile(
1487
+ join(canonicalGeneration, "manifest.json"),
1488
+ Buffer.from(description.manifestText, "utf8"),
1489
+ "Semantic projection manifest",
1490
+ );
1491
+ const notesByPath = new Map(notes.map((note) => [note.path, note]));
1492
+ if (notesByPath.size !== description.manifest.notes.length) {
1493
+ throw new Error("The immutable warm semantic projection note population drifted.");
1494
+ }
1495
+ for (const entry of description.manifest.notes) {
1496
+ const note = notesByPath.get(entry.path);
1497
+ if (note === undefined) {
1498
+ throw new Error(`Semantic projection note ${JSON.stringify(entry.path)} is absent.`);
1499
+ }
1500
+ const expected = Buffer.from(note.content, "utf8");
1501
+ if (
1502
+ expected.byteLength !== entry.bytes
1503
+ || createHash("sha256").update(expected).digest("hex") !== entry.sha256
1504
+ ) {
1505
+ throw new Error(`Semantic projection note ${JSON.stringify(entry.path)} drifted.`);
1506
+ }
1507
+ await assertExactWarmProjectionFile(
1508
+ resolve(canonicalGeneration, ...entry.path.split("/")),
1509
+ expected,
1510
+ `Semantic projection note ${JSON.stringify(entry.path)}`,
1511
+ );
1512
+ }
1513
+ }
1514
+
1515
+ async function openWarmSearchStore(
1516
+ database: string,
1517
+ databaseSnapshotSeal: SemanticDatabaseSnapshotSeal,
1518
+ embeddingModelSource: string,
1519
+ dependencies: SemanticDependencies,
1520
+ ): Promise<WarmSearchStore> {
1521
+ const snapshot = await createIsolatedQmdDatabaseSnapshot(database, databaseSnapshotSeal);
1522
+ try {
1523
+ const created = await (
1524
+ dependencies.createWarmSearchStore ?? defaultCreateWarmSearchStore
1525
+ )({
1526
+ dbPath: snapshot.database,
1527
+ embeddingModelSource,
1528
+ });
1529
+ const store = await openedWarmSearchStore(
1530
+ created,
1531
+ dependencies.now ?? performance.now.bind(performance),
1532
+ );
1533
+ return isolatedWarmSearchStore(store, snapshot);
1534
+ } catch (error: unknown) {
1535
+ await snapshot.cleanup();
1536
+ throw error;
1537
+ }
1538
+ }
1539
+
1540
+ async function openAttestationStore(
1541
+ database: string,
1542
+ databaseSnapshotSeal: SemanticDatabaseSnapshotSeal,
1543
+ dependencies: SemanticDependencies,
1544
+ ): Promise<SemanticAttestationStore> {
1545
+ const snapshot = await createIsolatedQmdDatabaseSnapshot(database, databaseSnapshotSeal);
1546
+ try {
1547
+ const created = await (
1548
+ dependencies.createAttestationStore ?? defaultCreateAttestationStore
1549
+ )({ dbPath: snapshot.database });
1550
+ const store = await openedSemanticAttestationStore(created);
1551
+ return isolatedAttestationStore(store, snapshot);
1552
+ } catch (error: unknown) {
1553
+ await snapshot.cleanup();
1554
+ throw error;
1555
+ }
1556
+ }
1557
+
1558
+ /**
1559
+ * Finish a private QMD writer generation before its bytes are sealed. This is
1560
+ * the only canonical SQLite mutation in the warm-reader composition: it
1561
+ * checkpoints WAL content into the main file and leaves journal mode DELETE so
1562
+ * every later reader can require canonical WAL/SHM/rollback-journal absence.
1563
+ */
1564
+ export async function checkpointSemanticWarmCache(
1565
+ options: Pick<SemanticWarmCacheAttestationOptions, "root" | "database">,
1566
+ dependencies: SemanticDependencies = {},
1567
+ ): Promise<SemanticWarmCacheCheckpointResult> {
1568
+ const root = await resolvedDirectory(options.root);
1569
+ const database = await resolveSemanticDatabase(
1570
+ databaseFor(root, options.database, dependencies),
1571
+ root,
1572
+ );
1573
+ const before = await lstat(database);
1574
+ if (before.isSymbolicLink() || !before.isFile() || before.nlink !== 1) {
1575
+ throw new TypeError("The semantic database to checkpoint must be a singly linked regular file.");
1576
+ }
1577
+ const opened = await (dependencies.openCheckpointDatabase ?? defaultOpenCheckpointDatabase)(
1578
+ database,
1579
+ );
1580
+ const checkpoint = await openedCheckpointDatabase(opened);
1581
+ try {
1582
+ const outcome = boundaryRecord(
1583
+ await checkpoint.checkpoint(),
1584
+ "Semantic checkpoint result",
1585
+ );
1586
+ const wal = boundaryRecord(outcome.wal, "Semantic checkpoint result.wal");
1587
+ const busy = boundaryCount(wal.busy, "Semantic checkpoint result.wal.busy");
1588
+ const log = boundaryCount(wal.log, "Semantic checkpoint result.wal.log");
1589
+ const checkpointed = boundaryCount(
1590
+ wal.checkpointed,
1591
+ "Semantic checkpoint result.wal.checkpointed",
1592
+ );
1593
+ const mode = boundaryRecord(outcome.mode, "Semantic checkpoint result.mode");
1594
+ if (busy !== 0 || log !== checkpointed) {
1595
+ throw new Error("Semantic WAL checkpoint did not copy every committed frame.");
1596
+ }
1597
+ if (boundaryString(mode.journal_mode, "Semantic checkpoint result.mode.journal_mode") !== "delete") {
1598
+ throw new Error("Semantic database did not leave WAL journal mode.");
1599
+ }
1600
+ } catch (error: unknown) {
1601
+ try {
1602
+ await checkpoint.close();
1603
+ } catch (cleanupError: unknown) {
1604
+ throw new AggregateError(
1605
+ [error, cleanupError],
1606
+ "Semantic database checkpoint and cleanup both failed.",
1607
+ { cause: error },
1608
+ );
1609
+ }
1610
+ throw error;
1611
+ }
1612
+ await checkpoint.close();
1613
+ for (const sidecar of [`${database}-wal`, `${database}-shm`]) {
1614
+ let metadata: Awaited<ReturnType<typeof lstat>>;
1615
+ try {
1616
+ metadata = await lstat(sidecar);
1617
+ } catch (error: unknown) {
1618
+ if (missingFile(error)) continue;
1619
+ throw error;
1620
+ }
1621
+ if (metadata.isSymbolicLink() || !metadata.isFile() || metadata.nlink !== 1) {
1622
+ throw new TypeError("Semantic checkpoint sidecars must be singly linked regular files.");
1623
+ }
1624
+ await rm(sidecar);
1625
+ }
1626
+ await Promise.all([
1627
+ assertAbsentSnapshotSidecar(`${database}-wal`, "Semantic database WAL"),
1628
+ assertAbsentSnapshotSidecar(`${database}-shm`, "Semantic database SHM"),
1629
+ assertAbsentSnapshotSidecar(
1630
+ `${database}-journal`,
1631
+ "Semantic database rollback journal",
1632
+ ),
1633
+ ]);
1634
+ return Object.freeze({ database, wal: null, shm: null, journal: null });
1635
+ }
1636
+
1637
+ /**
1638
+ * Prove that an existing QMD cache can serve strict local vector queries.
1639
+ * QMD opens a stable disposable snapshot because its public store constructor
1640
+ * always initializes SQLite. The attested cache itself is never opened,
1641
+ * locked, updated, embedded, searched, indexed, or repaired.
1642
+ */
1643
+ export async function attestSemanticWarmCache(
1644
+ options: SemanticWarmCacheAttestationOptions,
1645
+ dependencies: SemanticDependencies = {},
1646
+ ): Promise<SemanticWarmCacheReadiness> {
1647
+ const embeddingModel = retainVerifiedEmbeddingModelLease(options.embeddingModelLease);
1648
+ try {
1649
+ const root = await resolvedDirectory(options.root);
1650
+ const database = await resolveSemanticDatabase(
1651
+ databaseFor(root, options.database, dependencies),
1652
+ root,
1653
+ );
1654
+ if (Buffer.byteLength(database, "utf8") > MAX_SEMANTIC_DATABASE_IDENTITY_BYTES) {
1655
+ throw new RangeError(
1656
+ `Semantic database identity exceeds ${MAX_SEMANTIC_DATABASE_IDENTITY_BYTES.toLocaleString("en-US")} UTF-8 bytes.`,
1657
+ );
1658
+ }
1659
+ let databaseState: Awaited<ReturnType<typeof stat>>;
1660
+ try {
1661
+ databaseState = await stat(database);
1662
+ } catch (error: unknown) {
1663
+ throw new Error("The warm semantic database must already exist.", { cause: error });
1664
+ }
1665
+ if (!databaseState.isFile()) {
1666
+ throw new Error("The warm semantic database must already be a regular file.");
1667
+ }
1668
+ const store = await openAttestationStore(
1669
+ database,
1670
+ options.databaseSnapshotSeal,
1671
+ dependencies,
1672
+ );
1673
+ try {
1674
+ const pendingEmbeddings = await store.pendingEmbeddingCount();
1675
+ if (pendingEmbeddings !== 0) {
1676
+ throw new Error(
1677
+ `Warm semantic cache is not ready: ${pendingEmbeddings} embedding input(s) remain pending.`,
1678
+ );
1679
+ }
1680
+ return Object.freeze({
1681
+ model: recommendedEmbeddingModel,
1682
+ database,
1683
+ pendingEmbeddings: 0,
1684
+ });
1685
+ } finally {
1686
+ await store.close();
1687
+ }
1688
+ } finally {
1689
+ await embeddingModel.release();
1690
+ }
1691
+ }
1692
+
1693
+ /**
1694
+ * Open one query-only session over an already verified warm projection.
1695
+ * Establishment is eager: the projection, model lease, strict store-local
1696
+ * vector boundary, and zero pending-embedding state are all proven before the
1697
+ * session is returned. No writer lease, projection preparation, update, or
1698
+ * corpus embedding operation is reachable from this path.
1699
+ */
1700
+ export async function openSemanticWarmSearchSession(
1701
+ options: SemanticWarmSearchSessionOptions,
1702
+ dependencies: SemanticDependencies = {},
1703
+ ): Promise<SemanticSearchSession> {
1704
+ const embeddingModel = retainVerifiedEmbeddingModelLease(options.embeddingModelLease);
1705
+ let store: WarmSearchStore | undefined;
1706
+ try {
1707
+ const root = await resolvedDirectory(options.root);
1708
+ const database = await resolveSemanticDatabase(
1709
+ databaseFor(root, options.database, dependencies),
1710
+ root,
1711
+ );
1712
+ const databaseState = await stat(database).catch((error: unknown) => {
1713
+ throw new Error("The warm semantic database must already exist.", { cause: error });
1714
+ });
1715
+ if (!databaseState.isFile()) {
1716
+ throw new Error("The warm semantic database must already be a regular file.");
1717
+ }
1718
+ const snapshot = await semanticSnapshot(root, dependencies);
1719
+ const description = await describeSemanticProjection(
1720
+ database,
1721
+ root,
1722
+ snapshot.notes,
1723
+ semanticIndexIdentity,
1724
+ );
1725
+ await assertExistingWarmProjection(description, snapshot.notes);
1726
+ store = await openWarmSearchStore(
1727
+ database,
1728
+ options.databaseSnapshotSeal,
1729
+ embeddingModel.source,
1730
+ dependencies,
1731
+ );
1732
+ const pendingEmbeddings = await store.pendingEmbeddingCount();
1733
+ if (pendingEmbeddings !== 0) {
1734
+ throw new Error(
1735
+ `Warm semantic cache is not ready: ${pendingEmbeddings} embedding input(s) remain pending.`,
1736
+ );
1737
+ }
1738
+
1739
+ const notesByPath = new Map(snapshot.notes.map((note) => [note.path, note]));
1740
+ const contentHashesByPath = new Map(
1741
+ description.manifest.notes.map(({ path, sha256 }) => [path, sha256]),
1742
+ );
1743
+ const notesByQmdPath = qmdNoteLookup(snapshot.notes, contentHashesByPath);
1744
+ const connectionsById = new Map(
1745
+ snapshot.analysis.noteConnections.map((connection) => [connection.id, connection]),
1746
+ );
1747
+ const update: SemanticUpdateResult = Object.freeze({
1748
+ collections: 1,
1749
+ indexed: 0,
1750
+ updated: 0,
1751
+ unchanged: snapshot.notes.length,
1752
+ removed: 0,
1753
+ needsEmbedding: 0,
1754
+ });
1755
+ const context: SemanticSearchContext = {
1756
+ root,
1757
+ projectionRoot: description.generationPath,
1758
+ database,
1759
+ store,
1760
+ update,
1761
+ ensureEmbedding: () => Promise.resolve(null),
1762
+ notesByPath,
1763
+ notesByQmdPath,
1764
+ contentHashesByPath,
1765
+ connectionsById,
1766
+ };
1767
+ let tail: Promise<void> = Promise.resolve();
1768
+ let closeRequested = false;
1769
+ let closePromise: Promise<void> | undefined;
1770
+ const serialize = <Value>(operation: () => Promise<Value>): Promise<Value> => {
1771
+ const result = tail.then(operation);
1772
+ tail = result.then(() => undefined, () => undefined);
1773
+ return result;
1774
+ };
1775
+ const ownedStore = store;
1776
+ store = undefined;
1777
+ return Object.freeze({
1778
+ root,
1779
+ database,
1780
+ model: recommendedEmbeddingModel,
1781
+ update,
1782
+ search: (searchOptions: SemanticSessionSearchOptions) => {
1783
+ if (closeRequested) {
1784
+ return Promise.reject(new Error("Semantic warm search session is closed."));
1785
+ }
1786
+ return serialize(() => executeSemanticSearch(context, searchOptions));
1787
+ },
1788
+ close: () => {
1789
+ if (closePromise !== undefined) return closePromise;
1790
+ closeRequested = true;
1791
+ closePromise = serialize(async () => {
1792
+ try {
1793
+ await ownedStore.close();
1794
+ } finally {
1795
+ await embeddingModel.release();
1796
+ }
1797
+ });
1798
+ return closePromise;
1799
+ },
1800
+ });
1801
+ } catch (error: unknown) {
1802
+ try {
1803
+ await store?.close();
1804
+ } finally {
1805
+ await embeddingModel.release();
1806
+ }
1807
+ throw error;
1808
+ }
1809
+ }
1810
+
1811
+ function databaseFor(
1812
+ root: string,
1813
+ requested: string | undefined,
1814
+ dependencies: SemanticDependencies,
1815
+ ): string {
1816
+ if (requested === undefined) return semanticDatabasePath(root, dependencies);
1817
+ return resolve(requested);
1818
+ }
1819
+
1820
+ async function embedChanged(
1821
+ store: SearchStore,
1822
+ update: SemanticUpdateResult,
1823
+ force: boolean,
1824
+ ): Promise<SemanticEmbeddingResult | null> {
1825
+ if (!force && update.needsEmbedding === 0) return null;
1826
+ return await store.embed({
1827
+ collection: collectionName,
1828
+ force,
1829
+ model: recommendedEmbeddingModel,
1830
+ chunkStrategy: embeddingChunkStrategy,
1831
+ });
1832
+ }
1833
+
1834
+ async function semanticSnapshot(
1835
+ root: string,
1836
+ dependencies: SemanticDependencies,
1837
+ ): Promise<VaultSnapshot> {
1838
+ return await (dependencies.scanVault
1839
+ ?? ((vaultRoot: string) => scanVault(vaultRoot, { mentionScope: false })))(root);
1840
+ }
1841
+
1842
+ /** Build or incrementally refresh the local QMD vector index for one vault. */
1843
+ export async function indexSemanticVault(
1844
+ options: SemanticIndexOptions,
1845
+ dependencies: SemanticDependencies = {},
1846
+ ): Promise<SemanticIndexResult> {
1847
+ const embeddingModel = await verifiedIndexEmbeddingModelSource(
1848
+ options.embeddingModelFile,
1849
+ dependencies,
1850
+ );
1851
+ try {
1852
+ const root = await resolvedDirectory(options.root);
1853
+ const databaseCandidate = await resolveSemanticDatabase(
1854
+ databaseFor(root, options.database, dependencies),
1855
+ root,
1856
+ );
1857
+ const snapshot = await semanticSnapshot(root, dependencies);
1858
+ const description = await describeSemanticProjection(
1859
+ databaseCandidate,
1860
+ root,
1861
+ snapshot.notes,
1862
+ semanticIndexIdentity,
1863
+ );
1864
+ const database = description.database;
1865
+ return await withSemanticGenerationWriterLease(
1866
+ database,
1867
+ description.manifest.generation,
1868
+ async () => {
1869
+ const projection = await prepareSemanticProjection(description, snapshot.notes);
1870
+ let store: SearchStore | undefined;
1871
+ try {
1872
+ store = await openStore(
1873
+ projection.root,
1874
+ database,
1875
+ embeddingModel.source,
1876
+ dependencies,
1877
+ );
1878
+ const update = await store.update({ collections: [collectionName] });
1879
+ const embedding = await embedChanged(
1880
+ store,
1881
+ update,
1882
+ options.force ?? false,
1883
+ );
1884
+ return { root, database, model: recommendedEmbeddingModel, update, embedding };
1885
+ } finally {
1886
+ try {
1887
+ await store?.close();
1888
+ } finally {
1889
+ await projection.release();
1890
+ }
1891
+ }
1892
+ },
1893
+ {
1894
+ ...dependencies.writerLease,
1895
+ excludeReaders: options.force === true,
1896
+ },
1897
+ );
1898
+ } finally {
1899
+ await embeddingModel.release();
1900
+ }
1901
+ }
1902
+
1903
+ function qmdEmojiToHex(value: string): string {
1904
+ return value.replace(/(?:\p{So}\p{Mn}?|\p{Sk})+/gu, (run) =>
1905
+ [...run]
1906
+ .filter((character) => /\p{So}|\p{Sk}/u.test(character))
1907
+ .map((character) => character.codePointAt(0)?.toString(16) ?? "")
1908
+ .join("-"));
1909
+ }
1910
+
1911
+ /** Owned equivalent of QMD 2.5.3's pinned handelize path transform. */
1912
+ function qmdHandelize(path: string): string | null {
1913
+ if (path.trim() === "") return null;
1914
+ const segments = path.split("/").filter((segment) => segment !== "");
1915
+ const lastSegment = segments.at(-1) ?? "";
1916
+ const filenameWithoutExtension = lastSegment.replace(/\.[^.]+$/u, "");
1917
+ if (!/[\p{L}\p{N}\p{So}\p{Sk}$]/u.test(filenameWithoutExtension)) return null;
1918
+ const result = path
1919
+ .replaceAll("___", "/")
1920
+ .split("/")
1921
+ .map((rawSegment, index, allSegments) => {
1922
+ const segment = qmdEmojiToHex(rawSegment);
1923
+ if (index === allSegments.length - 1) {
1924
+ const extension = segment.match(/(\.[a-z0-9]+)$/iu)?.[1] ?? "";
1925
+ const name = extension === "" ? segment : segment.slice(0, -extension.length);
1926
+ return name
1927
+ .replace(/[^\p{L}\p{N}$]+/gu, "-")
1928
+ .replace(/^-+|-+$/gu, "") + extension;
1929
+ }
1930
+ return segment
1931
+ .replace(/[^\p{L}\p{N}$]+/gu, "-")
1932
+ .replace(/^-+|-+$/gu, "");
1933
+ })
1934
+ .filter((segment) => segment !== "")
1935
+ .join("/");
1936
+ return result === "" ? null : result;
1937
+ }
1938
+
1939
+ type QmdNoteLookup = ReadonlyMap<string, ReadonlyMap<string, readonly Note[]>>;
1940
+
1941
+ function qmdNoteLookup(
1942
+ notes: readonly Note[],
1943
+ contentHashesByPath: ReadonlyMap<string, string>,
1944
+ ): QmdNoteLookup {
1945
+ const lookup = new Map<string, Map<string, Note[]>>();
1946
+ for (const note of notes) {
1947
+ const qmdPath = qmdHandelize(note.path);
1948
+ if (qmdPath === null) continue;
1949
+ const contentHash = contentHashesByPath.get(note.path);
1950
+ if (contentHash === undefined) {
1951
+ throw new Error(`Semantic projection lost the hash for ${JSON.stringify(note.path)}.`);
1952
+ }
1953
+ const byHash = lookup.get(qmdPath) ?? new Map<string, Note[]>();
1954
+ const candidates = byHash.get(contentHash) ?? [];
1955
+ candidates.push(note);
1956
+ byHash.set(contentHash, candidates);
1957
+ lookup.set(qmdPath, byHash);
1958
+ }
1959
+ return lookup;
1960
+ }
1961
+
1962
+ /** Undefined means a filesystem result; null means a rejected virtual result. */
1963
+ function qmdVirtualNotePath(filepath: string): string | null | undefined {
1964
+ if (!filepath.startsWith("qmd://")) return undefined;
1965
+ const prefix = `qmd://${collectionName}/`;
1966
+ if (!filepath.startsWith(prefix)) return null;
1967
+ const path = filepath.slice(prefix.length);
1968
+ const segments = path.split("/");
1969
+ const hasControlCharacter = [...path].some((character) => {
1970
+ const codePoint = character.codePointAt(0) ?? 0;
1971
+ return codePoint <= 0x1f || codePoint === 0x7f;
1972
+ });
1973
+ if (path === ""
1974
+ || path.includes("\\")
1975
+ || path.includes("?")
1976
+ || path.includes("#")
1977
+ || path.includes("%")
1978
+ || hasControlCharacter
1979
+ || segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
1980
+ return null;
1981
+ }
1982
+ return path;
1983
+ }
1984
+
1985
+ async function resolvedSearchNote(
1986
+ projectionRoot: string,
1987
+ result: SemanticSearchDocument,
1988
+ notesByPath: ReadonlyMap<string, Note>,
1989
+ notesByQmdPath: QmdNoteLookup,
1990
+ contentHashesByPath: ReadonlyMap<string, string>,
1991
+ ): Promise<Note | null> {
1992
+ const virtualPath = qmdVirtualNotePath(result.filepath);
1993
+ if (virtualPath !== undefined) {
1994
+ if (virtualPath === null) return null;
1995
+ const candidates = notesByQmdPath.get(virtualPath)?.get(result.hash) ?? [];
1996
+ const candidate = candidates[0];
1997
+ return candidates.length === 1
1998
+ && candidate !== undefined
1999
+ && contentHashesByPath.get(candidate.path) === result.hash
2000
+ ? candidate
2001
+ : null;
2002
+ }
2003
+ if (!isAbsolute(result.filepath)) return null;
2004
+ let filepath: string;
2005
+ try {
2006
+ filepath = await realpath(resolve(result.filepath));
2007
+ } catch {
2008
+ return null;
2009
+ }
2010
+ const candidate = relative(projectionRoot, filepath);
2011
+ if (candidate === "" || candidate === ".." || candidate.startsWith(`..${sep}`) || isAbsolute(candidate)) {
2012
+ return null;
2013
+ }
2014
+ const note = notesByPath.get(candidate.split(sep).join("/"));
2015
+ return note !== undefined && contentHashesByPath.get(note.path) === result.hash ? note : null;
2016
+ }
2017
+
2018
+ function queryOffset(body: string, query: string, suggested: number | undefined): number {
2019
+ if (suggested !== undefined && Number.isSafeInteger(suggested) && suggested >= 0 && suggested <= body.length) {
2020
+ return suggested;
2021
+ }
2022
+ const terms = query.toLocaleLowerCase("en-US").match(/[\p{L}\p{N}]{2,}/gu) ?? [];
2023
+ const lowerBody = body.toLocaleLowerCase("en-US");
2024
+ for (const term of terms.toSorted((left, right) => right.length - left.length)) {
2025
+ const offset = lowerBody.indexOf(term);
2026
+ if (offset !== -1) return offset;
2027
+ }
2028
+ return 0;
2029
+ }
2030
+
2031
+ function boundedSnippet(body: string, offset: number): string {
2032
+ const normalized = body.replaceAll("\r\n", "\n").replaceAll("\r", "\n");
2033
+ const maximum = 600;
2034
+ const start = Math.max(0, Math.min(normalized.length, offset) - 180);
2035
+ const end = Math.min(normalized.length, start + maximum);
2036
+ const value = normalized.slice(start, end).replace(/\s+/gu, " ").trim();
2037
+ return `${start > 0 ? "…" : ""}${value}${end < normalized.length ? "…" : ""}`;
2038
+ }
2039
+
2040
+ async function searchHit(
2041
+ projectionRoot: string,
2042
+ query: string,
2043
+ result: SemanticSearchDocument,
2044
+ notesByPath: ReadonlyMap<string, Note>,
2045
+ notesByQmdPath: QmdNoteLookup,
2046
+ contentHashesByPath: ReadonlyMap<string, string>,
2047
+ connectionsById: ReadonlyMap<string, NoteConnections>,
2048
+ ): Promise<SemanticSearchHit | null> {
2049
+ const note = await resolvedSearchNote(
2050
+ projectionRoot,
2051
+ result,
2052
+ notesByPath,
2053
+ notesByQmdPath,
2054
+ contentHashesByPath,
2055
+ );
2056
+ if (note === null) return null;
2057
+ const connection = connectionsById.get(note.id);
2058
+ const body = note.content;
2059
+ const offset = queryOffset(body, query, result.chunkPos);
2060
+ return {
2061
+ path: note.path,
2062
+ title: note.title,
2063
+ score: result.score,
2064
+ source: result.source,
2065
+ docid: result.docid,
2066
+ modifiedAt: result.modifiedAt,
2067
+ ...(body === "" ? {} : { line: body.slice(0, offset).split("\n").length }),
2068
+ snippet: boundedSnippet(body, offset),
2069
+ tags: note.tags,
2070
+ metadata: note.metadata,
2071
+ inboundContextualCount: connection?.inboundContextualCount ?? 0,
2072
+ outboundContextualCount: connection?.outboundContextualCount ?? 0,
2073
+ backlinks: connection?.backlinks ?? [],
2074
+ };
2075
+ }
2076
+
2077
+ type FusedSemanticDocument = {
2078
+ readonly document: SemanticSearchDocument;
2079
+ readonly score: number;
2080
+ readonly signals: {
2081
+ readonly keyword: boolean;
2082
+ readonly semantic: boolean;
2083
+ };
2084
+ };
2085
+
2086
+ function semanticDocumentKey(document: SemanticSearchDocument): string {
2087
+ return JSON.stringify([document.filepath, document.hash]);
2088
+ }
2089
+
2090
+ function firstDocumentsByKey(
2091
+ documents: readonly SemanticSearchDocument[],
2092
+ ): ReadonlyMap<string, SemanticSearchDocument> {
2093
+ const byKey = new Map<string, SemanticSearchDocument>();
2094
+ for (const document of documents) {
2095
+ const key = semanticDocumentKey(document);
2096
+ if (!byKey.has(key)) byKey.set(key, document);
2097
+ }
2098
+ return byKey;
2099
+ }
2100
+
2101
+ function fusedHybridDocuments(
2102
+ lexical: readonly SemanticSearchDocument[],
2103
+ vector: readonly SemanticSearchDocument[],
2104
+ candidateLimit: number,
2105
+ ): readonly FusedSemanticDocument[] {
2106
+ const lexicalByKey = firstDocumentsByKey(lexical);
2107
+ const vectorByKey = firstDocumentsByKey(vector);
2108
+ return fuseRankedCandidates([
2109
+ { name: "keyword", weight: 1, ids: lexical.map(semanticDocumentKey) },
2110
+ { name: "semantic", weight: 1, ids: vector.map(semanticDocumentKey) },
2111
+ ]).slice(0, candidateLimit).map((candidate) => {
2112
+ const lexicalDocument = lexicalByKey.get(candidate.id);
2113
+ const vectorDocument = vectorByKey.get(candidate.id);
2114
+ const document = vectorDocument ?? lexicalDocument;
2115
+ if (document === undefined) {
2116
+ throw new Error("Fused QMD candidate lost its source document.");
2117
+ }
2118
+ return {
2119
+ document,
2120
+ score: candidate.score,
2121
+ signals: {
2122
+ keyword: candidate.contributions.some(({ lane }) => lane === "keyword"),
2123
+ semantic: candidate.contributions.some(({ lane }) => lane === "semantic"),
2124
+ },
2125
+ };
2126
+ });
2127
+ }
2128
+
2129
+ async function hybridSearchHit(
2130
+ projectionRoot: string,
2131
+ query: string,
2132
+ result: FusedSemanticDocument,
2133
+ notesByPath: ReadonlyMap<string, Note>,
2134
+ notesByQmdPath: QmdNoteLookup,
2135
+ contentHashesByPath: ReadonlyMap<string, string>,
2136
+ connectionsById: ReadonlyMap<string, NoteConnections>,
2137
+ ): Promise<SemanticSearchHit | null> {
2138
+ const hit = await searchHit(
2139
+ projectionRoot,
2140
+ query,
2141
+ result.document,
2142
+ notesByPath,
2143
+ notesByQmdPath,
2144
+ contentHashesByPath,
2145
+ connectionsById,
2146
+ );
2147
+ if (hit === null) return null;
2148
+ return {
2149
+ ...hit,
2150
+ score: result.score,
2151
+ source: "hybrid",
2152
+ signals: result.signals,
2153
+ };
2154
+ }
2155
+
2156
+ function boundedLimit(value: number | undefined, maximum: 100 | 500): number {
2157
+ if (value === undefined) return 10;
2158
+ if (!Number.isSafeInteger(value) || value < 1 || value > maximum) {
2159
+ throw new Error(`Search limit must be an integer from 1 through ${maximum}.`);
2160
+ }
2161
+ return value;
2162
+ }
2163
+
2164
+ function boundedCandidateLimit(
2165
+ value: number | undefined,
2166
+ resultLimit: number,
2167
+ ): number {
2168
+ if (value === undefined) return Math.max(40, resultLimit * 4);
2169
+ if (!Number.isSafeInteger(value) || value < resultLimit || value > 500) {
2170
+ throw new Error(
2171
+ `Search candidate limit must be an integer from ${resultLimit} through 500.`,
2172
+ );
2173
+ }
2174
+ return value;
2175
+ }
2176
+
2177
+ function boundedScore(value: number | undefined): number {
2178
+ if (value === undefined) return 0;
2179
+ if (!Number.isFinite(value) || value < 0 || value > 1) {
2180
+ throw new Error("Minimum score must be a number from 0 through 1.");
2181
+ }
2182
+ return value;
2183
+ }
2184
+
2185
+ function boundedMode(value: unknown): SemanticSearchMode {
2186
+ const mode = value ?? "semantic";
2187
+ if (mode !== "hybrid" && mode !== "keyword" && mode !== "semantic") {
2188
+ throw new Error('Search mode must be "hybrid", "keyword", or "semantic".');
2189
+ }
2190
+ return mode;
2191
+ }
2192
+
2193
+ type SemanticSearchContext = {
2194
+ readonly root: string;
2195
+ readonly projectionRoot: string;
2196
+ readonly database: string;
2197
+ readonly store: SemanticQueryStore;
2198
+ readonly update: SemanticUpdateResult;
2199
+ readonly ensureEmbedding: () => Promise<SemanticEmbeddingResult | null>;
2200
+ readonly notesByPath: ReadonlyMap<string, Note>;
2201
+ readonly notesByQmdPath: QmdNoteLookup;
2202
+ readonly contentHashesByPath: ReadonlyMap<string, string>;
2203
+ readonly connectionsById: ReadonlyMap<string, NoteConnections>;
2204
+ };
2205
+
2206
+ async function executeSemanticSearch(
2207
+ context: SemanticSearchContext,
2208
+ options: SemanticSessionSearchOptions,
2209
+ ): Promise<SemanticSearchResult> {
2210
+ const query = validateSearchQuery(options.query).query;
2211
+ const mode = boundedMode(options.mode);
2212
+ const limit = boundedLimit(options.limit, 500);
2213
+ const candidateLimit = boundedCandidateLimit(options.candidateLimit, limit);
2214
+ const minScore = boundedScore(options.minScore);
2215
+ const embedding = mode === "keyword"
2216
+ ? null
2217
+ : await context.ensureEmbedding();
2218
+ let queryEmbedding: SemanticQueryEmbeddingAccounting | null = mode === "keyword"
2219
+ ? Object.freeze({ calls: 0, inputTokens: 0, durationMs: 0 })
2220
+ : null;
2221
+ let hits: readonly (SemanticSearchHit | null)[];
2222
+ let rawRequested: number;
2223
+ let rawReturned: number;
2224
+ let rawDiscarded: number;
2225
+ let rawThresholdRejected: number;
2226
+ let rawExhausted: boolean;
2227
+ if (mode === "hybrid") {
2228
+ const lexical = await context.store.searchLex(query, {
2229
+ collection: collectionName,
2230
+ limit: candidateLimit,
2231
+ });
2232
+ const vectorSearch = await context.store.searchVector(query, {
2233
+ collection: collectionName,
2234
+ limit: candidateLimit,
2235
+ });
2236
+ queryEmbedding = vectorSearch.accounting;
2237
+ const fused = fusedHybridDocuments(lexical, vectorSearch.results, candidateLimit);
2238
+ const considered = fused.filter(({ score }) => score >= minScore);
2239
+ rawRequested = candidateLimit;
2240
+ rawReturned = fused.length;
2241
+ rawThresholdRejected = fused.length - considered.length;
2242
+ rawExhausted = fused.length < candidateLimit
2243
+ && lexical.length < candidateLimit
2244
+ && vectorSearch.results.length < candidateLimit;
2245
+ hits = await Promise.all(considered.map((result) =>
2246
+ hybridSearchHit(
2247
+ context.projectionRoot,
2248
+ query,
2249
+ result,
2250
+ context.notesByPath,
2251
+ context.notesByQmdPath,
2252
+ context.contentHashesByPath,
2253
+ context.connectionsById,
2254
+ )));
2255
+ rawDiscarded = considered.length - hits.filter((hit) => hit !== null).length;
2256
+ } else {
2257
+ const vectorSearch = mode === "semantic"
2258
+ ? await context.store.searchVector(query, {
2259
+ collection: collectionName,
2260
+ limit: candidateLimit,
2261
+ })
2262
+ : null;
2263
+ const matches = vectorSearch === null
2264
+ ? await context.store.searchLex(query, {
2265
+ collection: collectionName,
2266
+ limit: candidateLimit,
2267
+ })
2268
+ : vectorSearch.results;
2269
+ if (vectorSearch !== null) queryEmbedding = vectorSearch.accounting;
2270
+ rawRequested = candidateLimit;
2271
+ rawReturned = matches.length;
2272
+ const considered = matches.filter(({ score }) => score >= minScore);
2273
+ rawThresholdRejected = matches.length - considered.length;
2274
+ rawExhausted = matches.length < candidateLimit;
2275
+ hits = await Promise.all(considered
2276
+ .map((result) =>
2277
+ searchHit(
2278
+ context.projectionRoot,
2279
+ query,
2280
+ result,
2281
+ context.notesByPath,
2282
+ context.notesByQmdPath,
2283
+ context.contentHashesByPath,
2284
+ context.connectionsById,
2285
+ )));
2286
+ rawDiscarded = considered.length - hits.filter((hit) => hit !== null).length;
2287
+ }
2288
+ const verified = hits.filter((hit): hit is SemanticSearchHit => hit !== null);
2289
+ return {
2290
+ root: context.root,
2291
+ database: context.database,
2292
+ model: recommendedEmbeddingModel,
2293
+ mode,
2294
+ query,
2295
+ update: context.update,
2296
+ embedding,
2297
+ queryEmbedding,
2298
+ rawWindow: {
2299
+ requested: rawRequested,
2300
+ returned: rawReturned,
2301
+ discarded: rawDiscarded,
2302
+ thresholdRejected: rawThresholdRejected,
2303
+ exhausted: rawExhausted,
2304
+ },
2305
+ results: verified.slice(0, limit),
2306
+ };
2307
+ }
2308
+
2309
+ /** Open one serialized QMD search session over one immutable live vault snapshot. */
2310
+ export async function openSemanticSearchSession(
2311
+ options: SemanticSessionOptions,
2312
+ dependencies: SemanticDependencies = {},
2313
+ ): Promise<SemanticSearchSession> {
2314
+ const root = await resolvedDirectory(options.root);
2315
+ const databaseCandidate = await resolveSemanticDatabase(
2316
+ databaseFor(root, options.database, dependencies),
2317
+ root,
2318
+ );
2319
+ const snapshot = await semanticSnapshot(root, dependencies);
2320
+ const description = await describeSemanticProjection(
2321
+ databaseCandidate,
2322
+ root,
2323
+ snapshot.notes,
2324
+ semanticIndexIdentity,
2325
+ );
2326
+ const notesByPath = new Map(snapshot.notes.map((note) => [note.path, note]));
2327
+ const contentHashesByPath = new Map(
2328
+ description.manifest.notes.map(({ path, sha256 }) => [path, sha256]),
2329
+ );
2330
+ const notesByQmdPath = qmdNoteLookup(snapshot.notes, contentHashesByPath);
2331
+ const connectionsById = new Map(
2332
+ snapshot.analysis.noteConnections.map((connection) => [connection.id, connection]),
2333
+ );
2334
+ const requireStoreLocalVectorBoundary = requiredStoreLocalVectorBoundary(
2335
+ options.requireStoreLocalVectorBoundary,
2336
+ );
2337
+ const embeddingModel = await sessionEmbeddingModelSource(options, dependencies);
2338
+ const database = description.database;
2339
+ let retained: {
2340
+ readonly store: SearchStore;
2341
+ readonly projection: SemanticProjection;
2342
+ } | undefined;
2343
+ let initialized: {
2344
+ readonly store: SearchStore;
2345
+ readonly projection: SemanticProjection;
2346
+ readonly update: SemanticUpdateResult;
2347
+ };
2348
+ try {
2349
+ initialized = await withSemanticGenerationWriterLease(
2350
+ database,
2351
+ description.manifest.generation,
2352
+ async () => {
2353
+ const projection = await prepareSemanticProjection(description, snapshot.notes);
2354
+ let store: SearchStore | undefined;
2355
+ try {
2356
+ store = await openStore(
2357
+ projection.root,
2358
+ database,
2359
+ embeddingModel.source,
2360
+ dependencies,
2361
+ requireStoreLocalVectorBoundary,
2362
+ );
2363
+ retained = { store, projection };
2364
+ const update = await store.update({ collections: [collectionName] });
2365
+ return { store, projection, update };
2366
+ } catch (error: unknown) {
2367
+ try {
2368
+ await store?.close();
2369
+ } finally {
2370
+ await projection.release();
2371
+ retained = undefined;
2372
+ }
2373
+ throw error;
2374
+ }
2375
+ },
2376
+ dependencies.writerLease,
2377
+ );
2378
+ } catch (error: unknown) {
2379
+ await retained?.store.close().catch(() => undefined);
2380
+ await retained?.projection.release().catch(() => undefined);
2381
+ await embeddingModel.release().catch(() => undefined);
2382
+ throw error;
2383
+ }
2384
+ const { store, projection, update } = initialized;
2385
+ let embeddingPromise: Promise<SemanticEmbeddingResult | null> | undefined;
2386
+ const ensureEmbedding = (): Promise<SemanticEmbeddingResult | null> => {
2387
+ embeddingPromise ??= update.needsEmbedding === 0
2388
+ ? Promise.resolve(null)
2389
+ : withSemanticGenerationWriterLease(
2390
+ database,
2391
+ projection.manifest.generation,
2392
+ async () => {
2393
+ // Another process may have completed this generation's vectors while
2394
+ // the session waited. Refresh under the lease before writing.
2395
+ const refreshed = await store.update({ collections: [collectionName] });
2396
+ return await embedChanged(store, refreshed, false);
2397
+ },
2398
+ dependencies.writerLease,
2399
+ );
2400
+ return embeddingPromise;
2401
+ };
2402
+ let tail: Promise<void> = Promise.resolve();
2403
+ let closeRequested = false;
2404
+ let closePromise: Promise<void> | undefined;
2405
+ const serialize = <Value>(operation: () => Promise<Value>): Promise<Value> => {
2406
+ const result = tail.then(operation);
2407
+ tail = result.then(() => undefined, () => undefined);
2408
+ return result;
2409
+ };
2410
+ const context: SemanticSearchContext = {
2411
+ root,
2412
+ projectionRoot: projection.root,
2413
+ database,
2414
+ store,
2415
+ update,
2416
+ ensureEmbedding,
2417
+ notesByPath,
2418
+ notesByQmdPath,
2419
+ contentHashesByPath,
2420
+ connectionsById,
2421
+ };
2422
+ return {
2423
+ root,
2424
+ database,
2425
+ model: recommendedEmbeddingModel,
2426
+ update,
2427
+ search: (searchOptions) => {
2428
+ if (closeRequested) {
2429
+ return Promise.reject(new Error("Semantic search session is closed."));
2430
+ }
2431
+ return serialize(() => executeSemanticSearch(context, searchOptions));
2432
+ },
2433
+ close: () => {
2434
+ if (closePromise !== undefined) return closePromise;
2435
+ closeRequested = true;
2436
+ closePromise = serialize(async () => {
2437
+ try {
2438
+ await store.close();
2439
+ } finally {
2440
+ try {
2441
+ await projection.release();
2442
+ } finally {
2443
+ await embeddingModel.release();
2444
+ }
2445
+ }
2446
+ });
2447
+ return closePromise;
2448
+ },
2449
+ };
2450
+ }
2451
+
2452
+ /** Incrementally synchronize the vault, then run local hybrid, BM25, or embedding search. */
2453
+ export async function searchSemanticVault(
2454
+ options: SemanticSearchOptions,
2455
+ dependencies: SemanticDependencies = {},
2456
+ ): Promise<SemanticSearchResult> {
2457
+ const query = validateSearchQuery(options.query).query;
2458
+ const mode = boundedMode(options.mode);
2459
+ const limit = boundedLimit(options.limit, 100);
2460
+ const candidateLimit = boundedCandidateLimit(options.candidateLimit, limit);
2461
+ const minScore = boundedScore(options.minScore);
2462
+ const session = await openSemanticSearchSession(
2463
+ {
2464
+ root: options.root,
2465
+ ...(options.database === undefined ? {} : { database: options.database }),
2466
+ },
2467
+ dependencies,
2468
+ );
2469
+ try {
2470
+ return await session.search({
2471
+ query,
2472
+ mode,
2473
+ limit,
2474
+ candidateLimit,
2475
+ minScore,
2476
+ });
2477
+ } finally {
2478
+ await session.close();
2479
+ }
2480
+ }