@gmickel/gno 1.45.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/README.md +1 -1
  2. package/THIRD_PARTY_NOTICES.md +46 -0
  3. package/assets/skill/SKILL.md +7 -6
  4. package/assets/skill/cli-reference.md +14 -6
  5. package/assets/skill/mcp-reference.md +4 -1
  6. package/assets/spa-production.json.gz +0 -0
  7. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
  8. package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
  9. package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
  10. package/browser-extension/dist/manifest.json +1 -1
  11. package/browser-extension/dist/preview.html +1 -1
  12. package/browser-extension/dist/service-worker.js +32 -33
  13. package/bunfig.toml +2 -0
  14. package/package.json +40 -26
  15. package/spec/cli.md +30 -11
  16. package/spec/db/schema.sql +146 -1
  17. package/spec/mcp.md +26 -0
  18. package/src/app/context-runtime-types.ts +3 -0
  19. package/src/app/context-runtime.ts +2 -0
  20. package/src/cli/commands/ask.ts +6 -1
  21. package/src/cli/commands/daemon.ts +21 -8
  22. package/src/cli/commands/embed.ts +77 -41
  23. package/src/cli/commands/mcp/install.ts +20 -0
  24. package/src/cli/commands/mcp/paths.ts +25 -0
  25. package/src/cli/commands/mcp/status.ts +6 -0
  26. package/src/cli/detach.ts +3 -2
  27. package/src/cli/program.ts +6 -0
  28. package/src/config/types.ts +3 -3
  29. package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
  30. package/src/converters/adapters/officeparser/adapter.ts +1 -2
  31. package/src/converters/versions.ts +6 -8
  32. package/src/core/context-evidence.ts +8 -4
  33. package/src/core/job-manager.ts +95 -13
  34. package/src/core/network-boundary-inventory.ts +10 -0
  35. package/src/core/shutdown-budget.ts +45 -0
  36. package/src/embed/backlog.ts +107 -4
  37. package/src/embed/batch.ts +42 -2
  38. package/src/embed/fingerprint.ts +16 -0
  39. package/src/embed/retry.ts +113 -5
  40. package/src/embed/variant-backlog.ts +105 -0
  41. package/src/embed/variant-plan.ts +62 -0
  42. package/src/embed/variant-retry.ts +113 -0
  43. package/src/ingestion/graph-reconciliation.ts +327 -0
  44. package/src/ingestion/sync.ts +9 -272
  45. package/src/llm/http-inference.ts +6 -0
  46. package/src/llm/httpEmbedding.ts +37 -6
  47. package/src/llm/httpGeneration.ts +18 -3
  48. package/src/llm/httpRerank.ts +23 -5
  49. package/src/llm/inference-cancellation.ts +168 -0
  50. package/src/llm/inference-scope.ts +202 -0
  51. package/src/llm/lazy-ports.ts +115 -0
  52. package/src/llm/native-worker/client.ts +541 -0
  53. package/src/llm/native-worker/dispatcher.ts +228 -0
  54. package/src/llm/native-worker/embedding-identity.ts +33 -0
  55. package/src/llm/native-worker/entry.ts +173 -0
  56. package/src/llm/native-worker/errors.ts +32 -0
  57. package/src/llm/native-worker/evaluation.ts +16 -0
  58. package/src/llm/native-worker/owned-exit.ts +108 -0
  59. package/src/llm/native-worker/owner.ts +141 -0
  60. package/src/llm/native-worker/ports.ts +317 -0
  61. package/src/llm/native-worker/protocol.ts +442 -0
  62. package/src/llm/native-worker/runtime-config.ts +92 -0
  63. package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
  64. package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
  65. package/src/llm/nodeLlamaCpp/generation.ts +34 -5
  66. package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
  67. package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
  68. package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
  69. package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
  70. package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
  71. package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
  72. package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
  73. package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
  74. package/src/llm/types.ts +35 -5
  75. package/src/mcp/context.ts +27 -0
  76. package/src/mcp/http-transport.ts +12 -10
  77. package/src/mcp/server.ts +3 -0
  78. package/src/mcp/tool-profile.ts +30 -8
  79. package/src/mcp/tools/context.ts +8 -11
  80. package/src/mcp/tools/embed.ts +1 -1
  81. package/src/mcp/tools/index-cmd.ts +1 -1
  82. package/src/mcp/tools/index.ts +10 -8
  83. package/src/mcp/tools/query.ts +14 -30
  84. package/src/mcp/tools/vsearch.ts +1 -1
  85. package/src/pipeline/answer.ts +23 -3
  86. package/src/pipeline/claim-verifier.ts +6 -0
  87. package/src/pipeline/expansion.ts +43 -40
  88. package/src/pipeline/explain.ts +6 -2
  89. package/src/pipeline/filters.ts +63 -0
  90. package/src/pipeline/fusion.ts +29 -9
  91. package/src/pipeline/graph-retrieval.ts +29 -9
  92. package/src/pipeline/hybrid.ts +198 -55
  93. package/src/pipeline/hydration.ts +161 -0
  94. package/src/pipeline/owner-fusion.ts +87 -0
  95. package/src/pipeline/rerank.ts +35 -11
  96. package/src/pipeline/search.ts +13 -2
  97. package/src/pipeline/types.ts +5 -3
  98. package/src/pipeline/vsearch.ts +87 -7
  99. package/src/sdk/client.ts +47 -3
  100. package/src/sdk/embed.ts +63 -39
  101. package/src/serve/background-runtime.ts +1 -1
  102. package/src/serve/context.ts +41 -56
  103. package/src/serve/embed-scheduler.ts +58 -35
  104. package/src/serve/public/components/IndexingProgress.tsx +46 -60
  105. package/src/serve/public/globals.built.css +1 -1
  106. package/src/serve/public/lib/shiki-language-ids.ts +14 -0
  107. package/src/serve/resident-admission.ts +36 -36
  108. package/src/serve/resident-background-work.ts +20 -2
  109. package/src/serve/resident-request.ts +11 -5
  110. package/src/serve/resident-runtime.ts +97 -61
  111. package/src/serve/resident-shutdown.ts +153 -0
  112. package/src/serve/routes/api.ts +3 -1
  113. package/src/serve/server.ts +47 -26
  114. package/src/store/migrations/028-vector-variants.ts +54 -0
  115. package/src/store/migrations/029-graph-reference-state.ts +77 -0
  116. package/src/store/migrations/index.ts +4 -0
  117. package/src/store/sqlite/adapter.ts +251 -183
  118. package/src/store/sqlite/eligibility.ts +174 -0
  119. package/src/store/sqlite/graph-edge-application.ts +66 -0
  120. package/src/store/sqlite/graph-reference-state.ts +194 -0
  121. package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
  122. package/src/store/types.ts +80 -12
  123. package/src/store/vector/eligibility.ts +36 -0
  124. package/src/store/vector/freshness.ts +33 -6
  125. package/src/store/vector/lazy.ts +81 -0
  126. package/src/store/vector/sqlite-vec.ts +106 -54
  127. package/src/store/vector/stats.ts +14 -3
  128. package/src/store/vector/types.ts +35 -2
  129. package/src/store/vector/variant-search.ts +192 -0
  130. package/src/store/vector/variants.ts +451 -0
  131. package/vendor/converters/markitdown-ts/LICENSE +21 -0
  132. package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
  133. package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
  134. package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
  135. package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
  136. package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
  137. package/vendor/converters/markitdown-ts/package.json +77 -0
  138. package/vendor/converters/officeparser/LICENSE +21 -0
  139. package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
  140. package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
  141. package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
  142. package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
  143. package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
  144. package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
  145. package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
  146. package/vendor/converters/officeparser/dist/cli.js +381 -0
  147. package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
  148. package/vendor/converters/officeparser/dist/defaults.js +218 -0
  149. package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
  150. package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
  151. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
  152. package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
  153. package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
  154. package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
  155. package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
  156. package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
  157. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
  158. package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
  159. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
  160. package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
  161. package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
  162. package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
  163. package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
  164. package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
  165. package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
  166. package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
  167. package/vendor/converters/officeparser/dist/index.d.ts +60 -0
  168. package/vendor/converters/officeparser/dist/index.js +72 -0
  169. package/vendor/converters/officeparser/dist/index.mjs +18 -0
  170. package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
  171. package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
  172. package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
  173. package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
  174. package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
  175. package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
  176. package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
  177. package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
  178. package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
  179. package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
  180. package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
  181. package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
  182. package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
  183. package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
  184. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
  185. package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
  186. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
  187. package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
  188. package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
  189. package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
  190. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
  191. package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
  192. package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
  193. package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
  194. package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
  195. package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
  196. package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
  197. package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
  198. package/vendor/converters/officeparser/dist/types.js +107 -0
  199. package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
  200. package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
  201. package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
  202. package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
  203. package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
  204. package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
  205. package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
  206. package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
  207. package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
  208. package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
  209. package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
  210. package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
  211. package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
  212. package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
  213. package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
  214. package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
  215. package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
  216. package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
  217. package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
  218. package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
  219. package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
  220. package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
  221. package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
  222. package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
  223. package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
  224. package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
  225. package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
  226. package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
  227. package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
  228. package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
  229. package/vendor/converters/officeparser/package.json +147 -0
  230. package/vendor/converters/upstream-manifest.json +124 -0
  231. package/vendor/dependency-fixes/README.md +77 -0
  232. package/vendor/dependency-fixes/vendor-converters.py +83 -0
  233. package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip +0 -0
  234. package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip.sha256 +0 -1
  235. package/browser-extension/dist/chunk-627emwpj.js +0 -75
  236. /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
@@ -0,0 +1,161 @@
1
+ /** Immutable raw hydration owned by one retrieval request, never by a model.
2
+ * Create at the request boundary and release in finally (or on abort). No read
3
+ * transaction survives a store call. Already-started stages keep their values.
4
+ */
5
+ import type {
6
+ ChunkRow,
7
+ DocumentRow,
8
+ StorePort,
9
+ StoreResult,
10
+ } from "../store/types";
11
+
12
+ import { getContentBatch } from "../store/content-batch";
13
+ import { err, ok } from "../store/types";
14
+
15
+ type BatchCache<T> = Map<string, Promise<StoreResult<T | undefined>>>;
16
+ type DocumentOptions = Parameters<StorePort["getDocumentsByMirrorHashes"]>[1];
17
+
18
+ function freezeDeep<T>(value: T): T {
19
+ if (value && typeof value === "object") {
20
+ for (const child of Object.values(value)) freezeDeep(child);
21
+ Object.freeze(value);
22
+ }
23
+ return value;
24
+ }
25
+
26
+ /** Each caller owns its Map; cached rows are detached, recursively frozen data.
27
+ * The StorePort-compatible types let existing consumers read these snapshots.
28
+ * Consumers that need to mutate a row/array must copy it first.
29
+ */
30
+ async function loadBatch<T>(
31
+ cache: BatchCache<T>,
32
+ keys: string[],
33
+ load: (missing: string[]) => Promise<StoreResult<Map<string, T>>>
34
+ ): Promise<StoreResult<Map<string, T>>> {
35
+ const unique = [...new Set(keys)];
36
+ const missing = unique.filter((key) => !cache.has(key));
37
+ if (missing.length > 0) {
38
+ // Defer the store call until every pending key is published, including when
39
+ // a lightweight adapter throws synchronously instead of returning a result.
40
+ const batch = Promise.resolve().then(async () => {
41
+ const result = await load(missing);
42
+ if (!result.ok) return result;
43
+ const snapshots = new Map<string, T>();
44
+ for (const key of missing) {
45
+ if (result.value.has(key)) {
46
+ snapshots.set(
47
+ key,
48
+ freezeDeep(structuredClone(result.value.get(key)!))
49
+ );
50
+ }
51
+ }
52
+ return ok(snapshots);
53
+ });
54
+ for (const key of missing) {
55
+ const pending = batch.then(
56
+ (result): StoreResult<T | undefined> => {
57
+ if (!result.ok) {
58
+ if (cache.get(key) === pending) cache.delete(key);
59
+ return result;
60
+ }
61
+ return ok(result.value.get(key));
62
+ },
63
+ (error: unknown) => {
64
+ if (cache.get(key) === pending) cache.delete(key);
65
+ throw error;
66
+ }
67
+ );
68
+ cache.set(key, pending);
69
+ }
70
+ }
71
+ // Capture the promises before yielding: release clears ownership, not work.
72
+ const pending = unique.map((key) => cache.get(key)!);
73
+ const results = await Promise.all(pending);
74
+ const values = new Map<string, T>();
75
+ for (const [index, result] of results.entries()) {
76
+ if (!result.ok) return result;
77
+ if (result.value !== undefined) values.set(unique[index]!, result.value);
78
+ }
79
+ return ok(values);
80
+ }
81
+
82
+ export class RequestHydration {
83
+ private store: StorePort | null;
84
+ private readonly chunks: BatchCache<ChunkRow[]> = new Map();
85
+ private readonly content: BatchCache<string> = new Map();
86
+ private readonly documents: BatchCache<DocumentRow[]> = new Map();
87
+ private readonly signal?: AbortSignal;
88
+
89
+ constructor(store: StorePort, signal?: AbortSignal) {
90
+ this.store = store;
91
+ this.signal = signal;
92
+ if (signal?.aborted) this.release();
93
+ else signal?.addEventListener("abort", this.release, { once: true });
94
+ }
95
+
96
+ /** Idempotent. Returned snapshots and pending loads remain valid; new loads
97
+ * fail. Abort does not cancel a store operation another active stage needs.
98
+ */
99
+ readonly release = (): void => {
100
+ this.signal?.removeEventListener("abort", this.release);
101
+ this.store = null;
102
+ this.chunks.clear();
103
+ this.content.clear();
104
+ this.documents.clear();
105
+ };
106
+
107
+ getChunksBatch(
108
+ hashes: string[]
109
+ ): Promise<StoreResult<Map<string, ChunkRow[]>>> {
110
+ const store = this.store;
111
+ if (!store)
112
+ return Promise.resolve(err("QUERY_FAILED", "Hydration request released"));
113
+ return loadBatch(this.chunks, hashes, (missing) =>
114
+ store.getChunksBatch(missing)
115
+ );
116
+ }
117
+
118
+ getContentBatch(hashes: string[]): Promise<StoreResult<Map<string, string>>> {
119
+ const store = this.store;
120
+ if (!store)
121
+ return Promise.resolve(err("QUERY_FAILED", "Hydration request released"));
122
+ return loadBatch(this.content, hashes, (missing) =>
123
+ getContentBatch(store, missing)
124
+ );
125
+ }
126
+
127
+ async getContent(hash: string): Promise<StoreResult<string | null>> {
128
+ const store = this.store;
129
+ if (!store) return err("QUERY_FAILED", "Hydration request released");
130
+ const result = await loadBatch(this.content, [hash], async () => {
131
+ const content = await store.getContent(hash);
132
+ if (!content.ok) return content;
133
+ return ok(new Map(content.value === null ? [] : [[hash, content.value]]));
134
+ });
135
+ return result.ok ? ok(result.value.get(hash) ?? null) : result;
136
+ }
137
+
138
+ async getDocumentsByMirrorHashes(
139
+ hashes: string[],
140
+ options?: DocumentOptions
141
+ ): Promise<StoreResult<DocumentRow[]>> {
142
+ const store = this.store;
143
+ if (!store) return err("QUERY_FAILED", "Hydration request released");
144
+ // Capture caller-owned options before any asynchronous read. Collection
145
+ // and activity are part of the lookup identity; titles never key by hash.
146
+ const scope = { ...options };
147
+ // Cache the exact lookup, retaining the adapter's ordering (including SQL
148
+ // batch boundaries). Regrouping by content hash would reorder documents.
149
+ const requested = [...hashes];
150
+ const identity = JSON.stringify([
151
+ scope.collection ?? null,
152
+ scope.activeOnly ?? null,
153
+ requested,
154
+ ]);
155
+ const result = await loadBatch(this.documents, [identity], async () => {
156
+ const rows = await store.getDocumentsByMirrorHashes(requested, scope);
157
+ return rows.ok ? ok(new Map([[identity, rows.value]])) : rows;
158
+ });
159
+ return result.ok ? ok([...(result.value.get(identity) ?? [])]) : result;
160
+ }
161
+ }
@@ -0,0 +1,87 @@
1
+ import type { StorePort } from "../store/types";
2
+ import type { RankedInput } from "./fusion";
3
+ import type { RequestHydration } from "./hydration";
4
+ import type { HybridSearchOptions } from "./types";
5
+
6
+ import { evaluateRetrievalEligibility } from "./filters";
7
+
8
+ export class OwnerMetadataError extends Error {}
9
+
10
+ /** Materialize legacy lexical/graph owner domains only when variant retrieval is active. */
11
+ export async function resolveFusionOwners(
12
+ inputs: RankedInput[],
13
+ store: StorePort,
14
+ hydration: RequestHydration,
15
+ query: string,
16
+ options: HybridSearchOptions
17
+ ): Promise<RankedInput[]> {
18
+ if (
19
+ !inputs.some((input) =>
20
+ input.results.some((result) => result.documentId !== undefined)
21
+ )
22
+ )
23
+ return inputs;
24
+ const hashes = [
25
+ ...new Set(
26
+ inputs.flatMap((input) =>
27
+ input.results.map((result) => result.mirrorHash)
28
+ )
29
+ ),
30
+ ];
31
+ const documents = await hydration.getDocumentsByMirrorHashes(hashes, {
32
+ collection: options.collection,
33
+ activeOnly: true,
34
+ });
35
+ const chunks = await hydration.getChunksBatch(hashes);
36
+ if (!documents.ok || !chunks.ok)
37
+ throw new OwnerMetadataError("Vector owner metadata unavailable");
38
+ let memoryIds: Set<number> | undefined;
39
+ if (options.memoryFilter) {
40
+ memoryIds = new Set();
41
+ for (const collection of new Set(
42
+ documents.value.map((doc) => doc.collection)
43
+ )) {
44
+ const eligible = await store.listMemoryEligibleDocuments({
45
+ collection,
46
+ scopes: options.memoryFilter.scopes,
47
+ excludeSuperseded: options.memoryFilter.excludeSuperseded,
48
+ });
49
+ if (!eligible.ok)
50
+ throw new OwnerMetadataError("Memory owner metadata unavailable");
51
+ for (const doc of eligible.value) memoryIds.add(doc.id);
52
+ }
53
+ }
54
+ const docids = new Map(documents.value.map((doc) => [doc.id, doc.docid]));
55
+ const owners = new Map<string, number[]>();
56
+ for (const doc of documents.value) {
57
+ if (!doc.mirrorHash || (memoryIds && !memoryIds.has(doc.id))) continue;
58
+ const eligibility = await evaluateRetrievalEligibility(
59
+ store,
60
+ query,
61
+ doc,
62
+ chunks.value.get(doc.mirrorHash),
63
+ options
64
+ );
65
+ for (const chunk of eligibility.chunks) {
66
+ const key = `${doc.mirrorHash}:${chunk.seq}`;
67
+ const ids = owners.get(key) ?? [];
68
+ ids.push(doc.id);
69
+ owners.set(key, ids);
70
+ }
71
+ }
72
+ return inputs.map((input) => ({
73
+ ...input,
74
+ results: input.results.flatMap((result) => {
75
+ const ids = owners.get(`${result.mirrorHash}:${result.seq}`) ?? [];
76
+ return ids
77
+ .filter((id) => {
78
+ if (result.documentId !== undefined) return result.documentId === id;
79
+ if (result.sourceDocid !== undefined)
80
+ return result.sourceDocid === docids.get(id);
81
+ // Older custom stores without provenance cannot lend a rank to ambiguous owners.
82
+ return ids.length === 1;
83
+ })
84
+ .map((documentId) => ({ ...result, documentId }));
85
+ }),
86
+ }));
87
+ }
@@ -1,14 +1,18 @@
1
+ import type { RerankPort } from "../llm/types";
1
2
  /**
2
3
  * Reranking and position-aware blending.
3
4
  * Uses RerankPort to reorder candidates.
4
5
  *
5
6
  * @module src/pipeline/rerank
6
7
  */
7
-
8
- import type { RerankPort } from "../llm/types";
9
8
  import type { ChunkRow, StorePort } from "../store/types";
9
+ import type { RequestHydration } from "./hydration";
10
10
  import type { BlendingTier, FusionCandidate, RerankedCandidate } from "./types";
11
11
 
12
+ import {
13
+ assertInferenceActive,
14
+ assertInferenceResult,
15
+ } from "../llm/inference-scope";
12
16
  import {
13
17
  buildIntentAwareRerankQuery,
14
18
  selectBestChunkForSteering,
@@ -42,6 +46,8 @@ export interface RerankResult {
42
46
  export interface RerankDeps {
43
47
  rerankPort: RerankPort | null;
44
48
  store: StorePort;
49
+ /** Shared raw chunks only; model inputs are prepared per invocation. */
50
+ hydration?: RequestHydration;
45
51
  }
46
52
 
47
53
  // ─────────────────────────────────────────────────────────────────────────────
@@ -96,8 +102,12 @@ function isProtectedLexicalTopHit(candidate: FusionCandidate): boolean {
96
102
  /**
97
103
  * Fetch chunk texts for reranking.
98
104
  */
105
+ function rerankOwnerKey(candidate: FusionCandidate): string {
106
+ return `${candidate.mirrorHash}${candidate.documentId === undefined ? "" : `:${candidate.documentId}`}`;
107
+ }
108
+
99
109
  async function fetchChunkTexts(
100
- store: StorePort,
110
+ store: Pick<StorePort, "getChunksBatch">,
101
111
  toRerank: FusionCandidate[],
102
112
  query: string,
103
113
  intent: string | undefined
@@ -110,13 +120,21 @@ async function fetchChunkTexts(
110
120
  ? chunksBatchResult.value
111
121
  : new Map();
112
122
  const preferredSeqByHash = new Map<string, number>();
123
+ const ownerHashes = new Map(
124
+ toRerank.map((candidate) => [
125
+ rerankOwnerKey(candidate),
126
+ candidate.mirrorHash,
127
+ ])
128
+ );
113
129
 
114
130
  for (const candidate of toRerank) {
115
- const existingSeq = preferredSeqByHash.get(candidate.mirrorHash);
131
+ assertInferenceActive();
132
+ const existingSeq = preferredSeqByHash.get(rerankOwnerKey(candidate));
116
133
  if (existingSeq !== undefined) {
117
134
  const existingCandidate = toRerank.find(
118
135
  (entry) =>
119
- entry.mirrorHash === candidate.mirrorHash && entry.seq === existingSeq
136
+ rerankOwnerKey(entry) === rerankOwnerKey(candidate) &&
137
+ entry.seq === existingSeq
120
138
  );
121
139
  if (
122
140
  existingCandidate &&
@@ -125,12 +143,13 @@ async function fetchChunkTexts(
125
143
  continue;
126
144
  }
127
145
  }
128
- preferredSeqByHash.set(candidate.mirrorHash, candidate.seq);
146
+ preferredSeqByHash.set(rerankOwnerKey(candidate), candidate.seq);
129
147
  }
130
148
 
131
149
  const chunkTexts = new Map<string, string>();
132
- for (const hash of uniqueHashes) {
133
- const chunks = chunksByHash.get(hash);
150
+ for (const [hash, mirrorHash] of ownerHashes) {
151
+ assertInferenceActive();
152
+ const chunks = chunksByHash.get(mirrorHash);
134
153
  const bestChunk = selectBestChunkForSteering(chunks ?? [], query, intent, {
135
154
  preferredSeq: preferredSeqByHash.get(hash) ?? null,
136
155
  intentWeight: 0.5,
@@ -146,7 +165,8 @@ async function fetchChunkTexts(
146
165
 
147
166
  const hashToIndex = new Map<string, number>();
148
167
  const texts: string[] = [];
149
- for (const hash of uniqueHashes) {
168
+ for (const hash of ownerHashes.keys()) {
169
+ assertInferenceActive();
150
170
  hashToIndex.set(hash, texts.length);
151
171
  texts.push(chunkTexts.get(hash) ?? "");
152
172
  }
@@ -234,7 +254,7 @@ export async function rerankCandidates(
234
254
 
235
255
  // Extract best chunk per document for efficient reranking
236
256
  const { texts, hashToIndex } = await fetchChunkTexts(
237
- store,
257
+ deps.hydration ?? store,
238
258
  toRerank,
239
259
  query,
240
260
  options.intent
@@ -246,6 +266,7 @@ export async function rerankCandidates(
246
266
  const textToUniqueIndex = new Map<string, number>();
247
267
 
248
268
  for (const [docIndex, text] of texts.entries()) {
269
+ assertInferenceActive();
249
270
  const existingIndex = textToUniqueIndex.get(text);
250
271
  if (existingIndex !== undefined) {
251
272
  docIndexToUniqueIndex.set(docIndex, existingIndex);
@@ -268,6 +289,7 @@ export async function rerankCandidates(
268
289
  uniqueTexts
269
290
  );
270
291
 
292
+ assertInferenceResult(rerankResult);
271
293
  if (!rerankResult.ok) {
272
294
  return {
273
295
  candidates: sortAdjustedCandidates(
@@ -285,8 +307,10 @@ export async function rerankCandidates(
285
307
  // Normalize rerank scores using min-max
286
308
  const scoreByDocIndex = new Map<number, number>();
287
309
  for (const score of rerankResult.value) {
310
+ assertInferenceActive();
288
311
  const docIndices = uniqueIndexToDocIndices.get(score.index) ?? [];
289
312
  for (const docIndex of docIndices) {
313
+ assertInferenceActive();
290
314
  scoreByDocIndex.set(docIndex, score.score);
291
315
  }
292
316
  }
@@ -304,7 +328,7 @@ export async function rerankCandidates(
304
328
 
305
329
  // Build reranked candidates with blended scores
306
330
  const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
307
- const docIndex = hashToIndex.get(c.mirrorHash) ?? -1;
331
+ const docIndex = hashToIndex.get(rerankOwnerKey(c)) ?? -1;
308
332
  const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
309
333
  const normalizedRerankScore =
310
334
  rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
@@ -202,6 +202,8 @@ export async function searchBm25(
202
202
  const ftsResult = await store.searchFts(query, {
203
203
  limit: retrievalLimit,
204
204
  collection: options.collection,
205
+ allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
206
+ exclude: options.exclude,
205
207
  relPathPrefix: options.retrievalScope?.relPathPrefix,
206
208
  language: options.lang,
207
209
  snippet: !(options.full || options.lineNumbers),
@@ -244,13 +246,22 @@ export async function searchBm25(
244
246
  }
245
247
  >();
246
248
 
247
- // Pre-fetch all chunks in one batch query (eliminates N+1)
249
+ // Exact sequences suffice unless selection needs whole-document evidence.
248
250
  const uniqueHashes = [
249
251
  ...new Set(
250
252
  ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
251
253
  ),
252
254
  ];
253
- const chunksMapResult = await store.getChunksBatch(uniqueHashes);
255
+ const chunksMapResult =
256
+ store.getChunksBySequenceBatch &&
257
+ !options.intent &&
258
+ !options.exclude?.length
259
+ ? await store.getChunksBySequenceBatch(
260
+ ftsResult.value
261
+ .filter((row) => !!row.mirrorHash)
262
+ .map((row) => ({ mirrorHash: row.mirrorHash!, seq: row.seq }))
263
+ )
264
+ : await store.getChunksBatch(uniqueHashes);
254
265
  const getChunk = chunksMapResult.ok
255
266
  ? createChunkLookup(chunksMapResult.value)
256
267
  : () => undefined;
@@ -1,11 +1,10 @@
1
+ import type { NormalizedContentTypeRule } from "../config/content-types";
1
2
  /**
2
3
  * Search pipeline types.
3
4
  * Defines SearchPipelinePort and related types for search operations.
4
5
  *
5
6
  * @module src/pipeline/types
6
7
  */
7
-
8
- import type { NormalizedContentTypeRule } from "../config/content-types";
9
8
  import type {
10
9
  ContextCapsuleV1,
11
10
  ContextCapsuleVerification,
@@ -13,6 +12,7 @@ import type {
13
12
  import type { EgressLineage } from "../core/egress-provenance";
14
13
  import type { RecordEvidenceMetadata } from "../core/record-metadata";
15
14
  import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
15
+ import type { InferenceOptions } from "../llm/types";
16
16
  import type { StoreResult } from "../store/types";
17
17
  import type { ClaimVerificationResult } from "./claim-verification";
18
18
  import type { SemanticVerificationCapability } from "./claim-verifier";
@@ -172,7 +172,7 @@ export interface SearchResults {
172
172
  // ─────────────────────────────────────────────────────────────────────────────
173
173
 
174
174
  /** Common options for all search commands */
175
- export interface SearchOptions {
175
+ export interface SearchOptions extends InferenceOptions {
176
176
  /** Internal receipt seam; never serialized or included in public schemas. */
177
177
  traceSession?: RetrievalTraceSession;
178
178
  /** Trusted, already-resolved project affinity; never accepts raw roots. */
@@ -332,6 +332,8 @@ export type FusionSource =
332
332
 
333
333
  /** Fusion candidate with ranks from different sources */
334
334
  export interface FusionCandidate {
335
+ /** Internal exact document owner; canonical public chunk identity is unchanged. */
336
+ documentId?: number;
335
337
  mirrorHash: string;
336
338
  seq: number;
337
339
  bm25Rank: number | null;
@@ -1,11 +1,10 @@
1
+ import type { Config } from "../config/types";
1
2
  /**
2
3
  * Vector search pipeline.
3
4
  * Wraps VectorIndexPort.searchNearest() to produce SearchResults.
4
5
  *
5
6
  * @module src/pipeline/vsearch
6
7
  */
7
-
8
- import type { Config } from "../config/types";
9
8
  import type { EmbeddingPort } from "../llm/types";
10
9
  import type { DocumentRow, StorePort } from "../store/types";
11
10
  import type { VectorIndexPort } from "../store/vector/types";
@@ -13,8 +12,14 @@ import type { SearchOptions, SearchResult, SearchResults } from "./types";
13
12
 
14
13
  import { normalizeContentTypes } from "../config/content-types";
15
14
  import { projectRecordEvidenceMetadata } from "../core/record-metadata";
15
+ import {
16
+ assertInferenceActive,
17
+ assertInferenceResult,
18
+ withInferenceScope,
19
+ } from "../llm/inference-scope";
16
20
  import { getContentBatch } from "../store/content-batch";
17
21
  import { err, ok } from "../store/types";
22
+ import { resolveVectorSearchIdentity } from "../store/vector/variant-search";
18
23
  import { createChunkLookup } from "./chunk-lookup";
19
24
  import {
20
25
  applyContentTypeBoost,
@@ -87,6 +92,17 @@ export async function searchVectorWithEmbedding(
87
92
  query: string,
88
93
  queryEmbedding: Float32Array,
89
94
  options: SearchOptions = {}
95
+ ): Promise<ReturnType<typeof ok<SearchResults>>> {
96
+ return withInferenceScope(options, () =>
97
+ searchVectorWithEmbeddingOwned(deps, query, queryEmbedding, options)
98
+ );
99
+ }
100
+
101
+ async function searchVectorWithEmbeddingOwned(
102
+ deps: VectorSearchDeps,
103
+ query: string,
104
+ queryEmbedding: Float32Array,
105
+ options: SearchOptions = {}
90
106
  ): Promise<ReturnType<typeof ok<SearchResults>>> {
91
107
  const traceStartedAt = options.traceSession ? performance.now() : 0;
92
108
  const { store, vectorIndex } = deps;
@@ -118,11 +134,37 @@ export async function searchVectorWithEmbedding(
118
134
  return err("VEC_SEARCH_UNAVAILABLE", vectorUnavailableMessage(vectorIndex));
119
135
  }
120
136
 
137
+ let embeddingIdentity;
138
+ try {
139
+ embeddingIdentity = resolveVectorSearchIdentity(deps.embedPort);
140
+ } catch (cause) {
141
+ return err(
142
+ "QUERY_FAILED",
143
+ cause instanceof Error ? cause.message : String(cause)
144
+ );
145
+ }
121
146
  // Search nearest neighbors
122
147
  const searchResult = await vectorIndex.searchNearest(
123
148
  queryEmbedding,
124
149
  retrievalLimit,
125
150
  {
151
+ embeddingIdentity,
152
+ eligibility: {
153
+ excludeMetadata: true,
154
+ semanticMetadata: true,
155
+ collection: options.collection,
156
+ memoryScopesAny: options.memoryFilter?.scopes,
157
+ excludeSuperseded: options.memoryFilter?.excludeSuperseded,
158
+ relPathPrefix: options.retrievalScope?.relPathPrefix,
159
+ tagsAll: options.tagsAll,
160
+ tagsAny: options.tagsAny,
161
+ since: temporalRange.since,
162
+ until: temporalRange.until,
163
+ categories: options.categories,
164
+ author: options.author,
165
+ exclude: options.exclude,
166
+ language: options.lang,
167
+ },
126
168
  minScore: projectAffinityActive ? undefined : minScore,
127
169
  allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
128
170
  }
@@ -140,6 +182,7 @@ export async function searchVectorWithEmbedding(
140
182
  const collectionPaths = new Map<string, string>();
141
183
  if (collectionsResult.ok) {
142
184
  for (const c of collectionsResult.value) {
185
+ assertInferenceActive();
143
186
  collectionPaths.set(c.name, c.path);
144
187
  }
145
188
  }
@@ -158,8 +201,15 @@ export async function searchVectorWithEmbedding(
158
201
  mirrorHashes: uniqueHashes,
159
202
  });
160
203
 
161
- // Pre-fetch all chunks in one batch query (eliminates N+1)
162
- const chunksMapResult = await store.getChunksBatch(uniqueHashes);
204
+ // Exact sequences suffice unless selection needs whole-document evidence.
205
+ const chunksMapResult =
206
+ store.getChunksBySequenceBatch &&
207
+ !options.intent &&
208
+ !options.exclude?.length
209
+ ? await store.getChunksBySequenceBatch(
210
+ vecResults.map(({ mirrorHash, seq }) => ({ mirrorHash, seq }))
211
+ )
212
+ : await store.getChunksBatch(uniqueHashes);
163
213
  if (!chunksMapResult.ok) {
164
214
  return err("QUERY_FAILED", chunksMapResult.error.message);
165
215
  }
@@ -182,6 +232,7 @@ export async function searchVectorWithEmbedding(
182
232
  >();
183
233
 
184
234
  for (const vec of vecResults) {
235
+ assertInferenceActive();
185
236
  const baseScore = normalizeVectorScore(vec.distance);
186
237
  if (!projectAffinityActive && baseScore < minScore) {
187
238
  continue;
@@ -191,7 +242,9 @@ export async function searchVectorWithEmbedding(
191
242
  const rawChunk = getChunk(vec.mirrorHash, vec.seq);
192
243
  const chunk = options.intent
193
244
  ? (selectBestChunkForSteering(
194
- chunksMap.get(vec.mirrorHash) ?? [],
245
+ (chunksMap.get(vec.mirrorHash) ?? []).filter(
246
+ (chunk) => !options.lang || chunk.language === options.lang
247
+ ),
195
248
  query,
196
249
  options.intent,
197
250
  {
@@ -210,12 +263,21 @@ export async function searchVectorWithEmbedding(
210
263
  }
211
264
 
212
265
  // Get document (cached)
213
- const matchingDocs = docsByMirrorHash.get(vec.mirrorHash);
266
+ const matchingDocs = docsByMirrorHash
267
+ .get(vec.mirrorHash)
268
+ ?.filter(
269
+ (doc) =>
270
+ vec.documentIds === undefined || vec.documentIds.includes(doc.id)
271
+ );
214
272
  if (!matchingDocs || matchingDocs.length === 0) {
215
273
  continue;
216
274
  }
217
- const docs = auxiliaryRankingActive ? matchingDocs : [matchingDocs.at(-1)!];
275
+ const docs =
276
+ auxiliaryRankingActive || vec.documentIds !== undefined
277
+ ? matchingDocs
278
+ : [matchingDocs.at(-1)!];
218
279
  for (const doc of docs) {
280
+ assertInferenceActive();
219
281
  const collectionPath = collectionPaths.get(doc.collection);
220
282
  const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
221
283
  const excluded =
@@ -335,6 +397,7 @@ export async function searchVectorWithEmbedding(
335
397
  const fullContentByHash = fullContentResult.value;
336
398
 
337
399
  for (const { doc, chunk, rawDistance, score } of bestByDocid.values()) {
400
+ assertInferenceActive();
338
401
  const fullContent = doc.mirrorHash
339
402
  ? fullContentByHash.get(doc.mirrorHash)
340
403
  : undefined;
@@ -430,6 +493,7 @@ export async function searchVectorWithEmbedding(
430
493
 
431
494
  const finalResults = results.slice(0, limit);
432
495
  for (const [index, result] of finalResults.entries()) {
496
+ assertInferenceActive();
433
497
  const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
434
498
  if (metadata) metadata.retrievalRank = index + 1;
435
499
  }
@@ -487,6 +551,16 @@ export async function searchVector(
487
551
  deps: VectorSearchDeps,
488
552
  query: string,
489
553
  options: SearchOptions = {}
554
+ ): Promise<ReturnType<typeof ok<SearchResults>>> {
555
+ return withInferenceScope(options, () =>
556
+ searchVectorOwned(deps, query, options)
557
+ );
558
+ }
559
+
560
+ async function searchVectorOwned(
561
+ deps: VectorSearchDeps,
562
+ query: string,
563
+ options: SearchOptions = {}
490
564
  ): Promise<ReturnType<typeof ok<SearchResults>>> {
491
565
  const { vectorIndex, embedPort } = deps;
492
566
 
@@ -499,6 +573,7 @@ export async function searchVector(
499
573
  const embedResult = await embedPort.embed(
500
574
  formatQueryForEmbedding(query, embedPort.modelUri)
501
575
  );
576
+ assertInferenceResult(embedResult);
502
577
  if (!embedResult.ok) {
503
578
  return err(
504
579
  "QUERY_FAILED",
@@ -524,6 +599,7 @@ interface ChunkInfo {
524
599
  }
525
600
 
526
601
  interface DocumentInfo {
602
+ id: number;
527
603
  docid: string;
528
604
  uri: string;
529
605
  title: string | null;
@@ -625,11 +701,13 @@ async function buildDocumentMap(
625
701
  const docIds = activeDocs.map((d) => d.id);
626
702
  const tagsResult = await store.getTagsBatch(docIds);
627
703
 
704
+ if (!tagsResult.ok) return { documents, ownershipDocuments };
628
705
  if (tagsResult.ok) {
629
706
  allowedDocIds = new Set<number>();
630
707
  const tagsByDocId = tagsResult.value;
631
708
 
632
709
  for (const doc of activeDocs) {
710
+ assertInferenceActive();
633
711
  const docTags = new Set(
634
712
  (tagsByDocId.get(doc.id) ?? []).map((t) => t.tag)
635
713
  );
@@ -652,6 +730,7 @@ async function buildDocumentMap(
652
730
  }
653
731
 
654
732
  for (const doc of activeDocs) {
733
+ assertInferenceActive();
655
734
  const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
656
735
  if (
657
736
  options.relPathPrefix !== undefined &&
@@ -679,6 +758,7 @@ async function buildDocumentMap(
679
758
  }
680
759
 
681
760
  const documentInfo: DocumentInfo = {
761
+ id: doc.id,
682
762
  docid: doc.docid,
683
763
  uri: doc.uri,
684
764
  title: doc.title,