@gmickel/gno 1.45.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/skill/cli-reference.md +14 -6
- package/assets/skill/mcp-reference.md +4 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +30 -11
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/commands/mcp/install.ts +20 -0
- package/src/cli/commands/mcp/paths.ts +25 -0
- package/src/cli/commands/mcp/status.ts +6 -0
- package/src/cli/detach.ts +3 -2
- package/src/cli/program.ts +6 -0
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/** Immutable raw hydration owned by one retrieval request, never by a model.
|
|
2
|
+
* Create at the request boundary and release in finally (or on abort). No read
|
|
3
|
+
* transaction survives a store call. Already-started stages keep their values.
|
|
4
|
+
*/
|
|
5
|
+
import type {
|
|
6
|
+
ChunkRow,
|
|
7
|
+
DocumentRow,
|
|
8
|
+
StorePort,
|
|
9
|
+
StoreResult,
|
|
10
|
+
} from "../store/types";
|
|
11
|
+
|
|
12
|
+
import { getContentBatch } from "../store/content-batch";
|
|
13
|
+
import { err, ok } from "../store/types";
|
|
14
|
+
|
|
15
|
+
type BatchCache<T> = Map<string, Promise<StoreResult<T | undefined>>>;
|
|
16
|
+
type DocumentOptions = Parameters<StorePort["getDocumentsByMirrorHashes"]>[1];
|
|
17
|
+
|
|
18
|
+
function freezeDeep<T>(value: T): T {
|
|
19
|
+
if (value && typeof value === "object") {
|
|
20
|
+
for (const child of Object.values(value)) freezeDeep(child);
|
|
21
|
+
Object.freeze(value);
|
|
22
|
+
}
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Each caller owns its Map; cached rows are detached, recursively frozen data.
|
|
27
|
+
* The StorePort-compatible types let existing consumers read these snapshots.
|
|
28
|
+
* Consumers that need to mutate a row/array must copy it first.
|
|
29
|
+
*/
|
|
30
|
+
async function loadBatch<T>(
|
|
31
|
+
cache: BatchCache<T>,
|
|
32
|
+
keys: string[],
|
|
33
|
+
load: (missing: string[]) => Promise<StoreResult<Map<string, T>>>
|
|
34
|
+
): Promise<StoreResult<Map<string, T>>> {
|
|
35
|
+
const unique = [...new Set(keys)];
|
|
36
|
+
const missing = unique.filter((key) => !cache.has(key));
|
|
37
|
+
if (missing.length > 0) {
|
|
38
|
+
// Defer the store call until every pending key is published, including when
|
|
39
|
+
// a lightweight adapter throws synchronously instead of returning a result.
|
|
40
|
+
const batch = Promise.resolve().then(async () => {
|
|
41
|
+
const result = await load(missing);
|
|
42
|
+
if (!result.ok) return result;
|
|
43
|
+
const snapshots = new Map<string, T>();
|
|
44
|
+
for (const key of missing) {
|
|
45
|
+
if (result.value.has(key)) {
|
|
46
|
+
snapshots.set(
|
|
47
|
+
key,
|
|
48
|
+
freezeDeep(structuredClone(result.value.get(key)!))
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return ok(snapshots);
|
|
53
|
+
});
|
|
54
|
+
for (const key of missing) {
|
|
55
|
+
const pending = batch.then(
|
|
56
|
+
(result): StoreResult<T | undefined> => {
|
|
57
|
+
if (!result.ok) {
|
|
58
|
+
if (cache.get(key) === pending) cache.delete(key);
|
|
59
|
+
return result;
|
|
60
|
+
}
|
|
61
|
+
return ok(result.value.get(key));
|
|
62
|
+
},
|
|
63
|
+
(error: unknown) => {
|
|
64
|
+
if (cache.get(key) === pending) cache.delete(key);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
);
|
|
68
|
+
cache.set(key, pending);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Capture the promises before yielding: release clears ownership, not work.
|
|
72
|
+
const pending = unique.map((key) => cache.get(key)!);
|
|
73
|
+
const results = await Promise.all(pending);
|
|
74
|
+
const values = new Map<string, T>();
|
|
75
|
+
for (const [index, result] of results.entries()) {
|
|
76
|
+
if (!result.ok) return result;
|
|
77
|
+
if (result.value !== undefined) values.set(unique[index]!, result.value);
|
|
78
|
+
}
|
|
79
|
+
return ok(values);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export class RequestHydration {
|
|
83
|
+
private store: StorePort | null;
|
|
84
|
+
private readonly chunks: BatchCache<ChunkRow[]> = new Map();
|
|
85
|
+
private readonly content: BatchCache<string> = new Map();
|
|
86
|
+
private readonly documents: BatchCache<DocumentRow[]> = new Map();
|
|
87
|
+
private readonly signal?: AbortSignal;
|
|
88
|
+
|
|
89
|
+
constructor(store: StorePort, signal?: AbortSignal) {
|
|
90
|
+
this.store = store;
|
|
91
|
+
this.signal = signal;
|
|
92
|
+
if (signal?.aborted) this.release();
|
|
93
|
+
else signal?.addEventListener("abort", this.release, { once: true });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Idempotent. Returned snapshots and pending loads remain valid; new loads
|
|
97
|
+
* fail. Abort does not cancel a store operation another active stage needs.
|
|
98
|
+
*/
|
|
99
|
+
readonly release = (): void => {
|
|
100
|
+
this.signal?.removeEventListener("abort", this.release);
|
|
101
|
+
this.store = null;
|
|
102
|
+
this.chunks.clear();
|
|
103
|
+
this.content.clear();
|
|
104
|
+
this.documents.clear();
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
getChunksBatch(
|
|
108
|
+
hashes: string[]
|
|
109
|
+
): Promise<StoreResult<Map<string, ChunkRow[]>>> {
|
|
110
|
+
const store = this.store;
|
|
111
|
+
if (!store)
|
|
112
|
+
return Promise.resolve(err("QUERY_FAILED", "Hydration request released"));
|
|
113
|
+
return loadBatch(this.chunks, hashes, (missing) =>
|
|
114
|
+
store.getChunksBatch(missing)
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
getContentBatch(hashes: string[]): Promise<StoreResult<Map<string, string>>> {
|
|
119
|
+
const store = this.store;
|
|
120
|
+
if (!store)
|
|
121
|
+
return Promise.resolve(err("QUERY_FAILED", "Hydration request released"));
|
|
122
|
+
return loadBatch(this.content, hashes, (missing) =>
|
|
123
|
+
getContentBatch(store, missing)
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async getContent(hash: string): Promise<StoreResult<string | null>> {
|
|
128
|
+
const store = this.store;
|
|
129
|
+
if (!store) return err("QUERY_FAILED", "Hydration request released");
|
|
130
|
+
const result = await loadBatch(this.content, [hash], async () => {
|
|
131
|
+
const content = await store.getContent(hash);
|
|
132
|
+
if (!content.ok) return content;
|
|
133
|
+
return ok(new Map(content.value === null ? [] : [[hash, content.value]]));
|
|
134
|
+
});
|
|
135
|
+
return result.ok ? ok(result.value.get(hash) ?? null) : result;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async getDocumentsByMirrorHashes(
|
|
139
|
+
hashes: string[],
|
|
140
|
+
options?: DocumentOptions
|
|
141
|
+
): Promise<StoreResult<DocumentRow[]>> {
|
|
142
|
+
const store = this.store;
|
|
143
|
+
if (!store) return err("QUERY_FAILED", "Hydration request released");
|
|
144
|
+
// Capture caller-owned options before any asynchronous read. Collection
|
|
145
|
+
// and activity are part of the lookup identity; titles never key by hash.
|
|
146
|
+
const scope = { ...options };
|
|
147
|
+
// Cache the exact lookup, retaining the adapter's ordering (including SQL
|
|
148
|
+
// batch boundaries). Regrouping by content hash would reorder documents.
|
|
149
|
+
const requested = [...hashes];
|
|
150
|
+
const identity = JSON.stringify([
|
|
151
|
+
scope.collection ?? null,
|
|
152
|
+
scope.activeOnly ?? null,
|
|
153
|
+
requested,
|
|
154
|
+
]);
|
|
155
|
+
const result = await loadBatch(this.documents, [identity], async () => {
|
|
156
|
+
const rows = await store.getDocumentsByMirrorHashes(requested, scope);
|
|
157
|
+
return rows.ok ? ok(new Map([[identity, rows.value]])) : rows;
|
|
158
|
+
});
|
|
159
|
+
return result.ok ? ok([...(result.value.get(identity) ?? [])]) : result;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { StorePort } from "../store/types";
|
|
2
|
+
import type { RankedInput } from "./fusion";
|
|
3
|
+
import type { RequestHydration } from "./hydration";
|
|
4
|
+
import type { HybridSearchOptions } from "./types";
|
|
5
|
+
|
|
6
|
+
import { evaluateRetrievalEligibility } from "./filters";
|
|
7
|
+
|
|
8
|
+
export class OwnerMetadataError extends Error {}
|
|
9
|
+
|
|
10
|
+
/** Materialize legacy lexical/graph owner domains only when variant retrieval is active. */
|
|
11
|
+
export async function resolveFusionOwners(
|
|
12
|
+
inputs: RankedInput[],
|
|
13
|
+
store: StorePort,
|
|
14
|
+
hydration: RequestHydration,
|
|
15
|
+
query: string,
|
|
16
|
+
options: HybridSearchOptions
|
|
17
|
+
): Promise<RankedInput[]> {
|
|
18
|
+
if (
|
|
19
|
+
!inputs.some((input) =>
|
|
20
|
+
input.results.some((result) => result.documentId !== undefined)
|
|
21
|
+
)
|
|
22
|
+
)
|
|
23
|
+
return inputs;
|
|
24
|
+
const hashes = [
|
|
25
|
+
...new Set(
|
|
26
|
+
inputs.flatMap((input) =>
|
|
27
|
+
input.results.map((result) => result.mirrorHash)
|
|
28
|
+
)
|
|
29
|
+
),
|
|
30
|
+
];
|
|
31
|
+
const documents = await hydration.getDocumentsByMirrorHashes(hashes, {
|
|
32
|
+
collection: options.collection,
|
|
33
|
+
activeOnly: true,
|
|
34
|
+
});
|
|
35
|
+
const chunks = await hydration.getChunksBatch(hashes);
|
|
36
|
+
if (!documents.ok || !chunks.ok)
|
|
37
|
+
throw new OwnerMetadataError("Vector owner metadata unavailable");
|
|
38
|
+
let memoryIds: Set<number> | undefined;
|
|
39
|
+
if (options.memoryFilter) {
|
|
40
|
+
memoryIds = new Set();
|
|
41
|
+
for (const collection of new Set(
|
|
42
|
+
documents.value.map((doc) => doc.collection)
|
|
43
|
+
)) {
|
|
44
|
+
const eligible = await store.listMemoryEligibleDocuments({
|
|
45
|
+
collection,
|
|
46
|
+
scopes: options.memoryFilter.scopes,
|
|
47
|
+
excludeSuperseded: options.memoryFilter.excludeSuperseded,
|
|
48
|
+
});
|
|
49
|
+
if (!eligible.ok)
|
|
50
|
+
throw new OwnerMetadataError("Memory owner metadata unavailable");
|
|
51
|
+
for (const doc of eligible.value) memoryIds.add(doc.id);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
const docids = new Map(documents.value.map((doc) => [doc.id, doc.docid]));
|
|
55
|
+
const owners = new Map<string, number[]>();
|
|
56
|
+
for (const doc of documents.value) {
|
|
57
|
+
if (!doc.mirrorHash || (memoryIds && !memoryIds.has(doc.id))) continue;
|
|
58
|
+
const eligibility = await evaluateRetrievalEligibility(
|
|
59
|
+
store,
|
|
60
|
+
query,
|
|
61
|
+
doc,
|
|
62
|
+
chunks.value.get(doc.mirrorHash),
|
|
63
|
+
options
|
|
64
|
+
);
|
|
65
|
+
for (const chunk of eligibility.chunks) {
|
|
66
|
+
const key = `${doc.mirrorHash}:${chunk.seq}`;
|
|
67
|
+
const ids = owners.get(key) ?? [];
|
|
68
|
+
ids.push(doc.id);
|
|
69
|
+
owners.set(key, ids);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return inputs.map((input) => ({
|
|
73
|
+
...input,
|
|
74
|
+
results: input.results.flatMap((result) => {
|
|
75
|
+
const ids = owners.get(`${result.mirrorHash}:${result.seq}`) ?? [];
|
|
76
|
+
return ids
|
|
77
|
+
.filter((id) => {
|
|
78
|
+
if (result.documentId !== undefined) return result.documentId === id;
|
|
79
|
+
if (result.sourceDocid !== undefined)
|
|
80
|
+
return result.sourceDocid === docids.get(id);
|
|
81
|
+
// Older custom stores without provenance cannot lend a rank to ambiguous owners.
|
|
82
|
+
return ids.length === 1;
|
|
83
|
+
})
|
|
84
|
+
.map((documentId) => ({ ...result, documentId }));
|
|
85
|
+
}),
|
|
86
|
+
}));
|
|
87
|
+
}
|
package/src/pipeline/rerank.ts
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
|
+
import type { RerankPort } from "../llm/types";
|
|
1
2
|
/**
|
|
2
3
|
* Reranking and position-aware blending.
|
|
3
4
|
* Uses RerankPort to reorder candidates.
|
|
4
5
|
*
|
|
5
6
|
* @module src/pipeline/rerank
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
import type { RerankPort } from "../llm/types";
|
|
9
8
|
import type { ChunkRow, StorePort } from "../store/types";
|
|
9
|
+
import type { RequestHydration } from "./hydration";
|
|
10
10
|
import type { BlendingTier, FusionCandidate, RerankedCandidate } from "./types";
|
|
11
11
|
|
|
12
|
+
import {
|
|
13
|
+
assertInferenceActive,
|
|
14
|
+
assertInferenceResult,
|
|
15
|
+
} from "../llm/inference-scope";
|
|
12
16
|
import {
|
|
13
17
|
buildIntentAwareRerankQuery,
|
|
14
18
|
selectBestChunkForSteering,
|
|
@@ -42,6 +46,8 @@ export interface RerankResult {
|
|
|
42
46
|
export interface RerankDeps {
|
|
43
47
|
rerankPort: RerankPort | null;
|
|
44
48
|
store: StorePort;
|
|
49
|
+
/** Shared raw chunks only; model inputs are prepared per invocation. */
|
|
50
|
+
hydration?: RequestHydration;
|
|
45
51
|
}
|
|
46
52
|
|
|
47
53
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -96,8 +102,12 @@ function isProtectedLexicalTopHit(candidate: FusionCandidate): boolean {
|
|
|
96
102
|
/**
|
|
97
103
|
* Fetch chunk texts for reranking.
|
|
98
104
|
*/
|
|
105
|
+
function rerankOwnerKey(candidate: FusionCandidate): string {
|
|
106
|
+
return `${candidate.mirrorHash}${candidate.documentId === undefined ? "" : `:${candidate.documentId}`}`;
|
|
107
|
+
}
|
|
108
|
+
|
|
99
109
|
async function fetchChunkTexts(
|
|
100
|
-
store: StorePort,
|
|
110
|
+
store: Pick<StorePort, "getChunksBatch">,
|
|
101
111
|
toRerank: FusionCandidate[],
|
|
102
112
|
query: string,
|
|
103
113
|
intent: string | undefined
|
|
@@ -110,13 +120,21 @@ async function fetchChunkTexts(
|
|
|
110
120
|
? chunksBatchResult.value
|
|
111
121
|
: new Map();
|
|
112
122
|
const preferredSeqByHash = new Map<string, number>();
|
|
123
|
+
const ownerHashes = new Map(
|
|
124
|
+
toRerank.map((candidate) => [
|
|
125
|
+
rerankOwnerKey(candidate),
|
|
126
|
+
candidate.mirrorHash,
|
|
127
|
+
])
|
|
128
|
+
);
|
|
113
129
|
|
|
114
130
|
for (const candidate of toRerank) {
|
|
115
|
-
|
|
131
|
+
assertInferenceActive();
|
|
132
|
+
const existingSeq = preferredSeqByHash.get(rerankOwnerKey(candidate));
|
|
116
133
|
if (existingSeq !== undefined) {
|
|
117
134
|
const existingCandidate = toRerank.find(
|
|
118
135
|
(entry) =>
|
|
119
|
-
entry
|
|
136
|
+
rerankOwnerKey(entry) === rerankOwnerKey(candidate) &&
|
|
137
|
+
entry.seq === existingSeq
|
|
120
138
|
);
|
|
121
139
|
if (
|
|
122
140
|
existingCandidate &&
|
|
@@ -125,12 +143,13 @@ async function fetchChunkTexts(
|
|
|
125
143
|
continue;
|
|
126
144
|
}
|
|
127
145
|
}
|
|
128
|
-
preferredSeqByHash.set(candidate
|
|
146
|
+
preferredSeqByHash.set(rerankOwnerKey(candidate), candidate.seq);
|
|
129
147
|
}
|
|
130
148
|
|
|
131
149
|
const chunkTexts = new Map<string, string>();
|
|
132
|
-
for (const hash of
|
|
133
|
-
|
|
150
|
+
for (const [hash, mirrorHash] of ownerHashes) {
|
|
151
|
+
assertInferenceActive();
|
|
152
|
+
const chunks = chunksByHash.get(mirrorHash);
|
|
134
153
|
const bestChunk = selectBestChunkForSteering(chunks ?? [], query, intent, {
|
|
135
154
|
preferredSeq: preferredSeqByHash.get(hash) ?? null,
|
|
136
155
|
intentWeight: 0.5,
|
|
@@ -146,7 +165,8 @@ async function fetchChunkTexts(
|
|
|
146
165
|
|
|
147
166
|
const hashToIndex = new Map<string, number>();
|
|
148
167
|
const texts: string[] = [];
|
|
149
|
-
for (const hash of
|
|
168
|
+
for (const hash of ownerHashes.keys()) {
|
|
169
|
+
assertInferenceActive();
|
|
150
170
|
hashToIndex.set(hash, texts.length);
|
|
151
171
|
texts.push(chunkTexts.get(hash) ?? "");
|
|
152
172
|
}
|
|
@@ -234,7 +254,7 @@ export async function rerankCandidates(
|
|
|
234
254
|
|
|
235
255
|
// Extract best chunk per document for efficient reranking
|
|
236
256
|
const { texts, hashToIndex } = await fetchChunkTexts(
|
|
237
|
-
store,
|
|
257
|
+
deps.hydration ?? store,
|
|
238
258
|
toRerank,
|
|
239
259
|
query,
|
|
240
260
|
options.intent
|
|
@@ -246,6 +266,7 @@ export async function rerankCandidates(
|
|
|
246
266
|
const textToUniqueIndex = new Map<string, number>();
|
|
247
267
|
|
|
248
268
|
for (const [docIndex, text] of texts.entries()) {
|
|
269
|
+
assertInferenceActive();
|
|
249
270
|
const existingIndex = textToUniqueIndex.get(text);
|
|
250
271
|
if (existingIndex !== undefined) {
|
|
251
272
|
docIndexToUniqueIndex.set(docIndex, existingIndex);
|
|
@@ -268,6 +289,7 @@ export async function rerankCandidates(
|
|
|
268
289
|
uniqueTexts
|
|
269
290
|
);
|
|
270
291
|
|
|
292
|
+
assertInferenceResult(rerankResult);
|
|
271
293
|
if (!rerankResult.ok) {
|
|
272
294
|
return {
|
|
273
295
|
candidates: sortAdjustedCandidates(
|
|
@@ -285,8 +307,10 @@ export async function rerankCandidates(
|
|
|
285
307
|
// Normalize rerank scores using min-max
|
|
286
308
|
const scoreByDocIndex = new Map<number, number>();
|
|
287
309
|
for (const score of rerankResult.value) {
|
|
310
|
+
assertInferenceActive();
|
|
288
311
|
const docIndices = uniqueIndexToDocIndices.get(score.index) ?? [];
|
|
289
312
|
for (const docIndex of docIndices) {
|
|
313
|
+
assertInferenceActive();
|
|
290
314
|
scoreByDocIndex.set(docIndex, score.score);
|
|
291
315
|
}
|
|
292
316
|
}
|
|
@@ -304,7 +328,7 @@ export async function rerankCandidates(
|
|
|
304
328
|
|
|
305
329
|
// Build reranked candidates with blended scores
|
|
306
330
|
const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
|
|
307
|
-
const docIndex = hashToIndex.get(c
|
|
331
|
+
const docIndex = hashToIndex.get(rerankOwnerKey(c)) ?? -1;
|
|
308
332
|
const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
|
|
309
333
|
const normalizedRerankScore =
|
|
310
334
|
rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
|
package/src/pipeline/search.ts
CHANGED
|
@@ -202,6 +202,8 @@ export async function searchBm25(
|
|
|
202
202
|
const ftsResult = await store.searchFts(query, {
|
|
203
203
|
limit: retrievalLimit,
|
|
204
204
|
collection: options.collection,
|
|
205
|
+
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
206
|
+
exclude: options.exclude,
|
|
205
207
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
206
208
|
language: options.lang,
|
|
207
209
|
snippet: !(options.full || options.lineNumbers),
|
|
@@ -244,13 +246,22 @@ export async function searchBm25(
|
|
|
244
246
|
}
|
|
245
247
|
>();
|
|
246
248
|
|
|
247
|
-
//
|
|
249
|
+
// Exact sequences suffice unless selection needs whole-document evidence.
|
|
248
250
|
const uniqueHashes = [
|
|
249
251
|
...new Set(
|
|
250
252
|
ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
|
|
251
253
|
),
|
|
252
254
|
];
|
|
253
|
-
const chunksMapResult =
|
|
255
|
+
const chunksMapResult =
|
|
256
|
+
store.getChunksBySequenceBatch &&
|
|
257
|
+
!options.intent &&
|
|
258
|
+
!options.exclude?.length
|
|
259
|
+
? await store.getChunksBySequenceBatch(
|
|
260
|
+
ftsResult.value
|
|
261
|
+
.filter((row) => !!row.mirrorHash)
|
|
262
|
+
.map((row) => ({ mirrorHash: row.mirrorHash!, seq: row.seq }))
|
|
263
|
+
)
|
|
264
|
+
: await store.getChunksBatch(uniqueHashes);
|
|
254
265
|
const getChunk = chunksMapResult.ok
|
|
255
266
|
? createChunkLookup(chunksMapResult.value)
|
|
256
267
|
: () => undefined;
|
package/src/pipeline/types.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
+
import type { NormalizedContentTypeRule } from "../config/content-types";
|
|
1
2
|
/**
|
|
2
3
|
* Search pipeline types.
|
|
3
4
|
* Defines SearchPipelinePort and related types for search operations.
|
|
4
5
|
*
|
|
5
6
|
* @module src/pipeline/types
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
import type { NormalizedContentTypeRule } from "../config/content-types";
|
|
9
8
|
import type {
|
|
10
9
|
ContextCapsuleV1,
|
|
11
10
|
ContextCapsuleVerification,
|
|
@@ -13,6 +12,7 @@ import type {
|
|
|
13
12
|
import type { EgressLineage } from "../core/egress-provenance";
|
|
14
13
|
import type { RecordEvidenceMetadata } from "../core/record-metadata";
|
|
15
14
|
import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
|
|
15
|
+
import type { InferenceOptions } from "../llm/types";
|
|
16
16
|
import type { StoreResult } from "../store/types";
|
|
17
17
|
import type { ClaimVerificationResult } from "./claim-verification";
|
|
18
18
|
import type { SemanticVerificationCapability } from "./claim-verifier";
|
|
@@ -172,7 +172,7 @@ export interface SearchResults {
|
|
|
172
172
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
173
173
|
|
|
174
174
|
/** Common options for all search commands */
|
|
175
|
-
export interface SearchOptions {
|
|
175
|
+
export interface SearchOptions extends InferenceOptions {
|
|
176
176
|
/** Internal receipt seam; never serialized or included in public schemas. */
|
|
177
177
|
traceSession?: RetrievalTraceSession;
|
|
178
178
|
/** Trusted, already-resolved project affinity; never accepts raw roots. */
|
|
@@ -332,6 +332,8 @@ export type FusionSource =
|
|
|
332
332
|
|
|
333
333
|
/** Fusion candidate with ranks from different sources */
|
|
334
334
|
export interface FusionCandidate {
|
|
335
|
+
/** Internal exact document owner; canonical public chunk identity is unchanged. */
|
|
336
|
+
documentId?: number;
|
|
335
337
|
mirrorHash: string;
|
|
336
338
|
seq: number;
|
|
337
339
|
bm25Rank: number | null;
|
package/src/pipeline/vsearch.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
+
import type { Config } from "../config/types";
|
|
1
2
|
/**
|
|
2
3
|
* Vector search pipeline.
|
|
3
4
|
* Wraps VectorIndexPort.searchNearest() to produce SearchResults.
|
|
4
5
|
*
|
|
5
6
|
* @module src/pipeline/vsearch
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
import type { Config } from "../config/types";
|
|
9
8
|
import type { EmbeddingPort } from "../llm/types";
|
|
10
9
|
import type { DocumentRow, StorePort } from "../store/types";
|
|
11
10
|
import type { VectorIndexPort } from "../store/vector/types";
|
|
@@ -13,8 +12,14 @@ import type { SearchOptions, SearchResult, SearchResults } from "./types";
|
|
|
13
12
|
|
|
14
13
|
import { normalizeContentTypes } from "../config/content-types";
|
|
15
14
|
import { projectRecordEvidenceMetadata } from "../core/record-metadata";
|
|
15
|
+
import {
|
|
16
|
+
assertInferenceActive,
|
|
17
|
+
assertInferenceResult,
|
|
18
|
+
withInferenceScope,
|
|
19
|
+
} from "../llm/inference-scope";
|
|
16
20
|
import { getContentBatch } from "../store/content-batch";
|
|
17
21
|
import { err, ok } from "../store/types";
|
|
22
|
+
import { resolveVectorSearchIdentity } from "../store/vector/variant-search";
|
|
18
23
|
import { createChunkLookup } from "./chunk-lookup";
|
|
19
24
|
import {
|
|
20
25
|
applyContentTypeBoost,
|
|
@@ -87,6 +92,17 @@ export async function searchVectorWithEmbedding(
|
|
|
87
92
|
query: string,
|
|
88
93
|
queryEmbedding: Float32Array,
|
|
89
94
|
options: SearchOptions = {}
|
|
95
|
+
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
96
|
+
return withInferenceScope(options, () =>
|
|
97
|
+
searchVectorWithEmbeddingOwned(deps, query, queryEmbedding, options)
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function searchVectorWithEmbeddingOwned(
|
|
102
|
+
deps: VectorSearchDeps,
|
|
103
|
+
query: string,
|
|
104
|
+
queryEmbedding: Float32Array,
|
|
105
|
+
options: SearchOptions = {}
|
|
90
106
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
91
107
|
const traceStartedAt = options.traceSession ? performance.now() : 0;
|
|
92
108
|
const { store, vectorIndex } = deps;
|
|
@@ -118,11 +134,37 @@ export async function searchVectorWithEmbedding(
|
|
|
118
134
|
return err("VEC_SEARCH_UNAVAILABLE", vectorUnavailableMessage(vectorIndex));
|
|
119
135
|
}
|
|
120
136
|
|
|
137
|
+
let embeddingIdentity;
|
|
138
|
+
try {
|
|
139
|
+
embeddingIdentity = resolveVectorSearchIdentity(deps.embedPort);
|
|
140
|
+
} catch (cause) {
|
|
141
|
+
return err(
|
|
142
|
+
"QUERY_FAILED",
|
|
143
|
+
cause instanceof Error ? cause.message : String(cause)
|
|
144
|
+
);
|
|
145
|
+
}
|
|
121
146
|
// Search nearest neighbors
|
|
122
147
|
const searchResult = await vectorIndex.searchNearest(
|
|
123
148
|
queryEmbedding,
|
|
124
149
|
retrievalLimit,
|
|
125
150
|
{
|
|
151
|
+
embeddingIdentity,
|
|
152
|
+
eligibility: {
|
|
153
|
+
excludeMetadata: true,
|
|
154
|
+
semanticMetadata: true,
|
|
155
|
+
collection: options.collection,
|
|
156
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
157
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
158
|
+
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
159
|
+
tagsAll: options.tagsAll,
|
|
160
|
+
tagsAny: options.tagsAny,
|
|
161
|
+
since: temporalRange.since,
|
|
162
|
+
until: temporalRange.until,
|
|
163
|
+
categories: options.categories,
|
|
164
|
+
author: options.author,
|
|
165
|
+
exclude: options.exclude,
|
|
166
|
+
language: options.lang,
|
|
167
|
+
},
|
|
126
168
|
minScore: projectAffinityActive ? undefined : minScore,
|
|
127
169
|
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
128
170
|
}
|
|
@@ -140,6 +182,7 @@ export async function searchVectorWithEmbedding(
|
|
|
140
182
|
const collectionPaths = new Map<string, string>();
|
|
141
183
|
if (collectionsResult.ok) {
|
|
142
184
|
for (const c of collectionsResult.value) {
|
|
185
|
+
assertInferenceActive();
|
|
143
186
|
collectionPaths.set(c.name, c.path);
|
|
144
187
|
}
|
|
145
188
|
}
|
|
@@ -158,8 +201,15 @@ export async function searchVectorWithEmbedding(
|
|
|
158
201
|
mirrorHashes: uniqueHashes,
|
|
159
202
|
});
|
|
160
203
|
|
|
161
|
-
//
|
|
162
|
-
const chunksMapResult =
|
|
204
|
+
// Exact sequences suffice unless selection needs whole-document evidence.
|
|
205
|
+
const chunksMapResult =
|
|
206
|
+
store.getChunksBySequenceBatch &&
|
|
207
|
+
!options.intent &&
|
|
208
|
+
!options.exclude?.length
|
|
209
|
+
? await store.getChunksBySequenceBatch(
|
|
210
|
+
vecResults.map(({ mirrorHash, seq }) => ({ mirrorHash, seq }))
|
|
211
|
+
)
|
|
212
|
+
: await store.getChunksBatch(uniqueHashes);
|
|
163
213
|
if (!chunksMapResult.ok) {
|
|
164
214
|
return err("QUERY_FAILED", chunksMapResult.error.message);
|
|
165
215
|
}
|
|
@@ -182,6 +232,7 @@ export async function searchVectorWithEmbedding(
|
|
|
182
232
|
>();
|
|
183
233
|
|
|
184
234
|
for (const vec of vecResults) {
|
|
235
|
+
assertInferenceActive();
|
|
185
236
|
const baseScore = normalizeVectorScore(vec.distance);
|
|
186
237
|
if (!projectAffinityActive && baseScore < minScore) {
|
|
187
238
|
continue;
|
|
@@ -191,7 +242,9 @@ export async function searchVectorWithEmbedding(
|
|
|
191
242
|
const rawChunk = getChunk(vec.mirrorHash, vec.seq);
|
|
192
243
|
const chunk = options.intent
|
|
193
244
|
? (selectBestChunkForSteering(
|
|
194
|
-
chunksMap.get(vec.mirrorHash) ?? []
|
|
245
|
+
(chunksMap.get(vec.mirrorHash) ?? []).filter(
|
|
246
|
+
(chunk) => !options.lang || chunk.language === options.lang
|
|
247
|
+
),
|
|
195
248
|
query,
|
|
196
249
|
options.intent,
|
|
197
250
|
{
|
|
@@ -210,12 +263,21 @@ export async function searchVectorWithEmbedding(
|
|
|
210
263
|
}
|
|
211
264
|
|
|
212
265
|
// Get document (cached)
|
|
213
|
-
const matchingDocs = docsByMirrorHash
|
|
266
|
+
const matchingDocs = docsByMirrorHash
|
|
267
|
+
.get(vec.mirrorHash)
|
|
268
|
+
?.filter(
|
|
269
|
+
(doc) =>
|
|
270
|
+
vec.documentIds === undefined || vec.documentIds.includes(doc.id)
|
|
271
|
+
);
|
|
214
272
|
if (!matchingDocs || matchingDocs.length === 0) {
|
|
215
273
|
continue;
|
|
216
274
|
}
|
|
217
|
-
const docs =
|
|
275
|
+
const docs =
|
|
276
|
+
auxiliaryRankingActive || vec.documentIds !== undefined
|
|
277
|
+
? matchingDocs
|
|
278
|
+
: [matchingDocs.at(-1)!];
|
|
218
279
|
for (const doc of docs) {
|
|
280
|
+
assertInferenceActive();
|
|
219
281
|
const collectionPath = collectionPaths.get(doc.collection);
|
|
220
282
|
const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
|
|
221
283
|
const excluded =
|
|
@@ -335,6 +397,7 @@ export async function searchVectorWithEmbedding(
|
|
|
335
397
|
const fullContentByHash = fullContentResult.value;
|
|
336
398
|
|
|
337
399
|
for (const { doc, chunk, rawDistance, score } of bestByDocid.values()) {
|
|
400
|
+
assertInferenceActive();
|
|
338
401
|
const fullContent = doc.mirrorHash
|
|
339
402
|
? fullContentByHash.get(doc.mirrorHash)
|
|
340
403
|
: undefined;
|
|
@@ -430,6 +493,7 @@ export async function searchVectorWithEmbedding(
|
|
|
430
493
|
|
|
431
494
|
const finalResults = results.slice(0, limit);
|
|
432
495
|
for (const [index, result] of finalResults.entries()) {
|
|
496
|
+
assertInferenceActive();
|
|
433
497
|
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
434
498
|
if (metadata) metadata.retrievalRank = index + 1;
|
|
435
499
|
}
|
|
@@ -487,6 +551,16 @@ export async function searchVector(
|
|
|
487
551
|
deps: VectorSearchDeps,
|
|
488
552
|
query: string,
|
|
489
553
|
options: SearchOptions = {}
|
|
554
|
+
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
555
|
+
return withInferenceScope(options, () =>
|
|
556
|
+
searchVectorOwned(deps, query, options)
|
|
557
|
+
);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
async function searchVectorOwned(
|
|
561
|
+
deps: VectorSearchDeps,
|
|
562
|
+
query: string,
|
|
563
|
+
options: SearchOptions = {}
|
|
490
564
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
491
565
|
const { vectorIndex, embedPort } = deps;
|
|
492
566
|
|
|
@@ -499,6 +573,7 @@ export async function searchVector(
|
|
|
499
573
|
const embedResult = await embedPort.embed(
|
|
500
574
|
formatQueryForEmbedding(query, embedPort.modelUri)
|
|
501
575
|
);
|
|
576
|
+
assertInferenceResult(embedResult);
|
|
502
577
|
if (!embedResult.ok) {
|
|
503
578
|
return err(
|
|
504
579
|
"QUERY_FAILED",
|
|
@@ -524,6 +599,7 @@ interface ChunkInfo {
|
|
|
524
599
|
}
|
|
525
600
|
|
|
526
601
|
interface DocumentInfo {
|
|
602
|
+
id: number;
|
|
527
603
|
docid: string;
|
|
528
604
|
uri: string;
|
|
529
605
|
title: string | null;
|
|
@@ -625,11 +701,13 @@ async function buildDocumentMap(
|
|
|
625
701
|
const docIds = activeDocs.map((d) => d.id);
|
|
626
702
|
const tagsResult = await store.getTagsBatch(docIds);
|
|
627
703
|
|
|
704
|
+
if (!tagsResult.ok) return { documents, ownershipDocuments };
|
|
628
705
|
if (tagsResult.ok) {
|
|
629
706
|
allowedDocIds = new Set<number>();
|
|
630
707
|
const tagsByDocId = tagsResult.value;
|
|
631
708
|
|
|
632
709
|
for (const doc of activeDocs) {
|
|
710
|
+
assertInferenceActive();
|
|
633
711
|
const docTags = new Set(
|
|
634
712
|
(tagsByDocId.get(doc.id) ?? []).map((t) => t.tag)
|
|
635
713
|
);
|
|
@@ -652,6 +730,7 @@ async function buildDocumentMap(
|
|
|
652
730
|
}
|
|
653
731
|
|
|
654
732
|
for (const doc of activeDocs) {
|
|
733
|
+
assertInferenceActive();
|
|
655
734
|
const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
|
|
656
735
|
if (
|
|
657
736
|
options.relPathPrefix !== undefined &&
|
|
@@ -679,6 +758,7 @@ async function buildDocumentMap(
|
|
|
679
758
|
}
|
|
680
759
|
|
|
681
760
|
const documentInfo: DocumentInfo = {
|
|
761
|
+
id: doc.id,
|
|
682
762
|
docid: doc.docid,
|
|
683
763
|
uri: doc.uri,
|
|
684
764
|
title: doc.title,
|