@gmickel/gno 1.45.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/THIRD_PARTY_NOTICES.md +46 -0
- package/assets/skill/SKILL.md +7 -6
- package/assets/skill/cli-reference.md +14 -6
- package/assets/skill/mcp-reference.md +4 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.0.0.zip.sha256 +1 -0
- package/browser-extension/dist/chunk-4tc9v0ja.js +74 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/browser-extension/dist/preview.html +1 -1
- package/browser-extension/dist/service-worker.js +32 -33
- package/bunfig.toml +2 -0
- package/package.json +40 -26
- package/spec/cli.md +30 -11
- package/spec/db/schema.sql +146 -1
- package/spec/mcp.md +26 -0
- package/src/app/context-runtime-types.ts +3 -0
- package/src/app/context-runtime.ts +2 -0
- package/src/cli/commands/ask.ts +6 -1
- package/src/cli/commands/daemon.ts +21 -8
- package/src/cli/commands/embed.ts +77 -41
- package/src/cli/commands/mcp/install.ts +20 -0
- package/src/cli/commands/mcp/paths.ts +25 -0
- package/src/cli/commands/mcp/status.ts +6 -0
- package/src/cli/detach.ts +3 -2
- package/src/cli/program.ts +6 -0
- package/src/config/types.ts +3 -3
- package/src/converters/adapters/markitdownTs/adapter.ts +1 -2
- package/src/converters/adapters/officeparser/adapter.ts +1 -2
- package/src/converters/versions.ts +6 -8
- package/src/core/context-evidence.ts +8 -4
- package/src/core/job-manager.ts +95 -13
- package/src/core/network-boundary-inventory.ts +10 -0
- package/src/core/shutdown-budget.ts +45 -0
- package/src/embed/backlog.ts +107 -4
- package/src/embed/batch.ts +42 -2
- package/src/embed/fingerprint.ts +16 -0
- package/src/embed/retry.ts +113 -5
- package/src/embed/variant-backlog.ts +105 -0
- package/src/embed/variant-plan.ts +62 -0
- package/src/embed/variant-retry.ts +113 -0
- package/src/ingestion/graph-reconciliation.ts +327 -0
- package/src/ingestion/sync.ts +9 -272
- package/src/llm/http-inference.ts +6 -0
- package/src/llm/httpEmbedding.ts +37 -6
- package/src/llm/httpGeneration.ts +18 -3
- package/src/llm/httpRerank.ts +23 -5
- package/src/llm/inference-cancellation.ts +168 -0
- package/src/llm/inference-scope.ts +202 -0
- package/src/llm/lazy-ports.ts +115 -0
- package/src/llm/native-worker/client.ts +541 -0
- package/src/llm/native-worker/dispatcher.ts +228 -0
- package/src/llm/native-worker/embedding-identity.ts +33 -0
- package/src/llm/native-worker/entry.ts +173 -0
- package/src/llm/native-worker/errors.ts +32 -0
- package/src/llm/native-worker/evaluation.ts +16 -0
- package/src/llm/native-worker/owned-exit.ts +108 -0
- package/src/llm/native-worker/owner.ts +141 -0
- package/src/llm/native-worker/ports.ts +317 -0
- package/src/llm/native-worker/protocol.ts +442 -0
- package/src/llm/native-worker/runtime-config.ts +92 -0
- package/src/llm/nodeLlamaCpp/adapter.ts +77 -20
- package/src/llm/nodeLlamaCpp/embedding.ts +130 -46
- package/src/llm/nodeLlamaCpp/generation.ts +34 -5
- package/src/llm/nodeLlamaCpp/lifecycle-options.ts +99 -0
- package/src/llm/nodeLlamaCpp/lifecycle.ts +209 -204
- package/src/llm/nodeLlamaCpp/rerank-capacity.ts +111 -0
- package/src/llm/nodeLlamaCpp/rerank.ts +118 -27
- package/src/llm/nodeLlamaCpp/simulator-handle.ts +73 -0
- package/src/llm/nodeLlamaCpp/simulator-install.ts +124 -0
- package/src/llm/nodeLlamaCpp/simulator-session.ts +240 -0
- package/src/llm/nodeLlamaCpp/simulator-types.ts +80 -0
- package/src/llm/types.ts +35 -5
- package/src/mcp/context.ts +27 -0
- package/src/mcp/http-transport.ts +12 -10
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-profile.ts +30 -8
- package/src/mcp/tools/context.ts +8 -11
- package/src/mcp/tools/embed.ts +1 -1
- package/src/mcp/tools/index-cmd.ts +1 -1
- package/src/mcp/tools/index.ts +10 -8
- package/src/mcp/tools/query.ts +14 -30
- package/src/mcp/tools/vsearch.ts +1 -1
- package/src/pipeline/answer.ts +23 -3
- package/src/pipeline/claim-verifier.ts +6 -0
- package/src/pipeline/expansion.ts +43 -40
- package/src/pipeline/explain.ts +6 -2
- package/src/pipeline/filters.ts +63 -0
- package/src/pipeline/fusion.ts +29 -9
- package/src/pipeline/graph-retrieval.ts +29 -9
- package/src/pipeline/hybrid.ts +198 -55
- package/src/pipeline/hydration.ts +161 -0
- package/src/pipeline/owner-fusion.ts +87 -0
- package/src/pipeline/rerank.ts +35 -11
- package/src/pipeline/search.ts +13 -2
- package/src/pipeline/types.ts +5 -3
- package/src/pipeline/vsearch.ts +87 -7
- package/src/sdk/client.ts +47 -3
- package/src/sdk/embed.ts +63 -39
- package/src/serve/background-runtime.ts +1 -1
- package/src/serve/context.ts +41 -56
- package/src/serve/embed-scheduler.ts +58 -35
- package/src/serve/public/components/IndexingProgress.tsx +46 -60
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/shiki-language-ids.ts +14 -0
- package/src/serve/resident-admission.ts +36 -36
- package/src/serve/resident-background-work.ts +20 -2
- package/src/serve/resident-request.ts +11 -5
- package/src/serve/resident-runtime.ts +97 -61
- package/src/serve/resident-shutdown.ts +153 -0
- package/src/serve/routes/api.ts +3 -1
- package/src/serve/server.ts +47 -26
- package/src/store/migrations/028-vector-variants.ts +54 -0
- package/src/store/migrations/029-graph-reference-state.ts +77 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +251 -183
- package/src/store/sqlite/eligibility.ts +174 -0
- package/src/store/sqlite/graph-edge-application.ts +66 -0
- package/src/store/sqlite/graph-reference-state.ts +194 -0
- package/src/store/sqlite/legacy-vector-ownership.ts +79 -0
- package/src/store/types.ts +80 -12
- package/src/store/vector/eligibility.ts +36 -0
- package/src/store/vector/freshness.ts +33 -6
- package/src/store/vector/lazy.ts +81 -0
- package/src/store/vector/sqlite-vec.ts +106 -54
- package/src/store/vector/stats.ts +14 -3
- package/src/store/vector/types.ts +35 -2
- package/src/store/vector/variant-search.ts +192 -0
- package/src/store/vector/variants.ts +451 -0
- package/vendor/converters/markitdown-ts/LICENSE +21 -0
- package/vendor/converters/markitdown-ts/dist/index.cjs +1180 -0
- package/vendor/converters/markitdown-ts/dist/index.d.cts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.mts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.d.ts +46 -0
- package/vendor/converters/markitdown-ts/dist/index.mjs +1152 -0
- package/vendor/converters/markitdown-ts/package.json +77 -0
- package/vendor/converters/officeparser/LICENSE +21 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.d.ts +47 -0
- package/vendor/converters/officeparser/dist/OfficeConverter.js +76 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.d.ts +23 -0
- package/vendor/converters/officeparser/dist/OfficeGenerator.js +73 -0
- package/vendor/converters/officeparser/dist/OfficeParser.d.ts +106 -0
- package/vendor/converters/officeparser/dist/OfficeParser.js +332 -0
- package/vendor/converters/officeparser/dist/cli.d.ts +28 -0
- package/vendor/converters/officeparser/dist/cli.js +381 -0
- package/vendor/converters/officeparser/dist/defaults.d.ts +41 -0
- package/vendor/converters/officeparser/dist/defaults.js +218 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.d.ts +107 -0
- package/vendor/converters/officeparser/dist/generators/BaseGenerator.js +248 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.d.ts +82 -0
- package/vendor/converters/officeparser/dist/generators/ChunkingGenerator.js +797 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.d.ts +38 -0
- package/vendor/converters/officeparser/dist/generators/CsvGenerator.js +245 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.d.ts +43 -0
- package/vendor/converters/officeparser/dist/generators/EpubGenerator.js +315 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.d.ts +59 -0
- package/vendor/converters/officeparser/dist/generators/HtmlGenerator.js +1942 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.d.ts +96 -0
- package/vendor/converters/officeparser/dist/generators/MarkdownGenerator.js +1175 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.d.ts +22 -0
- package/vendor/converters/officeparser/dist/generators/PdfGenerator.js +194 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.d.ts +29 -0
- package/vendor/converters/officeparser/dist/generators/RtfGenerator.js +316 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.d.ts +13 -0
- package/vendor/converters/officeparser/dist/generators/TextGenerator.js +201 -0
- package/vendor/converters/officeparser/dist/index.d.ts +60 -0
- package/vendor/converters/officeparser/dist/index.js +72 -0
- package/vendor/converters/officeparser/dist/index.mjs +18 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.mjs +1335 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.d.ts +2621 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.iife.js +1336 -0
- package/vendor/converters/officeparser/dist/officeparser.browser.slim.mjs +1335 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.d.ts +9 -0
- package/vendor/converters/officeparser/dist/parsers/CsvParser.js +115 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.d.ts +8 -0
- package/vendor/converters/officeparser/dist/parsers/EpubParser.js +217 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/ExcelParser.js +736 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/HtmlParser.js +1287 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.d.ts +2 -0
- package/vendor/converters/officeparser/dist/parsers/MarkdownParser.js +1272 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.d.ts +31 -0
- package/vendor/converters/officeparser/dist/parsers/OpenOfficeParser.js +1819 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.d.ts +67 -0
- package/vendor/converters/officeparser/dist/parsers/PdfParser.js +848 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.d.ts +32 -0
- package/vendor/converters/officeparser/dist/parsers/PowerPointParser.js +950 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.d.ts +187 -0
- package/vendor/converters/officeparser/dist/parsers/RtfParser.js +1801 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.d.ts +79 -0
- package/vendor/converters/officeparser/dist/parsers/WordParser.js +1177 -0
- package/vendor/converters/officeparser/dist/sbom.cdx.json +1763 -0
- package/vendor/converters/officeparser/dist/types.d.ts +2507 -0
- package/vendor/converters/officeparser/dist/types.js +107 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.d.ts +16 -0
- package/vendor/converters/officeparser/dist/utils/astUtils.js +33 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.d.ts +6 -0
- package/vendor/converters/officeparser/dist/utils/chartUtils.js +257 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.d.ts +44 -0
- package/vendor/converters/officeparser/dist/utils/configUtils.js +315 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.d.ts +17 -0
- package/vendor/converters/officeparser/dist/utils/dateUtils.js +69 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.d.ts +29 -0
- package/vendor/converters/officeparser/dist/utils/envUtils.js +152 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.d.ts +72 -0
- package/vendor/converters/officeparser/dist/utils/errorUtils.js +245 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.d.ts +66 -0
- package/vendor/converters/officeparser/dist/utils/imageUtils.js +133 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/mathUtils.js +385 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.d.ts +18 -0
- package/vendor/converters/officeparser/dist/utils/moduleLoader.js +106 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.d.ts +42 -0
- package/vendor/converters/officeparser/dist/utils/ocrUtils.js +428 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.d.ts +148 -0
- package/vendor/converters/officeparser/dist/utils/sanitize.js +344 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.d.ts +7 -0
- package/vendor/converters/officeparser/dist/utils/sheetUtils.js +35 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.d.ts +36 -0
- package/vendor/converters/officeparser/dist/utils/styleMapper.js +224 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.d.ts +163 -0
- package/vendor/converters/officeparser/dist/utils/xmlUtils.js +461 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.d.ts +134 -0
- package/vendor/converters/officeparser/dist/utils/zipUtils.js +337 -0
- package/vendor/converters/officeparser/package.json +147 -0
- package/vendor/converters/upstream-manifest.json +124 -0
- package/vendor/dependency-fixes/README.md +77 -0
- package/vendor/dependency-fixes/vendor-converters.py +83 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.45.1.zip.sha256 +0 -1
- package/browser-extension/dist/chunk-627emwpj.js +0 -75
- /package/browser-extension/dist/{chunk-ydfx5d7p.css → chunk-z74y8n8c.css} +0 -0
|
@@ -26,7 +26,7 @@ export interface GraphRetrievalMeta {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export interface GraphRetrievalResult {
|
|
29
|
-
candidates: Array<{ mirrorHash: string; seq: number }>;
|
|
29
|
+
candidates: Array<{ mirrorHash: string; seq: number; sourceDocid?: string }>;
|
|
30
30
|
meta: GraphRetrievalMeta;
|
|
31
31
|
}
|
|
32
32
|
|
|
@@ -246,7 +246,11 @@ export async function expandGraphCandidates(
|
|
|
246
246
|
categories?: string[];
|
|
247
247
|
author?: string;
|
|
248
248
|
relPathPrefix?: string;
|
|
249
|
-
} = {}
|
|
249
|
+
} = {},
|
|
250
|
+
hydration: Pick<
|
|
251
|
+
StorePort,
|
|
252
|
+
"getChunksBatch" | "getDocumentsByMirrorHashes"
|
|
253
|
+
> = store
|
|
250
254
|
): Promise<GraphRetrievalResult> {
|
|
251
255
|
const maxCandidates = Math.max(
|
|
252
256
|
1,
|
|
@@ -282,10 +286,13 @@ export async function expandGraphCandidates(
|
|
|
282
286
|
preferredSeqByHash.set(candidate.mirrorHash, candidate.seq);
|
|
283
287
|
}
|
|
284
288
|
}
|
|
285
|
-
const seedDocsResult = await
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
+
const seedDocsResult = await hydration.getDocumentsByMirrorHashes(
|
|
290
|
+
seedHashes,
|
|
291
|
+
{
|
|
292
|
+
collection: options.collection,
|
|
293
|
+
activeOnly: true,
|
|
294
|
+
}
|
|
295
|
+
);
|
|
289
296
|
if (!seedDocsResult.ok || seedDocsResult.value.length === 0) {
|
|
290
297
|
meta.fallbackReasons.push("graph_seed_lookup_empty");
|
|
291
298
|
return { candidates: [], meta };
|
|
@@ -305,7 +312,10 @@ export async function expandGraphCandidates(
|
|
|
305
312
|
}
|
|
306
313
|
const rank =
|
|
307
314
|
seedCandidates.findIndex(
|
|
308
|
-
(candidate) =>
|
|
315
|
+
(candidate) =>
|
|
316
|
+
candidate.mirrorHash === doc.mirrorHash &&
|
|
317
|
+
(candidate.documentId === undefined ||
|
|
318
|
+
candidate.documentId === doc.id)
|
|
309
319
|
) + 1;
|
|
310
320
|
if (rank <= 0) {
|
|
311
321
|
continue;
|
|
@@ -409,7 +419,7 @@ export async function expandGraphCandidates(
|
|
|
409
419
|
const hashes = rankedDocs
|
|
410
420
|
.map((doc) => doc.mirrorHash)
|
|
411
421
|
.filter((hash): hash is string => Boolean(hash));
|
|
412
|
-
const chunksResult = await
|
|
422
|
+
const chunksResult = await hydration.getChunksBatch(hashes);
|
|
413
423
|
if (!chunksResult.ok) {
|
|
414
424
|
meta.fallbackReasons.push("graph_neighbor_chunks_failed");
|
|
415
425
|
return { candidates: [], meta };
|
|
@@ -424,7 +434,17 @@ export async function expandGraphCandidates(
|
|
|
424
434
|
preferredSeqByHash,
|
|
425
435
|
options.lang
|
|
426
436
|
);
|
|
427
|
-
return seq === null
|
|
437
|
+
return seq === null
|
|
438
|
+
? null
|
|
439
|
+
: {
|
|
440
|
+
mirrorHash,
|
|
441
|
+
seq,
|
|
442
|
+
...(fusedCandidates.some(
|
|
443
|
+
(candidate) => candidate.documentId !== undefined
|
|
444
|
+
)
|
|
445
|
+
? { sourceDocid: doc.docid }
|
|
446
|
+
: {}),
|
|
447
|
+
};
|
|
428
448
|
})
|
|
429
449
|
.filter(
|
|
430
450
|
(candidate): candidate is { mirrorHash: string; seq: number } =>
|
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
|
+
import type { Config } from "../config/types";
|
|
1
2
|
/**
|
|
2
3
|
* Hybrid search orchestrator.
|
|
3
4
|
* Combines BM25, vector search, expansion, fusion, and reranking.
|
|
4
5
|
*
|
|
5
6
|
* @module src/pipeline/hybrid
|
|
6
7
|
*/
|
|
7
|
-
|
|
8
|
-
import type { Config } from "../config/types";
|
|
9
8
|
import type { EmbeddingPort, GenerationPort, RerankPort } from "../llm/types";
|
|
10
9
|
import type { DocumentRow, StorePort } from "../store/types";
|
|
11
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
VectorIndexPort,
|
|
12
|
+
VectorSearchOptions,
|
|
13
|
+
} from "../store/vector/types";
|
|
12
14
|
import type {
|
|
13
15
|
ExpansionResult,
|
|
14
16
|
ExplainLine,
|
|
@@ -23,7 +25,13 @@ import type {
|
|
|
23
25
|
import { normalizeContentTypes } from "../config/content-types";
|
|
24
26
|
import { projectRecordEvidenceMetadata } from "../core/record-metadata";
|
|
25
27
|
import { embedTextsWithRecovery } from "../embed/batch";
|
|
28
|
+
import {
|
|
29
|
+
assertInferenceActive,
|
|
30
|
+
assertInferenceResult,
|
|
31
|
+
withInferenceScope,
|
|
32
|
+
} from "../llm/inference-scope";
|
|
26
33
|
import { err, ok } from "../store/types";
|
|
34
|
+
import { resolveVectorSearchIdentity } from "../store/vector/variant-search";
|
|
27
35
|
import { createChunkLookup } from "./chunk-lookup";
|
|
28
36
|
import {
|
|
29
37
|
attachAuxiliaryScoreMetadata,
|
|
@@ -49,7 +57,9 @@ import {
|
|
|
49
57
|
import { evaluateDocumentChunkFilters } from "./filters";
|
|
50
58
|
import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
|
|
51
59
|
import { expandGraphCandidates } from "./graph-retrieval";
|
|
60
|
+
import { RequestHydration } from "./hydration";
|
|
52
61
|
import { selectBestChunkForSteering } from "./intent";
|
|
62
|
+
import { OwnerMetadataError, resolveFusionOwners } from "./owner-fusion";
|
|
53
63
|
import { hasProjectAffinity } from "./project-affinity";
|
|
54
64
|
import { detectQueryLanguage } from "./query-language";
|
|
55
65
|
import {
|
|
@@ -86,6 +96,8 @@ export interface HybridSearchDeps {
|
|
|
86
96
|
expandPort: GenerationPort | null;
|
|
87
97
|
rerankPort: RerankPort | null;
|
|
88
98
|
pipelineConfig?: PipelineConfig;
|
|
99
|
+
/** Internal request owner; the creating caller releases it. */
|
|
100
|
+
hydration?: RequestHydration;
|
|
89
101
|
}
|
|
90
102
|
|
|
91
103
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -137,6 +149,10 @@ async function checkBm25Strength(
|
|
|
137
149
|
categories?: string[];
|
|
138
150
|
author?: string;
|
|
139
151
|
relPathPrefix?: string;
|
|
152
|
+
allowedMirrorHashes?: string[];
|
|
153
|
+
exclude?: string[];
|
|
154
|
+
memoryScopesAny?: string[];
|
|
155
|
+
excludeSuperseded?: boolean;
|
|
140
156
|
}
|
|
141
157
|
): Promise<boolean> {
|
|
142
158
|
const result = await store.searchFts(query, {
|
|
@@ -144,6 +160,13 @@ async function checkBm25Strength(
|
|
|
144
160
|
collection: options?.collection,
|
|
145
161
|
relPathPrefix: options?.relPathPrefix,
|
|
146
162
|
language: options?.lang,
|
|
163
|
+
chunkLanguage: options?.lang,
|
|
164
|
+
allowedMirrorHashes: options?.allowedMirrorHashes,
|
|
165
|
+
exclude: options?.exclude,
|
|
166
|
+
excludeMetadata: true,
|
|
167
|
+
semanticMetadata: true,
|
|
168
|
+
memoryScopesAny: options?.memoryScopesAny,
|
|
169
|
+
excludeSuperseded: options?.excludeSuperseded,
|
|
147
170
|
tagsAll: options?.tagsAll,
|
|
148
171
|
tagsAny: options?.tagsAny,
|
|
149
172
|
since: options?.since,
|
|
@@ -174,6 +197,8 @@ async function checkBm25Strength(
|
|
|
174
197
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
175
198
|
|
|
176
199
|
interface ChunkId {
|
|
200
|
+
sourceDocid?: string;
|
|
201
|
+
documentIds?: number[];
|
|
177
202
|
mirrorHash: string;
|
|
178
203
|
seq: number;
|
|
179
204
|
score?: number;
|
|
@@ -197,6 +222,10 @@ async function searchFtsChunks(
|
|
|
197
222
|
categories?: string[];
|
|
198
223
|
author?: string;
|
|
199
224
|
relPathPrefix?: string;
|
|
225
|
+
allowedMirrorHashes?: string[];
|
|
226
|
+
exclude?: string[];
|
|
227
|
+
memoryScopesAny?: string[];
|
|
228
|
+
excludeSuperseded?: boolean;
|
|
200
229
|
}
|
|
201
230
|
): Promise<FtsChunksResult> {
|
|
202
231
|
const result = await store.searchFts(query, {
|
|
@@ -204,6 +233,13 @@ async function searchFtsChunks(
|
|
|
204
233
|
collection: options.collection,
|
|
205
234
|
relPathPrefix: options.relPathPrefix,
|
|
206
235
|
language: options.lang,
|
|
236
|
+
chunkLanguage: options.lang,
|
|
237
|
+
excludeMetadata: true,
|
|
238
|
+
semanticMetadata: true,
|
|
239
|
+
memoryScopesAny: options.memoryScopesAny,
|
|
240
|
+
excludeSuperseded: options.excludeSuperseded,
|
|
241
|
+
exclude: options.exclude,
|
|
242
|
+
allowedMirrorHashes: options.allowedMirrorHashes,
|
|
207
243
|
tagsAll: options.tagsAll,
|
|
208
244
|
tagsAny: options.tagsAny,
|
|
209
245
|
since: options.since,
|
|
@@ -220,6 +256,7 @@ async function searchFtsChunks(
|
|
|
220
256
|
return {
|
|
221
257
|
ok: true,
|
|
222
258
|
chunks: result.value.map((r) => ({
|
|
259
|
+
sourceDocid: r.docid,
|
|
223
260
|
mirrorHash: r.mirrorHash,
|
|
224
261
|
seq: r.seq,
|
|
225
262
|
score: r.score,
|
|
@@ -239,18 +276,20 @@ async function searchVectorChunks(
|
|
|
239
276
|
limit: number;
|
|
240
277
|
minScore?: number;
|
|
241
278
|
allowedMirrorHashes?: string[];
|
|
279
|
+
eligibility?: VectorSearchOptions["eligibility"];
|
|
242
280
|
}
|
|
243
|
-
): Promise<ChunkId[]> {
|
|
281
|
+
): Promise<{ ok: true; chunks: ChunkId[] } | { ok: false; reason: string }> {
|
|
244
282
|
if (!vectorIndex.searchAvailable) {
|
|
245
|
-
return
|
|
283
|
+
return { ok: false, reason: "vector_unavailable" };
|
|
246
284
|
}
|
|
247
285
|
|
|
248
286
|
// Embed query with contextual formatting
|
|
249
287
|
const embedResult = await embedPort.embed(
|
|
250
288
|
formatQueryForEmbedding(query, embedPort.modelUri)
|
|
251
289
|
);
|
|
290
|
+
assertInferenceResult(embedResult);
|
|
252
291
|
if (!embedResult.ok) {
|
|
253
|
-
return
|
|
292
|
+
return { ok: false, reason: "vector_embed_error" };
|
|
254
293
|
}
|
|
255
294
|
|
|
256
295
|
const queryEmbedding = new Float32Array(embedResult.value);
|
|
@@ -258,20 +297,26 @@ async function searchVectorChunks(
|
|
|
258
297
|
queryEmbedding,
|
|
259
298
|
options.limit,
|
|
260
299
|
{
|
|
300
|
+
embeddingIdentity: resolveVectorSearchIdentity(embedPort),
|
|
261
301
|
minScore: options.minScore,
|
|
262
302
|
allowedMirrorHashes: options.allowedMirrorHashes,
|
|
303
|
+
eligibility: options.eligibility,
|
|
263
304
|
}
|
|
264
305
|
);
|
|
265
306
|
|
|
266
307
|
if (!searchResult.ok) {
|
|
267
|
-
return
|
|
308
|
+
return { ok: false, reason: "vector_search_error" };
|
|
268
309
|
}
|
|
269
310
|
|
|
270
|
-
return
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
311
|
+
return {
|
|
312
|
+
ok: true,
|
|
313
|
+
chunks: searchResult.value.map((r) => ({
|
|
314
|
+
documentIds: r.documentIds,
|
|
315
|
+
mirrorHash: r.mirrorHash,
|
|
316
|
+
seq: r.seq,
|
|
317
|
+
score: r.distance,
|
|
318
|
+
})),
|
|
319
|
+
};
|
|
275
320
|
}
|
|
276
321
|
|
|
277
322
|
function toTraceCandidates(chunks: ChunkId[]): QueryDiagnoseTraceCandidate[] {
|
|
@@ -306,11 +351,31 @@ function candidatesToTrace(
|
|
|
306
351
|
/**
|
|
307
352
|
* Execute hybrid search with full pipeline.
|
|
308
353
|
*/
|
|
309
|
-
// oxlint-disable-next-line max-lines-per-function -- search orchestration with BM25, vector, fusion, reranking
|
|
310
354
|
export async function searchHybrid(
|
|
311
355
|
deps: HybridSearchDeps,
|
|
312
356
|
query: string,
|
|
313
357
|
options: HybridSearchOptions = {}
|
|
358
|
+
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
359
|
+
const hydration = deps.hydration ?? new RequestHydration(deps.store);
|
|
360
|
+
try {
|
|
361
|
+
return await withInferenceScope(options, () =>
|
|
362
|
+
searchHybridWithHydration(deps, query, options, hydration)
|
|
363
|
+
);
|
|
364
|
+
} catch (cause) {
|
|
365
|
+
if (cause instanceof OwnerMetadataError)
|
|
366
|
+
return err("QUERY_FAILED", cause.message);
|
|
367
|
+
throw cause;
|
|
368
|
+
} finally {
|
|
369
|
+
if (!deps.hydration) hydration.release();
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// oxlint-disable-next-line max-lines-per-function -- search orchestration with BM25, vector, fusion, reranking
|
|
374
|
+
async function searchHybridWithHydration(
|
|
375
|
+
deps: HybridSearchDeps,
|
|
376
|
+
query: string,
|
|
377
|
+
options: HybridSearchOptions,
|
|
378
|
+
hydration: RequestHydration
|
|
314
379
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
315
380
|
const runStartedAt = performance.now();
|
|
316
381
|
const { store, vectorIndex, embedPort, expandPort, rerankPort } = deps;
|
|
@@ -408,6 +473,10 @@ export async function searchHybrid(
|
|
|
408
473
|
: await checkBm25Strength(store, query, {
|
|
409
474
|
collection: options.collection,
|
|
410
475
|
lang: options.lang,
|
|
476
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
477
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
478
|
+
exclude: options.exclude,
|
|
479
|
+
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
411
480
|
tagsAll: options.tagsAll,
|
|
412
481
|
tagsAny: options.tagsAny,
|
|
413
482
|
since: temporalRange.since,
|
|
@@ -451,11 +520,31 @@ export async function searchHybrid(
|
|
|
451
520
|
? { stages: [] }
|
|
452
521
|
: undefined;
|
|
453
522
|
|
|
523
|
+
const vectorEligibility: VectorSearchOptions["eligibility"] = {
|
|
524
|
+
collection: options.collection,
|
|
525
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
526
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
527
|
+
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
528
|
+
tagsAll: options.tagsAll,
|
|
529
|
+
tagsAny: options.tagsAny,
|
|
530
|
+
since: temporalRange.since,
|
|
531
|
+
until: temporalRange.until,
|
|
532
|
+
categories: options.categories,
|
|
533
|
+
author: options.author,
|
|
534
|
+
exclude: options.exclude,
|
|
535
|
+
excludeMetadata: true,
|
|
536
|
+
semanticMetadata: true,
|
|
537
|
+
language: options.lang,
|
|
538
|
+
};
|
|
454
539
|
const bm25StartedAt = performance.now();
|
|
455
540
|
|
|
456
541
|
// BM25: original query
|
|
457
542
|
const bm25Result = await searchFtsChunks(store, query, {
|
|
458
543
|
limit: limit * 2 * retrievalMultiplier,
|
|
544
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
545
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
546
|
+
exclude: options.exclude,
|
|
547
|
+
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
459
548
|
collection: options.collection,
|
|
460
549
|
lang: options.lang,
|
|
461
550
|
tagsAll: options.tagsAll,
|
|
@@ -491,6 +580,10 @@ export async function searchHybrid(
|
|
|
491
580
|
expansion.lexicalQueries.map((variant) =>
|
|
492
581
|
searchFtsChunks(store, variant, {
|
|
493
582
|
limit: limit * retrievalMultiplier,
|
|
583
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
584
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
585
|
+
exclude: options.exclude,
|
|
586
|
+
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
494
587
|
collection: options.collection,
|
|
495
588
|
lang: options.lang,
|
|
496
589
|
tagsAll: options.tagsAll,
|
|
@@ -505,6 +598,7 @@ export async function searchHybrid(
|
|
|
505
598
|
);
|
|
506
599
|
|
|
507
600
|
for (const settled of lexicalVariantResults) {
|
|
601
|
+
assertInferenceActive();
|
|
508
602
|
if (settled.status !== "fulfilled") {
|
|
509
603
|
continue;
|
|
510
604
|
}
|
|
@@ -520,6 +614,7 @@ export async function searchHybrid(
|
|
|
520
614
|
|
|
521
615
|
// Vector search
|
|
522
616
|
let vecCount = 0;
|
|
617
|
+
let vectorsUsed = false;
|
|
523
618
|
const vectorAvailable =
|
|
524
619
|
(vectorIndex?.searchAvailable && embedPort !== null) ?? false;
|
|
525
620
|
if (!vectorAvailable) {
|
|
@@ -541,16 +636,20 @@ export async function searchHybrid(
|
|
|
541
636
|
];
|
|
542
637
|
|
|
543
638
|
if (vectorVariantQueries.length === 0) {
|
|
544
|
-
const
|
|
639
|
+
const vectorResult = await searchVectorChunks(
|
|
545
640
|
vectorIndex,
|
|
546
641
|
embedPort,
|
|
547
642
|
query,
|
|
548
643
|
{
|
|
549
644
|
limit: limit * 2 * retrievalMultiplier,
|
|
550
645
|
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
646
|
+
eligibility: vectorEligibility,
|
|
551
647
|
}
|
|
552
648
|
);
|
|
553
649
|
|
|
650
|
+
if (!vectorResult.ok) counters.fallbackEvents.push(vectorResult.reason);
|
|
651
|
+
else vectorsUsed = true;
|
|
652
|
+
const vecChunks = vectorResult.ok ? vectorResult.chunks : [];
|
|
554
653
|
vecCount = vecChunks.length;
|
|
555
654
|
vectorTraceChunks.push(...vecChunks);
|
|
556
655
|
if (vecCount > 0) {
|
|
@@ -576,6 +675,7 @@ export async function searchHybrid(
|
|
|
576
675
|
)
|
|
577
676
|
);
|
|
578
677
|
|
|
678
|
+
assertInferenceResult(embedResult);
|
|
579
679
|
if (!embedResult.ok) {
|
|
580
680
|
counters.fallbackEvents.push("vector_embed_error");
|
|
581
681
|
} else {
|
|
@@ -584,6 +684,7 @@ export async function searchHybrid(
|
|
|
584
684
|
}
|
|
585
685
|
|
|
586
686
|
for (const [index, variant] of batchedQueries.entries()) {
|
|
687
|
+
assertInferenceActive();
|
|
587
688
|
const embedding = embedResult.value.vectors[index];
|
|
588
689
|
if (!embedding || !variant) {
|
|
589
690
|
continue;
|
|
@@ -593,14 +694,20 @@ export async function searchHybrid(
|
|
|
593
694
|
new Float32Array(embedding),
|
|
594
695
|
variant.limit,
|
|
595
696
|
{
|
|
697
|
+
embeddingIdentity: resolveVectorSearchIdentity(embedPort),
|
|
596
698
|
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
699
|
+
eligibility: vectorEligibility,
|
|
597
700
|
}
|
|
598
701
|
);
|
|
599
|
-
if (!searchResult.ok
|
|
702
|
+
if (!searchResult.ok) {
|
|
703
|
+
counters.fallbackEvents.push("vector_search_error");
|
|
600
704
|
continue;
|
|
601
705
|
}
|
|
706
|
+
vectorsUsed = true;
|
|
707
|
+
if (searchResult.value.length === 0) continue;
|
|
602
708
|
|
|
603
709
|
const chunks = searchResult.value.map((item) => ({
|
|
710
|
+
documentIds: item.documentIds,
|
|
604
711
|
mirrorHash: item.mirrorHash,
|
|
605
712
|
seq: item.seq,
|
|
606
713
|
}));
|
|
@@ -641,7 +748,10 @@ export async function searchHybrid(
|
|
|
641
748
|
const fusionStartedAt = performance.now();
|
|
642
749
|
const candidateLimit =
|
|
643
750
|
options.candidateLimit ?? pipelineConfig.rerankCandidates;
|
|
644
|
-
let fusedCandidates = rrfFuse(
|
|
751
|
+
let fusedCandidates = rrfFuse(
|
|
752
|
+
await resolveFusionOwners(rankedInputs, store, hydration, query, options),
|
|
753
|
+
pipelineConfig.rrf
|
|
754
|
+
);
|
|
645
755
|
diagnoseTrace?.stages.push({
|
|
646
756
|
id: "fusion",
|
|
647
757
|
status: "active",
|
|
@@ -651,26 +761,34 @@ export async function searchHybrid(
|
|
|
651
761
|
|
|
652
762
|
timings.fusionMs = performance.now() - fusionStartedAt;
|
|
653
763
|
const graphStartedAt = performance.now();
|
|
654
|
-
const graphExpansion = await expandGraphCandidates(
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
764
|
+
const graphExpansion = await expandGraphCandidates(
|
|
765
|
+
store,
|
|
766
|
+
fusedCandidates,
|
|
767
|
+
{
|
|
768
|
+
collection: options.collection,
|
|
769
|
+
includeSimilar: vectorAvailable,
|
|
770
|
+
limit,
|
|
771
|
+
candidateLimit,
|
|
772
|
+
disabled: options.graph === false || options.noGraph === true,
|
|
773
|
+
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
774
|
+
lang: options.lang,
|
|
775
|
+
tagsAll: options.tagsAll,
|
|
776
|
+
tagsAny: options.tagsAny,
|
|
777
|
+
since: temporalRange.since,
|
|
778
|
+
until: temporalRange.until,
|
|
779
|
+
categories: options.categories,
|
|
780
|
+
author: options.author,
|
|
781
|
+
},
|
|
782
|
+
hydration
|
|
783
|
+
);
|
|
669
784
|
timings.graphMs = performance.now() - graphStartedAt;
|
|
670
785
|
if (graphExpansion.candidates.length > 0) {
|
|
671
786
|
const graphFusionStartedAt = performance.now();
|
|
672
787
|
rankedInputs.push(toRankedInput("graph", graphExpansion.candidates));
|
|
673
|
-
fusedCandidates = rrfFuse(
|
|
788
|
+
fusedCandidates = rrfFuse(
|
|
789
|
+
await resolveFusionOwners(rankedInputs, store, hydration, query, options),
|
|
790
|
+
pipelineConfig.rrf
|
|
791
|
+
);
|
|
674
792
|
timings.fusionMs += performance.now() - graphFusionStartedAt;
|
|
675
793
|
}
|
|
676
794
|
diagnoseTrace?.stages.push({
|
|
@@ -713,13 +831,14 @@ export async function searchHybrid(
|
|
|
713
831
|
) => number)
|
|
714
832
|
| undefined;
|
|
715
833
|
if (auxiliaryRankingActive) {
|
|
716
|
-
const prefetchedDocumentsResult =
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
834
|
+
const prefetchedDocumentsResult =
|
|
835
|
+
await hydration.getDocumentsByMirrorHashes(
|
|
836
|
+
[...new Set(fusedCandidates.map((candidate) => candidate.mirrorHash))],
|
|
837
|
+
{
|
|
838
|
+
collection: options.collection,
|
|
839
|
+
activeOnly: true,
|
|
840
|
+
}
|
|
841
|
+
);
|
|
723
842
|
if (!prefetchedDocumentsResult.ok) {
|
|
724
843
|
return err("QUERY_FAILED", prefetchedDocumentsResult.error.message);
|
|
725
844
|
}
|
|
@@ -735,9 +854,15 @@ export async function searchHybrid(
|
|
|
735
854
|
scoringDocumentsByHash.set(document.mirrorHash, documents);
|
|
736
855
|
}
|
|
737
856
|
adjustNormalizedFusionScore = (candidate, normalizedScore) => {
|
|
738
|
-
const candidateKey = `${candidate.mirrorHash}:${candidate.seq}`;
|
|
857
|
+
const candidateKey = `${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}`;
|
|
739
858
|
auxiliaryBaseScores.set(candidateKey, normalizedScore);
|
|
740
|
-
const documents = scoringDocumentsByHash
|
|
859
|
+
const documents = scoringDocumentsByHash
|
|
860
|
+
.get(candidate.mirrorHash)
|
|
861
|
+
?.filter(
|
|
862
|
+
(doc) =>
|
|
863
|
+
candidate.documentId === undefined ||
|
|
864
|
+
candidate.documentId === doc.id
|
|
865
|
+
);
|
|
741
866
|
if (!documents?.length) return normalizedScore;
|
|
742
867
|
const projectedScores = documents.map(
|
|
743
868
|
(document) =>
|
|
@@ -767,7 +892,7 @@ export async function searchHybrid(
|
|
|
767
892
|
// ─────────────────────────────────────────────────────────────────────────
|
|
768
893
|
const rerankStartedAt = performance.now();
|
|
769
894
|
const rerankResult = await rerankCandidates(
|
|
770
|
-
{ rerankPort: options.noRerank ? null : rerankPort, store },
|
|
895
|
+
{ rerankPort: options.noRerank ? null : rerankPort, store, hydration },
|
|
771
896
|
query,
|
|
772
897
|
fusedCandidates,
|
|
773
898
|
{
|
|
@@ -819,7 +944,7 @@ export async function searchHybrid(
|
|
|
819
944
|
// Fetch only needed documents and collections.
|
|
820
945
|
let documents = prefetchedDocuments;
|
|
821
946
|
if (!documents) {
|
|
822
|
-
const docsResult = await
|
|
947
|
+
const docsResult = await hydration.getDocumentsByMirrorHashes(
|
|
823
948
|
[...neededHashes],
|
|
824
949
|
{
|
|
825
950
|
collection: options.collection,
|
|
@@ -881,6 +1006,7 @@ export async function searchHybrid(
|
|
|
881
1006
|
const candidateDocs: DocumentRow[] = [];
|
|
882
1007
|
|
|
883
1008
|
for (const doc of documents) {
|
|
1009
|
+
assertInferenceActive();
|
|
884
1010
|
if (!doc.mirrorHash) {
|
|
885
1011
|
continue;
|
|
886
1012
|
}
|
|
@@ -900,6 +1026,7 @@ export async function searchHybrid(
|
|
|
900
1026
|
if (tagsResult.ok) {
|
|
901
1027
|
const tagsByDocId = tagsResult.value;
|
|
902
1028
|
for (const doc of candidateDocs) {
|
|
1029
|
+
assertInferenceActive();
|
|
903
1030
|
const docTags = new Set(
|
|
904
1031
|
(tagsByDocId.get(doc.id) ?? []).map((t) => t.tag)
|
|
905
1032
|
);
|
|
@@ -924,6 +1051,7 @@ export async function searchHybrid(
|
|
|
924
1051
|
}
|
|
925
1052
|
|
|
926
1053
|
for (const docs of docsByMirrorHash.values()) {
|
|
1054
|
+
assertInferenceActive();
|
|
927
1055
|
docs.sort((left, right) => {
|
|
928
1056
|
if (left.uri < right.uri) return -1;
|
|
929
1057
|
if (left.uri > right.uri) return 1;
|
|
@@ -934,12 +1062,13 @@ export async function searchHybrid(
|
|
|
934
1062
|
const collectionPaths = new Map<string, string>();
|
|
935
1063
|
if (collectionsResult.ok) {
|
|
936
1064
|
for (const c of collectionsResult.value) {
|
|
1065
|
+
assertInferenceActive();
|
|
937
1066
|
collectionPaths.set(c.name, c.path);
|
|
938
1067
|
}
|
|
939
1068
|
}
|
|
940
1069
|
|
|
941
1070
|
// Pre-fetch all chunks in one batch query (eliminates N+1)
|
|
942
|
-
const chunksMapResult = await
|
|
1071
|
+
const chunksMapResult = await hydration.getChunksBatch([...neededHashes]);
|
|
943
1072
|
if (!chunksMapResult.ok) {
|
|
944
1073
|
return err("QUERY_FAILED", chunksMapResult.error.message);
|
|
945
1074
|
}
|
|
@@ -960,13 +1089,19 @@ export async function searchHybrid(
|
|
|
960
1089
|
|
|
961
1090
|
// Iterate until we have enough results (don't slice early - deduping may skip candidates)
|
|
962
1091
|
for (const [candidateIndex, candidate] of filteredCandidates.entries()) {
|
|
1092
|
+
assertInferenceActive();
|
|
963
1093
|
// Stop when we have enough results
|
|
964
1094
|
if (!auxiliaryRankingActive && results.length >= assemblyLimit) {
|
|
965
1095
|
break;
|
|
966
1096
|
}
|
|
967
1097
|
|
|
968
1098
|
// Find document from pre-fetched map
|
|
969
|
-
const candidateDocs =
|
|
1099
|
+
const candidateDocs = (
|
|
1100
|
+
docsByMirrorHash.get(candidate.mirrorHash) ?? []
|
|
1101
|
+
).filter(
|
|
1102
|
+
(doc) =>
|
|
1103
|
+
candidate.documentId === undefined || candidate.documentId === doc.id
|
|
1104
|
+
);
|
|
970
1105
|
if (candidateDocs.length === 0) {
|
|
971
1106
|
continue;
|
|
972
1107
|
}
|
|
@@ -994,7 +1129,9 @@ export async function searchHybrid(
|
|
|
994
1129
|
options.full || !options.intent?.trim()
|
|
995
1130
|
? chunk
|
|
996
1131
|
: (selectBestChunkForSteering(
|
|
997
|
-
chunksMap.get(candidate.mirrorHash) ?? []
|
|
1132
|
+
(chunksMap.get(candidate.mirrorHash) ?? []).filter(
|
|
1133
|
+
(chunk) => !options.lang || chunk.language === options.lang
|
|
1134
|
+
),
|
|
998
1135
|
query,
|
|
999
1136
|
options.intent,
|
|
1000
1137
|
{
|
|
@@ -1014,7 +1151,7 @@ export async function searchHybrid(
|
|
|
1014
1151
|
// Get or fetch full content for this mirrorHash
|
|
1015
1152
|
let contentResult = contentCache.get(candidate.mirrorHash);
|
|
1016
1153
|
if (!contentResult) {
|
|
1017
|
-
contentResult = await
|
|
1154
|
+
contentResult = await hydration.getContent(candidate.mirrorHash);
|
|
1018
1155
|
contentCache.set(candidate.mirrorHash, contentResult);
|
|
1019
1156
|
}
|
|
1020
1157
|
|
|
@@ -1038,6 +1175,7 @@ export async function searchHybrid(
|
|
|
1038
1175
|
}
|
|
1039
1176
|
|
|
1040
1177
|
for (const doc of candidateDocs) {
|
|
1178
|
+
assertInferenceActive();
|
|
1041
1179
|
if (!auxiliaryRankingActive && results.length >= assemblyLimit) break;
|
|
1042
1180
|
const filterEval = evaluateDocumentChunkFilters(
|
|
1043
1181
|
query,
|
|
@@ -1051,7 +1189,7 @@ export async function searchHybrid(
|
|
|
1051
1189
|
) {
|
|
1052
1190
|
continue;
|
|
1053
1191
|
}
|
|
1054
|
-
const docidKey = `${candidate.mirrorHash}:${candidate.seq}`;
|
|
1192
|
+
const docidKey = `${candidate.mirrorHash}:${candidate.seq}${candidate.documentId === undefined ? "" : `:${candidate.documentId}`}`;
|
|
1055
1193
|
if (!docidMap.has(docidKey)) docidMap.set(docidKey, doc.docid);
|
|
1056
1194
|
const collectionPath = collectionPaths.get(doc.collection);
|
|
1057
1195
|
if (options.full && !auxiliaryRankingActive) {
|
|
@@ -1088,9 +1226,10 @@ export async function searchHybrid(
|
|
|
1088
1226
|
record: projectRecordEvidenceMetadata(doc),
|
|
1089
1227
|
};
|
|
1090
1228
|
const auxiliaryBaseScore =
|
|
1091
|
-
auxiliaryBaseScores.get(
|
|
1092
|
-
|
|
1093
|
-
|
|
1229
|
+
auxiliaryBaseScores.get(
|
|
1230
|
+
`${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}`
|
|
1231
|
+
) ?? candidate.blendedScore;
|
|
1232
|
+
const candidateKey = `${candidate.mirrorHash}:${candidate.seq}:${candidate.documentId ?? ""}`;
|
|
1094
1233
|
const composedBeforeRerank =
|
|
1095
1234
|
preRerankAdjustedCandidates.has(candidateKey);
|
|
1096
1235
|
const scoringBaseScore =
|
|
@@ -1172,6 +1311,7 @@ export async function searchHybrid(
|
|
|
1172
1311
|
|
|
1173
1312
|
const finalResults = dedupedResults.slice(0, limit);
|
|
1174
1313
|
for (const [index, result] of finalResults.entries()) {
|
|
1314
|
+
assertInferenceActive();
|
|
1175
1315
|
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
1176
1316
|
if (metadata) metadata.retrievalRank = index + 1;
|
|
1177
1317
|
}
|
|
@@ -1202,10 +1342,10 @@ export async function searchHybrid(
|
|
|
1202
1342
|
results: finalResults,
|
|
1203
1343
|
meta: {
|
|
1204
1344
|
query,
|
|
1205
|
-
mode:
|
|
1345
|
+
mode: vectorsUsed ? "hybrid" : "bm25_only",
|
|
1206
1346
|
expanded: expansion !== null,
|
|
1207
1347
|
reranked: rerankResult.reranked,
|
|
1208
|
-
vectorsUsed
|
|
1348
|
+
vectorsUsed,
|
|
1209
1349
|
totalResults: finalResults.length,
|
|
1210
1350
|
intent: options.intent,
|
|
1211
1351
|
exclude: options.exclude,
|
|
@@ -1234,11 +1374,13 @@ export async function searchHybrid(
|
|
|
1234
1374
|
const capabilityOutcomes = [
|
|
1235
1375
|
{ capability: "lexical_search", status: "used" as const },
|
|
1236
1376
|
vectorAvailable
|
|
1237
|
-
?
|
|
1377
|
+
? !vectorsUsed
|
|
1238
1378
|
? {
|
|
1239
1379
|
capability: "semantic_search",
|
|
1240
1380
|
status: "failed" as const,
|
|
1241
|
-
reasonCode: "vector_embed_error"
|
|
1381
|
+
reasonCode: fallbackCodes.includes("vector_embed_error")
|
|
1382
|
+
? "vector_embed_error"
|
|
1383
|
+
: "vector_search_error",
|
|
1242
1384
|
}
|
|
1243
1385
|
: { capability: "semantic_search", status: "used" as const }
|
|
1244
1386
|
: {
|
|
@@ -1304,6 +1446,7 @@ export async function searchHybrid(
|
|
|
1304
1446
|
function dedupeFullResultsByDocid(results: SearchResult[]): SearchResult[] {
|
|
1305
1447
|
const bestByDocid = new Map<string, SearchResult>();
|
|
1306
1448
|
for (const result of results) {
|
|
1449
|
+
assertInferenceActive();
|
|
1307
1450
|
const existing = bestByDocid.get(result.docid);
|
|
1308
1451
|
if (!existing || result.score > existing.score) {
|
|
1309
1452
|
bestByDocid.set(result.docid, result);
|