@gmickel/gno 1.26.0 → 1.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/assets/skill/SKILL.md +8 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v1.26.0.zip → gno-browser-clipper-v1.27.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v1.27.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +15 -2
- package/spec/evals-agentic.md +17 -0
- package/spec/mcp.md +25 -6
- package/spec/output-schemas/ask.schema.json +3 -0
- package/spec/output-schemas/query-diagnose.schema.json +61 -4
- package/spec/output-schemas/search-results.schema.json +87 -1
- package/spec/output-schemas/status.schema.json +24 -0
- package/spec/project-profile.schema.json +3 -1
- package/src/app/context-runtime-contract.ts +4 -1
- package/src/app/context-runtime-types.ts +2 -0
- package/src/app/context-runtime.ts +26 -0
- package/src/app/verified-ask.ts +6 -1
- package/src/cli/commands/ask.ts +8 -1
- package/src/cli/commands/query.ts +6 -3
- package/src/cli/commands/search.ts +6 -1
- package/src/cli/commands/status.ts +43 -7
- package/src/cli/program.ts +2 -0
- package/src/config/content-types.ts +82 -0
- package/src/config/index.ts +8 -0
- package/src/config/project-profile.ts +8 -1
- package/src/config/types.ts +11 -2
- package/src/core/context-compiler.ts +38 -1
- package/src/core/retrieval-replay-candidate.ts +6 -1
- package/src/ingestion/sync-options.ts +6 -2
- package/src/ingestion/sync.ts +21 -29
- package/src/ingestion/types.ts +1 -1
- package/src/mcp/tools/ask.ts +1 -0
- package/src/mcp/tools/index.ts +4 -0
- package/src/mcp/tools/query.ts +4 -2
- package/src/mcp/tools/search.ts +3 -0
- package/src/mcp/tools/status.ts +4 -0
- package/src/pipeline/content-type-boost.ts +264 -0
- package/src/pipeline/diagnose.ts +46 -19
- package/src/pipeline/explain.ts +15 -2
- package/src/pipeline/hybrid.ts +170 -74
- package/src/pipeline/rerank.ts +45 -15
- package/src/pipeline/search.ts +29 -11
- package/src/pipeline/types.ts +13 -4
- package/src/pipeline/vsearch.ts +30 -10
- package/src/sdk/client.ts +19 -3
- package/src/sdk/index.ts +1 -0
- package/src/sdk/types.ts +21 -5
- package/src/serve/routes/api.ts +17 -3
- package/src/serve/status-model.ts +2 -0
- package/src/serve/status.ts +4 -0
- package/src/store/sqlite/adapter.ts +3 -0
- package/src/store/types.ts +3 -2
- package/browser-extension/artifacts/gno-browser-clipper-v1.26.0.zip.sha256 +0 -1
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { Config } from "../config/types";
|
|
9
9
|
import type { EmbeddingPort, GenerationPort, RerankPort } from "../llm/types";
|
|
10
|
-
import type { StorePort } from "../store/types";
|
|
10
|
+
import type { DocumentRow, StorePort } from "../store/types";
|
|
11
11
|
import type { VectorIndexPort } from "../store/vector/types";
|
|
12
12
|
import type {
|
|
13
13
|
ExpansionResult,
|
|
@@ -20,9 +20,16 @@ import type {
|
|
|
20
20
|
SearchResults,
|
|
21
21
|
} from "./types";
|
|
22
22
|
|
|
23
|
+
import { normalizeContentTypes } from "../config/content-types";
|
|
23
24
|
import { embedTextsWithRecovery } from "../embed/batch";
|
|
24
25
|
import { err, ok } from "../store/types";
|
|
25
26
|
import { createChunkLookup } from "./chunk-lookup";
|
|
27
|
+
import {
|
|
28
|
+
attachAuxiliaryScoreMetadata,
|
|
29
|
+
hasAuxiliaryRanking,
|
|
30
|
+
scoreContentTypeBoost,
|
|
31
|
+
sortByFinalScoreStable,
|
|
32
|
+
} from "./content-type-boost";
|
|
26
33
|
import { formatQueryForEmbedding } from "./contextual";
|
|
27
34
|
import { expandQuery } from "./expansion";
|
|
28
35
|
import {
|
|
@@ -41,11 +48,7 @@ import { evaluateDocumentChunkFilters } from "./filters";
|
|
|
41
48
|
import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
|
|
42
49
|
import { expandGraphCandidates } from "./graph-retrieval";
|
|
43
50
|
import { selectBestChunkForSteering } from "./intent";
|
|
44
|
-
import {
|
|
45
|
-
applyProjectAffinity,
|
|
46
|
-
getProjectAffinityMetadata,
|
|
47
|
-
hasProjectAffinity,
|
|
48
|
-
} from "./project-affinity";
|
|
51
|
+
import { hasProjectAffinity } from "./project-affinity";
|
|
49
52
|
import { detectQueryLanguage } from "./query-language";
|
|
50
53
|
import {
|
|
51
54
|
buildExpansionFromQueryModes,
|
|
@@ -63,7 +66,10 @@ import {
|
|
|
63
66
|
attachSearchResultPlannerMetadata,
|
|
64
67
|
attachSearchResultsTraceMetadata,
|
|
65
68
|
} from "./trace-metadata";
|
|
66
|
-
import {
|
|
69
|
+
import {
|
|
70
|
+
DEFAULT_PIPELINE_CONFIG,
|
|
71
|
+
SEARCH_RESULT_PLANNER_METADATA,
|
|
72
|
+
} from "./types";
|
|
67
73
|
|
|
68
74
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
69
75
|
// Dependencies
|
|
@@ -306,7 +312,13 @@ export async function searchHybrid(
|
|
|
306
312
|
const runStartedAt = performance.now();
|
|
307
313
|
const { store, vectorIndex, embedPort, expandPort, rerankPort } = deps;
|
|
308
314
|
const pipelineConfig = deps.pipelineConfig ?? DEFAULT_PIPELINE_CONFIG;
|
|
309
|
-
const
|
|
315
|
+
const contentTypeRules =
|
|
316
|
+
options.contentTypeRules ??
|
|
317
|
+
normalizeContentTypes(deps.config.contentTypes ?? []).rules;
|
|
318
|
+
const auxiliaryRankingActive = hasAuxiliaryRanking(
|
|
319
|
+
options.projectAffinity,
|
|
320
|
+
contentTypeRules
|
|
321
|
+
);
|
|
310
322
|
|
|
311
323
|
const limit = options.limit ?? 20;
|
|
312
324
|
const recencySort = shouldSortByRecency(query);
|
|
@@ -685,6 +697,68 @@ export async function searchHybrid(
|
|
|
685
697
|
explainFusion(pipelineConfig.rrf.k, fusedCandidates.length)
|
|
686
698
|
);
|
|
687
699
|
|
|
700
|
+
// Auxiliary scores enter after fusion normalization and before rerank
|
|
701
|
+
// blending. The reranker therefore remains the final ordering authority,
|
|
702
|
+
// including its lexical top-hit guardrail.
|
|
703
|
+
let prefetchedDocuments: DocumentRow[] | undefined;
|
|
704
|
+
const auxiliaryBaseScores = new Map<string, number>();
|
|
705
|
+
const preRerankAdjustedCandidates = new Set<string>();
|
|
706
|
+
let adjustNormalizedFusionScore:
|
|
707
|
+
| ((
|
|
708
|
+
candidate: (typeof fusedCandidates)[number],
|
|
709
|
+
normalizedScore: number
|
|
710
|
+
) => number)
|
|
711
|
+
| undefined;
|
|
712
|
+
if (auxiliaryRankingActive) {
|
|
713
|
+
const prefetchedDocumentsResult = await store.getDocumentsByMirrorHashes(
|
|
714
|
+
[...new Set(fusedCandidates.map((candidate) => candidate.mirrorHash))],
|
|
715
|
+
{
|
|
716
|
+
collection: options.collection,
|
|
717
|
+
activeOnly: true,
|
|
718
|
+
}
|
|
719
|
+
);
|
|
720
|
+
if (!prefetchedDocumentsResult.ok) {
|
|
721
|
+
return err("QUERY_FAILED", prefetchedDocumentsResult.error.message);
|
|
722
|
+
}
|
|
723
|
+
prefetchedDocuments = prefetchedDocumentsResult.value;
|
|
724
|
+
const scoringDocumentsByHash = new Map<string, DocumentRow[]>();
|
|
725
|
+
for (const document of [...prefetchedDocuments].sort((left, right) => {
|
|
726
|
+
if (left.uri !== right.uri) return left.uri.localeCompare(right.uri);
|
|
727
|
+
return left.docid.localeCompare(right.docid);
|
|
728
|
+
})) {
|
|
729
|
+
if (!document.mirrorHash) continue;
|
|
730
|
+
const documents = scoringDocumentsByHash.get(document.mirrorHash) ?? [];
|
|
731
|
+
documents.push(document);
|
|
732
|
+
scoringDocumentsByHash.set(document.mirrorHash, documents);
|
|
733
|
+
}
|
|
734
|
+
adjustNormalizedFusionScore = (candidate, normalizedScore) => {
|
|
735
|
+
const candidateKey = `${candidate.mirrorHash}:${candidate.seq}`;
|
|
736
|
+
auxiliaryBaseScores.set(candidateKey, normalizedScore);
|
|
737
|
+
const documents = scoringDocumentsByHash.get(candidate.mirrorHash);
|
|
738
|
+
if (!documents?.length) return normalizedScore;
|
|
739
|
+
const projectedScores = documents.map(
|
|
740
|
+
(document) =>
|
|
741
|
+
scoreContentTypeBoost(
|
|
742
|
+
normalizedScore,
|
|
743
|
+
document.contentType ?? undefined,
|
|
744
|
+
document.contentTypeSource,
|
|
745
|
+
document.relPath,
|
|
746
|
+
document.collection,
|
|
747
|
+
contentTypeRules,
|
|
748
|
+
options.projectAffinity,
|
|
749
|
+
{ kind: "hybrid_blended", score: normalizedScore }
|
|
750
|
+
).projectAffinity.finalScore
|
|
751
|
+
);
|
|
752
|
+
const agreedScore = projectedScores[0] ?? normalizedScore;
|
|
753
|
+
const projectionsAgree = projectedScores.every(
|
|
754
|
+
(score) => Math.abs(score - agreedScore) < 1e-9
|
|
755
|
+
);
|
|
756
|
+
if (!projectionsAgree) return normalizedScore;
|
|
757
|
+
preRerankAdjustedCandidates.add(candidateKey);
|
|
758
|
+
return agreedScore;
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
|
|
688
762
|
// ─────────────────────────────────────────────────────────────────────────
|
|
689
763
|
// 4. Reranking
|
|
690
764
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -697,6 +771,7 @@ export async function searchHybrid(
|
|
|
697
771
|
maxCandidates: candidateLimit,
|
|
698
772
|
blendingSchedule: pipelineConfig.blendingSchedule,
|
|
699
773
|
intent: options.intent,
|
|
774
|
+
adjustNormalizedFusionScore,
|
|
700
775
|
}
|
|
701
776
|
);
|
|
702
777
|
if (rerankResult.fallbackReason === "disabled") {
|
|
@@ -725,7 +800,7 @@ export async function searchHybrid(
|
|
|
725
800
|
// ─────────────────────────────────────────────────────────────────────────
|
|
726
801
|
const minScore = options.minScore ?? 0;
|
|
727
802
|
const filteredCandidates =
|
|
728
|
-
minScore > 0 && !
|
|
803
|
+
minScore > 0 && !auxiliaryRankingActive
|
|
729
804
|
? rerankResult.candidates.filter((c) => c.blendedScore >= minScore)
|
|
730
805
|
: rerankResult.candidates;
|
|
731
806
|
|
|
@@ -739,30 +814,31 @@ export async function searchHybrid(
|
|
|
739
814
|
const neededHashes = new Set(filteredCandidates.map((c) => c.mirrorHash));
|
|
740
815
|
|
|
741
816
|
// Fetch only needed documents and collections.
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
817
|
+
let documents = prefetchedDocuments;
|
|
818
|
+
if (!documents) {
|
|
819
|
+
const docsResult = await store.getDocumentsByMirrorHashes(
|
|
820
|
+
[...neededHashes],
|
|
821
|
+
{
|
|
822
|
+
collection: options.collection,
|
|
823
|
+
activeOnly: true,
|
|
824
|
+
}
|
|
825
|
+
);
|
|
826
|
+
if (!docsResult.ok) {
|
|
827
|
+
return err("QUERY_FAILED", docsResult.error.message);
|
|
828
|
+
}
|
|
829
|
+
documents = docsResult.value;
|
|
750
830
|
}
|
|
831
|
+
const collectionsResult = await store.getCollections();
|
|
751
832
|
|
|
752
833
|
// Build lookup maps.
|
|
753
|
-
const docsByMirrorHash = new Map<
|
|
754
|
-
|
|
755
|
-
(typeof docsResult.value)[number][]
|
|
756
|
-
>();
|
|
757
|
-
const addDocument = (doc: (typeof docsResult.value)[number]): void => {
|
|
834
|
+
const docsByMirrorHash = new Map<string, DocumentRow[]>();
|
|
835
|
+
const addDocument = (doc: DocumentRow): void => {
|
|
758
836
|
if (!doc.mirrorHash) return;
|
|
759
837
|
const docs = docsByMirrorHash.get(doc.mirrorHash) ?? [];
|
|
760
838
|
docs.push(doc);
|
|
761
839
|
docsByMirrorHash.set(doc.mirrorHash, docs);
|
|
762
840
|
};
|
|
763
|
-
const matchesMetadataFilters = (
|
|
764
|
-
doc: (typeof docsResult.value)[number]
|
|
765
|
-
): boolean => {
|
|
841
|
+
const matchesMetadataFilters = (doc: DocumentRow): boolean => {
|
|
766
842
|
const relPathPrefix = options.retrievalScope?.relPathPrefix;
|
|
767
843
|
if (
|
|
768
844
|
relPathPrefix !== undefined &&
|
|
@@ -798,9 +874,9 @@ export async function searchHybrid(
|
|
|
798
874
|
// Collect doc IDs that need tag filtering
|
|
799
875
|
const needsTagFilter = options.tagsAll?.length || options.tagsAny?.length;
|
|
800
876
|
const docIdsForTagCheck: number[] = [];
|
|
801
|
-
const candidateDocs:
|
|
877
|
+
const candidateDocs: DocumentRow[] = [];
|
|
802
878
|
|
|
803
|
-
for (const doc of
|
|
879
|
+
for (const doc of documents) {
|
|
804
880
|
if (!doc.mirrorHash) {
|
|
805
881
|
continue;
|
|
806
882
|
}
|
|
@@ -881,7 +957,7 @@ export async function searchHybrid(
|
|
|
881
957
|
// Iterate until we have enough results (don't slice early - deduping may skip candidates)
|
|
882
958
|
for (const [candidateIndex, candidate] of filteredCandidates.entries()) {
|
|
883
959
|
// Stop when we have enough results
|
|
884
|
-
if (!
|
|
960
|
+
if (!auxiliaryRankingActive && results.length >= assemblyLimit) {
|
|
885
961
|
break;
|
|
886
962
|
}
|
|
887
963
|
|
|
@@ -945,7 +1021,7 @@ export async function searchHybrid(
|
|
|
945
1021
|
}
|
|
946
1022
|
|
|
947
1023
|
for (const doc of candidateDocs) {
|
|
948
|
-
if (!
|
|
1024
|
+
if (!auxiliaryRankingActive && results.length >= assemblyLimit) break;
|
|
949
1025
|
const filterEval = evaluateDocumentChunkFilters(
|
|
950
1026
|
query,
|
|
951
1027
|
doc,
|
|
@@ -954,49 +1030,72 @@ export async function searchHybrid(
|
|
|
954
1030
|
);
|
|
955
1031
|
if (
|
|
956
1032
|
!filterEval.matches ||
|
|
957
|
-
(options.full && !
|
|
1033
|
+
(options.full && !auxiliaryRankingActive && seenDocids.has(doc.docid))
|
|
958
1034
|
) {
|
|
959
1035
|
continue;
|
|
960
1036
|
}
|
|
961
1037
|
const docidKey = `${candidate.mirrorHash}:${candidate.seq}`;
|
|
962
1038
|
if (!docidMap.has(docidKey)) docidMap.set(docidKey, doc.docid);
|
|
963
1039
|
const collectionPath = collectionPaths.get(doc.collection);
|
|
964
|
-
if (options.full && !
|
|
1040
|
+
if (options.full && !auxiliaryRankingActive) {
|
|
965
1041
|
seenDocids.add(doc.docid);
|
|
966
1042
|
}
|
|
967
|
-
const
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
sourceHash: doc.sourceHash,
|
|
990
|
-
},
|
|
991
|
-
conversion: {
|
|
992
|
-
mirrorHash: candidate.mirrorHash,
|
|
993
|
-
converterId: doc.converterId ?? undefined,
|
|
994
|
-
converterVersion: doc.converterVersion ?? undefined,
|
|
995
|
-
},
|
|
1043
|
+
const baseResult: SearchResult = {
|
|
1044
|
+
docid: doc.docid,
|
|
1045
|
+
score: candidate.blendedScore,
|
|
1046
|
+
uri: doc.uri,
|
|
1047
|
+
title: doc.title ?? undefined,
|
|
1048
|
+
contentType: doc.contentType ?? undefined,
|
|
1049
|
+
categories: doc.categories ?? undefined,
|
|
1050
|
+
line: snippetChunk.startLine,
|
|
1051
|
+
snippet,
|
|
1052
|
+
snippetLanguage: chunk.language ?? undefined,
|
|
1053
|
+
snippetRange,
|
|
1054
|
+
source: {
|
|
1055
|
+
relPath: doc.relPath,
|
|
1056
|
+
absPath: collectionPath
|
|
1057
|
+
? `${collectionPath}/${doc.relPath}`
|
|
1058
|
+
: undefined,
|
|
1059
|
+
mime: doc.sourceMime,
|
|
1060
|
+
ext: doc.sourceExt,
|
|
1061
|
+
modifiedAt: doc.sourceMtime,
|
|
1062
|
+
documentDate: doc.frontmatterDate ?? undefined,
|
|
1063
|
+
sizeBytes: doc.sourceSize,
|
|
1064
|
+
sourceHash: doc.sourceHash,
|
|
996
1065
|
},
|
|
1066
|
+
conversion: {
|
|
1067
|
+
mirrorHash: candidate.mirrorHash,
|
|
1068
|
+
converterId: doc.converterId ?? undefined,
|
|
1069
|
+
converterVersion: doc.converterVersion ?? undefined,
|
|
1070
|
+
},
|
|
1071
|
+
};
|
|
1072
|
+
const auxiliaryBaseScore =
|
|
1073
|
+
auxiliaryBaseScores.get(`${candidate.mirrorHash}:${candidate.seq}`) ??
|
|
1074
|
+
candidate.blendedScore;
|
|
1075
|
+
const candidateKey = `${candidate.mirrorHash}:${candidate.seq}`;
|
|
1076
|
+
const composedBeforeRerank =
|
|
1077
|
+
preRerankAdjustedCandidates.has(candidateKey);
|
|
1078
|
+
const scoringBaseScore =
|
|
1079
|
+
rerankResult.reranked && !composedBeforeRerank
|
|
1080
|
+
? candidate.blendedScore
|
|
1081
|
+
: auxiliaryBaseScore;
|
|
1082
|
+
const scored = scoreContentTypeBoost(
|
|
1083
|
+
scoringBaseScore,
|
|
1084
|
+
doc.contentType ?? undefined,
|
|
1085
|
+
doc.contentTypeSource,
|
|
1086
|
+
doc.relPath,
|
|
997
1087
|
doc.collection,
|
|
1088
|
+
contentTypeRules,
|
|
998
1089
|
options.projectAffinity,
|
|
999
|
-
{ kind: "hybrid_blended", score:
|
|
1090
|
+
{ kind: "hybrid_blended", score: scoringBaseScore }
|
|
1091
|
+
);
|
|
1092
|
+
const scoredResult = attachAuxiliaryScoreMetadata(
|
|
1093
|
+
baseResult,
|
|
1094
|
+
scored,
|
|
1095
|
+
rerankResult.reranked && composedBeforeRerank
|
|
1096
|
+
? candidate.blendedScore
|
|
1097
|
+
: scored.projectAffinity.finalScore,
|
|
1098
|
+
hasProjectAffinity(options.projectAffinity)
|
|
1000
1099
|
);
|
|
1001
1100
|
if (scoredResult.score < minScore) continue;
|
|
1002
1101
|
results.push(
|
|
@@ -1004,7 +1103,7 @@ export async function searchHybrid(
|
|
|
1004
1103
|
retrievalRank: candidateIndex + 1,
|
|
1005
1104
|
mirrorHash: candidate.mirrorHash,
|
|
1006
1105
|
seq: snippetChunk.seq,
|
|
1007
|
-
...(
|
|
1106
|
+
...(auxiliaryRankingActive && snippetChunk.seq !== candidate.seq
|
|
1008
1107
|
? { retrievalSeq: candidate.seq }
|
|
1009
1108
|
: {}),
|
|
1010
1109
|
sources: [...candidate.sources].sort(),
|
|
@@ -1030,7 +1129,7 @@ export async function searchHybrid(
|
|
|
1030
1129
|
// 7. Return results
|
|
1031
1130
|
// ─────────────────────────────────────────────────────────────────────────
|
|
1032
1131
|
const dedupedResults =
|
|
1033
|
-
options.full &&
|
|
1132
|
+
options.full && auxiliaryRankingActive
|
|
1034
1133
|
? dedupeFullResultsByDocid(results)
|
|
1035
1134
|
: results;
|
|
1036
1135
|
|
|
@@ -1049,23 +1148,20 @@ export async function searchHybrid(
|
|
|
1049
1148
|
}
|
|
1050
1149
|
return b.score - a.score;
|
|
1051
1150
|
});
|
|
1052
|
-
} else if (
|
|
1053
|
-
dedupedResults
|
|
1151
|
+
} else if (auxiliaryRankingActive && !rerankResult.reranked) {
|
|
1152
|
+
sortByFinalScoreStable(dedupedResults);
|
|
1054
1153
|
}
|
|
1055
1154
|
|
|
1056
1155
|
const finalResults = dedupedResults.slice(0, limit);
|
|
1156
|
+
for (const [index, result] of finalResults.entries()) {
|
|
1157
|
+
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
1158
|
+
if (metadata) metadata.retrievalRank = index + 1;
|
|
1159
|
+
}
|
|
1057
1160
|
const explainData = options.explain
|
|
1058
1161
|
? {
|
|
1059
1162
|
lines: explainLines,
|
|
1060
|
-
results:
|
|
1061
|
-
? buildExplainResults(filteredCandidates, docidMap, finalResults)
|
|
1062
|
-
(result, index) => ({
|
|
1063
|
-
...result,
|
|
1064
|
-
projectAffinity: getProjectAffinityMetadata(
|
|
1065
|
-
finalResults[index]!
|
|
1066
|
-
),
|
|
1067
|
-
})
|
|
1068
|
-
)
|
|
1163
|
+
results: auxiliaryRankingActive
|
|
1164
|
+
? buildExplainResults(filteredCandidates, docidMap, finalResults)
|
|
1069
1165
|
: buildExplainResults(filteredCandidates.slice(0, limit), docidMap),
|
|
1070
1166
|
}
|
|
1071
1167
|
: undefined;
|
package/src/pipeline/rerank.ts
CHANGED
|
@@ -26,6 +26,11 @@ export interface RerankOptions {
|
|
|
26
26
|
blendingSchedule?: BlendingTier[];
|
|
27
27
|
/** Optional disambiguating context for reranking */
|
|
28
28
|
intent?: string;
|
|
29
|
+
/** Apply bounded auxiliary scoring after fusion normalization, before blend. */
|
|
30
|
+
adjustNormalizedFusionScore?: (
|
|
31
|
+
candidate: FusionCandidate,
|
|
32
|
+
normalizedScore: number
|
|
33
|
+
) => number;
|
|
29
34
|
}
|
|
30
35
|
|
|
31
36
|
export interface RerankResult {
|
|
@@ -184,15 +189,41 @@ export async function rerankCandidates(
|
|
|
184
189
|
const v = (score - minFusionAll) / fusionRangeAll;
|
|
185
190
|
return Math.max(0, Math.min(1, v));
|
|
186
191
|
};
|
|
192
|
+
const adjustedFusionScore = (candidate: FusionCandidate): number => {
|
|
193
|
+
const normalized = normalizeFusionScore(candidate.fusionScore);
|
|
194
|
+
const adjusted = options.adjustNormalizedFusionScore?.(
|
|
195
|
+
candidate,
|
|
196
|
+
normalized
|
|
197
|
+
);
|
|
198
|
+
return adjusted === undefined
|
|
199
|
+
? normalized
|
|
200
|
+
: Math.max(0, Math.min(1, adjusted));
|
|
201
|
+
};
|
|
202
|
+
const sortAdjustedCandidates = (
|
|
203
|
+
adjustedCandidates: RerankedCandidate[]
|
|
204
|
+
): RerankedCandidate[] => {
|
|
205
|
+
if (!options.adjustNormalizedFusionScore) {
|
|
206
|
+
return adjustedCandidates;
|
|
207
|
+
}
|
|
208
|
+
return adjustedCandidates.sort(
|
|
209
|
+
(left, right) =>
|
|
210
|
+
right.blendedScore - left.blendedScore ||
|
|
211
|
+
`${left.mirrorHash}:${left.seq}`.localeCompare(
|
|
212
|
+
`${right.mirrorHash}:${right.seq}`
|
|
213
|
+
)
|
|
214
|
+
);
|
|
215
|
+
};
|
|
187
216
|
|
|
188
217
|
// No reranker: return candidates with normalized fusion scores
|
|
189
218
|
if (!rerankPort) {
|
|
190
219
|
return {
|
|
191
|
-
candidates:
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
220
|
+
candidates: sortAdjustedCandidates(
|
|
221
|
+
candidates.map((c) => ({
|
|
222
|
+
...c,
|
|
223
|
+
rerankScore: null,
|
|
224
|
+
blendedScore: adjustedFusionScore(c),
|
|
225
|
+
}))
|
|
226
|
+
),
|
|
196
227
|
reranked: false,
|
|
197
228
|
fallbackReason: "disabled",
|
|
198
229
|
};
|
|
@@ -239,11 +270,13 @@ export async function rerankCandidates(
|
|
|
239
270
|
|
|
240
271
|
if (!rerankResult.ok) {
|
|
241
272
|
return {
|
|
242
|
-
candidates:
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
273
|
+
candidates: sortAdjustedCandidates(
|
|
274
|
+
candidates.map((c) => ({
|
|
275
|
+
...c,
|
|
276
|
+
rerankScore: null,
|
|
277
|
+
blendedScore: adjustedFusionScore(c),
|
|
278
|
+
}))
|
|
279
|
+
),
|
|
247
280
|
reranked: false,
|
|
248
281
|
fallbackReason: "error",
|
|
249
282
|
};
|
|
@@ -277,7 +310,7 @@ export async function rerankCandidates(
|
|
|
277
310
|
rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
|
|
278
311
|
|
|
279
312
|
const position = i + 1;
|
|
280
|
-
const normalizedFusion =
|
|
313
|
+
const normalizedFusion = adjustedFusionScore(c);
|
|
281
314
|
const blendedScore =
|
|
282
315
|
normalizedRerankScore !== null
|
|
283
316
|
? blend(normalizedFusion, normalizedRerankScore, position, schedule)
|
|
@@ -292,10 +325,7 @@ export async function rerankCandidates(
|
|
|
292
325
|
...remaining.map((c) => ({
|
|
293
326
|
...c,
|
|
294
327
|
rerankScore: null,
|
|
295
|
-
blendedScore: Math.max(
|
|
296
|
-
0,
|
|
297
|
-
Math.min(1, normalizeFusionScore(c.fusionScore) * 0.5)
|
|
298
|
-
),
|
|
328
|
+
blendedScore: Math.max(0, Math.min(1, adjustedFusionScore(c) * 0.5)),
|
|
299
329
|
})),
|
|
300
330
|
];
|
|
301
331
|
|
package/src/pipeline/search.ts
CHANGED
|
@@ -18,9 +18,14 @@ import type {
|
|
|
18
18
|
import { getContentBatch } from "../store/content-batch";
|
|
19
19
|
import { err, ok } from "../store/types";
|
|
20
20
|
import { createChunkLookup } from "./chunk-lookup";
|
|
21
|
+
import {
|
|
22
|
+
applyContentTypeBoost,
|
|
23
|
+
hasAuxiliaryRanking,
|
|
24
|
+
sortByFinalScoreStable,
|
|
25
|
+
} from "./content-type-boost";
|
|
21
26
|
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
22
27
|
import { selectBestChunkForSteering } from "./intent";
|
|
23
|
-
import {
|
|
28
|
+
import { hasProjectAffinity } from "./project-affinity";
|
|
24
29
|
import { detectQueryLanguage } from "./query-language";
|
|
25
30
|
import { attachSearchResultContexts } from "./result-context";
|
|
26
31
|
import {
|
|
@@ -161,9 +166,14 @@ export async function searchBm25(
|
|
|
161
166
|
const traceStartedAt = options.traceSession ? performance.now() : 0;
|
|
162
167
|
const limit = options.limit ?? 20;
|
|
163
168
|
const minScore = options.minScore ?? 0;
|
|
164
|
-
const
|
|
169
|
+
const auxiliaryRankingActive = hasAuxiliaryRanking(
|
|
170
|
+
options.projectAffinity,
|
|
171
|
+
options.contentTypeRules
|
|
172
|
+
);
|
|
173
|
+
const projectAffinityActive = hasProjectAffinity(options.projectAffinity);
|
|
165
174
|
const recencySort = shouldSortByRecency(query);
|
|
166
|
-
const retrievalLimit =
|
|
175
|
+
const retrievalLimit =
|
|
176
|
+
recencySort || projectAffinityActive ? limit * 3 : limit;
|
|
167
177
|
const temporalRange = resolveTemporalRange(
|
|
168
178
|
query,
|
|
169
179
|
options.since,
|
|
@@ -212,7 +222,11 @@ export async function searchBm25(
|
|
|
212
222
|
const results: SearchResult[] = [];
|
|
213
223
|
const scoringByResult = new WeakMap<
|
|
214
224
|
SearchResult,
|
|
215
|
-
{
|
|
225
|
+
{
|
|
226
|
+
collection: string;
|
|
227
|
+
contentTypeSource?: string;
|
|
228
|
+
rawScore: number;
|
|
229
|
+
}
|
|
216
230
|
>();
|
|
217
231
|
|
|
218
232
|
// Pre-fetch all chunks in one batch query (eliminates N+1)
|
|
@@ -289,7 +303,7 @@ export async function searchBm25(
|
|
|
289
303
|
// For --full, de-dupe by docid (keep best scoring chunk per doc)
|
|
290
304
|
// Raw BM25: smaller (more negative) is better
|
|
291
305
|
if (options.full) {
|
|
292
|
-
if (
|
|
306
|
+
if (auxiliaryRankingActive) {
|
|
293
307
|
fullAffinityEntries.push({ fts, chunk, score: fts.score });
|
|
294
308
|
continue;
|
|
295
309
|
}
|
|
@@ -308,6 +322,7 @@ export async function searchBm25(
|
|
|
308
322
|
if (fts.collection) {
|
|
309
323
|
scoringByResult.set(result, {
|
|
310
324
|
collection: fts.collection,
|
|
325
|
+
contentTypeSource: fts.contentTypeSource,
|
|
311
326
|
rawScore: fts.score,
|
|
312
327
|
});
|
|
313
328
|
}
|
|
@@ -318,7 +333,7 @@ export async function searchBm25(
|
|
|
318
333
|
if (options.full) {
|
|
319
334
|
// Sort by raw BM25 score (smaller = better) before building results
|
|
320
335
|
const sortedEntries = (
|
|
321
|
-
|
|
336
|
+
auxiliaryRankingActive ? fullAffinityEntries : [...bestByDocid.values()]
|
|
322
337
|
).sort((a, b) => a.score - b.score);
|
|
323
338
|
const fullContentResult = await getContentBatch(
|
|
324
339
|
store,
|
|
@@ -348,6 +363,7 @@ export async function searchBm25(
|
|
|
348
363
|
if (fts.collection) {
|
|
349
364
|
scoringByResult.set(result, {
|
|
350
365
|
collection: fts.collection,
|
|
366
|
+
contentTypeSource: fts.contentTypeSource,
|
|
351
367
|
rawScore: fts.score,
|
|
352
368
|
});
|
|
353
369
|
}
|
|
@@ -358,14 +374,16 @@ export async function searchBm25(
|
|
|
358
374
|
// Normalize scores to 0-1 range (batch min-max)
|
|
359
375
|
normalizeBm25Scores(results);
|
|
360
376
|
|
|
361
|
-
if (
|
|
377
|
+
if (auxiliaryRankingActive) {
|
|
362
378
|
for (const result of results) {
|
|
363
379
|
const scoring = scoringByResult.get(result);
|
|
364
380
|
if (scoring) {
|
|
365
|
-
|
|
381
|
+
applyContentTypeBoost(
|
|
366
382
|
result,
|
|
367
383
|
scoring.collection,
|
|
384
|
+
options.contentTypeRules,
|
|
368
385
|
options.projectAffinity,
|
|
386
|
+
scoring.contentTypeSource,
|
|
369
387
|
{ kind: "bm25", score: scoring.rawScore }
|
|
370
388
|
);
|
|
371
389
|
}
|
|
@@ -373,7 +391,7 @@ export async function searchBm25(
|
|
|
373
391
|
}
|
|
374
392
|
|
|
375
393
|
const dedupedResults =
|
|
376
|
-
options.full &&
|
|
394
|
+
options.full && auxiliaryRankingActive
|
|
377
395
|
? dedupeFullResultsByDocid(results)
|
|
378
396
|
: results;
|
|
379
397
|
|
|
@@ -398,8 +416,8 @@ export async function searchBm25(
|
|
|
398
416
|
}
|
|
399
417
|
return b.score - a.score;
|
|
400
418
|
});
|
|
401
|
-
} else if (
|
|
402
|
-
filteredResults
|
|
419
|
+
} else if (auxiliaryRankingActive) {
|
|
420
|
+
sortByFinalScoreStable(filteredResults);
|
|
403
421
|
}
|
|
404
422
|
|
|
405
423
|
const finalResults = filteredResults.slice(0, limit);
|
package/src/pipeline/types.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* @module src/pipeline/types
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
+
import type { NormalizedContentTypeRule } from "../config/content-types";
|
|
8
9
|
import type {
|
|
9
10
|
ContextCapsuleV1,
|
|
10
11
|
ContextCapsuleVerification,
|
|
@@ -13,6 +14,7 @@ import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
|
|
|
13
14
|
import type { StoreResult } from "../store/types";
|
|
14
15
|
import type { ClaimVerificationResult } from "./claim-verification";
|
|
15
16
|
import type { SemanticVerificationCapability } from "./claim-verifier";
|
|
17
|
+
import type { ContentTypeBoostScoreMetadata } from "./content-type-boost";
|
|
16
18
|
import type {
|
|
17
19
|
ProjectAffinityScoreMetadata,
|
|
18
20
|
ProjectAffinityScoringInput,
|
|
@@ -132,10 +134,7 @@ export interface SearchMeta {
|
|
|
132
134
|
/** Explicit exclusion terms applied */
|
|
133
135
|
exclude?: string[];
|
|
134
136
|
/** Explain data (when --explain is used) */
|
|
135
|
-
explain?:
|
|
136
|
-
lines: ExplainLine[];
|
|
137
|
-
results: ExplainResult[];
|
|
138
|
-
};
|
|
137
|
+
explain?: SearchExplain;
|
|
139
138
|
/** Internal diagnose trace, only populated when diagnoseTrace is enabled */
|
|
140
139
|
trace?: QueryDiagnoseTrace;
|
|
141
140
|
}
|
|
@@ -172,6 +171,8 @@ export interface SearchOptions {
|
|
|
172
171
|
traceSession?: RetrievalTraceSession;
|
|
173
172
|
/** Trusted, already-resolved project affinity; never accepts raw roots. */
|
|
174
173
|
projectAffinity?: ProjectAffinityScoringInput;
|
|
174
|
+
/** Internal normalized rules used by bounded content-type scoring. */
|
|
175
|
+
contentTypeRules?: NormalizedContentTypeRule[];
|
|
175
176
|
/** Max results */
|
|
176
177
|
limit?: number;
|
|
177
178
|
/** Min score threshold (0-1) */
|
|
@@ -459,6 +460,8 @@ export interface AskMeta {
|
|
|
459
460
|
answerGenerated?: boolean;
|
|
460
461
|
totalResults?: number;
|
|
461
462
|
answerContext?: AnswerContextExplain;
|
|
463
|
+
/** Optional retrieval scoring explanation; absent from normal output. */
|
|
464
|
+
explain?: SearchExplain;
|
|
462
465
|
verificationRequested?: boolean;
|
|
463
466
|
abstained?: boolean;
|
|
464
467
|
}
|
|
@@ -545,4 +548,10 @@ export interface ExplainResult {
|
|
|
545
548
|
vecScore?: number;
|
|
546
549
|
rerankScore?: number;
|
|
547
550
|
projectAffinity?: ProjectAffinityScoreMetadata;
|
|
551
|
+
contentTypeBoost?: ContentTypeBoostScoreMetadata;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
export interface SearchExplain {
|
|
555
|
+
lines: ExplainLine[];
|
|
556
|
+
results: ExplainResult[];
|
|
548
557
|
}
|