@retinue/agentkit 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -277
- package/dist/adapters/embeddings/openai.d.ts +45 -0
- package/dist/adapters/embeddings/openai.js +109 -0
- package/dist/agents/agent.d.ts +22 -1
- package/dist/agents/agent.js +97 -11
- package/dist/agents/engine.d.ts +28 -0
- package/dist/agents/engine.js +194 -8
- package/dist/capabilities/index.d.ts +5 -1
- package/dist/capabilities/index.js +23 -0
- package/dist/capabilities/runtime.d.ts +8 -0
- package/dist/core/budget.d.ts +55 -0
- package/dist/core/budget.js +56 -0
- package/dist/core/content-parts.d.ts +8 -0
- package/dist/core/events.d.ts +68 -2
- package/dist/core/events.js +2 -0
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/documents/index.d.ts +14 -0
- package/dist/documents/parsers/text.d.ts +16 -0
- package/dist/documents/parsers/text.js +54 -2
- package/dist/entries/guardrails.d.ts +14 -0
- package/dist/entries/guardrails.js +14 -0
- package/dist/entries/knowledge.d.ts +9 -0
- package/dist/entries/knowledge.js +8 -0
- package/dist/graphql/resolvers.d.ts +4 -0
- package/dist/graphql/resolvers.js +6 -0
- package/dist/graphql/schema.d.ts +1 -1
- package/dist/graphql/schema.js +44 -0
- package/dist/guardrails/index.d.ts +115 -0
- package/dist/guardrails/index.js +108 -0
- package/dist/guardrails/moderation.d.ts +53 -0
- package/dist/guardrails/moderation.js +75 -0
- package/dist/guardrails/pii.d.ts +75 -0
- package/dist/guardrails/pii.js +193 -0
- package/dist/knowledge/index.d.ts +1 -0
- package/dist/knowledge/index.js +1 -0
- package/dist/knowledge/navigate.d.ts +89 -0
- package/dist/knowledge/navigate.js +107 -0
- package/dist/knowledge/retrieval.d.ts +73 -5
- package/dist/knowledge/retrieval.js +82 -28
- package/dist/models/streaming.d.ts +22 -1
- package/dist/models/streaming.js +5 -1
- package/dist/security/checklist.js +9 -0
- package/dist/security/findings.js +18 -9
- package/dist/skills/catalogue.d.ts +49 -0
- package/dist/skills/catalogue.js +61 -0
- package/dist/skills/index.d.ts +1 -0
- package/dist/skills/index.js +1 -0
- package/dist/telemetry/spans.js +12 -0
- package/dist/toolkit/files.d.ts +125 -0
- package/dist/toolkit/files.js +320 -0
- package/dist/toolkit/index.d.ts +4 -0
- package/dist/toolkit/index.js +2 -0
- package/dist/toolkit/sandbox.d.ts +119 -0
- package/dist/toolkit/sandbox.js +239 -0
- package/dist/toolkit/web.d.ts +13 -0
- package/dist/toolkit/web.js +7 -1
- package/dist/tools/budget.d.ts +28 -0
- package/dist/tools/budget.js +35 -0
- package/dist/tools/credentials.d.ts +57 -0
- package/dist/tools/credentials.js +54 -0
- package/dist/tools/define.d.ts +31 -0
- package/dist/tools/define.js +23 -0
- package/dist/tools/find.d.ts +109 -0
- package/dist/tools/find.js +210 -0
- package/dist/tools/index.d.ts +14 -2
- package/dist/tools/index.js +4 -0
- package/dist/tools/library/fs.d.ts +24 -0
- package/dist/tools/library/fs.js +102 -0
- package/dist/tools/library/index.d.ts +29 -2
- package/dist/tools/library/index.js +40 -0
- package/dist/tools/library/shell.d.ts +45 -0
- package/dist/tools/library/shell.js +70 -0
- package/dist/tools/meta-tools.js +8 -0
- package/dist/tools/registry.d.ts +113 -0
- package/dist/tools/registry.js +180 -4
- package/package.json +5 -1
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieval without vectors — REQ-050 (#209), task #219, AC-4.
|
|
3
|
+
*
|
|
4
|
+
* A **spike**, and the deliverable is a decision with numbers rather than a subsystem. See
|
|
5
|
+
* `docs/26-retrieval-quality.md` for what it scored.
|
|
6
|
+
*
|
|
7
|
+
* ## The idea being tested
|
|
8
|
+
*
|
|
9
|
+
* Embedding-based retrieval matches a query against fragments of text and hopes the fragments it surfaces are the
|
|
10
|
+
* ones that answer it. A person looking something up in a manual does something else entirely: they read the
|
|
11
|
+
* table of contents, decide which chapter is relevant, and then read it. That needs no index, no embedding cost
|
|
12
|
+
* and no re-indexing when a document changes — and its citations name a *document* somebody chose rather than a
|
|
13
|
+
* fragment a cosine distance surfaced.
|
|
14
|
+
*
|
|
15
|
+
* The cost is a model call per query, and latency measured in seconds rather than milliseconds.
|
|
16
|
+
*
|
|
17
|
+
* ## Two things this prototype found immediately
|
|
18
|
+
*
|
|
19
|
+
* **`KnowledgeStore` cannot enumerate its sources.** There is `listBySource`, `get`, `deleteSource` and
|
|
20
|
+
* `staleSources`, and no way to ask "what documents are in here". That is correct for a vector-based design —
|
|
21
|
+
* nothing needed it — and it is exactly what a navigating retriever needs first. So the outline arrives through
|
|
22
|
+
* a port the *host* supplies (`OutlineCatalogue`), which is honest but means this mode is not a drop-in for a
|
|
23
|
+
* deployment that already has hybrid retrieval working.
|
|
24
|
+
*
|
|
25
|
+
* **It does fit behind `RetrievalMode`**, which the issue asked to test. `createRetriever` gains one optional
|
|
26
|
+
* dependency and a fourth mode; every caller — `search_knowledge` included — is unchanged, and a deployment that
|
|
27
|
+
* has not wired a navigator gets a named refusal rather than a silent fall back to semantic search. If it had
|
|
28
|
+
* *not* fit, that would have been the finding; it fits.
|
|
29
|
+
*/
|
|
30
|
+
export const DEFAULT_MAX_SOURCES = 3;
|
|
31
|
+
export const DEFAULT_MAX_CHUNKS_PER_SOURCE = 40;
|
|
32
|
+
const reference = (chunk) => ({
|
|
33
|
+
sourceType: chunk.sourceType,
|
|
34
|
+
sourceId: chunk.sourceId,
|
|
35
|
+
chunkIndex: chunk.chunkIndex,
|
|
36
|
+
chunkId: chunk.id,
|
|
37
|
+
...(chunk.locator === undefined ? {} : { locator: chunk.locator }),
|
|
38
|
+
});
|
|
39
|
+
/**
|
|
40
|
+
* Query terms, for ordering chunks *within* the documents the chooser picked.
|
|
41
|
+
*
|
|
42
|
+
* Six lines of tokenisation rather than a second ranker: there is no fusion here and no relevance floor, because
|
|
43
|
+
* the relevance decision was already made — by a model, over titles and headings. What remains is "which part of
|
|
44
|
+
* this chapter", and term overlap answers that well enough to measure. If the eval had shown this mode worth
|
|
45
|
+
* shipping, this is the first thing to replace.
|
|
46
|
+
*/
|
|
47
|
+
const terms = (text) => [
|
|
48
|
+
...new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((word) => word.length > 2)),
|
|
49
|
+
];
|
|
50
|
+
export const createNavigator = (deps) => {
|
|
51
|
+
const maxSources = deps.maxSources ?? DEFAULT_MAX_SOURCES;
|
|
52
|
+
const maxChunks = deps.maxChunksPerSource ?? DEFAULT_MAX_CHUNKS_PER_SOURCE;
|
|
53
|
+
return {
|
|
54
|
+
id: `navigate:${deps.chooser.id}`,
|
|
55
|
+
async navigate(context, input) {
|
|
56
|
+
if (input.authSubjects.length === 0)
|
|
57
|
+
return { found: false, reason: "no-access", message: "There is no material you have access to.", mode: "navigate" };
|
|
58
|
+
const catalogue = await deps.catalogue.list({ tenantId: context.tenantId, authSubjects: input.authSubjects });
|
|
59
|
+
if (catalogue.length === 0)
|
|
60
|
+
return { found: false, reason: "nothing-indexed", message: "There is no indexed material to search yet.", mode: "navigate" };
|
|
61
|
+
const chosen = await deps.chooser.choose({ query: input.query, catalogue, limit: maxSources });
|
|
62
|
+
/**
|
|
63
|
+
* An empty choice is a real answer, and a distinct one.
|
|
64
|
+
*
|
|
65
|
+
* A model that has read the table of contents and concluded nothing there is relevant has told you
|
|
66
|
+
* something a cosine distance cannot: `no-match` rather than the least-bad chapter. This is the mode's most
|
|
67
|
+
* attractive property and the reason it is worth measuring at all.
|
|
68
|
+
*/
|
|
69
|
+
if (chosen.length === 0)
|
|
70
|
+
return { found: false, reason: "no-match", message: "Nothing in the available material covers that.", mode: "navigate" };
|
|
71
|
+
const known = new Map(catalogue.map((outline) => [outline.sourceId, outline]));
|
|
72
|
+
const wanted = input.query.toLowerCase();
|
|
73
|
+
const queryTerms = terms(input.query);
|
|
74
|
+
const hits = [];
|
|
75
|
+
for (const sourceId of chosen.slice(0, maxSources)) {
|
|
76
|
+
const outline = known.get(sourceId);
|
|
77
|
+
// A chooser naming a document that is not in the catalogue it was given is a chooser that hallucinated
|
|
78
|
+
// one. Skipped rather than fetched: fetching would be a model choosing which document to read.
|
|
79
|
+
if (outline === undefined)
|
|
80
|
+
continue;
|
|
81
|
+
const page = await deps.store.listBySource({
|
|
82
|
+
tenantId: context.tenantId,
|
|
83
|
+
sourceType: outline.sourceType,
|
|
84
|
+
sourceId,
|
|
85
|
+
limit: maxChunks,
|
|
86
|
+
});
|
|
87
|
+
for (const chunk of page.items) {
|
|
88
|
+
const content = chunk.content.toLowerCase();
|
|
89
|
+
const overlap = queryTerms.filter((term) => content.includes(term)).length;
|
|
90
|
+
hits.push({
|
|
91
|
+
chunk,
|
|
92
|
+
// Not comparable with a fused score, and deliberately so: this number orders chunks inside a chosen
|
|
93
|
+
// document and means nothing outside one.
|
|
94
|
+
score: queryTerms.length === 0 ? 0 : overlap / queryTerms.length + (content.includes(wanted) ? 1 : 0),
|
|
95
|
+
signals: ["navigate"],
|
|
96
|
+
reference: reference(chunk),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
if (hits.length === 0)
|
|
101
|
+
return { found: false, reason: "no-match", message: "The chosen documents had no readable content.", mode: "navigate" };
|
|
102
|
+
const ordered = [...hits].sort((a, b) => b.score !== a.score ? b.score - a.score : a.reference.chunkId.localeCompare(b.reference.chunkId));
|
|
103
|
+
return { found: true, hits: ordered.slice(0, input.limit), mode: "navigate" };
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
//# sourceMappingURL=navigate.js.map
|
|
@@ -14,9 +14,19 @@
|
|
|
14
14
|
*
|
|
15
15
|
* score(d) = Σ over signals of 1 / (K + rank(d))
|
|
16
16
|
*
|
|
17
|
-
* A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which is
|
|
18
|
+
* the behaviour hybrid exists for — the exact-term hit surfaces even though the semantic signal never saw it.
|
|
19
|
+
*
|
|
20
|
+
* **And on one real corpus it is worse than semantic alone.** This comment used to say hybrid "measurably beats"
|
|
21
|
+
* both parts; #219 measured it over 56 documents of technical prose and hybrid lost 11.1 points of success@5 to
|
|
22
|
+
* semantic-only — two cases out of eighteen. The mechanism is the same paragraph read the other way: RRF weights both signals equally by
|
|
23
|
+
* construction, so fusing a weak list with a strong one demotes the strong list's top hits wherever the weak one
|
|
24
|
+
* disagrees. The lexical signal is weak on natural-language questions over prose.
|
|
25
|
+
*
|
|
26
|
+
* Hybrid remains the default, deliberately — that dataset has 18 queries, one author, and **no identifier
|
|
27
|
+
* queries**, which is precisely the case hybrid exists for. But the claim in this comment was untested for months
|
|
28
|
+
* and turned out to be false where it was finally tested, so it is stated with its evidence now. See
|
|
29
|
+
* `docs/26-retrieval-quality.md`.
|
|
20
30
|
*
|
|
21
31
|
* **`K = 60`** is the value from Cormack, Clarke and Buettcher's original TREC work and the one every
|
|
22
32
|
* implementation since has used. It is large relative to the ranks that matter, which flattens the difference
|
|
@@ -30,6 +40,7 @@
|
|
|
30
40
|
import type { TenantId } from "../core/ids.js";
|
|
31
41
|
import type { KeywordIndex, KnowledgeChunk, KnowledgeSourceType, VectorIndex } from "../persistence/index.js";
|
|
32
42
|
import type { EmbeddingProvider } from "./index.js";
|
|
43
|
+
import type { Navigator } from "./navigate.js";
|
|
33
44
|
/** The rank-fusion constant. See the note above on why 60 and why rank rather than score. */
|
|
34
45
|
export declare const RRF_K = 60;
|
|
35
46
|
/** How many candidates each signal contributes before fusion. */
|
|
@@ -42,7 +53,18 @@ export declare const DEFAULT_CANDIDATES = 20;
|
|
|
42
53
|
* second-best answer and tight enough to reject a corpus that simply has nothing to say.
|
|
43
54
|
*/
|
|
44
55
|
export declare const DEFAULT_RELEVANCE_FLOOR = 0.4;
|
|
45
|
-
|
|
56
|
+
/**
|
|
57
|
+
* `navigate` is a **spike** — REQ-050 (#209), task #219, AC-4.
|
|
58
|
+
*
|
|
59
|
+
* The issue asked whether retrieval without vectors can be expressed as a fourth mode behind this interface, and
|
|
60
|
+
* "if it cannot, that is itself the finding". It can: `createRetriever` gains one optional dependency, and every
|
|
61
|
+
* caller — `search_knowledge` included — is unchanged. A deployment that has not wired a navigator gets a named
|
|
62
|
+
* refusal rather than a silent fall back to semantic search, which is the failure that would have made the mode
|
|
63
|
+
* dangerous rather than merely unused.
|
|
64
|
+
*
|
|
65
|
+
* See `navigate.ts` for what it is and `docs/26-retrieval-quality.md` for what it scored.
|
|
66
|
+
*/
|
|
67
|
+
export type RetrievalMode = "semantic" | "keyword" | "hybrid" | "navigate";
|
|
46
68
|
/** What a citation needs, derived from a hit so there is one shape rather than each caller's own (AC-6). */
|
|
47
69
|
export type SourceReference = {
|
|
48
70
|
readonly sourceType: KnowledgeSourceType;
|
|
@@ -67,7 +89,7 @@ export type RetrievalHit = {
|
|
|
67
89
|
* different answer from "you have no indexed documents", and telling a user the first when the second is true
|
|
68
90
|
* sends them looking for content they never uploaded.
|
|
69
91
|
*/
|
|
70
|
-
export declare const NO_RESULT_REASONS: readonly ["nothing-indexed", "no-match", "below-threshold", "no-access"];
|
|
92
|
+
export declare const NO_RESULT_REASONS: readonly ["nothing-indexed", "no-match", "below-threshold", "no-access", "not-configured"];
|
|
71
93
|
export type NoResultReason = (typeof NO_RESULT_REASONS)[number];
|
|
72
94
|
export type RetrievalOutcome = {
|
|
73
95
|
readonly found: true;
|
|
@@ -85,6 +107,14 @@ export type RetrievalOutcome = {
|
|
|
85
107
|
* A port, and **switchable**, because a reranker's value is a claim that has to be provable. A cross-encoder is
|
|
86
108
|
* materially more expensive than the retrieval it reorders, so "we rerank" without a measured contribution is
|
|
87
109
|
* a cost nobody justified. Absent means fusion order stands, which is the honest default.
|
|
110
|
+
*
|
|
111
|
+
* **`createExactTermReranker`'s measured contribution is negative** — #219, over 56 documents of technical prose:
|
|
112
|
+
* −5.6 points of success@5, −5.6 of recall, −0.028 MRR, and no latency saving. It promotes chunks containing query
|
|
113
|
+
* terms verbatim, which on prose queries promotes chunks that happen to repeat a common word. Leave it off.
|
|
114
|
+
*
|
|
115
|
+
* That is a result about *that* reranker, not about reranking: a cross-encoder is a different mechanism and might
|
|
116
|
+
* well earn its cost. This port is how it would be measured, and `docs/26-retrieval-quality.md` is the baseline to
|
|
117
|
+
* measure it against.
|
|
88
118
|
*/
|
|
89
119
|
export interface Reranker {
|
|
90
120
|
readonly id: string;
|
|
@@ -111,6 +141,14 @@ export type RetrieverDeps = {
|
|
|
111
141
|
* "no match"), the best of them normalises to 1.0, and every query finds something.
|
|
112
142
|
*/
|
|
113
143
|
readonly semanticFloor?: number;
|
|
144
|
+
/**
|
|
145
|
+
* Serves `mode: "navigate"` — task #219, AC-4.
|
|
146
|
+
*
|
|
147
|
+
* Optional, and its absence is a *named refusal* for that mode rather than a fall back to semantic search: a
|
|
148
|
+
* caller that asked for navigation and silently got embeddings would attribute the results to the wrong
|
|
149
|
+
* mechanism, which is the only way this spike could have done harm.
|
|
150
|
+
*/
|
|
151
|
+
readonly navigator?: Navigator;
|
|
114
152
|
};
|
|
115
153
|
export type RetrieveInput = {
|
|
116
154
|
readonly query: string;
|
|
@@ -126,6 +164,36 @@ export type RetrieveInput = {
|
|
|
126
164
|
/** Defaults to `hybrid`. The other two exist so the hybrid claim can be measured against them. */
|
|
127
165
|
readonly mode?: RetrievalMode;
|
|
128
166
|
};
|
|
167
|
+
/**
|
|
168
|
+
* Reciprocal rank fusion, extracted so there is exactly one of it — REQ-045 (#204), task #210, AC-2.
|
|
169
|
+
*
|
|
170
|
+
* `find_tools` fuses two signals over tool descriptors and this fuses two signals over knowledge chunks. Those
|
|
171
|
+
* are the same algorithm with a different corpus, and writing it twice is the shape this repository keeps
|
|
172
|
+
* finding defects in: the second copy drifts, usually in the tie-break or the normalisation, and the drift
|
|
173
|
+
* shows up as one ranker being subtly worse with nothing pointing at why.
|
|
174
|
+
*
|
|
175
|
+
* Generic over the item and its key. The **key** is what merges an item found by both signals; without it a
|
|
176
|
+
* chunk in both lists would fuse with itself and score twice.
|
|
177
|
+
*
|
|
178
|
+
* Scores come back normalised against the best fused score, because a raw RRF sum means nothing on its own —
|
|
179
|
+
* `2/61` is not "poor", it is "found first by both signals". Normalising is what lets one relevance floor apply
|
|
180
|
+
* to any corpus, tools included.
|
|
181
|
+
*/
|
|
182
|
+
export type FusedEntry<T, S extends string> = {
|
|
183
|
+
readonly item: T;
|
|
184
|
+
/** 0–1, relative to the best entry in this fusion. Comparable within one query, never across two. */
|
|
185
|
+
readonly score: number;
|
|
186
|
+
readonly signals: readonly S[];
|
|
187
|
+
};
|
|
188
|
+
export declare const fuseByRank: <T, S extends string>(input: {
|
|
189
|
+
readonly lists: readonly {
|
|
190
|
+
readonly signal: S;
|
|
191
|
+
readonly items: readonly T[];
|
|
192
|
+
}[];
|
|
193
|
+
readonly keyOf: (item: T) => string;
|
|
194
|
+
/** The rank-fusion constant. Defaults to `RRF_K`; a caller changing it should say why. */
|
|
195
|
+
readonly k?: number;
|
|
196
|
+
}) => readonly FusedEntry<T, S>[];
|
|
129
197
|
export declare const createRetriever: (deps: RetrieverDeps) => {
|
|
130
198
|
rerankerId: string | null;
|
|
131
199
|
retrieve(context: {
|
|
@@ -14,9 +14,19 @@
|
|
|
14
14
|
*
|
|
15
15
|
* score(d) = Σ over signals of 1 / (K + rank(d))
|
|
16
16
|
*
|
|
17
|
-
* A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which
|
|
18
|
-
*
|
|
19
|
-
*
|
|
17
|
+
* A document ranked first by one signal and absent from the other still beats one ranked fifth by both, which is
|
|
18
|
+
* the behaviour hybrid exists for — the exact-term hit surfaces even though the semantic signal never saw it.
|
|
19
|
+
*
|
|
20
|
+
* **And on one real corpus it is worse than semantic alone.** This comment used to say hybrid "measurably beats"
|
|
21
|
+
* both parts; #219 measured it over 56 documents of technical prose and hybrid lost 11.1 points of success@5 to
|
|
22
|
+
* semantic-only — two cases out of eighteen. The mechanism is the same paragraph read the other way: RRF weights both signals equally by
|
|
23
|
+
* construction, so fusing a weak list with a strong one demotes the strong list's top hits wherever the weak one
|
|
24
|
+
* disagrees. The lexical signal is weak on natural-language questions over prose.
|
|
25
|
+
*
|
|
26
|
+
* Hybrid remains the default, deliberately — that dataset has 18 queries, one author, and **no identifier
|
|
27
|
+
* queries**, which is precisely the case hybrid exists for. But the claim in this comment was untested for months
|
|
28
|
+
* and turned out to be false where it was finally tested, so it is stated with its evidence now. See
|
|
29
|
+
* `docs/26-retrieval-quality.md`.
|
|
20
30
|
*
|
|
21
31
|
* **`K = 60`** is the value from Cormack, Clarke and Buettcher's original TREC work and the one every
|
|
22
32
|
* implementation since has used. It is large relative to the ranks that matter, which flattens the difference
|
|
@@ -47,7 +57,46 @@ export const DEFAULT_RELEVANCE_FLOOR = 0.4;
|
|
|
47
57
|
* different answer from "you have no indexed documents", and telling a user the first when the second is true
|
|
48
58
|
* sends them looking for content they never uploaded.
|
|
49
59
|
*/
|
|
50
|
-
export const NO_RESULT_REASONS = [
|
|
60
|
+
export const NO_RESULT_REASONS = [
|
|
61
|
+
"nothing-indexed",
|
|
62
|
+
"no-match",
|
|
63
|
+
"below-threshold",
|
|
64
|
+
"no-access",
|
|
65
|
+
/**
|
|
66
|
+
* The mode asked for is not wired — task #219.
|
|
67
|
+
*
|
|
68
|
+
* Its own reason rather than `no-match`, because the two want opposite responses: one says rephrase, this says
|
|
69
|
+
* a deployment has not configured what you asked for. Falling back to another mode silently would be worse
|
|
70
|
+
* than either, since the caller would attribute the results to the mode it named.
|
|
71
|
+
*/
|
|
72
|
+
"not-configured",
|
|
73
|
+
];
|
|
74
|
+
export const fuseByRank = (input) => {
|
|
75
|
+
const k = input.k ?? RRF_K;
|
|
76
|
+
const fused = new Map();
|
|
77
|
+
for (const list of input.lists) {
|
|
78
|
+
list.items.forEach((item, rank) => {
|
|
79
|
+
const key = input.keyOf(item);
|
|
80
|
+
const increment = 1 / (k + rank + 1);
|
|
81
|
+
const existing = fused.get(key);
|
|
82
|
+
if (existing === undefined)
|
|
83
|
+
fused.set(key, { item, key, score: increment, signals: new Set([list.signal]) });
|
|
84
|
+
else {
|
|
85
|
+
existing.score += increment;
|
|
86
|
+
existing.signals.add(list.signal);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
// Key order breaks ties, so two runs over the same corpus produce the same ranking. A `Map` iteration order
|
|
91
|
+
// tie-break would depend on which signal happened to return first.
|
|
92
|
+
const ordered = [...fused.values()].sort((a, b) => (b.score !== a.score ? b.score - a.score : a.key.localeCompare(b.key)));
|
|
93
|
+
const best = ordered[0]?.score ?? 0;
|
|
94
|
+
return ordered.map((entry) => ({
|
|
95
|
+
item: entry.item,
|
|
96
|
+
score: best === 0 ? 0 : entry.score / best,
|
|
97
|
+
signals: [...entry.signals],
|
|
98
|
+
}));
|
|
99
|
+
};
|
|
51
100
|
const referenceFor = (chunk) => ({
|
|
52
101
|
sourceType: chunk.sourceType,
|
|
53
102
|
sourceId: chunk.sourceId,
|
|
@@ -60,6 +109,7 @@ const NO_RESULT_MESSAGES = {
|
|
|
60
109
|
"no-match": "Nothing in the available material matches that.",
|
|
61
110
|
"below-threshold": "Nothing in the available material is a close enough match to rely on.",
|
|
62
111
|
"no-access": "There is no material you have access to that matches that.",
|
|
112
|
+
"not-configured": "That retrieval mode is not configured for this deployment.",
|
|
63
113
|
};
|
|
64
114
|
export const createRetriever = (deps) => {
|
|
65
115
|
const candidateCount = deps.candidates ?? DEFAULT_CANDIDATES;
|
|
@@ -69,6 +119,21 @@ export const createRetriever = (deps) => {
|
|
|
69
119
|
rerankerId: deps.reranker?.id ?? null,
|
|
70
120
|
async retrieve(context, input) {
|
|
71
121
|
const mode = input.mode ?? "hybrid";
|
|
122
|
+
// The spike's mode, delegated whole: it shares no step with the fusion path below.
|
|
123
|
+
if (mode === "navigate") {
|
|
124
|
+
if (deps.navigator === undefined)
|
|
125
|
+
return {
|
|
126
|
+
found: false,
|
|
127
|
+
reason: "not-configured",
|
|
128
|
+
message: NO_RESULT_MESSAGES["not-configured"],
|
|
129
|
+
mode,
|
|
130
|
+
};
|
|
131
|
+
return deps.navigator.navigate(context, {
|
|
132
|
+
query: input.query,
|
|
133
|
+
authSubjects: input.authSubjects,
|
|
134
|
+
limit: input.limit,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
72
137
|
// Checked before either index is asked. An empty subject list is not a query with no results — it is a
|
|
73
138
|
// caller with no access, and the two want different sentences.
|
|
74
139
|
if (input.authSubjects.length === 0)
|
|
@@ -93,30 +158,19 @@ export const createRetriever = (deps) => {
|
|
|
93
158
|
const lexical = mode === "semantic" ? [] : await deps.keyword.search({ ...scope, query: input.query });
|
|
94
159
|
if (semantic.length === 0 && lexical.length === 0)
|
|
95
160
|
return { found: false, reason: "no-match", message: NO_RESULT_MESSAGES["no-match"], mode };
|
|
96
|
-
// RRF. Rank, not score: see the note at the top on why adding two
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
};
|
|
110
|
-
contribute(semantic, "semantic");
|
|
111
|
-
contribute(lexical, "keyword");
|
|
112
|
-
const ordered = [...fused.values()].sort((a, b) => b.score !== a.score ? b.score - a.score : a.chunk.id.localeCompare(b.chunk.id));
|
|
113
|
-
const best = ordered[0]?.score ?? 0;
|
|
114
|
-
const candidates = ordered.map((entry) => ({
|
|
115
|
-
chunk: entry.chunk,
|
|
116
|
-
// Normalised against the best fused score, so the floor means the same thing whatever the corpus.
|
|
117
|
-
score: best === 0 ? 0 : entry.score / best,
|
|
118
|
-
signals: [...entry.signals],
|
|
119
|
-
reference: referenceFor(entry.chunk),
|
|
161
|
+
// RRF, through the shared implementation. Rank, not score: see the note at the top on why adding two
|
|
162
|
+
// incomparable scales fails silently, and `fuseByRank` on why there is only one of these.
|
|
163
|
+
const candidates = fuseByRank({
|
|
164
|
+
lists: [
|
|
165
|
+
{ signal: "semantic", items: semantic },
|
|
166
|
+
{ signal: "keyword", items: lexical },
|
|
167
|
+
],
|
|
168
|
+
keyOf: (hit) => hit.chunk.id,
|
|
169
|
+
}).map((entry) => ({
|
|
170
|
+
chunk: entry.item.chunk,
|
|
171
|
+
score: entry.score,
|
|
172
|
+
signals: entry.signals,
|
|
173
|
+
reference: referenceFor(entry.item.chunk),
|
|
120
174
|
}));
|
|
121
175
|
const relevant = candidates.filter((hit) => hit.score >= floor);
|
|
122
176
|
if (relevant.length === 0)
|
|
@@ -61,13 +61,34 @@ export declare const nonTextCounts: (messages: readonly TurnMessage[]) => {
|
|
|
61
61
|
readonly imageCount?: number;
|
|
62
62
|
};
|
|
63
63
|
export declare const modalitiesOf: (messages: readonly TurnMessage[]) => readonly InputModality[];
|
|
64
|
+
/**
|
|
65
|
+
* What the provider tells us about a call it is making.
|
|
66
|
+
*
|
|
67
|
+
* Only the id, and only because a wrapper needs a key: an execution that resolved to a *different* tool than the
|
|
68
|
+
* one the model named — `execute_tool` — has to be able to say which call it was, or the run event log records
|
|
69
|
+
* the indirection and loses the action.
|
|
70
|
+
*/
|
|
71
|
+
export type ModelToolCallOptions = {
|
|
72
|
+
readonly toolCallId?: string;
|
|
73
|
+
/**
|
|
74
|
+
* Report what actually ran, when it is not the tool the model named.
|
|
75
|
+
*
|
|
76
|
+
* Best effort by construction: the platform's execution path knows the fact and the *host's* `execute` closure
|
|
77
|
+
* is the only thing standing between the two, so a host that does not call this leaves the field absent. Absent
|
|
78
|
+
* therefore means "nobody reported an indirection", not "there was none" — which is why the run event log keeps
|
|
79
|
+
* the model's own tool name as the primary record and treats this as an addition to it.
|
|
80
|
+
*/
|
|
81
|
+
readonly report?: (fact: {
|
|
82
|
+
readonly ranToolName: string;
|
|
83
|
+
}) => void;
|
|
84
|
+
};
|
|
64
85
|
/** A tool the model may call this turn. `execute` is the platform's guarded execution path. */
|
|
65
86
|
export type ModelTurnTool = {
|
|
66
87
|
readonly name: string;
|
|
67
88
|
readonly description?: string;
|
|
68
89
|
/** Zod schema or JSON-schema object; a permissive object schema is used when absent. */
|
|
69
90
|
readonly inputSchema?: unknown;
|
|
70
|
-
execute(input: unknown): Promise<unknown>;
|
|
91
|
+
execute(input: unknown, options?: ModelToolCallOptions): Promise<unknown>;
|
|
71
92
|
};
|
|
72
93
|
export type ModelTurnRequest = {
|
|
73
94
|
readonly model: ResolvedModel;
|
package/dist/models/streaming.js
CHANGED
|
@@ -87,7 +87,11 @@ const toToolSet = (tools) => {
|
|
|
87
87
|
: isJsonSchema(t.inputSchema)
|
|
88
88
|
? jsonSchema(t.inputSchema)
|
|
89
89
|
: jsonSchema({ type: "object", additionalProperties: true }),
|
|
90
|
-
|
|
90
|
+
// The options are *forwarded*, not dropped. Without the call id a wrapper cannot attribute what it ran,
|
|
91
|
+
// which is how a tool executed through `execute_tool` became an unattributable entry in the audit trail.
|
|
92
|
+
// The options are *forwarded*, not dropped. Without the call id a wrapper cannot attribute what it ran,
|
|
93
|
+
// which is how a tool executed through `execute_tool` became an unattributable entry in the audit trail.
|
|
94
|
+
execute: (input, options) => t.execute(input, { toolCallId: options?.toolCallId }),
|
|
91
95
|
});
|
|
92
96
|
}
|
|
93
97
|
return set;
|
|
@@ -198,6 +198,15 @@ export const CREDENTIAL_FIELD_EXEMPTIONS = [
|
|
|
198
198
|
"credential must exist in memory to authenticate; what AC-1 forbids is storing, passing, returning or " +
|
|
199
199
|
"logging one, and none of those happen here.",
|
|
200
200
|
},
|
|
201
|
+
{
|
|
202
|
+
file: "adapters/embeddings/openai.ts",
|
|
203
|
+
reason: "`OpenAiEmbeddingsConfig.apiKey` is the same shape as the model provider's, for the same reason and with " +
|
|
204
|
+
"the same limits: a key the host supplies at wiring time and this adapter puts in one Authorization " +
|
|
205
|
+
"header. Process-local — never written to a table, never in a message part or a result envelope, and no " +
|
|
206
|
+
"allowlisted log field could carry it. It is also never reachable from a model: an embedding is computed " +
|
|
207
|
+
"for a chunk the platform is indexing, not for text a model asked about, so there is no input path that " +
|
|
208
|
+
"could name or read it.",
|
|
209
|
+
},
|
|
201
210
|
];
|
|
202
211
|
/** Checks with no automated backing. The set a person must actually walk at each release. */
|
|
203
212
|
export const manualChecks = () => SECURITY_CHECKS.filter((check) => check.verifiedBy === "manual");
|
|
@@ -134,19 +134,28 @@ export const FINDINGS = [
|
|
|
134
134
|
id: "SEC-006",
|
|
135
135
|
area: "egress",
|
|
136
136
|
severity: "informational",
|
|
137
|
-
title: "
|
|
137
|
+
title: "The web-fetch path exists now, and the audit had been passing it for the wrong reason",
|
|
138
138
|
impact: "AC-2 asks for the allow-list to be enforced at a single point covered by *both* the research and MCP paths. " +
|
|
139
|
-
"
|
|
140
|
-
"
|
|
141
|
-
"
|
|
142
|
-
|
|
139
|
+
"When this was written the research path did not exist and the property held trivially. It exists now — " +
|
|
140
|
+
"REQ-039 (#188) shipped `fetch_url`, `fetch_json`, `http_request` and `http_write`, all through " +
|
|
141
|
+
"`toolkit/http.ts`, and #219 added an embedding adapter — so the property is real rather than trivial: the " +
|
|
142
|
+
"model-directed path goes through `validateHttpEgress`, which refuses private ranges, cloud metadata, " +
|
|
143
|
+
"non-https schemes, credentials in the URL and followed redirects.\n\n" +
|
|
144
|
+
"The finding that replaces the original: the audit asserting 'no other outbound call' matched `fetch(` and " +
|
|
145
|
+
"not `?? fetch`, and every injectable client in this codebase captures the global that way. So " +
|
|
146
|
+
"`toolkit/http.ts` had never appeared in that check at all. Not a vulnerability — its destination was " +
|
|
147
|
+
"policed throughout — but the check was passing it by accident, and would have passed the next such file " +
|
|
148
|
+
"the same way.",
|
|
149
|
+
foundBy: "adding a third outbound path in #219 and asking why the audit's allow-list had only ever had three " +
|
|
150
|
+
"entries when the tree had four outbound sites",
|
|
143
151
|
resolution: {
|
|
144
152
|
kind: "accepted",
|
|
145
153
|
owner: "azeem@snipe-solutions.de",
|
|
146
|
-
reason: "
|
|
147
|
-
"
|
|
148
|
-
"
|
|
149
|
-
|
|
154
|
+
reason: "The audit now matches a captured `fetch` as well as a called one, and its allow-list names five paths " +
|
|
155
|
+
"with the reason each destination is auditable. Adding a sixth fails the test and forces its author to " +
|
|
156
|
+
"say which policy governs the destination — which is the durable version of this guarantee, and is what " +
|
|
157
|
+
"the original entry was reaching for before there was anything to govern.",
|
|
158
|
+
revisitBy: "2027-06-30",
|
|
150
159
|
},
|
|
151
160
|
},
|
|
152
161
|
];
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A token ceiling on the skill catalogue — REQ-045 (#204), task #210, AC-5.
|
|
3
|
+
*
|
|
4
|
+
* Skills have exactly the tool catalogue's problem: a compact entry per skill sits in context on every turn, and
|
|
5
|
+
* the bodies already load on demand, so what is left is linear in how many skills a tenant has. 25 skills at
|
|
6
|
+
* `descriptionMaxLength` is a page of prompt before the conversation starts.
|
|
7
|
+
*
|
|
8
|
+
* The budget is `core/budget.ts` — the same one the tool catalogue uses, not a second copy.
|
|
9
|
+
*
|
|
10
|
+
* ## The notice is part of the catalogue, not part of the log
|
|
11
|
+
*
|
|
12
|
+
* The tool path can be loud in a run event because the engine assembles the tool list and the engine owns the
|
|
13
|
+
* event stream. The skill catalogue is assembled by a *context provider*, which has no event stream, so the same
|
|
14
|
+
* report reaches the same place by a different route: `truncationNotice` puts it in the text the model reads.
|
|
15
|
+
*
|
|
16
|
+
* That is arguably the stronger channel of the two. A run event tells whoever reviews the run afterwards; this
|
|
17
|
+
* tells the model *during* the turn, so it can say "there are more skills than I was shown" instead of
|
|
18
|
+
* confidently reporting that no skill exists for the job. Callers still get the report, and a host with an event
|
|
19
|
+
* stream to hand should log it too.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* The specific core modules, **not** `core/index.js`.
|
|
23
|
+
*
|
|
24
|
+
* The barrel re-exports `core/validation.ts`, which imports `zod`. Importing it from here put zod into the
|
|
25
|
+
* dependency graph of `@retinue/agentkit/persistence` — a subpath whose whole claim is that it reaches nothing
|
|
26
|
+
* outside the standard library, so a test or a prototype needs no install beyond the package. Caught by
|
|
27
|
+
* `root-import-weight.test.ts`, which walks the graph transitively; a barrel import is how that guarantee gets
|
|
28
|
+
* lost, and it is invisible in review.
|
|
29
|
+
*/
|
|
30
|
+
import { type BudgetOutcome, type TokenBudget } from "../core/budget.js";
|
|
31
|
+
import type { SkillCatalogEntry } from "./index.js";
|
|
32
|
+
/**
|
|
33
|
+
* Per-entry scaffolding: the bullet, the name emphasis, the version.
|
|
34
|
+
*
|
|
35
|
+
* Slightly larger than the tool catalogue's because a skill entry is rendered as prose in Markdown rather than
|
|
36
|
+
* as a JSON tool definition.
|
|
37
|
+
*/
|
|
38
|
+
export declare const SKILL_ENTRY_OVERHEAD_TOKENS = 8;
|
|
39
|
+
export declare const skillEntryTokens: (entry: SkillCatalogEntry) => number;
|
|
40
|
+
export declare const budgetSkillCatalogue: (entries: readonly SkillCatalogEntry[], budget: TokenBudget) => BudgetOutcome<SkillCatalogEntry>;
|
|
41
|
+
/**
|
|
42
|
+
* What the model is told when the catalogue was shortened.
|
|
43
|
+
*
|
|
44
|
+
* Names the skills rather than counting them, for the same reason the run event does: "3 more skills exist" is
|
|
45
|
+
* something a model can only ignore, while a name is something it can ask for. Empty string when nothing was
|
|
46
|
+
* dropped, so a caller can concatenate unconditionally.
|
|
47
|
+
*/
|
|
48
|
+
export declare const truncationNotice: (outcome: BudgetOutcome<SkillCatalogEntry>) => string;
|
|
49
|
+
//# sourceMappingURL=catalogue.d.ts.map
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A token ceiling on the skill catalogue — REQ-045 (#204), task #210, AC-5.
|
|
3
|
+
*
|
|
4
|
+
* Skills have exactly the tool catalogue's problem: a compact entry per skill sits in context on every turn, and
|
|
5
|
+
* the bodies already load on demand, so what is left is linear in how many skills a tenant has. 25 skills at
|
|
6
|
+
* `descriptionMaxLength` is a page of prompt before the conversation starts.
|
|
7
|
+
*
|
|
8
|
+
* The budget is `core/budget.ts` — the same one the tool catalogue uses, not a second copy.
|
|
9
|
+
*
|
|
10
|
+
* ## The notice is part of the catalogue, not part of the log
|
|
11
|
+
*
|
|
12
|
+
* The tool path can be loud in a run event because the engine assembles the tool list and the engine owns the
|
|
13
|
+
* event stream. The skill catalogue is assembled by a *context provider*, which has no event stream, so the same
|
|
14
|
+
* report reaches the same place by a different route: `truncationNotice` puts it in the text the model reads.
|
|
15
|
+
*
|
|
16
|
+
* That is arguably the stronger channel of the two. A run event tells whoever reviews the run afterwards; this
|
|
17
|
+
* tells the model *during* the turn, so it can say "there are more skills than I was shown" instead of
|
|
18
|
+
* confidently reporting that no skill exists for the job. Callers still get the report, and a host with an event
|
|
19
|
+
* stream to hand should log it too.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* The specific core modules, **not** `core/index.js`.
|
|
23
|
+
*
|
|
24
|
+
* The barrel re-exports `core/validation.ts`, which imports `zod`. Importing it from here put zod into the
|
|
25
|
+
* dependency graph of `@retinue/agentkit/persistence` — a subpath whose whole claim is that it reaches nothing
|
|
26
|
+
* outside the standard library, so a test or a prototype needs no install beyond the package. Caught by
|
|
27
|
+
* `root-import-weight.test.ts`, which walks the graph transitively; a barrel import is how that guarantee gets
|
|
28
|
+
* lost, and it is invisible in review.
|
|
29
|
+
*/
|
|
30
|
+
import { applyTokenBudget } from "../core/budget.js";
|
|
31
|
+
import { estimateTokens } from "../core/tokens.js";
|
|
32
|
+
/**
|
|
33
|
+
* Per-entry scaffolding: the bullet, the name emphasis, the version.
|
|
34
|
+
*
|
|
35
|
+
* Slightly larger than the tool catalogue's because a skill entry is rendered as prose in Markdown rather than
|
|
36
|
+
* as a JSON tool definition.
|
|
37
|
+
*/
|
|
38
|
+
export const SKILL_ENTRY_OVERHEAD_TOKENS = 8;
|
|
39
|
+
export const skillEntryTokens = (entry) => estimateTokens(`${entry.name} v${entry.version} ${entry.description}`) + SKILL_ENTRY_OVERHEAD_TOKENS;
|
|
40
|
+
export const budgetSkillCatalogue = (entries, budget) => applyTokenBudget({
|
|
41
|
+
items: entries,
|
|
42
|
+
budget,
|
|
43
|
+
tokensOf: skillEntryTokens,
|
|
44
|
+
nameOf: (entry) => entry.name,
|
|
45
|
+
});
|
|
46
|
+
/**
|
|
47
|
+
* What the model is told when the catalogue was shortened.
|
|
48
|
+
*
|
|
49
|
+
* Names the skills rather than counting them, for the same reason the run event does: "3 more skills exist" is
|
|
50
|
+
* something a model can only ignore, while a name is something it can ask for. Empty string when nothing was
|
|
51
|
+
* dropped, so a caller can concatenate unconditionally.
|
|
52
|
+
*/
|
|
53
|
+
export const truncationNotice = (outcome) => outcome.dropped.length === 0
|
|
54
|
+
? ""
|
|
55
|
+
: [
|
|
56
|
+
"",
|
|
57
|
+
`Not every skill is listed above: ${outcome.dropped.length} more exist but did not fit this turn's`,
|
|
58
|
+
`catalogue budget (${outcome.dropped.join(", ")}). If one of them is what the task needs, say so rather`,
|
|
59
|
+
"than concluding there is no skill for it.",
|
|
60
|
+
].join("\n");
|
|
61
|
+
//# sourceMappingURL=catalogue.js.map
|
package/dist/skills/index.d.ts
CHANGED
package/dist/skills/index.js
CHANGED
package/dist/telemetry/spans.js
CHANGED
|
@@ -38,6 +38,18 @@ export const SPAN_FOR_RUN_EVENT = {
|
|
|
38
38
|
"approval.decided": "hitl.approval",
|
|
39
39
|
"usage.updated": "run.step",
|
|
40
40
|
"context.compacted": "context.compact",
|
|
41
|
+
// Its own span, not `run.step`: a guardrail verdict is the boundary of a decision somebody will need to find
|
|
42
|
+
// later — "what stopped this turn" is the question a trace gets opened to answer.
|
|
43
|
+
"guardrail.verdict": "guardrail.inspect",
|
|
44
|
+
/**
|
|
45
|
+
* Its own span, and the reason is a rule this repository enforces: a span shares its first word with its
|
|
46
|
+
* event unless it is deliberately folded into `run.step`.
|
|
47
|
+
*
|
|
48
|
+
* `context.compact` was the first choice — dropping tools to fit a budget is the same *kind* of act as
|
|
49
|
+
* dropping history to fit one — and `telemetry.test.ts` rejected it. Correctly: an operator searching traces
|
|
50
|
+
* for why a turn was short of tools would not find it filed under context compaction.
|
|
51
|
+
*/
|
|
52
|
+
"catalog.truncated": "catalog.truncate",
|
|
41
53
|
};
|
|
42
54
|
/**
|
|
43
55
|
* The spans that are *not* run events.
|