@retinue/agentkit 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -277
- package/dist/adapters/audio/openai.d.ts +36 -0
- package/dist/adapters/audio/openai.js +136 -0
- package/dist/adapters/bullmq/lock.d.ts +26 -2
- package/dist/adapters/embeddings/openai.d.ts +45 -0
- package/dist/adapters/embeddings/openai.js +109 -0
- package/dist/adapters/memory/connections.d.ts +10 -0
- package/dist/adapters/memory/connections.js +87 -0
- package/dist/adapters/memory/graph.d.ts +17 -0
- package/dist/adapters/memory/graph.js +267 -0
- package/dist/adapters/memory/index.d.ts +3 -0
- package/dist/adapters/memory/index.js +3 -0
- package/dist/adapters/memory/rate-limit.d.ts +14 -0
- package/dist/adapters/memory/rate-limit.js +28 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/postgres/connections.d.ts +15 -0
- package/dist/adapters/postgres/connections.js +134 -0
- package/dist/adapters/postgres/graph.d.ts +24 -0
- package/dist/adapters/postgres/graph.js +325 -0
- package/dist/adapters/postgres/index.d.ts +2 -0
- package/dist/adapters/postgres/index.js +2 -0
- package/dist/adapters/postgres/migrations.js +198 -0
- package/dist/adapters/postgres/run-store.js +26 -2
- package/dist/adapters/postgres/schema.d.ts +61 -2
- package/dist/adapters/postgres/schema.js +95 -10
- package/dist/adapters/redis/index.d.ts +1 -0
- package/dist/adapters/redis/index.js +1 -0
- package/dist/adapters/redis/rate-limit.d.ts +46 -0
- package/dist/adapters/redis/rate-limit.js +55 -0
- package/dist/adapters/supabase/index.d.ts +15 -0
- package/dist/adapters/supabase/index.js +15 -0
- package/dist/adapters/supabase/rls.js +33 -0
- package/dist/agents/agent.d.ts +99 -1
- package/dist/agents/agent.js +215 -16
- package/dist/agents/define.js +24 -4
- package/dist/agents/engine.d.ts +48 -0
- package/dist/agents/engine.js +370 -13
- package/dist/audio/index.d.ts +151 -0
- package/dist/audio/index.js +183 -0
- package/dist/capabilities/index.d.ts +5 -1
- package/dist/capabilities/index.js +23 -0
- package/dist/capabilities/runtime.d.ts +8 -0
- package/dist/connections/cipher.d.ts +103 -0
- package/dist/connections/cipher.js +141 -0
- package/dist/connections/index.d.ts +156 -0
- package/dist/connections/index.js +72 -0
- package/dist/connections/oauth/client.d.ts +84 -0
- package/dist/connections/oauth/client.js +141 -0
- package/dist/connections/oauth/index.d.ts +166 -0
- package/dist/connections/oauth/index.js +226 -0
- package/dist/connections/oauth/service.d.ts +90 -0
- package/dist/connections/oauth/service.js +132 -0
- package/dist/connections/pause.d.ts +68 -0
- package/dist/connections/pause.js +97 -0
- package/dist/connections/resolver.d.ts +59 -0
- package/dist/connections/resolver.js +138 -0
- package/dist/connections/resume.d.ts +64 -0
- package/dist/connections/resume.js +99 -0
- package/dist/connections/vault-cipher.d.ts +95 -0
- package/dist/connections/vault-cipher.js +206 -0
- package/dist/core/budget.d.ts +55 -0
- package/dist/core/budget.js +56 -0
- package/dist/core/content-parts.d.ts +26 -2
- package/dist/core/content-parts.js +1 -0
- package/dist/core/context.d.ts +25 -0
- package/dist/core/errors.d.ts +1 -1
- package/dist/core/errors.js +15 -0
- package/dist/core/events.d.ts +110 -2
- package/dist/core/events.js +33 -2
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/core/validation.js +19 -0
- package/dist/documents/index.d.ts +14 -0
- package/dist/documents/parsers/text.d.ts +16 -0
- package/dist/documents/parsers/text.js +54 -2
- package/dist/entries/connections.d.ts +10 -0
- package/dist/entries/connections.js +10 -0
- package/dist/entries/guardrails.d.ts +14 -0
- package/dist/entries/guardrails.js +14 -0
- package/dist/entries/knowledge.d.ts +17 -0
- package/dist/entries/knowledge.js +16 -0
- package/dist/entries/mcp-server.d.ts +9 -0
- package/dist/entries/mcp-server.js +9 -0
- package/dist/entries/runtime.d.ts +10 -0
- package/dist/entries/runtime.js +8 -0
- package/dist/entries/testing.d.ts +28 -0
- package/dist/entries/testing.js +28 -0
- package/dist/graphql/resolvers.d.ts +7 -1
- package/dist/graphql/resolvers.js +20 -0
- package/dist/graphql/schema.d.ts +1 -1
- package/dist/graphql/schema.js +44 -0
- package/dist/guardrails/index.d.ts +115 -0
- package/dist/guardrails/index.js +108 -0
- package/dist/guardrails/moderation.d.ts +53 -0
- package/dist/guardrails/moderation.js +75 -0
- package/dist/guardrails/pii.d.ts +75 -0
- package/dist/guardrails/pii.js +193 -0
- package/dist/knowledge/communities.d.ts +166 -0
- package/dist/knowledge/communities.js +377 -0
- package/dist/knowledge/graph-global.d.ts +153 -0
- package/dist/knowledge/graph-global.js +246 -0
- package/dist/knowledge/graph-retrieval.d.ts +150 -0
- package/dist/knowledge/graph-retrieval.js +303 -0
- package/dist/knowledge/graph.d.ts +223 -0
- package/dist/knowledge/graph.js +369 -0
- package/dist/knowledge/index.d.ts +25 -0
- package/dist/knowledge/index.js +37 -1
- package/dist/knowledge/navigate.d.ts +89 -0
- package/dist/knowledge/navigate.js +107 -0
- package/dist/knowledge/retrieval.d.ts +98 -5
- package/dist/knowledge/retrieval.js +158 -28
- package/dist/mcp-server/index.d.ts +168 -0
- package/dist/mcp-server/index.js +175 -0
- package/dist/models/extraction.d.ts +73 -0
- package/dist/models/extraction.js +118 -0
- package/dist/models/index.d.ts +40 -1
- package/dist/models/index.js +22 -1
- package/dist/models/pricing.d.ts +2 -0
- package/dist/models/pricing.js +17 -1
- package/dist/models/provider-factory.js +9 -7
- package/dist/models/streaming.d.ts +111 -1
- package/dist/models/streaming.js +184 -3
- package/dist/persistence/index.d.ts +230 -0
- package/dist/persistence/index.js +2 -0
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +12 -0
- package/dist/runtime/worker.d.ts +36 -1
- package/dist/runtime/worker.js +37 -4
- package/dist/security/checklist.js +55 -0
- package/dist/security/findings.js +18 -9
- package/dist/server/bin.d.ts +19 -0
- package/dist/server/bin.js +226 -0
- package/dist/server/boot.d.ts +20 -2
- package/dist/server/boot.js +29 -3
- package/dist/server/cli.d.ts +13 -0
- package/dist/server/cli.js +7 -4
- package/dist/server/doctor.d.ts +74 -0
- package/dist/server/doctor.js +280 -0
- package/dist/skills/catalogue.d.ts +49 -0
- package/dist/skills/catalogue.js +61 -0
- package/dist/skills/context.d.ts +61 -0
- package/dist/skills/context.js +107 -0
- package/dist/skills/index.d.ts +2 -0
- package/dist/skills/index.js +2 -0
- package/dist/telemetry/spans.js +17 -0
- package/dist/testing/conformance/artifact-exports.d.ts +20 -0
- package/dist/testing/conformance/artifact-exports.js +172 -0
- package/dist/testing/conformance/artifacts.d.ts +11 -0
- package/dist/testing/conformance/artifacts.js +316 -0
- package/dist/testing/conformance/audio.d.ts +38 -0
- package/dist/testing/conformance/audio.js +145 -0
- package/dist/testing/conformance/capability.d.ts +23 -0
- package/dist/testing/conformance/capability.js +27 -0
- package/dist/testing/conformance/checkpoint-store.d.ts +15 -0
- package/dist/testing/conformance/checkpoint-store.js +72 -0
- package/dist/testing/conformance/connections.d.ts +14 -0
- package/dist/testing/conformance/connections.js +171 -0
- package/dist/testing/conformance/conversation-store.d.ts +10 -0
- package/dist/testing/conformance/conversation-store.js +60 -0
- package/dist/testing/conformance/evaluation.d.ts +10 -0
- package/dist/testing/conformance/evaluation.js +295 -0
- package/dist/testing/conformance/files.d.ts +19 -0
- package/dist/testing/conformance/files.js +454 -0
- package/dist/testing/conformance/flows.d.ts +16 -0
- package/dist/testing/conformance/flows.js +193 -0
- package/dist/testing/conformance/graph.d.ts +22 -0
- package/dist/testing/conformance/graph.js +500 -0
- package/dist/testing/conformance/hitl.d.ts +25 -0
- package/dist/testing/conformance/hitl.js +523 -0
- package/dist/testing/conformance/index.d.ts +163 -0
- package/dist/testing/conformance/index.js +321 -0
- package/dist/testing/conformance/invariants.d.ts +23 -0
- package/dist/testing/conformance/invariants.js +80 -0
- package/dist/testing/conformance/knowledge.d.ts +41 -0
- package/dist/testing/conformance/knowledge.js +592 -0
- package/dist/testing/conformance/parents.d.ts +50 -0
- package/dist/testing/conformance/parents.js +39 -0
- package/dist/testing/conformance/rate-limit.d.ts +25 -0
- package/dist/testing/conformance/rate-limit.js +71 -0
- package/dist/testing/conformance/records.d.ts +36 -0
- package/dist/testing/conformance/records.js +400 -0
- package/dist/testing/conformance/rollups.d.ts +30 -0
- package/dist/testing/conformance/rollups.js +425 -0
- package/dist/testing/conformance/run-coordinator.d.ts +27 -0
- package/dist/testing/conformance/run-coordinator.js +150 -0
- package/dist/testing/conformance/run-event-log.d.ts +9 -0
- package/dist/testing/conformance/run-event-log.js +138 -0
- package/dist/testing/conformance/run-store.d.ts +12 -0
- package/dist/testing/conformance/run-store.js +294 -0
- package/dist/testing/conformance/session-state.d.ts +38 -0
- package/dist/testing/conformance/session-state.js +198 -0
- package/dist/testing/conformance/usage-limits.d.ts +10 -0
- package/dist/testing/conformance/usage-limits.js +309 -0
- package/dist/testing/conformance.d.ts +8 -0
- package/dist/testing/conformance.js +8 -0
- package/dist/testing/memory-backend.d.ts +39 -0
- package/dist/testing/memory-backend.js +41 -0
- package/dist/testing/pglite.d.ts +55 -0
- package/dist/testing/pglite.js +78 -0
- package/dist/testing/stub-model.d.ts +58 -0
- package/dist/testing/stub-model.js +71 -0
- package/dist/testing/supabase-storage-double.d.ts +35 -0
- package/dist/testing/supabase-storage-double.js +101 -0
- package/dist/toolkit/files.d.ts +125 -0
- package/dist/toolkit/files.js +320 -0
- package/dist/toolkit/http.d.ts +27 -0
- package/dist/toolkit/http.js +48 -0
- package/dist/toolkit/index.d.ts +31 -1
- package/dist/toolkit/index.js +26 -1
- package/dist/toolkit/mime.d.ts +97 -0
- package/dist/toolkit/mime.js +233 -0
- package/dist/toolkit/sandbox.d.ts +119 -0
- package/dist/toolkit/sandbox.js +239 -0
- package/dist/toolkit/ssrf.d.ts +139 -0
- package/dist/toolkit/ssrf.js +349 -0
- package/dist/toolkit/vendor.d.ts +91 -0
- package/dist/toolkit/vendor.js +153 -0
- package/dist/toolkit/web.d.ts +13 -0
- package/dist/toolkit/web.js +7 -1
- package/dist/tools/budget.d.ts +28 -0
- package/dist/tools/budget.js +35 -0
- package/dist/tools/credentials.d.ts +316 -0
- package/dist/tools/credentials.js +369 -0
- package/dist/tools/define.d.ts +45 -0
- package/dist/tools/define.js +26 -0
- package/dist/tools/find.d.ts +109 -0
- package/dist/tools/find.js +210 -0
- package/dist/tools/index.d.ts +35 -2
- package/dist/tools/index.js +11 -0
- package/dist/tools/library/audio.d.ts +63 -0
- package/dist/tools/library/audio.js +148 -0
- package/dist/tools/library/fs.d.ts +24 -0
- package/dist/tools/library/fs.js +102 -0
- package/dist/tools/library/index.d.ts +37 -2
- package/dist/tools/library/index.js +47 -0
- package/dist/tools/library/shell.d.ts +45 -0
- package/dist/tools/library/shell.js +70 -0
- package/dist/tools/meta-tools.js +8 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +250 -4
- package/dist/usage/index.d.ts +21 -0
- package/dist/usage/index.js +1 -0
- package/dist/usage/rate-limit.d.ts +122 -0
- package/dist/usage/rate-limit.js +131 -0
- package/dist/usage/recorder.js +3 -0
- package/package.json +33 -2
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `graph-local` — entity-centric retrieval. REQ-064 (#270), task #273.
|
|
3
|
+
*
|
|
4
|
+
* The mode that answers *"which teams depend on the retry budget?"* — a question whose answer exists in no
|
|
5
|
+
* single chunk. Semantic search finds chunks that look like the question; this finds the chunks that are
|
|
6
|
+
* *connected to what the question is about*.
|
|
7
|
+
*
|
|
8
|
+
* Four steps, and the interesting decisions are in the first and the last:
|
|
9
|
+
*
|
|
10
|
+
* 1. **Resolve** the entities the question names.
|
|
11
|
+
* 2. **Traverse** their neighbourhood, bounded.
|
|
12
|
+
* 3. **Gather** the chunks those entities and edges came from.
|
|
13
|
+
* 4. **Filter and rank**, returning ordinary `RetrievalHit`s.
|
|
14
|
+
*
|
|
15
|
+
* ## Why the query is swept for n-grams rather than read by a model
|
|
16
|
+
*
|
|
17
|
+
* The obvious way to find the entities in a question is to ask a model. It costs a call per query, it is not
|
|
18
|
+
* reproducible across model versions, and #275 has to measure this against a fixed baseline.
|
|
19
|
+
*
|
|
20
|
+
* So instead every 1-to-4 word window of the question is normalised and looked up, and **the graph itself is
|
|
21
|
+
* the filter** — a window that is not an entity simply matches nothing. That is deterministic, free, and uses
|
|
22
|
+
* exactly the normalisation the index used, which is what makes AC-3 hold by construction rather than by two
|
|
23
|
+
* functions agreeing.
|
|
24
|
+
*
|
|
25
|
+
* It will miss an entity the question refers to obliquely ("the budget thing"). That is a real limit, it is
|
|
26
|
+
* measurable, and #275 is where it gets measured.
|
|
27
|
+
*
|
|
28
|
+
* ## Why there is no fallback to semantic search
|
|
29
|
+
*
|
|
30
|
+
* When nothing resolves, this returns an honest empty result. Falling back would mean a caller attributing
|
|
31
|
+
* semantic results to `graph-local` — which is precisely why `not-configured` exists as its own reason rather
|
|
32
|
+
* than as `no-match`, and the same reasoning applies one level down.
|
|
33
|
+
*/
|
|
34
|
+
import { normaliseName } from "./graph.js";
|
|
35
|
+
/**
|
|
36
|
+
* How far a traversal goes by default.
|
|
37
|
+
*
|
|
38
|
+
* One hop is often too few — "which teams depend on the retry budget" is one hop, but "what does the retry
|
|
39
|
+
* budget affect downstream" is two. Three is a large fraction of a connected graph, and the cost is not the
|
|
40
|
+
* traversal but the chunks it drags in. Two is the useful middle, and it is a parameter because the right
|
|
41
|
+
* answer is corpus-shaped.
|
|
42
|
+
*/
|
|
43
|
+
export const DEFAULT_GRAPH_DEPTH = 2;
|
|
44
|
+
/** Edges considered per hop. A hub entity can have hundreds; taking all of them buries the specific ones. */
|
|
45
|
+
export const DEFAULT_NEIGHBOUR_LIMIT = 50;
|
|
46
|
+
/** The longest phrase treated as a possible entity name. Four words covers "the retry budget policy". */
|
|
47
|
+
export const MAX_ENTITY_NGRAM = 4;
|
|
48
|
+
/**
|
|
49
|
+
* Words never worth looking up on their own.
|
|
50
|
+
*
|
|
51
|
+
* Not a linguistic stopword list — a *lookup* filter. Its only job is to stop the sweep asking the store about
|
|
52
|
+
* "the" and "which" fifty times per query. A word here can still appear *inside* a longer n-gram, so "the
|
|
53
|
+
* retry budget" resolves even though "the" does not.
|
|
54
|
+
*/
|
|
55
|
+
const SKIP_ALONE = new Set([
|
|
56
|
+
"the", "a", "an", "of", "in", "on", "at", "to", "for", "by", "with", "and", "or", "is", "are", "was", "were",
|
|
57
|
+
"be", "been", "do", "does", "did", "which", "what", "who", "whom", "whose", "when", "where", "why", "how",
|
|
58
|
+
"that", "this", "these", "those", "it", "its", "we", "our", "you", "your", "they", "their", "from", "as",
|
|
59
|
+
]);
|
|
60
|
+
/**
|
|
61
|
+
* Every phrase in the query that could name an entity, normalised and deduplicated.
|
|
62
|
+
*
|
|
63
|
+
* Longest first, which matters for ranking rather than correctness: "retry budget policy" is a more specific
|
|
64
|
+
* match than "retry budget", and a caller that stops early should see the specific one.
|
|
65
|
+
*
|
|
66
|
+
* Exported for its own test — this is the function that decides whether a real question finds anything, and
|
|
67
|
+
* its failure mode is silent (an empty result that looks like an empty corpus).
|
|
68
|
+
*/
|
|
69
|
+
export const entityCandidates = (query) => {
|
|
70
|
+
const words = normaliseName(query).split(" ").filter((word) => word !== "");
|
|
71
|
+
const seen = new Set();
|
|
72
|
+
const out = [];
|
|
73
|
+
for (let size = Math.min(MAX_ENTITY_NGRAM, words.length); size >= 1; size -= 1) {
|
|
74
|
+
for (let start = 0; start + size <= words.length; start += 1) {
|
|
75
|
+
const phrase = words.slice(start, start + size).join(" ");
|
|
76
|
+
if (size === 1 && SKIP_ALONE.has(phrase))
|
|
77
|
+
continue;
|
|
78
|
+
if (seen.has(phrase))
|
|
79
|
+
continue;
|
|
80
|
+
seen.add(phrase);
|
|
81
|
+
out.push(phrase);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
};
|
|
86
|
+
/** Gathered wide, then ranked. Below this a re-rank has too little to choose between. */
|
|
87
|
+
export const DEFAULT_POOL_FACTOR = 8;
|
|
88
|
+
export const MIN_POOL = 40;
|
|
89
|
+
/**
|
|
90
|
+
* How much an entity's breadth discounts it — the hub problem, in one line.
|
|
91
|
+
*
|
|
92
|
+
* `docs/01-architecture.md` and `concepts/architecture.md` appeared in miss after miss. Generic entities
|
|
93
|
+
* ("run", "tool", "platform") are extracted from nearly every chunk, link to nearly everything, and drag the
|
|
94
|
+
* architecture overview into every traversal. An entity that touches half the corpus tells you almost nothing
|
|
95
|
+
* about *which* chunk you want, and counting it equally with a specific one is the mechanism by which the
|
|
96
|
+
* overview outranks the page that answers the question.
|
|
97
|
+
*
|
|
98
|
+
* A logarithmic discount rather than an IDF proper, because IDF needs the corpus size and this needs only what
|
|
99
|
+
* is already in hand. An entity in 2 chunks keeps most of its weight; one in 200 keeps about a seventh of it.
|
|
100
|
+
*/
|
|
101
|
+
export const hubDiscount = (provenanceSize) => 1 / Math.log2(2 + Math.max(0, provenanceSize));
|
|
102
|
+
/** Cosine similarity. Vectors from one provider share a dimension, and a mismatch is a bug worth surfacing. */
|
|
103
|
+
export const cosine = (a, b) => {
|
|
104
|
+
if (a.length !== b.length)
|
|
105
|
+
return 0;
|
|
106
|
+
let dot = 0;
|
|
107
|
+
let normA = 0;
|
|
108
|
+
let normB = 0;
|
|
109
|
+
for (let index = 0; index < a.length; index += 1) {
|
|
110
|
+
const x = a[index];
|
|
111
|
+
const y = b[index];
|
|
112
|
+
dot += x * y;
|
|
113
|
+
normA += x * x;
|
|
114
|
+
normB += y * y;
|
|
115
|
+
}
|
|
116
|
+
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
|
117
|
+
return magnitude === 0 ? 0 : dot / magnitude;
|
|
118
|
+
};
|
|
119
|
+
export const createGraphLocalSearch = (deps) => {
|
|
120
|
+
const maxDepth = Math.max(1, deps.depth ?? DEFAULT_GRAPH_DEPTH);
|
|
121
|
+
const neighbourLimit = Math.max(1, deps.neighbourLimit ?? DEFAULT_NEIGHBOUR_LIMIT);
|
|
122
|
+
return {
|
|
123
|
+
async search(context, input) {
|
|
124
|
+
const rankedBy = deps.embeddings === undefined ? "connectivity" : "relevance";
|
|
125
|
+
const empty = {
|
|
126
|
+
hits: [],
|
|
127
|
+
matchedEntities: [],
|
|
128
|
+
relationships: [],
|
|
129
|
+
truncated: false,
|
|
130
|
+
rankedBy,
|
|
131
|
+
};
|
|
132
|
+
if (input.authSubjects.length === 0)
|
|
133
|
+
return empty;
|
|
134
|
+
const candidates = entityCandidates(input.query);
|
|
135
|
+
if (candidates.length === 0)
|
|
136
|
+
return empty;
|
|
137
|
+
const matched = await deps.graph.resolveEntities({
|
|
138
|
+
tenantId: context.tenantId,
|
|
139
|
+
normalisedNames: candidates,
|
|
140
|
+
});
|
|
141
|
+
// No fallback. See the header: semantic results wearing this mode's label is the failure to avoid.
|
|
142
|
+
if (matched.length === 0)
|
|
143
|
+
return empty;
|
|
144
|
+
/**
|
|
145
|
+
* Breadth-first, bounded on both axes.
|
|
146
|
+
*
|
|
147
|
+
* `truncated` is set when a hop returned exactly its limit, which is the honest signal: the store
|
|
148
|
+
* returned as much as it was allowed to, so there may be more. Inferring truncation from a short page
|
|
149
|
+
* would be wrong in the other direction.
|
|
150
|
+
*/
|
|
151
|
+
const seedIds = matched.map((entity) => entity.id);
|
|
152
|
+
const reached = new Set(seedIds);
|
|
153
|
+
const edges = new Map();
|
|
154
|
+
let frontier = seedIds;
|
|
155
|
+
let truncated = false;
|
|
156
|
+
for (let hop = 0; hop < maxDepth && frontier.length > 0; hop += 1) {
|
|
157
|
+
const found = await deps.graph.neighbours({
|
|
158
|
+
tenantId: context.tenantId,
|
|
159
|
+
entityIds: frontier,
|
|
160
|
+
limit: neighbourLimit,
|
|
161
|
+
});
|
|
162
|
+
if (found.length >= neighbourLimit)
|
|
163
|
+
truncated = true;
|
|
164
|
+
const next = [];
|
|
165
|
+
for (const edge of found) {
|
|
166
|
+
edges.set(edge.id, edge);
|
|
167
|
+
for (const id of [edge.fromId, edge.toId]) {
|
|
168
|
+
if (reached.has(id))
|
|
169
|
+
continue;
|
|
170
|
+
reached.add(id);
|
|
171
|
+
next.push(id);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
// Sorted so a traversal is reproducible: `neighbours` is ordered, but the set of *new* ids depends on
|
|
175
|
+
// what was already reached, and an unsorted frontier makes the next hop's `limit` cut differently.
|
|
176
|
+
frontier = next.sort();
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Chunks, and **which entities led to each** — the scoring signal.
|
|
180
|
+
*
|
|
181
|
+
* A chunk named by three of the question's entities is more relevant than one named by a single
|
|
182
|
+
* neighbour two hops out, and that is knowable without embeddings. Seeds count for more than entities
|
|
183
|
+
* discovered by traversal, because the question actually mentioned them.
|
|
184
|
+
*/
|
|
185
|
+
const SEED_WEIGHT = 3;
|
|
186
|
+
const chunkScores = new Map();
|
|
187
|
+
const credit = (chunkId, entityName, weight) => {
|
|
188
|
+
const existing = chunkScores.get(chunkId);
|
|
189
|
+
if (existing === undefined)
|
|
190
|
+
chunkScores.set(chunkId, { score: weight, via: new Set([entityName]) });
|
|
191
|
+
else {
|
|
192
|
+
existing.score += weight;
|
|
193
|
+
existing.via.add(entityName);
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
const seedIdSet = new Set(seedIds);
|
|
197
|
+
const reachedEntities = await deps.graph.getEntities({ tenantId: context.tenantId, ids: [...reached].sort() });
|
|
198
|
+
for (const entity of reachedEntities) {
|
|
199
|
+
/**
|
|
200
|
+
* Discounted by how many chunks the entity appears in — #277, AC-2.
|
|
201
|
+
*
|
|
202
|
+
* Without it, an entity extracted from nearly every chunk contributes to nearly every chunk's score,
|
|
203
|
+
* and the documents that mention everything — the architecture overviews — win by breadth. The
|
|
204
|
+
* discount is applied to seeds too: a question that names "run" has named a hub, and the hub is no
|
|
205
|
+
* more discriminating for having been asked about.
|
|
206
|
+
*/
|
|
207
|
+
const weight = (seedIdSet.has(entity.id) ? SEED_WEIGHT : 1) * hubDiscount(entity.provenance.length);
|
|
208
|
+
for (const chunkId of entity.provenance)
|
|
209
|
+
credit(chunkId, entity.name, weight);
|
|
210
|
+
}
|
|
211
|
+
for (const edge of edges.values()) {
|
|
212
|
+
// An edge's own provenance is where the *relationship* was stated, which is usually the chunk that
|
|
213
|
+
// answers a "how are these connected" question — so it is credited too.
|
|
214
|
+
for (const chunkId of edge.provenance)
|
|
215
|
+
credit(chunkId, `${edge.fromId} → ${edge.toId}`, 1);
|
|
216
|
+
}
|
|
217
|
+
const poolSize = Math.max(input.limit, MIN_POOL, input.limit * Math.max(1, deps.poolFactor ?? DEFAULT_POOL_FACTOR));
|
|
218
|
+
const wantedTypes = input.sourceTypes === undefined ? null : new Set(input.sourceTypes);
|
|
219
|
+
const allowed = new Set(input.authSubjects);
|
|
220
|
+
const gathered = [];
|
|
221
|
+
// Sorted before fetching, so the chunks fetched under a cap are the same ones every run.
|
|
222
|
+
const ranked = [...chunkScores.entries()].sort((a, b) => b[1].score - a[1].score || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
|
|
223
|
+
for (const [chunkId, scored] of ranked) {
|
|
224
|
+
const chunk = await deps.knowledge.get({ tenantId: context.tenantId, id: chunkId });
|
|
225
|
+
if (chunk === null)
|
|
226
|
+
continue;
|
|
227
|
+
/**
|
|
228
|
+
* **AC-8, and the whole reason this loop does not trust the graph.**
|
|
229
|
+
*
|
|
230
|
+
* `KnowledgeStore.get` returns a chunk by id with no permission filter — it is a primary-key read, and
|
|
231
|
+
* the vector index is where filtering normally lives. A graph traversal can arrive at a chunk from an
|
|
232
|
+
* entity mentioned by a document the principal *can* read, so without this check an edge becomes a way
|
|
233
|
+
* to reach text they cannot. That is the most plausible way a graph leaks, and it is one line.
|
|
234
|
+
*/
|
|
235
|
+
if (!allowed.has(chunk.authSubject))
|
|
236
|
+
continue;
|
|
237
|
+
if (wantedTypes !== null && !wantedTypes.has(chunk.sourceType))
|
|
238
|
+
continue;
|
|
239
|
+
gathered.push({ chunk, score: scored.score, via: [...scored.via].sort() });
|
|
240
|
+
/**
|
|
241
|
+
* A **pool**, not the answer — #277.
|
|
242
|
+
*
|
|
243
|
+
* This used to stop at `input.limit`, which meant the top-k by connectivity were the only chunks that
|
|
244
|
+
* could ever be returned and no re-ranking could reach past them. Gathering wider costs a few more
|
|
245
|
+
* primary-key reads and is what makes ranking able to change anything.
|
|
246
|
+
*/
|
|
247
|
+
if (gathered.length >= poolSize) {
|
|
248
|
+
if (ranked.length > gathered.length)
|
|
249
|
+
truncated = true;
|
|
250
|
+
break;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (gathered.length === 0)
|
|
254
|
+
return { ...empty, matchedEntities: matched.map((entity) => entity.name) };
|
|
255
|
+
/**
|
|
256
|
+
* **The graph selected; embeddings rank** — #277, AC-1.
|
|
257
|
+
*
|
|
258
|
+
* The candidate set is exactly what the traversal found: this changes the order, never the membership.
|
|
259
|
+
* That is the whole claim of the mode — these chunks are connected to what you asked about — and
|
|
260
|
+
* replacing the candidates with a semantic search would be a different mode wearing this one's label.
|
|
261
|
+
*
|
|
262
|
+
* **The vector-privacy constraint is respected rather than worked around.** `KnowledgeStore` read paths
|
|
263
|
+
* deliberately never return vectors, so the stored embeddings are not available here and the port is not
|
|
264
|
+
* widened to leak them. The chunk *text* is already in hand — it was fetched to be returned — so the
|
|
265
|
+
* candidates are embedded from their text, in one batched call alongside the query.
|
|
266
|
+
*
|
|
267
|
+
* The cost is one extra embedding call per query, and it is real: see `docs/29`. A mode whose selling
|
|
268
|
+
* point was two milliseconds does not get to spend that silently.
|
|
269
|
+
*/
|
|
270
|
+
let ordered = gathered;
|
|
271
|
+
if (deps.embeddings !== undefined) {
|
|
272
|
+
const vectors = await deps.embeddings.embed([input.query, ...gathered.map((entry) => entry.chunk.content)]);
|
|
273
|
+
const queryVector = vectors[0];
|
|
274
|
+
if (queryVector !== undefined && vectors.length === gathered.length + 1) {
|
|
275
|
+
ordered = gathered
|
|
276
|
+
.map((entry, index) => ({ ...entry, score: cosine(queryVector, vectors[index + 1]) }))
|
|
277
|
+
// Ties broken by chunk id, so the order is reproducible across runs like every other mode's.
|
|
278
|
+
.sort((a, b) => b.score - a.score || (a.chunk.id < b.chunk.id ? -1 : a.chunk.id > b.chunk.id ? 1 : 0));
|
|
279
|
+
}
|
|
280
|
+
// A provider that returned the wrong count is a bug, and the connectivity order is a worse answer
|
|
281
|
+
// rather than a wrong one — so the traversal still returns something rather than throwing.
|
|
282
|
+
}
|
|
283
|
+
const hits = ordered.slice(0, input.limit);
|
|
284
|
+
if (ordered.length > hits.length)
|
|
285
|
+
truncated = true;
|
|
286
|
+
// Normalised against the best, so the score means the same thing it does in every other mode.
|
|
287
|
+
const best = hits[0]?.score ?? 1;
|
|
288
|
+
return {
|
|
289
|
+
hits: hits.map((entry) => ({
|
|
290
|
+
chunk: entry.chunk,
|
|
291
|
+
score: best === 0 ? 0 : entry.score / best,
|
|
292
|
+
viaEntities: entry.via,
|
|
293
|
+
})),
|
|
294
|
+
matchedEntities: matched.map((entity) => entity.name),
|
|
295
|
+
// Only the edges whose endpoints were both reached, sorted so the output is stable.
|
|
296
|
+
relationships: [...edges.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)),
|
|
297
|
+
truncated,
|
|
298
|
+
rankedBy,
|
|
299
|
+
};
|
|
300
|
+
},
|
|
301
|
+
};
|
|
302
|
+
};
|
|
303
|
+
//# sourceMappingURL=graph-retrieval.js.map
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The knowledge graph — REQ-064 (#270), task #271.
|
|
3
|
+
*
|
|
4
|
+
* Retrieval finds the chunks that *look like* the question. That cannot answer "which teams depend on the retry
|
|
5
|
+
* budget?", because no chunk says so: the fact is spread across a dozen documents, each naming one team and one
|
|
6
|
+
* dependency. This module builds the structure that does — entities, the relationships between them, and where
|
|
7
|
+
* each came from.
|
|
8
|
+
*
|
|
9
|
+
* Nothing here retrieves anything. #273 and #274 do that; this produces the graph and, just as importantly,
|
|
10
|
+
* proves it costs nothing when nobody asked for it.
|
|
11
|
+
*
|
|
12
|
+
* ## Three properties everything else rests on
|
|
13
|
+
*
|
|
14
|
+
* **Off costs nothing.** The extractor is optional in the dependencies and gated twice more at run time. A
|
|
15
|
+
* deployment that never enables GraphRAG supplies no extractor, makes no model calls, and writes no graph rows
|
|
16
|
+
* — measured by a test that counts calls rather than by reading this paragraph.
|
|
17
|
+
*
|
|
18
|
+
* **Determinism.** The same corpus produces byte-identical graphs twice. This is not tidiness: #275 measures
|
|
19
|
+
* GraphRAG against a fixed baseline, and a graph that shifts between runs cannot be measured, cannot be
|
|
20
|
+
* debugged, and cannot have its incremental rebuild (#272) verified. Every ordering here is explicit, and
|
|
21
|
+
* entity resolution is rule-based rather than similarity-based for exactly this reason — embedding similarity
|
|
22
|
+
* is not stable across model versions.
|
|
23
|
+
*
|
|
24
|
+
* **Provenance.** Every entity and edge records its chunks. The retriever will present graph-derived material
|
|
25
|
+
* as citable, so an untraceable claim is one the model states as though a document said it.
|
|
26
|
+
*
|
|
27
|
+
* ## What extraction is allowed to do to us
|
|
28
|
+
*
|
|
29
|
+
* Extraction is a model call, so it can return prose, malformed JSON, entities with no names, or an edge
|
|
30
|
+
* between two things it never mentioned. **None of that may corrupt the graph or fail the index.** A chunk
|
|
31
|
+
* whose extraction is unusable simply contributes nothing — it stays embedded and findable by every existing
|
|
32
|
+
* mode, and the graph is the only thing missing. `sanitiseExtraction` is where that judgement lives, and it is
|
|
33
|
+
* deliberately strict: an edge naming an entity that was not extracted is dropped rather than repaired,
|
|
34
|
+
* because repairing it would invent a provenance nobody asserted.
|
|
35
|
+
*/
|
|
36
|
+
import type { TenantId } from "../core/ids.js";
|
|
37
|
+
import type { GraphContribution, GraphStore, KnowledgeSourceType } from "../persistence/index.js";
|
|
38
|
+
/**
|
|
39
|
+
* What an extractor returns for one chunk, before validation.
|
|
40
|
+
*
|
|
41
|
+
* Loose on purpose — it is the output of a language model, and typing it strictly here would only mean casting
|
|
42
|
+
* somewhere less visible. `sanitiseExtraction` is the boundary where it becomes trustworthy.
|
|
43
|
+
*/
|
|
44
|
+
export type RawExtraction = {
|
|
45
|
+
readonly entities?: readonly {
|
|
46
|
+
readonly name?: unknown;
|
|
47
|
+
readonly type?: unknown;
|
|
48
|
+
readonly description?: unknown;
|
|
49
|
+
}[];
|
|
50
|
+
readonly relationships?: readonly {
|
|
51
|
+
readonly from?: unknown;
|
|
52
|
+
readonly to?: unknown;
|
|
53
|
+
readonly type?: unknown;
|
|
54
|
+
readonly description?: unknown;
|
|
55
|
+
}[];
|
|
56
|
+
};
|
|
57
|
+
export type ExtractionChunk = {
|
|
58
|
+
readonly id: string;
|
|
59
|
+
readonly content: string;
|
|
60
|
+
};
|
|
61
|
+
/**
|
|
62
|
+
* Turns a chunk into entities and relationships.
|
|
63
|
+
*
|
|
64
|
+
* A port separate from `EmbeddingProvider`, not a widening of it. Embedding is cheap, deterministic and
|
|
65
|
+
* batchable; extraction is none of those. Keeping them apart is what lets a deployment with GraphRAG off supply
|
|
66
|
+
* an embedder and nothing else — if this were one port, every deployment would have to configure a language
|
|
67
|
+
* model to index a document.
|
|
68
|
+
*
|
|
69
|
+
* `usage` is optional and reported when the provider gives it, because AC-9 needs the bill to be visible before
|
|
70
|
+
* it arrives rather than reconstructed from an invoice.
|
|
71
|
+
*/
|
|
72
|
+
export interface EntityExtractor {
|
|
73
|
+
readonly id: string;
|
|
74
|
+
extract(chunk: ExtractionChunk): Promise<{
|
|
75
|
+
readonly extraction: RawExtraction;
|
|
76
|
+
readonly usage?: {
|
|
77
|
+
readonly inputTokens?: number;
|
|
78
|
+
readonly outputTokens?: number;
|
|
79
|
+
};
|
|
80
|
+
}>;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* A name reduced to what makes two spellings the same thing.
|
|
84
|
+
*
|
|
85
|
+
* Case, whitespace, separators and a leading article. That gets "the retry budget", "Retry Budget" and
|
|
86
|
+
* "retry-budget" to one entity, which is the merge AC-4 names.
|
|
87
|
+
*
|
|
88
|
+
* **What it deliberately does not do** is anything cleverer. Stemming would merge "billing" with "bill";
|
|
89
|
+
* embedding similarity would merge whatever the model of the week thinks is close and would stop being
|
|
90
|
+
* reproducible the moment that model changed. Both would make the graph unmeasurable, and the second would
|
|
91
|
+
* make it undebuggable too.
|
|
92
|
+
*
|
|
93
|
+
* Parenthetical qualifiers survive as words, so "Ana (engineering)" and "Ana (sales)" stay distinct — which
|
|
94
|
+
* AC-4 asks for, and which a punctuation-stripping rule gets right only by accident. It is worth being explicit
|
|
95
|
+
* that this is *why* punctuation becomes a space rather than nothing.
|
|
96
|
+
*/
|
|
97
|
+
export declare const normaliseName: (raw: string) => string;
|
|
98
|
+
/** An entity type, normalised the same way but without the article rule — a type is one word in practice. */
|
|
99
|
+
export declare const normaliseType: (raw: string) => string;
|
|
100
|
+
/**
|
|
101
|
+
* The identity of an entity: its type and its normalised name.
|
|
102
|
+
*
|
|
103
|
+
* Readable rather than hashed, deliberately. `concept:retry budget` in a log, an error message or a database
|
|
104
|
+
* row tells whoever is reading it what went wrong; `sha256:9f2a…` tells them to go and write a script. The id
|
|
105
|
+
* is not a secret and its length is not a problem at this scale.
|
|
106
|
+
*
|
|
107
|
+
* Type participates because a person and a project can share a name and are not the same thing.
|
|
108
|
+
*/
|
|
109
|
+
export declare const entityId: (type: string, name: string) => string;
|
|
110
|
+
/** The identity of an edge. Direction is part of it: "A depends on B" is not "B depends on A". */
|
|
111
|
+
export declare const relationshipId: (fromId: string, type: string, toId: string) => string;
|
|
112
|
+
/**
|
|
113
|
+
* How long a description may be before it is a chunk rather than a description.
|
|
114
|
+
*
|
|
115
|
+
* A model asked for a short description sometimes returns the paragraph. Truncating rather than refusing keeps
|
|
116
|
+
* the entity — the name and the edges are the valuable part — and stops one verbose extraction from dominating
|
|
117
|
+
* the storage of a whole corpus.
|
|
118
|
+
*/
|
|
119
|
+
export declare const MAX_DESCRIPTION_CHARS = 480;
|
|
120
|
+
/**
|
|
121
|
+
* One chunk's raw extraction, turned into rows that can be trusted — or into nothing.
|
|
122
|
+
*
|
|
123
|
+
* Total: any input produces a valid contribution, possibly empty. That is what makes AC-7 hold everywhere at
|
|
124
|
+
* once rather than at each call site.
|
|
125
|
+
*
|
|
126
|
+
* The strict part is edges. An extractor routinely names an endpoint it did not extract — a pronoun it
|
|
127
|
+
* resolved, a thing it inferred — and the tempting fix is to create the missing entity. That invents an entity
|
|
128
|
+
* with a provenance nobody asserted, which is the exact failure provenance exists to prevent. So the edge is
|
|
129
|
+
* dropped, and the chunk keeps whatever else it gave us.
|
|
130
|
+
*/
|
|
131
|
+
export declare const sanitiseExtraction: (chunkId: string, raw: RawExtraction) => GraphContribution;
|
|
132
|
+
/**
|
|
133
|
+
* Many chunks' contributions, merged into one source's contribution.
|
|
134
|
+
*
|
|
135
|
+
* Where surface forms accumulate and edge weights are counted. The canonical `name` is the one that sorts
|
|
136
|
+
* first among everything merged — an arbitrary rule, chosen because it is *stable*: "first seen" would depend
|
|
137
|
+
* on chunk order, and chunk order is a thing that can change without the document changing.
|
|
138
|
+
*/
|
|
139
|
+
export declare const mergeContributions: (parts: readonly GraphContribution[]) => GraphContribution;
|
|
140
|
+
/** What indexing a source into the graph cost and produced — AC-9. */
|
|
141
|
+
export type GraphIndexResult = {
|
|
142
|
+
/** `false` when GraphRAG is off for the tenant or the source is not flagged. Nothing was called. */
|
|
143
|
+
readonly ran: boolean;
|
|
144
|
+
readonly entities: number;
|
|
145
|
+
readonly relationships: number;
|
|
146
|
+
readonly pruned: number;
|
|
147
|
+
/** Model calls made. The number an operator needs before enabling this on a large corpus. */
|
|
148
|
+
readonly extractionCalls: number;
|
|
149
|
+
readonly inputTokens: number;
|
|
150
|
+
readonly outputTokens: number;
|
|
151
|
+
/** Chunks whose extraction was unusable. Nonzero is a signal about the extractor, not a failure. */
|
|
152
|
+
readonly unusableChunks: number;
|
|
153
|
+
readonly elapsedMs: number;
|
|
154
|
+
};
|
|
155
|
+
export type GraphIndexerDeps = {
|
|
156
|
+
readonly store: GraphStore;
|
|
157
|
+
readonly extractor: EntityExtractor;
|
|
158
|
+
readonly now?: () => number;
|
|
159
|
+
readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
|
|
160
|
+
};
|
|
161
|
+
/**
|
|
162
|
+
* Extraction runs one chunk at a time by default.
|
|
163
|
+
*
|
|
164
|
+
* Sequential rather than parallel because the cost is the point: a corpus of 603 chunks is 603 model calls, and
|
|
165
|
+
* firing them concurrently is how a re-index becomes a rate-limit incident. A deployment that wants throughput
|
|
166
|
+
* raises this deliberately, having read the number.
|
|
167
|
+
*/
|
|
168
|
+
export declare const DEFAULT_EXTRACTION_CONCURRENCY = 1;
|
|
169
|
+
export declare const createGraphIndexer: (deps: GraphIndexerDeps) => {
|
|
170
|
+
extractorId: string;
|
|
171
|
+
/**
|
|
172
|
+
* Whether this source would be graph-indexed — both switches, in the cheap order.
|
|
173
|
+
*
|
|
174
|
+
* The tenant switch first, so a deployment with GraphRAG off does one settings read and stops, rather than
|
|
175
|
+
* one read per source. AC-2's "flag is inert while the tenant switch is off" is this ordering: the source
|
|
176
|
+
* flag is never even consulted, and it stays stored for the day the tenant switch flips.
|
|
177
|
+
*/
|
|
178
|
+
shouldIndex(context: {
|
|
179
|
+
readonly tenantId: TenantId;
|
|
180
|
+
}, source: {
|
|
181
|
+
readonly sourceType: KnowledgeSourceType;
|
|
182
|
+
readonly sourceId: string;
|
|
183
|
+
}): Promise<boolean>;
|
|
184
|
+
/**
|
|
185
|
+
* Extract one source's chunks into the graph, replacing whatever that source contributed before.
|
|
186
|
+
*
|
|
187
|
+
* The caller has already decided this should run — `indexSource` calls `shouldIndex` first — so this does
|
|
188
|
+
* not re-check. Keeping the decision in one place is what makes "off costs nothing" a property of one
|
|
189
|
+
* function rather than a convention two functions share.
|
|
190
|
+
*/
|
|
191
|
+
indexSource(context: {
|
|
192
|
+
readonly tenantId: TenantId;
|
|
193
|
+
}, input: {
|
|
194
|
+
readonly sourceType: KnowledgeSourceType;
|
|
195
|
+
readonly sourceId: string;
|
|
196
|
+
readonly chunks: readonly ExtractionChunk[];
|
|
197
|
+
readonly concurrency?: number;
|
|
198
|
+
}): Promise<GraphIndexResult>;
|
|
199
|
+
};
|
|
200
|
+
export type GraphIndexer = ReturnType<typeof createGraphIndexer>;
|
|
201
|
+
/** What `indexSource` reports when the graph was not touched. Every counter zero, and `ran` says why. */
|
|
202
|
+
export declare const GRAPH_NOT_RUN: GraphIndexResult;
|
|
203
|
+
/**
|
|
204
|
+
* The default `EntityExtractor`: one model call per chunk — AC-2.
|
|
205
|
+
*
|
|
206
|
+
* Thin on purpose. The provider call lives in `models/extraction.ts` behind boundary rule R3, so this is the
|
|
207
|
+
* adapter between that function and the port — which means a deployment can replace the extractor entirely
|
|
208
|
+
* (a smaller model, a local one, a rules-based one for a known corpus) without touching the pipeline.
|
|
209
|
+
*
|
|
210
|
+
* `model` is a factory rather than a value because a `LanguageModel` is resolved from a policy per tenant, and
|
|
211
|
+
* a single resolved model captured at construction would be the same one for every tenant in the deployment.
|
|
212
|
+
*/
|
|
213
|
+
export declare const createModelEntityExtractor: (deps: {
|
|
214
|
+
readonly id?: string;
|
|
215
|
+
readonly extract: (text: string) => Promise<{
|
|
216
|
+
readonly extraction: RawExtraction;
|
|
217
|
+
readonly usage: {
|
|
218
|
+
readonly inputTokens: number;
|
|
219
|
+
readonly outputTokens: number;
|
|
220
|
+
};
|
|
221
|
+
}>;
|
|
222
|
+
}) => EntityExtractor;
|
|
223
|
+
//# sourceMappingURL=graph.d.ts.map
|