@retinue/agentkit 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -277
- package/dist/adapters/audio/openai.d.ts +36 -0
- package/dist/adapters/audio/openai.js +136 -0
- package/dist/adapters/bullmq/lock.d.ts +26 -2
- package/dist/adapters/embeddings/openai.d.ts +45 -0
- package/dist/adapters/embeddings/openai.js +109 -0
- package/dist/adapters/memory/connections.d.ts +10 -0
- package/dist/adapters/memory/connections.js +87 -0
- package/dist/adapters/memory/graph.d.ts +17 -0
- package/dist/adapters/memory/graph.js +267 -0
- package/dist/adapters/memory/index.d.ts +3 -0
- package/dist/adapters/memory/index.js +3 -0
- package/dist/adapters/memory/rate-limit.d.ts +14 -0
- package/dist/adapters/memory/rate-limit.js +28 -0
- package/dist/adapters/memory/runtime.js +0 -0
- package/dist/adapters/postgres/connections.d.ts +15 -0
- package/dist/adapters/postgres/connections.js +134 -0
- package/dist/adapters/postgres/graph.d.ts +24 -0
- package/dist/adapters/postgres/graph.js +325 -0
- package/dist/adapters/postgres/index.d.ts +2 -0
- package/dist/adapters/postgres/index.js +2 -0
- package/dist/adapters/postgres/migrations.js +198 -0
- package/dist/adapters/postgres/run-store.js +26 -2
- package/dist/adapters/postgres/schema.d.ts +61 -2
- package/dist/adapters/postgres/schema.js +95 -10
- package/dist/adapters/redis/index.d.ts +1 -0
- package/dist/adapters/redis/index.js +1 -0
- package/dist/adapters/redis/rate-limit.d.ts +46 -0
- package/dist/adapters/redis/rate-limit.js +55 -0
- package/dist/adapters/supabase/index.d.ts +15 -0
- package/dist/adapters/supabase/index.js +15 -0
- package/dist/adapters/supabase/rls.js +33 -0
- package/dist/agents/agent.d.ts +99 -1
- package/dist/agents/agent.js +215 -16
- package/dist/agents/define.js +24 -4
- package/dist/agents/engine.d.ts +48 -0
- package/dist/agents/engine.js +370 -13
- package/dist/audio/index.d.ts +151 -0
- package/dist/audio/index.js +183 -0
- package/dist/capabilities/index.d.ts +5 -1
- package/dist/capabilities/index.js +23 -0
- package/dist/capabilities/runtime.d.ts +8 -0
- package/dist/connections/cipher.d.ts +103 -0
- package/dist/connections/cipher.js +141 -0
- package/dist/connections/index.d.ts +156 -0
- package/dist/connections/index.js +72 -0
- package/dist/connections/oauth/client.d.ts +84 -0
- package/dist/connections/oauth/client.js +141 -0
- package/dist/connections/oauth/index.d.ts +166 -0
- package/dist/connections/oauth/index.js +226 -0
- package/dist/connections/oauth/service.d.ts +90 -0
- package/dist/connections/oauth/service.js +132 -0
- package/dist/connections/pause.d.ts +68 -0
- package/dist/connections/pause.js +97 -0
- package/dist/connections/resolver.d.ts +59 -0
- package/dist/connections/resolver.js +138 -0
- package/dist/connections/resume.d.ts +64 -0
- package/dist/connections/resume.js +99 -0
- package/dist/connections/vault-cipher.d.ts +95 -0
- package/dist/connections/vault-cipher.js +206 -0
- package/dist/core/budget.d.ts +55 -0
- package/dist/core/budget.js +56 -0
- package/dist/core/content-parts.d.ts +26 -2
- package/dist/core/content-parts.js +1 -0
- package/dist/core/context.d.ts +25 -0
- package/dist/core/errors.d.ts +1 -1
- package/dist/core/errors.js +15 -0
- package/dist/core/events.d.ts +110 -2
- package/dist/core/events.js +33 -2
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.js +1 -0
- package/dist/core/validation.js +19 -0
- package/dist/documents/index.d.ts +14 -0
- package/dist/documents/parsers/text.d.ts +16 -0
- package/dist/documents/parsers/text.js +54 -2
- package/dist/entries/connections.d.ts +10 -0
- package/dist/entries/connections.js +10 -0
- package/dist/entries/guardrails.d.ts +14 -0
- package/dist/entries/guardrails.js +14 -0
- package/dist/entries/knowledge.d.ts +17 -0
- package/dist/entries/knowledge.js +16 -0
- package/dist/entries/mcp-server.d.ts +9 -0
- package/dist/entries/mcp-server.js +9 -0
- package/dist/entries/runtime.d.ts +10 -0
- package/dist/entries/runtime.js +8 -0
- package/dist/entries/testing.d.ts +28 -0
- package/dist/entries/testing.js +28 -0
- package/dist/graphql/resolvers.d.ts +7 -1
- package/dist/graphql/resolvers.js +20 -0
- package/dist/graphql/schema.d.ts +1 -1
- package/dist/graphql/schema.js +44 -0
- package/dist/guardrails/index.d.ts +115 -0
- package/dist/guardrails/index.js +108 -0
- package/dist/guardrails/moderation.d.ts +53 -0
- package/dist/guardrails/moderation.js +75 -0
- package/dist/guardrails/pii.d.ts +75 -0
- package/dist/guardrails/pii.js +193 -0
- package/dist/knowledge/communities.d.ts +166 -0
- package/dist/knowledge/communities.js +377 -0
- package/dist/knowledge/graph-global.d.ts +153 -0
- package/dist/knowledge/graph-global.js +246 -0
- package/dist/knowledge/graph-retrieval.d.ts +150 -0
- package/dist/knowledge/graph-retrieval.js +303 -0
- package/dist/knowledge/graph.d.ts +223 -0
- package/dist/knowledge/graph.js +369 -0
- package/dist/knowledge/index.d.ts +25 -0
- package/dist/knowledge/index.js +37 -1
- package/dist/knowledge/navigate.d.ts +89 -0
- package/dist/knowledge/navigate.js +107 -0
- package/dist/knowledge/retrieval.d.ts +98 -5
- package/dist/knowledge/retrieval.js +158 -28
- package/dist/mcp-server/index.d.ts +168 -0
- package/dist/mcp-server/index.js +175 -0
- package/dist/models/extraction.d.ts +73 -0
- package/dist/models/extraction.js +118 -0
- package/dist/models/index.d.ts +40 -1
- package/dist/models/index.js +22 -1
- package/dist/models/pricing.d.ts +2 -0
- package/dist/models/pricing.js +17 -1
- package/dist/models/provider-factory.js +9 -7
- package/dist/models/streaming.d.ts +111 -1
- package/dist/models/streaming.js +184 -3
- package/dist/persistence/index.d.ts +230 -0
- package/dist/persistence/index.js +2 -0
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +12 -0
- package/dist/runtime/worker.d.ts +36 -1
- package/dist/runtime/worker.js +37 -4
- package/dist/security/checklist.js +55 -0
- package/dist/security/findings.js +18 -9
- package/dist/server/bin.d.ts +19 -0
- package/dist/server/bin.js +226 -0
- package/dist/server/boot.d.ts +20 -2
- package/dist/server/boot.js +29 -3
- package/dist/server/cli.d.ts +13 -0
- package/dist/server/cli.js +7 -4
- package/dist/server/doctor.d.ts +74 -0
- package/dist/server/doctor.js +280 -0
- package/dist/skills/catalogue.d.ts +49 -0
- package/dist/skills/catalogue.js +61 -0
- package/dist/skills/context.d.ts +61 -0
- package/dist/skills/context.js +107 -0
- package/dist/skills/index.d.ts +2 -0
- package/dist/skills/index.js +2 -0
- package/dist/telemetry/spans.js +17 -0
- package/dist/testing/conformance/artifact-exports.d.ts +20 -0
- package/dist/testing/conformance/artifact-exports.js +172 -0
- package/dist/testing/conformance/artifacts.d.ts +11 -0
- package/dist/testing/conformance/artifacts.js +316 -0
- package/dist/testing/conformance/audio.d.ts +38 -0
- package/dist/testing/conformance/audio.js +145 -0
- package/dist/testing/conformance/capability.d.ts +23 -0
- package/dist/testing/conformance/capability.js +27 -0
- package/dist/testing/conformance/checkpoint-store.d.ts +15 -0
- package/dist/testing/conformance/checkpoint-store.js +72 -0
- package/dist/testing/conformance/connections.d.ts +14 -0
- package/dist/testing/conformance/connections.js +171 -0
- package/dist/testing/conformance/conversation-store.d.ts +10 -0
- package/dist/testing/conformance/conversation-store.js +60 -0
- package/dist/testing/conformance/evaluation.d.ts +10 -0
- package/dist/testing/conformance/evaluation.js +295 -0
- package/dist/testing/conformance/files.d.ts +19 -0
- package/dist/testing/conformance/files.js +454 -0
- package/dist/testing/conformance/flows.d.ts +16 -0
- package/dist/testing/conformance/flows.js +193 -0
- package/dist/testing/conformance/graph.d.ts +22 -0
- package/dist/testing/conformance/graph.js +500 -0
- package/dist/testing/conformance/hitl.d.ts +25 -0
- package/dist/testing/conformance/hitl.js +523 -0
- package/dist/testing/conformance/index.d.ts +163 -0
- package/dist/testing/conformance/index.js +321 -0
- package/dist/testing/conformance/invariants.d.ts +23 -0
- package/dist/testing/conformance/invariants.js +80 -0
- package/dist/testing/conformance/knowledge.d.ts +41 -0
- package/dist/testing/conformance/knowledge.js +592 -0
- package/dist/testing/conformance/parents.d.ts +50 -0
- package/dist/testing/conformance/parents.js +39 -0
- package/dist/testing/conformance/rate-limit.d.ts +25 -0
- package/dist/testing/conformance/rate-limit.js +71 -0
- package/dist/testing/conformance/records.d.ts +36 -0
- package/dist/testing/conformance/records.js +400 -0
- package/dist/testing/conformance/rollups.d.ts +30 -0
- package/dist/testing/conformance/rollups.js +425 -0
- package/dist/testing/conformance/run-coordinator.d.ts +27 -0
- package/dist/testing/conformance/run-coordinator.js +150 -0
- package/dist/testing/conformance/run-event-log.d.ts +9 -0
- package/dist/testing/conformance/run-event-log.js +138 -0
- package/dist/testing/conformance/run-store.d.ts +12 -0
- package/dist/testing/conformance/run-store.js +294 -0
- package/dist/testing/conformance/session-state.d.ts +38 -0
- package/dist/testing/conformance/session-state.js +198 -0
- package/dist/testing/conformance/usage-limits.d.ts +10 -0
- package/dist/testing/conformance/usage-limits.js +309 -0
- package/dist/testing/conformance.d.ts +8 -0
- package/dist/testing/conformance.js +8 -0
- package/dist/testing/memory-backend.d.ts +39 -0
- package/dist/testing/memory-backend.js +41 -0
- package/dist/testing/pglite.d.ts +55 -0
- package/dist/testing/pglite.js +78 -0
- package/dist/testing/stub-model.d.ts +58 -0
- package/dist/testing/stub-model.js +71 -0
- package/dist/testing/supabase-storage-double.d.ts +35 -0
- package/dist/testing/supabase-storage-double.js +101 -0
- package/dist/toolkit/files.d.ts +125 -0
- package/dist/toolkit/files.js +320 -0
- package/dist/toolkit/http.d.ts +27 -0
- package/dist/toolkit/http.js +48 -0
- package/dist/toolkit/index.d.ts +31 -1
- package/dist/toolkit/index.js +26 -1
- package/dist/toolkit/mime.d.ts +97 -0
- package/dist/toolkit/mime.js +233 -0
- package/dist/toolkit/sandbox.d.ts +119 -0
- package/dist/toolkit/sandbox.js +239 -0
- package/dist/toolkit/ssrf.d.ts +139 -0
- package/dist/toolkit/ssrf.js +349 -0
- package/dist/toolkit/vendor.d.ts +91 -0
- package/dist/toolkit/vendor.js +153 -0
- package/dist/toolkit/web.d.ts +13 -0
- package/dist/toolkit/web.js +7 -1
- package/dist/tools/budget.d.ts +28 -0
- package/dist/tools/budget.js +35 -0
- package/dist/tools/credentials.d.ts +316 -0
- package/dist/tools/credentials.js +369 -0
- package/dist/tools/define.d.ts +45 -0
- package/dist/tools/define.js +26 -0
- package/dist/tools/find.d.ts +109 -0
- package/dist/tools/find.js +210 -0
- package/dist/tools/index.d.ts +35 -2
- package/dist/tools/index.js +11 -0
- package/dist/tools/library/audio.d.ts +63 -0
- package/dist/tools/library/audio.js +148 -0
- package/dist/tools/library/fs.d.ts +24 -0
- package/dist/tools/library/fs.js +102 -0
- package/dist/tools/library/index.d.ts +37 -2
- package/dist/tools/library/index.js +47 -0
- package/dist/tools/library/shell.d.ts +45 -0
- package/dist/tools/library/shell.js +70 -0
- package/dist/tools/meta-tools.js +8 -0
- package/dist/tools/registry.d.ts +146 -0
- package/dist/tools/registry.js +250 -4
- package/dist/usage/index.d.ts +21 -0
- package/dist/usage/index.js +1 -0
- package/dist/usage/rate-limit.d.ts +122 -0
- package/dist/usage/rate-limit.js +131 -0
- package/dist/usage/recorder.js +3 -0
- package/package.json +33 -2
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Community detection — REQ-064 (#270), task #272.
|
|
3
|
+
*
|
|
4
|
+
* `graph-global` answers "what are the main themes?" by reducing over summaries of clusters. This produces the
|
|
5
|
+
* clusters.
|
|
6
|
+
*
|
|
7
|
+
* ## Why this is implemented here rather than imported
|
|
8
|
+
*
|
|
9
|
+
* `backend` has exactly two runtime dependencies, `ai` and `zod`, and that is a property worth keeping. But the
|
|
10
|
+
* dependency count is the smaller argument. The larger one is **determinism**: Microsoft's GraphRAG uses
|
|
11
|
+
* hierarchical Leiden, and the widely-used implementations of Leiden and Louvain are seeded-random — they
|
|
12
|
+
* shuffle node order to escape local optima, which is good for modularity and fatal here. #275 measures
|
|
13
|
+
* GraphRAG against a fixed baseline, #272's own incremental rebuild is verified by comparing against a
|
|
14
|
+
* previous clustering, and both need the same graph to cluster the same way twice.
|
|
15
|
+
*
|
|
16
|
+
* So this is Louvain with every source of nondeterminism removed:
|
|
17
|
+
*
|
|
18
|
+
* - Nodes are visited in **sorted id order**, never shuffled.
|
|
19
|
+
* - A tie in modularity gain is broken by the **smallest community id**, so equal options resolve the same way.
|
|
20
|
+
* - Aggregation preserves that ordering into the next level.
|
|
21
|
+
*
|
|
22
|
+
* The cost is some modularity — a shuffled Louvain finds slightly better partitions on average. That is a
|
|
23
|
+
* trade worth making for a result that can be measured, diffed and debugged, and it is stated here rather than
|
|
24
|
+
* discovered later by somebody wondering why two runs disagree.
|
|
25
|
+
*
|
|
26
|
+
* Louvain rather than label propagation, which was the other candidate: label propagation is simpler and
|
|
27
|
+
* roughly as fast, but it produces one flat partition and #272 needs **hierarchy**. Louvain's aggregation
|
|
28
|
+
* phases *are* the levels — level 0 is fine-grained, each subsequent level coarser — which is exactly the
|
|
29
|
+
* granularity choice `graph-global` has to make between "what are the main themes" and "what are the themes in
|
|
30
|
+
* observability".
|
|
31
|
+
*/
|
|
32
|
+
import type { TenantId } from "../core/ids.js";
|
|
33
|
+
import type { GraphStore, KnowledgeRelationship, KnowledgeStore } from "../persistence/index.js";
|
|
34
|
+
/** A cluster of entities at one level of the hierarchy. */
|
|
35
|
+
export type Community = {
|
|
36
|
+
/**
|
|
37
|
+
* Deterministic and readable: `L<level>:<smallest member id>`.
|
|
38
|
+
*
|
|
39
|
+
* Derived from the membership rather than assigned by a counter, so the same cluster keeps the same id
|
|
40
|
+
* across runs — which is what lets a rebuild compare against the previous clustering at all. A counter would
|
|
41
|
+
* renumber everything whenever one entity moved.
|
|
42
|
+
*/
|
|
43
|
+
readonly id: string;
|
|
44
|
+
readonly level: number;
|
|
45
|
+
/** Sorted. */
|
|
46
|
+
readonly entityIds: readonly string[];
|
|
47
|
+
/** Edges with both endpoints inside this community, sorted. */
|
|
48
|
+
readonly relationshipIds: readonly string[];
|
|
49
|
+
/** Chunks behind those entities and edges, sorted. The provenance a summary inherits. */
|
|
50
|
+
readonly chunkIds: readonly string[];
|
|
51
|
+
/**
|
|
52
|
+
* A fingerprint of exactly what this community contains.
|
|
53
|
+
*
|
|
54
|
+
* The mechanism behind incremental summarisation and visible staleness: a summary records the fingerprint it
|
|
55
|
+
* was written against, and a community whose fingerprint no longer matches is stale — knowable without
|
|
56
|
+
* re-reading the summary or trusting a timestamp.
|
|
57
|
+
*/
|
|
58
|
+
readonly fingerprint: string;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* How many levels of hierarchy to build.
|
|
62
|
+
*
|
|
63
|
+
* Two is the minimum AC-1 asks for and the useful default: level 0 is fine-grained clusters and level 1 groups
|
|
64
|
+
* them. More levels on a small graph collapse to the same partition repeated, which costs summarisation calls
|
|
65
|
+
* for no new information — so the loop stops early when a level stops merging anything.
|
|
66
|
+
*/
|
|
67
|
+
export declare const DEFAULT_COMMUNITY_LEVELS = 2;
|
|
68
|
+
/**
|
|
69
|
+
* Cluster a graph into a hierarchy of communities.
|
|
70
|
+
*
|
|
71
|
+
* Pure and deterministic: the same entities and relationships always produce the same result, which every test
|
|
72
|
+
* of incremental rebuild depends on.
|
|
73
|
+
*
|
|
74
|
+
* Singletons are kept rather than dropped. An entity nothing links to is a real part of the corpus and a
|
|
75
|
+
* `graph-global` answer that silently omitted every unconnected concept would be quietly wrong about what the
|
|
76
|
+
* corpus contains.
|
|
77
|
+
*/
|
|
78
|
+
export declare const detectCommunities: (input: {
|
|
79
|
+
readonly entityIds: readonly string[];
|
|
80
|
+
readonly relationships: readonly KnowledgeRelationship[];
|
|
81
|
+
readonly chunksOf: (entityId: string) => readonly string[];
|
|
82
|
+
readonly levels?: number;
|
|
83
|
+
}) => readonly Community[];
|
|
84
|
+
/**
|
|
85
|
+
* Writes a community's summary — REQ-064 (#270), task #272.
|
|
86
|
+
*
|
|
87
|
+
* A port, like `EntityExtractor`, and optional for the same reason: a deployment that clusters but never runs
|
|
88
|
+
* `graph-global` needs no language model for it. Given the community's entities, its relationships and the
|
|
89
|
+
* text behind them, and asked for a short description of what this cluster is *about*.
|
|
90
|
+
*/
|
|
91
|
+
export interface CommunitySummariser {
|
|
92
|
+
readonly id: string;
|
|
93
|
+
summarise(input: {
|
|
94
|
+
readonly community: Community;
|
|
95
|
+
/** The chunk text behind the community, already permission-free — it is all one tenant's own material. */
|
|
96
|
+
readonly excerpts: readonly string[];
|
|
97
|
+
readonly entityNames: readonly string[];
|
|
98
|
+
readonly relationshipDescriptions: readonly string[];
|
|
99
|
+
}): Promise<{
|
|
100
|
+
readonly summary: string;
|
|
101
|
+
readonly usage?: {
|
|
102
|
+
readonly inputTokens?: number;
|
|
103
|
+
readonly outputTokens?: number;
|
|
104
|
+
};
|
|
105
|
+
}>;
|
|
106
|
+
}
|
|
107
|
+
export type CommunityRebuildResult = {
|
|
108
|
+
readonly communities: number;
|
|
109
|
+
readonly levels: number;
|
|
110
|
+
/** Communities whose summary was reused because their membership did not change. The incremental saving. */
|
|
111
|
+
readonly summariesKept: number;
|
|
112
|
+
readonly summariesWritten: number;
|
|
113
|
+
/** Summarisation calls that failed. The community survives without a summary — AC-7. */
|
|
114
|
+
readonly summariesFailed: number;
|
|
115
|
+
readonly inputTokens: number;
|
|
116
|
+
readonly outputTokens: number;
|
|
117
|
+
};
|
|
118
|
+
export type CommunityBuilderDeps = {
|
|
119
|
+
readonly store: GraphStore;
|
|
120
|
+
/** Absent means cluster but do not summarise. Useful, and the honest default for a deployment without `graph-global`. */
|
|
121
|
+
readonly summariser?: CommunitySummariser;
|
|
122
|
+
readonly knowledge?: KnowledgeStore;
|
|
123
|
+
readonly levels?: number;
|
|
124
|
+
/** Chunk excerpts handed to the summariser per community. A ceiling, because a community can span hundreds. */
|
|
125
|
+
readonly maxExcerpts?: number;
|
|
126
|
+
/**
|
|
127
|
+
* Which levels to summarise. Absent means all of them.
|
|
128
|
+
*
|
|
129
|
+
* **Worth setting, because summarising a level nobody queries is pure cost.** `graph-global` reads *one*
|
|
130
|
+
* level per query — the coarsest by default — and a two-level hierarchy over a real corpus has far more
|
|
131
|
+
* fine-grained communities than coarse ones. Summarising every level can therefore multiply the bill several
|
|
132
|
+
* times over for content no query will ever reduce across.
|
|
133
|
+
*
|
|
134
|
+
* Not defaulted to the coarsest, deliberately: a deployment that lets callers ask at a finer granularity
|
|
135
|
+
* needs those summaries, and silently not writing them would make `graph-global` at that level return
|
|
136
|
+
* nothing with no explanation. The cost is real either way; this makes it a decision.
|
|
137
|
+
*/
|
|
138
|
+
readonly summariseLevels?: readonly number[];
|
|
139
|
+
readonly clock?: () => string;
|
|
140
|
+
readonly log?: (message: string, detail?: Readonly<Record<string, unknown>>) => void;
|
|
141
|
+
};
|
|
142
|
+
export declare const DEFAULT_MAX_EXCERPTS = 20;
|
|
143
|
+
/**
|
|
144
|
+
* Rebuilds the community hierarchy, and re-summarises **only what changed**.
|
|
145
|
+
*
|
|
146
|
+
* This is where AC-5's "incremental" honestly lives, and it is worth being precise about the split rather than
|
|
147
|
+
* claiming more than is true:
|
|
148
|
+
*
|
|
149
|
+
* - **Clustering is global and cheap.** Community detection is global by nature — one entity moving can change
|
|
150
|
+
* every level — so the whole graph is re-clustered. It is arithmetic over ids, with no model calls, and
|
|
151
|
+
* pretending to do it incrementally would mean a partition that disagrees with itself.
|
|
152
|
+
* - **Summarisation is incremental and expensive.** One model call per community, and that is the cost worth
|
|
153
|
+
* avoiding. A community whose membership fingerprint is unchanged keeps its summary untouched.
|
|
154
|
+
*
|
|
155
|
+
* So changing one source re-clusters everything and re-summarises the handful of communities that actually
|
|
156
|
+
* moved. The staleness window is therefore *zero for structure* and *one rebuild for summaries*, and a
|
|
157
|
+
* community caught between the two is visibly stale rather than silently wrong.
|
|
158
|
+
*/
|
|
159
|
+
export declare const createCommunityBuilder: (deps: CommunityBuilderDeps) => {
|
|
160
|
+
summariserId: string | null;
|
|
161
|
+
rebuild(context: {
|
|
162
|
+
readonly tenantId: TenantId;
|
|
163
|
+
}): Promise<CommunityRebuildResult>;
|
|
164
|
+
};
|
|
165
|
+
export type CommunityBuilder = ReturnType<typeof createCommunityBuilder>;
|
|
166
|
+
//# sourceMappingURL=communities.d.ts.map
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Community detection — REQ-064 (#270), task #272.
|
|
3
|
+
*
|
|
4
|
+
* `graph-global` answers "what are the main themes?" by reducing over summaries of clusters. This produces the
|
|
5
|
+
* clusters.
|
|
6
|
+
*
|
|
7
|
+
* ## Why this is implemented here rather than imported
|
|
8
|
+
*
|
|
9
|
+
* `backend` has exactly two runtime dependencies, `ai` and `zod`, and that is a property worth keeping. But the
|
|
10
|
+
* dependency count is the smaller argument. The larger one is **determinism**: Microsoft's GraphRAG uses
|
|
11
|
+
* hierarchical Leiden, and the widely-used implementations of Leiden and Louvain are seeded-random — they
|
|
12
|
+
* shuffle node order to escape local optima, which is good for modularity and fatal here. #275 measures
|
|
13
|
+
* GraphRAG against a fixed baseline, #272's own incremental rebuild is verified by comparing against a
|
|
14
|
+
* previous clustering, and both need the same graph to cluster the same way twice.
|
|
15
|
+
*
|
|
16
|
+
* So this is Louvain with every source of nondeterminism removed:
|
|
17
|
+
*
|
|
18
|
+
* - Nodes are visited in **sorted id order**, never shuffled.
|
|
19
|
+
* - A tie in modularity gain is broken by the **smallest community id**, so equal options resolve the same way.
|
|
20
|
+
* - Aggregation preserves that ordering into the next level.
|
|
21
|
+
*
|
|
22
|
+
* The cost is some modularity — a shuffled Louvain finds slightly better partitions on average. That is a
|
|
23
|
+
* trade worth making for a result that can be measured, diffed and debugged, and it is stated here rather than
|
|
24
|
+
* discovered later by somebody wondering why two runs disagree.
|
|
25
|
+
*
|
|
26
|
+
* Louvain rather than label propagation, which was the other candidate: label propagation is simpler and
|
|
27
|
+
* roughly as fast, but it produces one flat partition and #272 needs **hierarchy**. Louvain's aggregation
|
|
28
|
+
* phases *are* the levels — level 0 is fine-grained, each subsequent level coarser — which is exactly the
|
|
29
|
+
* granularity choice `graph-global` has to make between "what are the main themes" and "what are the themes in
|
|
30
|
+
* observability".
|
|
31
|
+
*/
|
|
32
|
+
/**
|
|
33
|
+
* One Louvain pass: move nodes between communities while modularity improves.
|
|
34
|
+
*
|
|
35
|
+
* Returns a map from node id to community label. Deterministic throughout — see the header.
|
|
36
|
+
*/
|
|
37
|
+
const localMoving = (nodes, edges) => {
|
|
38
|
+
const community = new Map(nodes.map((node) => [node, node]));
|
|
39
|
+
const adjacency = new Map(nodes.map((node) => [node, []]));
|
|
40
|
+
const degree = new Map(nodes.map((node) => [node, 0]));
|
|
41
|
+
let totalWeight = 0;
|
|
42
|
+
for (const edge of edges) {
|
|
43
|
+
adjacency.get(edge.a)?.push({ neighbour: edge.b, weight: edge.weight });
|
|
44
|
+
adjacency.get(edge.b)?.push({ neighbour: edge.a, weight: edge.weight });
|
|
45
|
+
degree.set(edge.a, (degree.get(edge.a) ?? 0) + edge.weight);
|
|
46
|
+
degree.set(edge.b, (degree.get(edge.b) ?? 0) + edge.weight);
|
|
47
|
+
totalWeight += edge.weight;
|
|
48
|
+
}
|
|
49
|
+
// A graph with no edges is a graph of singletons, and dividing by its total weight would be a NaN that
|
|
50
|
+
// propagates into every gain comparison and produces an arbitrary partition.
|
|
51
|
+
if (totalWeight === 0)
|
|
52
|
+
return community;
|
|
53
|
+
const m2 = 2 * totalWeight;
|
|
54
|
+
const communityDegree = new Map();
|
|
55
|
+
for (const node of nodes)
|
|
56
|
+
communityDegree.set(node, degree.get(node) ?? 0);
|
|
57
|
+
// Bounded rather than "until no change": a cycle between two equal-gain configurations would otherwise spin
|
|
58
|
+
// forever, and the bound costs at most a slightly worse partition.
|
|
59
|
+
const MAX_PASSES = 20;
|
|
60
|
+
for (let pass = 0; pass < MAX_PASSES; pass += 1) {
|
|
61
|
+
let moved = false;
|
|
62
|
+
// Sorted order, every pass. This is the single most important line for determinism.
|
|
63
|
+
for (const node of nodes) {
|
|
64
|
+
const current = community.get(node);
|
|
65
|
+
const nodeDegree = degree.get(node) ?? 0;
|
|
66
|
+
// Weight from this node into each candidate community.
|
|
67
|
+
const into = new Map();
|
|
68
|
+
for (const { neighbour, weight } of adjacency.get(node) ?? []) {
|
|
69
|
+
if (neighbour === node)
|
|
70
|
+
continue;
|
|
71
|
+
const target = community.get(neighbour);
|
|
72
|
+
into.set(target, (into.get(target) ?? 0) + weight);
|
|
73
|
+
}
|
|
74
|
+
// Remove the node from its own community before comparing, or it competes with itself.
|
|
75
|
+
communityDegree.set(current, (communityDegree.get(current) ?? 0) - nodeDegree);
|
|
76
|
+
let best = current;
|
|
77
|
+
let bestGain = (into.get(current) ?? 0) - ((communityDegree.get(current) ?? 0) * nodeDegree) / m2;
|
|
78
|
+
/**
|
|
79
|
+
* Sorted, and **belt-and-braces rather than load-bearing** — worth saying, because a comment claiming
|
|
80
|
+
* more than it does is how a redundant line survives a refactor that made it necessary.
|
|
81
|
+
*
|
|
82
|
+
* Given sorted nodes and sorted edges, adjacency lists are built in a deterministic order, so `into`
|
|
83
|
+
* already iterates deterministically. Sorting here makes the tie-break independent of *that* fact, so a
|
|
84
|
+
* future change to how adjacency is assembled cannot silently reintroduce order sensitivity. Removing it
|
|
85
|
+
* today breaks no test, which is exactly what one would expect.
|
|
86
|
+
*/
|
|
87
|
+
for (const target of [...into.keys()].sort()) {
|
|
88
|
+
if (target === current)
|
|
89
|
+
continue;
|
|
90
|
+
const gain = (into.get(target) ?? 0) - ((communityDegree.get(target) ?? 0) * nodeDegree) / m2;
|
|
91
|
+
// Strictly greater, then smallest id: an equal-gain move must resolve the same way every run, and
|
|
92
|
+
// `>` alone would take whichever candidate happened to be visited first.
|
|
93
|
+
if (gain > bestGain || (gain === bestGain && target < best)) {
|
|
94
|
+
best = target;
|
|
95
|
+
bestGain = gain;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
communityDegree.set(best, (communityDegree.get(best) ?? 0) + nodeDegree);
|
|
99
|
+
if (best !== current) {
|
|
100
|
+
community.set(node, best);
|
|
101
|
+
moved = true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (!moved)
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
return community;
|
|
108
|
+
};
|
|
109
|
+
/** Relabels communities by their smallest member, so labels do not depend on which node was seen first. */
|
|
110
|
+
const canonicalise = (assignment) => {
|
|
111
|
+
const members = new Map();
|
|
112
|
+
for (const [node, label] of assignment) {
|
|
113
|
+
const group = members.get(label);
|
|
114
|
+
if (group === undefined)
|
|
115
|
+
members.set(label, [node]);
|
|
116
|
+
else
|
|
117
|
+
group.push(node);
|
|
118
|
+
}
|
|
119
|
+
const canonical = new Map();
|
|
120
|
+
for (const [, group] of members) {
|
|
121
|
+
const label = [...group].sort()[0];
|
|
122
|
+
for (const node of group)
|
|
123
|
+
canonical.set(node, label);
|
|
124
|
+
}
|
|
125
|
+
return canonical;
|
|
126
|
+
};
|
|
127
|
+
const fingerprintOf = (level, entityIds, relationshipIds) => `L${level}|${[...entityIds].sort().join(",")}|${[...relationshipIds].sort().join(",")}`;
|
|
128
|
+
/**
|
|
129
|
+
* How many levels of hierarchy to build.
|
|
130
|
+
*
|
|
131
|
+
* Two is the minimum AC-1 asks for and the useful default: level 0 is fine-grained clusters and level 1 groups
|
|
132
|
+
* them. More levels on a small graph collapse to the same partition repeated, which costs summarisation calls
|
|
133
|
+
* for no new information — so the loop stops early when a level stops merging anything.
|
|
134
|
+
*/
|
|
135
|
+
export const DEFAULT_COMMUNITY_LEVELS = 2;
|
|
136
|
+
/**
|
|
137
|
+
* Cluster a graph into a hierarchy of communities.
|
|
138
|
+
*
|
|
139
|
+
* Pure and deterministic: the same entities and relationships always produce the same result, which every test
|
|
140
|
+
* of incremental rebuild depends on.
|
|
141
|
+
*
|
|
142
|
+
* Singletons are kept rather than dropped. An entity nothing links to is a real part of the corpus and a
|
|
143
|
+
* `graph-global` answer that silently omitted every unconnected concept would be quietly wrong about what the
|
|
144
|
+
* corpus contains.
|
|
145
|
+
*/
|
|
146
|
+
export const detectCommunities = (input) => {
|
|
147
|
+
const levels = Math.max(1, input.levels ?? DEFAULT_COMMUNITY_LEVELS);
|
|
148
|
+
const out = [];
|
|
149
|
+
// Sorted once, and every derived list keeps that order.
|
|
150
|
+
let nodes = [...input.entityIds].sort();
|
|
151
|
+
if (nodes.length === 0)
|
|
152
|
+
return [];
|
|
153
|
+
const known = new Set(nodes);
|
|
154
|
+
let edges = input.relationships
|
|
155
|
+
.filter((edge) => known.has(edge.fromId) && known.has(edge.toId))
|
|
156
|
+
// Undirected for clustering: "A depends on B" and "B is depended on by A" are the same association, and
|
|
157
|
+
// direction would split a community that is obviously one.
|
|
158
|
+
.map((edge) => ({ a: edge.fromId < edge.toId ? edge.fromId : edge.toId, b: edge.fromId < edge.toId ? edge.toId : edge.fromId, weight: edge.weight }))
|
|
159
|
+
.sort((x, y) => (x.a === y.a ? (x.b < y.b ? -1 : 1) : x.a < y.a ? -1 : 1));
|
|
160
|
+
/** Which original entities each current node stands for. Grows as levels aggregate. */
|
|
161
|
+
let expansion = new Map(nodes.map((node) => [node, [node]]));
|
|
162
|
+
for (let level = 0; level < levels; level += 1) {
|
|
163
|
+
const assignment = canonicalise(localMoving(nodes, edges));
|
|
164
|
+
const grouped = new Map();
|
|
165
|
+
for (const node of nodes) {
|
|
166
|
+
const label = assignment.get(node);
|
|
167
|
+
const group = grouped.get(label);
|
|
168
|
+
if (group === undefined)
|
|
169
|
+
grouped.set(label, [node]);
|
|
170
|
+
else
|
|
171
|
+
group.push(node);
|
|
172
|
+
}
|
|
173
|
+
// A level that changed nothing adds no information and would cost a summarisation call per community.
|
|
174
|
+
if (level > 0 && grouped.size === nodes.length)
|
|
175
|
+
break;
|
|
176
|
+
for (const [label, group] of [...grouped.entries()].sort()) {
|
|
177
|
+
const entityIds = group.flatMap((node) => expansion.get(node) ?? [node]).sort();
|
|
178
|
+
const inside = new Set(entityIds);
|
|
179
|
+
const relationshipIds = input.relationships
|
|
180
|
+
.filter((edge) => inside.has(edge.fromId) && inside.has(edge.toId))
|
|
181
|
+
.map((edge) => edge.id)
|
|
182
|
+
.sort();
|
|
183
|
+
const chunkIds = [
|
|
184
|
+
...new Set([
|
|
185
|
+
...entityIds.flatMap((id) => input.chunksOf(id)),
|
|
186
|
+
...input.relationships.filter((edge) => relationshipIds.includes(edge.id)).flatMap((edge) => edge.provenance),
|
|
187
|
+
]),
|
|
188
|
+
].sort();
|
|
189
|
+
out.push({
|
|
190
|
+
// `label` is already the smallest member at this level, but the *expanded* smallest is what identifies
|
|
191
|
+
// the cluster across levels — otherwise a level-1 community and the level-0 one it contains could
|
|
192
|
+
// share an id.
|
|
193
|
+
id: `L${level}:${entityIds[0] ?? label}`,
|
|
194
|
+
level,
|
|
195
|
+
entityIds,
|
|
196
|
+
relationshipIds,
|
|
197
|
+
chunkIds,
|
|
198
|
+
fingerprint: fingerprintOf(level, entityIds, relationshipIds),
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
if (grouped.size <= 1)
|
|
202
|
+
break; // Everything is one community; there is nothing coarser to build.
|
|
203
|
+
// Aggregate for the next level: each community becomes a node, edges between them are summed.
|
|
204
|
+
const nextNodes = [...grouped.keys()].sort();
|
|
205
|
+
const nextExpansion = new Map(nextNodes.map((label) => [label, (grouped.get(label) ?? []).flatMap((node) => expansion.get(node) ?? [node]).sort()]));
|
|
206
|
+
const merged = new Map();
|
|
207
|
+
for (const edge of edges) {
|
|
208
|
+
const a = assignment.get(edge.a);
|
|
209
|
+
const b = assignment.get(edge.b);
|
|
210
|
+
if (a === b)
|
|
211
|
+
continue; // Internal edges do not connect communities.
|
|
212
|
+
const key = a < b ? `${a}${b}` : `${b}${a}`;
|
|
213
|
+
merged.set(key, (merged.get(key) ?? 0) + edge.weight);
|
|
214
|
+
}
|
|
215
|
+
nodes = nextNodes;
|
|
216
|
+
expansion = nextExpansion;
|
|
217
|
+
edges = [...merged.entries()]
|
|
218
|
+
.map(([key, weight]) => {
|
|
219
|
+
const [a, b] = key.split("");
|
|
220
|
+
return { a, b, weight };
|
|
221
|
+
})
|
|
222
|
+
.sort((x, y) => (x.a === y.a ? (x.b < y.b ? -1 : 1) : x.a < y.a ? -1 : 1));
|
|
223
|
+
}
|
|
224
|
+
return out;
|
|
225
|
+
};
|
|
226
|
+
export const DEFAULT_MAX_EXCERPTS = 20;
|
|
227
|
+
/**
|
|
228
|
+
* Rebuilds the community hierarchy, and re-summarises **only what changed**.
|
|
229
|
+
*
|
|
230
|
+
* This is where AC-5's "incremental" honestly lives, and it is worth being precise about the split rather than
|
|
231
|
+
* claiming more than is true:
|
|
232
|
+
*
|
|
233
|
+
* - **Clustering is global and cheap.** Community detection is global by nature — one entity moving can change
|
|
234
|
+
* every level — so the whole graph is re-clustered. It is arithmetic over ids, with no model calls, and
|
|
235
|
+
* pretending to do it incrementally would mean a partition that disagrees with itself.
|
|
236
|
+
* - **Summarisation is incremental and expensive.** One model call per community, and that is the cost worth
|
|
237
|
+
* avoiding. A community whose membership fingerprint is unchanged keeps its summary untouched.
|
|
238
|
+
*
|
|
239
|
+
* So changing one source re-clusters everything and re-summarises the handful of communities that actually
|
|
240
|
+
* moved. The staleness window is therefore *zero for structure* and *one rebuild for summaries*, and a
|
|
241
|
+
* community caught between the two is visibly stale rather than silently wrong.
|
|
242
|
+
*/
|
|
243
|
+
export const createCommunityBuilder = (deps) => {
|
|
244
|
+
const clock = deps.clock ?? (() => new Date().toISOString());
|
|
245
|
+
const log = deps.log ?? (() => { });
|
|
246
|
+
const maxExcerpts = Math.max(1, deps.maxExcerpts ?? DEFAULT_MAX_EXCERPTS);
|
|
247
|
+
return {
|
|
248
|
+
summariserId: deps.summariser?.id ?? null,
|
|
249
|
+
async rebuild(context) {
|
|
250
|
+
// Everything, because clustering is global. Paged out rather than assumed to fit in one call.
|
|
251
|
+
const entities = [];
|
|
252
|
+
let cursor;
|
|
253
|
+
do {
|
|
254
|
+
const page = await deps.store.listEntities({
|
|
255
|
+
tenantId: context.tenantId,
|
|
256
|
+
limit: 500,
|
|
257
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
258
|
+
});
|
|
259
|
+
entities.push(...page.items);
|
|
260
|
+
cursor = page.nextCursor;
|
|
261
|
+
} while (cursor !== undefined);
|
|
262
|
+
const empty = {
|
|
263
|
+
communities: 0,
|
|
264
|
+
levels: 0,
|
|
265
|
+
summariesKept: 0,
|
|
266
|
+
summariesWritten: 0,
|
|
267
|
+
summariesFailed: 0,
|
|
268
|
+
inputTokens: 0,
|
|
269
|
+
outputTokens: 0,
|
|
270
|
+
};
|
|
271
|
+
if (entities.length === 0) {
|
|
272
|
+
await deps.store.replaceCommunities({ tenantId: context.tenantId, communities: [] });
|
|
273
|
+
return empty;
|
|
274
|
+
}
|
|
275
|
+
const entityIds = entities.map((entity) => entity.id).sort();
|
|
276
|
+
const relationships = await deps.store.neighbours({
|
|
277
|
+
tenantId: context.tenantId,
|
|
278
|
+
entityIds,
|
|
279
|
+
// Every edge, because clustering on a sample of the graph would produce a partition of a graph that
|
|
280
|
+
// does not exist. The bound is generous rather than absent so a runaway corpus fails loudly.
|
|
281
|
+
limit: Math.max(1000, entityIds.length * 20),
|
|
282
|
+
});
|
|
283
|
+
const chunksByEntity = new Map(entities.map((entity) => [entity.id, entity.provenance]));
|
|
284
|
+
const communities = detectCommunities({
|
|
285
|
+
entityIds,
|
|
286
|
+
relationships,
|
|
287
|
+
chunksOf: (id) => chunksByEntity.get(id) ?? [],
|
|
288
|
+
...(deps.levels === undefined ? {} : { levels: deps.levels }),
|
|
289
|
+
});
|
|
290
|
+
const written = await deps.store.replaceCommunities({ tenantId: context.tenantId, communities });
|
|
291
|
+
const levels = new Set(communities.map((community) => community.level)).size;
|
|
292
|
+
if (deps.summariser === undefined) {
|
|
293
|
+
return { ...empty, communities: communities.length, levels, summariesKept: written.summariesKept };
|
|
294
|
+
}
|
|
295
|
+
const namesById = new Map(entities.map((entity) => [entity.id, entity.name]));
|
|
296
|
+
const byId = new Map(relationships.map((edge) => [edge.id, edge]));
|
|
297
|
+
let summariesWritten = 0;
|
|
298
|
+
let summariesFailed = 0;
|
|
299
|
+
let inputTokens = 0;
|
|
300
|
+
let outputTokens = 0;
|
|
301
|
+
const wantedLevels = deps.summariseLevels === undefined ? null : new Set(deps.summariseLevels);
|
|
302
|
+
for (const community of communities) {
|
|
303
|
+
// A level nobody queries costs nothing — see `summariseLevels`.
|
|
304
|
+
if (wantedLevels !== null && !wantedLevels.has(community.level))
|
|
305
|
+
continue;
|
|
306
|
+
// Only what changed. `replaceCommunities` already carried over the summaries that survived, so a
|
|
307
|
+
// community that still has one is one this pass must not pay for again.
|
|
308
|
+
const stored = await deps.store.getCommunity({ tenantId: context.tenantId, id: community.id });
|
|
309
|
+
if (stored?.summary !== undefined && stored.summaryFingerprint === community.fingerprint)
|
|
310
|
+
continue;
|
|
311
|
+
const excerpts = [];
|
|
312
|
+
if (deps.knowledge !== undefined) {
|
|
313
|
+
for (const chunkId of community.chunkIds.slice(0, maxExcerpts)) {
|
|
314
|
+
const chunk = await deps.knowledge.get({ tenantId: context.tenantId, id: chunkId });
|
|
315
|
+
if (chunk !== null)
|
|
316
|
+
excerpts.push(chunk.content);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
try {
|
|
320
|
+
const { summary, usage } = await deps.summariser.summarise({
|
|
321
|
+
community,
|
|
322
|
+
excerpts,
|
|
323
|
+
entityNames: community.entityIds.map((id) => namesById.get(id) ?? id),
|
|
324
|
+
relationshipDescriptions: community.relationshipIds.map((id) => {
|
|
325
|
+
const edge = byId.get(id);
|
|
326
|
+
return edge === undefined
|
|
327
|
+
? id
|
|
328
|
+
: `${namesById.get(edge.fromId) ?? edge.fromId} ${edge.type} ${namesById.get(edge.toId) ?? edge.toId}`;
|
|
329
|
+
}),
|
|
330
|
+
});
|
|
331
|
+
inputTokens += usage?.inputTokens ?? 0;
|
|
332
|
+
outputTokens += usage?.outputTokens ?? 0;
|
|
333
|
+
if (summary.trim() === "") {
|
|
334
|
+
// An empty summary is a failure wearing a success's shape: `graph-global` would reduce over it and
|
|
335
|
+
// silently under-report the corpus.
|
|
336
|
+
summariesFailed += 1;
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
await deps.store.setCommunitySummary({
|
|
340
|
+
tenantId: context.tenantId,
|
|
341
|
+
id: community.id,
|
|
342
|
+
summary,
|
|
343
|
+
fingerprint: community.fingerprint,
|
|
344
|
+
at: clock(),
|
|
345
|
+
});
|
|
346
|
+
summariesWritten += 1;
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
/**
|
|
350
|
+
* AC-7. A failed summarisation leaves the community **without** a summary, never with a wrong one.
|
|
351
|
+
*
|
|
352
|
+
* Swallowed rather than propagated because the clustering succeeded and is useful on its own —
|
|
353
|
+
* failing the rebuild would discard a correct hierarchy over one model call. `graph-global` sees a
|
|
354
|
+
* community with no summary and can say so.
|
|
355
|
+
*/
|
|
356
|
+
summariesFailed += 1;
|
|
357
|
+
log("community summarisation failed", {
|
|
358
|
+
communityId: community.id,
|
|
359
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (summariesFailed > 0)
|
|
364
|
+
log("some communities have no summary", { failed: summariesFailed, of: communities.length });
|
|
365
|
+
return {
|
|
366
|
+
communities: communities.length,
|
|
367
|
+
levels,
|
|
368
|
+
summariesKept: written.summariesKept,
|
|
369
|
+
summariesWritten,
|
|
370
|
+
summariesFailed,
|
|
371
|
+
inputTokens,
|
|
372
|
+
outputTokens,
|
|
373
|
+
};
|
|
374
|
+
},
|
|
375
|
+
};
|
|
376
|
+
};
|
|
377
|
+
//# sourceMappingURL=communities.js.map
|