@botbotgo/agent-harness 0.0.18 → 0.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -491
- package/dist/api.d.ts +3 -0
- package/dist/api.js +7 -0
- package/dist/config/agent-context.md +8 -0
- package/dist/config/orchestra.yaml +11 -8
- package/dist/config/{runtime.yaml → workspace.yaml} +23 -0
- package/dist/contracts/types.d.ts +14 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +2 -1
- package/dist/mcp.d.ts +12 -0
- package/dist/mcp.js +112 -0
- package/dist/resource/isolation.d.ts +2 -0
- package/dist/resource/isolation.js +79 -0
- package/dist/resource/resource-impl.d.ts +18 -0
- package/dist/resource/resource-impl.js +179 -11
- package/dist/resource/sources.d.ts +3 -0
- package/dist/resource/sources.js +105 -25
- package/dist/runtime/checkpoint-maintenance.d.ts +36 -0
- package/dist/runtime/checkpoint-maintenance.js +223 -0
- package/dist/runtime/harness.d.ts +10 -1
- package/dist/runtime/harness.js +38 -2
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/sqlite-maintained-checkpoint-saver.d.ts +9 -0
- package/dist/runtime/sqlite-maintained-checkpoint-saver.js +39 -0
- package/dist/runtime/support/runtime-factories.js +3 -1
- package/dist/tool-modules.d.ts +17 -0
- package/dist/tool-modules.js +143 -0
- package/dist/tools.d.ts +20 -0
- package/dist/tools.js +17 -0
- package/dist/workspace/compile.js +124 -5
- package/dist/workspace/object-loader.js +90 -24
- package/dist/workspace/resource-compilers.d.ts +3 -1
- package/dist/workspace/resource-compilers.js +72 -5
- package/package.json +10 -3
package/dist/resource/sources.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { execFile } from "node:child_process";
|
|
2
2
|
import { existsSync, statSync } from "node:fs";
|
|
3
|
-
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
|
-
import { createHash } from "node:crypto";
|
|
6
|
+
import { createHash, createPublicKey, verify } from "node:crypto";
|
|
7
7
|
import { promisify } from "node:util";
|
|
8
8
|
const execFileAsync = promisify(execFile);
|
|
9
9
|
const sourceCache = new Map();
|
|
@@ -38,14 +38,22 @@ export function parseExternalSourceLocator(locator) {
|
|
|
38
38
|
const separator = locator.indexOf("::");
|
|
39
39
|
const rawBase = separator >= 0 ? locator.slice(0, separator) : locator;
|
|
40
40
|
const subpath = separator >= 0 ? locator.slice(separator + 2) : undefined;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
41
|
+
const [baseWithoutQuery, rawQuery = ""] = rawBase.split("?", 2);
|
|
42
|
+
const query = new URLSearchParams(rawQuery);
|
|
43
|
+
const shared = {
|
|
44
|
+
subpath,
|
|
45
|
+
integrity: query.get("integrity") ?? undefined,
|
|
46
|
+
signature: query.get("signature") ?? undefined,
|
|
47
|
+
publicKey: query.get("publicKey") ?? query.get("key") ?? undefined,
|
|
48
|
+
};
|
|
49
|
+
if (baseWithoutQuery.startsWith("npm:")) {
|
|
50
|
+
return { kind: "npm", spec: baseWithoutQuery.slice(4), ...shared };
|
|
51
|
+
}
|
|
52
|
+
if (baseWithoutQuery.startsWith("tgz:")) {
|
|
53
|
+
return { kind: "tgz", spec: baseWithoutQuery.slice(4), ...shared };
|
|
54
|
+
}
|
|
55
|
+
if (baseWithoutQuery.startsWith("file:")) {
|
|
56
|
+
return { kind: "file", spec: baseWithoutQuery.slice(5), ...shared };
|
|
49
57
|
}
|
|
50
58
|
throw new Error(`Unsupported external source locator ${locator}. Use npm:, tgz:, or file:.`);
|
|
51
59
|
}
|
|
@@ -59,8 +67,13 @@ async function packAndExtract(packageSpec, cacheKey) {
|
|
|
59
67
|
const root = sourceCacheDir(cacheKey);
|
|
60
68
|
const extractRoot = path.join(root, "extract");
|
|
61
69
|
const packageRoot = path.join(extractRoot, "package");
|
|
70
|
+
const tarballMarker = path.join(root, ".tarball-path");
|
|
62
71
|
if (existsSync(path.join(packageRoot, "package.json"))) {
|
|
63
|
-
|
|
72
|
+
const tarballPath = existsSync(tarballMarker) ? (await readFile(tarballMarker, "utf8")).trim() : undefined;
|
|
73
|
+
return {
|
|
74
|
+
basePath: packageRoot,
|
|
75
|
+
tarballPath: tarballPath || undefined,
|
|
76
|
+
};
|
|
64
77
|
}
|
|
65
78
|
await mkdir(root, { recursive: true });
|
|
66
79
|
await mkdir(extractRoot, { recursive: true });
|
|
@@ -72,41 +85,108 @@ async function packAndExtract(packageSpec, cacheKey) {
|
|
|
72
85
|
if (!tarballName) {
|
|
73
86
|
throw new Error(`Failed to pack external source ${cacheKey}`);
|
|
74
87
|
}
|
|
88
|
+
const tarballPath = path.join(root, tarballName);
|
|
75
89
|
await execFileAsync("tar", ["-xzf", tarballName, "-C", extractRoot], {
|
|
76
90
|
cwd: root,
|
|
77
91
|
maxBuffer: 1024 * 1024 * 10,
|
|
78
92
|
});
|
|
79
|
-
|
|
93
|
+
await writeFile(tarballMarker, tarballPath, "utf8");
|
|
94
|
+
return {
|
|
95
|
+
basePath: packageRoot,
|
|
96
|
+
tarballPath,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
function decodeIntegrity(integrity) {
|
|
100
|
+
const trimmed = integrity.trim();
|
|
101
|
+
if (!trimmed) {
|
|
102
|
+
throw new Error("External source integrity must not be empty.");
|
|
103
|
+
}
|
|
104
|
+
const [algorithm, encoded] = trimmed.includes("-")
|
|
105
|
+
? trimmed.split("-", 2)
|
|
106
|
+
: trimmed.includes(":")
|
|
107
|
+
? trimmed.split(":", 2)
|
|
108
|
+
: [trimmed, ""];
|
|
109
|
+
if (!encoded) {
|
|
110
|
+
throw new Error(`External source integrity ${integrity} must use <algorithm>-<digest> or <algorithm>:<digest>.`);
|
|
111
|
+
}
|
|
112
|
+
const normalizedAlgorithm = algorithm.toLowerCase();
|
|
113
|
+
const encoding = /^[a-f0-9]+$/i.test(encoded) ? "hex" : "base64";
|
|
114
|
+
return {
|
|
115
|
+
algorithm: normalizedAlgorithm,
|
|
116
|
+
expected: Buffer.from(encoded, encoding),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
async function readVerificationPublicKey(publicKey, workspaceRoot) {
|
|
120
|
+
const trimmed = publicKey.trim();
|
|
121
|
+
if (trimmed.startsWith("-----BEGIN")) {
|
|
122
|
+
return trimmed;
|
|
123
|
+
}
|
|
124
|
+
const resolved = trimmed.startsWith("file:")
|
|
125
|
+
? resolveFileSpec(trimmed.slice("file:".length), workspaceRoot)
|
|
126
|
+
: resolveFileSpec(trimmed, workspaceRoot);
|
|
127
|
+
return readFile(resolved);
|
|
128
|
+
}
|
|
129
|
+
async function verifyExternalSource(locator, parsed, cached, workspaceRoot) {
|
|
130
|
+
if (!parsed.integrity && !parsed.signature) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (!cached.tarballPath) {
|
|
134
|
+
throw new Error(`External source ${locator} does not provide a tarball to verify.`);
|
|
135
|
+
}
|
|
136
|
+
const tarball = await readFile(cached.tarballPath);
|
|
137
|
+
if (parsed.integrity) {
|
|
138
|
+
const { algorithm, expected } = decodeIntegrity(parsed.integrity);
|
|
139
|
+
const actual = createHash(algorithm).update(tarball).digest();
|
|
140
|
+
if (!actual.equals(expected)) {
|
|
141
|
+
throw new Error(`External source ${locator} failed integrity verification.`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (parsed.signature) {
|
|
145
|
+
if (!parsed.publicKey) {
|
|
146
|
+
throw new Error(`External source ${locator} signature verification requires publicKey=.`);
|
|
147
|
+
}
|
|
148
|
+
const key = createPublicKey(await readVerificationPublicKey(parsed.publicKey, workspaceRoot));
|
|
149
|
+
const signature = Buffer.from(parsed.signature, "base64");
|
|
150
|
+
const verified = verify(null, tarball, key, signature) ||
|
|
151
|
+
verify("sha256", tarball, key, signature);
|
|
152
|
+
if (!verified) {
|
|
153
|
+
throw new Error(`External source ${locator} failed signature verification.`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
80
156
|
}
|
|
81
157
|
export async function ensureExternalSource(locator, workspaceRoot) {
|
|
82
158
|
const parsed = parseExternalSourceLocator(locator);
|
|
83
159
|
const cacheKey = `${parsed.kind}:${parsed.spec}`;
|
|
84
160
|
const cached = sourceCache.get(cacheKey);
|
|
85
161
|
if (cached) {
|
|
86
|
-
|
|
162
|
+
await verifyExternalSource(locator, parsed, cached, workspaceRoot);
|
|
163
|
+
return parsed.subpath ? path.join(cached.basePath, parsed.subpath) : cached.basePath;
|
|
87
164
|
}
|
|
88
|
-
let
|
|
165
|
+
let resolved;
|
|
89
166
|
if (parsed.kind === "npm") {
|
|
90
|
-
|
|
167
|
+
resolved = await packAndExtract(parsed.spec, cacheKey);
|
|
91
168
|
}
|
|
92
169
|
else if (parsed.kind === "tgz") {
|
|
93
|
-
const
|
|
94
|
-
|
|
170
|
+
const tgzPath = resolveFileSpec(parsed.spec, workspaceRoot);
|
|
171
|
+
resolved = await packAndExtract(tgzPath, cacheKey);
|
|
172
|
+
resolved.tarballPath ??= tgzPath;
|
|
95
173
|
}
|
|
96
174
|
else {
|
|
97
|
-
const
|
|
98
|
-
if (!existsSync(
|
|
175
|
+
const filePath = resolveFileSpec(parsed.spec, workspaceRoot);
|
|
176
|
+
if (!existsSync(filePath)) {
|
|
99
177
|
throw new Error(`External file source ${locator} does not exist`);
|
|
100
178
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
179
|
+
if (filePath.endsWith(".tgz") || filePath.endsWith(".tar.gz")) {
|
|
180
|
+
resolved = await packAndExtract(filePath, cacheKey);
|
|
181
|
+
resolved.tarballPath ??= filePath;
|
|
103
182
|
}
|
|
104
183
|
else {
|
|
105
|
-
|
|
184
|
+
resolved = { basePath: filePath };
|
|
106
185
|
}
|
|
107
186
|
}
|
|
108
|
-
sourceCache.set(cacheKey,
|
|
109
|
-
|
|
187
|
+
sourceCache.set(cacheKey, resolved);
|
|
188
|
+
await verifyExternalSource(locator, parsed, resolved, workspaceRoot);
|
|
189
|
+
return parsed.subpath ? path.join(resolved.basePath, parsed.subpath) : resolved.basePath;
|
|
110
190
|
}
|
|
111
191
|
export async function ensureExternalResourceSource(locator, workspaceRoot) {
|
|
112
192
|
const parsed = parseExternalSourceLocator(locator);
|
|
@@ -134,7 +214,7 @@ export function resolveExternalSourcePath(locator, workspaceRoot) {
|
|
|
134
214
|
const cacheKey = `${parsed.kind}:${parsed.spec}`;
|
|
135
215
|
const cached = sourceCache.get(cacheKey);
|
|
136
216
|
if (cached) {
|
|
137
|
-
return parsed.subpath ? path.join(cached, parsed.subpath) : cached;
|
|
217
|
+
return parsed.subpath ? path.join(cached.basePath, parsed.subpath) : cached.basePath;
|
|
138
218
|
}
|
|
139
219
|
if (parsed.kind === "file") {
|
|
140
220
|
const resolved = resolveFileSpec(parsed.spec, workspaceRoot);
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { WorkspaceBundle } from "../contracts/types.js";
|
|
2
|
+
type CheckpointMaintenanceConfig = {
|
|
3
|
+
enabled: boolean;
|
|
4
|
+
schedule: {
|
|
5
|
+
intervalSeconds: number;
|
|
6
|
+
runOnStartup: boolean;
|
|
7
|
+
};
|
|
8
|
+
policies: {
|
|
9
|
+
maxAgeSeconds?: number;
|
|
10
|
+
maxBytes?: number;
|
|
11
|
+
};
|
|
12
|
+
sqlite: {
|
|
13
|
+
sweepBatchSize: number;
|
|
14
|
+
vacuum: boolean;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
type CheckpointMaintenanceTarget = {
|
|
18
|
+
agentId: string;
|
|
19
|
+
dbPath: string;
|
|
20
|
+
};
|
|
21
|
+
export declare function readCheckpointMaintenanceConfig(workspace: WorkspaceBundle): CheckpointMaintenanceConfig | null;
|
|
22
|
+
export declare function discoverCheckpointMaintenanceTargets(workspace: WorkspaceBundle): CheckpointMaintenanceTarget[];
|
|
23
|
+
export declare function maintainSqliteCheckpoints(dbPath: string, config: CheckpointMaintenanceConfig, nowMs?: number): {
|
|
24
|
+
deletedCount: number;
|
|
25
|
+
};
|
|
26
|
+
export declare class CheckpointMaintenanceLoop {
|
|
27
|
+
private readonly targets;
|
|
28
|
+
private readonly config;
|
|
29
|
+
private timer;
|
|
30
|
+
private running;
|
|
31
|
+
constructor(targets: CheckpointMaintenanceTarget[], config: CheckpointMaintenanceConfig);
|
|
32
|
+
runOnce(nowMs?: number): Promise<void>;
|
|
33
|
+
start(): Promise<void>;
|
|
34
|
+
stop(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
export {};
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite";
|
|
3
|
+
import { getRuntimeDefaults } from "../workspace/support/workspace-ref-utils.js";
|
|
4
|
+
import { ManagedSqliteSaver } from "./sqlite-maintained-checkpoint-saver.js";
|
|
5
|
+
function asObject(value) {
|
|
6
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : undefined;
|
|
7
|
+
}
|
|
8
|
+
function readPositiveNumber(value, label, allowUndefined = true) {
|
|
9
|
+
if (value === undefined && allowUndefined) {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
13
|
+
throw new Error(`${label} must be a positive number`);
|
|
14
|
+
}
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
export function readCheckpointMaintenanceConfig(workspace) {
|
|
18
|
+
const runtimeDefaults = getRuntimeDefaults(workspace.refs);
|
|
19
|
+
const maintenance = asObject(runtimeDefaults?.maintenance);
|
|
20
|
+
const checkpoints = asObject(maintenance?.checkpoints);
|
|
21
|
+
if (!checkpoints || checkpoints.enabled !== true) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
const schedule = asObject(checkpoints.schedule);
|
|
25
|
+
const policies = asObject(checkpoints.policies);
|
|
26
|
+
const sqlite = asObject(checkpoints.sqlite);
|
|
27
|
+
const config = {
|
|
28
|
+
enabled: true,
|
|
29
|
+
schedule: {
|
|
30
|
+
intervalSeconds: readPositiveNumber(schedule?.intervalSeconds, "runtime.maintenance.checkpoints.schedule.intervalSeconds") ?? 3600,
|
|
31
|
+
runOnStartup: schedule?.runOnStartup !== false,
|
|
32
|
+
},
|
|
33
|
+
policies: {
|
|
34
|
+
maxAgeSeconds: readPositiveNumber(policies?.maxAgeSeconds, "runtime.maintenance.checkpoints.policies.maxAgeSeconds"),
|
|
35
|
+
maxBytes: readPositiveNumber(policies?.maxBytes, "runtime.maintenance.checkpoints.policies.maxBytes"),
|
|
36
|
+
},
|
|
37
|
+
sqlite: {
|
|
38
|
+
sweepBatchSize: readPositiveNumber(sqlite?.sweepBatchSize, "runtime.maintenance.checkpoints.sqlite.sweepBatchSize") ?? 200,
|
|
39
|
+
vacuum: sqlite?.vacuum === true,
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
if (config.policies.maxAgeSeconds === undefined && config.policies.maxBytes === undefined) {
|
|
43
|
+
throw new Error("runtime.maintenance.checkpoints.enabled requires at least one cleanup policy");
|
|
44
|
+
}
|
|
45
|
+
return config;
|
|
46
|
+
}
|
|
47
|
+
function resolveSqliteCheckpointPath(binding) {
|
|
48
|
+
const config = binding.harnessRuntime.checkpointer;
|
|
49
|
+
if (!config) {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
const kind = typeof config.kind === "string" ? config.kind : "FileCheckpointer";
|
|
53
|
+
if (kind !== "SqliteSaver") {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
const configuredPath = typeof config.path === "string" ? String(config.path) : "checkpoints.sqlite";
|
|
57
|
+
return path.isAbsolute(configuredPath) ? configuredPath : path.join(binding.harnessRuntime.runRoot, configuredPath);
|
|
58
|
+
}
|
|
59
|
+
export function discoverCheckpointMaintenanceTargets(workspace) {
|
|
60
|
+
const deduped = new Map();
|
|
61
|
+
for (const binding of workspace.bindings.values()) {
|
|
62
|
+
const dbPath = resolveSqliteCheckpointPath(binding);
|
|
63
|
+
if (!dbPath) {
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
deduped.set(dbPath, {
|
|
67
|
+
agentId: binding.agent.id,
|
|
68
|
+
dbPath,
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return Array.from(deduped.values());
|
|
72
|
+
}
|
|
73
|
+
function backfillCheckpointMetadata(db, nowMs = Date.now()) {
|
|
74
|
+
db.exec(`
|
|
75
|
+
CREATE TABLE IF NOT EXISTS checkpoint_maintenance_meta (
|
|
76
|
+
thread_id TEXT NOT NULL,
|
|
77
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
78
|
+
checkpoint_id TEXT NOT NULL,
|
|
79
|
+
created_at_ms INTEGER NOT NULL,
|
|
80
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id)
|
|
81
|
+
);`);
|
|
82
|
+
db.prepare(`INSERT OR IGNORE INTO checkpoint_maintenance_meta
|
|
83
|
+
(thread_id, checkpoint_ns, checkpoint_id, created_at_ms)
|
|
84
|
+
SELECT thread_id, checkpoint_ns, checkpoint_id, ?
|
|
85
|
+
FROM checkpoints`).run(nowMs);
|
|
86
|
+
}
|
|
87
|
+
function deleteCheckpointRows(db, rows) {
|
|
88
|
+
if (rows.length === 0) {
|
|
89
|
+
return 0;
|
|
90
|
+
}
|
|
91
|
+
const deleteCheckpoint = db.prepare(`DELETE FROM checkpoints
|
|
92
|
+
WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?`);
|
|
93
|
+
const deleteWrites = db.prepare(`DELETE FROM writes
|
|
94
|
+
WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?`);
|
|
95
|
+
const deleteMeta = db.prepare(`DELETE FROM checkpoint_maintenance_meta
|
|
96
|
+
WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?`);
|
|
97
|
+
db.transaction((items) => {
|
|
98
|
+
for (const row of items) {
|
|
99
|
+
deleteCheckpoint.run(row.thread_id, row.checkpoint_ns, row.checkpoint_id);
|
|
100
|
+
deleteWrites.run(row.thread_id, row.checkpoint_ns, row.checkpoint_id);
|
|
101
|
+
deleteMeta.run(row.thread_id, row.checkpoint_ns, row.checkpoint_id);
|
|
102
|
+
}
|
|
103
|
+
})(rows);
|
|
104
|
+
return rows.length;
|
|
105
|
+
}
|
|
106
|
+
function selectOldestRows(db, limit) {
|
|
107
|
+
return db
|
|
108
|
+
.prepare(`SELECT
|
|
109
|
+
meta.thread_id,
|
|
110
|
+
meta.checkpoint_ns,
|
|
111
|
+
meta.checkpoint_id,
|
|
112
|
+
meta.created_at_ms,
|
|
113
|
+
COALESCE(LENGTH(cp.checkpoint), 0) + COALESCE(LENGTH(cp.metadata), 0) + COALESCE(SUM(LENGTH(w.value)), 0) AS size_bytes
|
|
114
|
+
FROM checkpoint_maintenance_meta AS meta
|
|
115
|
+
JOIN checkpoints AS cp
|
|
116
|
+
ON cp.thread_id = meta.thread_id
|
|
117
|
+
AND cp.checkpoint_ns = meta.checkpoint_ns
|
|
118
|
+
AND cp.checkpoint_id = meta.checkpoint_id
|
|
119
|
+
LEFT JOIN writes AS w
|
|
120
|
+
ON w.thread_id = meta.thread_id
|
|
121
|
+
AND w.checkpoint_ns = meta.checkpoint_ns
|
|
122
|
+
AND w.checkpoint_id = meta.checkpoint_id
|
|
123
|
+
GROUP BY meta.thread_id, meta.checkpoint_ns, meta.checkpoint_id, meta.created_at_ms, cp.checkpoint, cp.metadata
|
|
124
|
+
ORDER BY meta.created_at_ms ASC, meta.checkpoint_id ASC
|
|
125
|
+
LIMIT ?`)
|
|
126
|
+
.all(limit);
|
|
127
|
+
}
|
|
128
|
+
function totalCheckpointBytes(db) {
|
|
129
|
+
const checkpointsBytes = db
|
|
130
|
+
.prepare(`SELECT COALESCE(SUM(LENGTH(checkpoint) + LENGTH(metadata)), 0) AS total FROM checkpoints`)
|
|
131
|
+
.get();
|
|
132
|
+
const writesBytes = db
|
|
133
|
+
.prepare(`SELECT COALESCE(SUM(LENGTH(value)), 0) AS total FROM writes`)
|
|
134
|
+
.get();
|
|
135
|
+
return Number(checkpointsBytes.total ?? 0) + Number(writesBytes.total ?? 0);
|
|
136
|
+
}
|
|
137
|
+
export function maintainSqliteCheckpoints(dbPath, config, nowMs = Date.now()) {
|
|
138
|
+
const saver = new ManagedSqliteSaver(SqliteSaver.fromConnString(dbPath).db);
|
|
139
|
+
const db = saver.db;
|
|
140
|
+
try {
|
|
141
|
+
saver.prepareMaintenance();
|
|
142
|
+
backfillCheckpointMetadata(db, nowMs);
|
|
143
|
+
let deletedCount = 0;
|
|
144
|
+
if (config.policies.maxAgeSeconds !== undefined) {
|
|
145
|
+
const cutoffMs = nowMs - config.policies.maxAgeSeconds * 1000;
|
|
146
|
+
const expired = db
|
|
147
|
+
.prepare(`SELECT
|
|
148
|
+
meta.thread_id,
|
|
149
|
+
meta.checkpoint_ns,
|
|
150
|
+
meta.checkpoint_id,
|
|
151
|
+
meta.created_at_ms,
|
|
152
|
+
0 AS size_bytes
|
|
153
|
+
FROM checkpoint_maintenance_meta AS meta
|
|
154
|
+
WHERE meta.created_at_ms <= ?
|
|
155
|
+
ORDER BY meta.created_at_ms ASC, meta.checkpoint_id ASC
|
|
156
|
+
LIMIT ?`)
|
|
157
|
+
.all(cutoffMs, config.sqlite.sweepBatchSize);
|
|
158
|
+
deletedCount += deleteCheckpointRows(db, expired);
|
|
159
|
+
}
|
|
160
|
+
if (config.policies.maxBytes !== undefined) {
|
|
161
|
+
let currentBytes = totalCheckpointBytes(db);
|
|
162
|
+
while (currentBytes > config.policies.maxBytes) {
|
|
163
|
+
const oldest = selectOldestRows(db, config.sqlite.sweepBatchSize);
|
|
164
|
+
if (oldest.length === 0) {
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
let reclaimed = 0;
|
|
168
|
+
const toDelete = [];
|
|
169
|
+
for (const row of oldest) {
|
|
170
|
+
toDelete.push(row);
|
|
171
|
+
reclaimed += Number(row.size_bytes ?? 0);
|
|
172
|
+
if (currentBytes - reclaimed <= config.policies.maxBytes) {
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
deletedCount += deleteCheckpointRows(db, toDelete);
|
|
177
|
+
currentBytes = totalCheckpointBytes(db);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if (deletedCount > 0 && config.sqlite.vacuum) {
|
|
181
|
+
db.exec("VACUUM");
|
|
182
|
+
}
|
|
183
|
+
return { deletedCount };
|
|
184
|
+
}
|
|
185
|
+
finally {
|
|
186
|
+
db.close();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
export class CheckpointMaintenanceLoop {
|
|
190
|
+
targets;
|
|
191
|
+
config;
|
|
192
|
+
timer = null;
|
|
193
|
+
running = false;
|
|
194
|
+
constructor(targets, config) {
|
|
195
|
+
this.targets = targets;
|
|
196
|
+
this.config = config;
|
|
197
|
+
}
|
|
198
|
+
async runOnce(nowMs = Date.now()) {
|
|
199
|
+
for (const target of this.targets) {
|
|
200
|
+
maintainSqliteCheckpoints(target.dbPath, this.config, nowMs);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
async start() {
|
|
204
|
+
if (this.running) {
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
this.running = true;
|
|
208
|
+
if (this.config.schedule.runOnStartup) {
|
|
209
|
+
await this.runOnce();
|
|
210
|
+
}
|
|
211
|
+
this.timer = setInterval(() => {
|
|
212
|
+
void this.runOnce();
|
|
213
|
+
}, this.config.schedule.intervalSeconds * 1000);
|
|
214
|
+
this.timer.unref?.();
|
|
215
|
+
}
|
|
216
|
+
async stop() {
|
|
217
|
+
if (this.timer) {
|
|
218
|
+
clearInterval(this.timer);
|
|
219
|
+
this.timer = null;
|
|
220
|
+
}
|
|
221
|
+
this.running = false;
|
|
222
|
+
}
|
|
223
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ApprovalRecord, HarnessEvent, HarnessStreamItem, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
|
|
1
|
+
import type { ApprovalRecord, CompiledTool, HarnessEvent, HarnessStreamItem, RunStartOptions, RestartConversationOptions, RuntimeAdapterOptions, ResumeOptions, RunOptions, RunResult, ThreadSummary, ThreadRecord, WorkspaceBundle } from "../contracts/types.js";
|
|
2
2
|
export declare class AgentHarness {
|
|
3
3
|
private readonly workspace;
|
|
4
4
|
private readonly runtimeAdapterOptions;
|
|
@@ -14,6 +14,8 @@ export declare class AgentHarness {
|
|
|
14
14
|
private readonly routingSystemPrompt?;
|
|
15
15
|
private readonly threadMemorySync;
|
|
16
16
|
private readonly unsubscribeThreadMemorySync;
|
|
17
|
+
private readonly resolvedRuntimeAdapterOptions;
|
|
18
|
+
private readonly checkpointMaintenance;
|
|
17
19
|
private listHostBindings;
|
|
18
20
|
private defaultRunRoot;
|
|
19
21
|
private heuristicRoute;
|
|
@@ -25,6 +27,13 @@ export declare class AgentHarness {
|
|
|
25
27
|
constructor(workspace: WorkspaceBundle, runtimeAdapterOptions?: RuntimeAdapterOptions);
|
|
26
28
|
initialize(): Promise<void>;
|
|
27
29
|
subscribe(listener: (event: HarnessEvent) => void): () => void;
|
|
30
|
+
getWorkspace(): WorkspaceBundle;
|
|
31
|
+
getBinding(agentId: string): WorkspaceBundle["bindings"] extends Map<any, infer T> ? T | undefined : never;
|
|
32
|
+
listAgentTools(agentId: string): CompiledTool[];
|
|
33
|
+
resolveAgentTools(agentId: string): Array<{
|
|
34
|
+
compiledTool: CompiledTool;
|
|
35
|
+
resolvedTool: unknown;
|
|
36
|
+
}>;
|
|
28
37
|
listSessions(filter?: {
|
|
29
38
|
agentId?: string;
|
|
30
39
|
}): Promise<ThreadSummary[]>;
|
package/dist/runtime/harness.js
CHANGED
|
@@ -12,6 +12,7 @@ import { resolveCompiledEmbeddingModel, resolveCompiledEmbeddingModelRef } from
|
|
|
12
12
|
import { resolveCompiledVectorStore, resolveCompiledVectorStoreRef } from "./support/vector-stores.js";
|
|
13
13
|
import { ThreadMemorySync } from "./thread-memory-sync.js";
|
|
14
14
|
import { FileBackedStore } from "./store.js";
|
|
15
|
+
import { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
|
|
15
16
|
export class AgentHarness {
|
|
16
17
|
workspace;
|
|
17
18
|
runtimeAdapterOptions;
|
|
@@ -27,6 +28,8 @@ export class AgentHarness {
|
|
|
27
28
|
routingSystemPrompt;
|
|
28
29
|
threadMemorySync;
|
|
29
30
|
unsubscribeThreadMemorySync;
|
|
31
|
+
resolvedRuntimeAdapterOptions;
|
|
32
|
+
checkpointMaintenance;
|
|
30
33
|
listHostBindings() {
|
|
31
34
|
return inferRoutingBindings(this.workspace).hostBindings;
|
|
32
35
|
}
|
|
@@ -112,7 +115,7 @@ export class AgentHarness {
|
|
|
112
115
|
this.persistence = new FilePersistence(runRoot);
|
|
113
116
|
const defaultStoreConfig = this.listHostBindings()[0]?.harnessRuntime.store;
|
|
114
117
|
this.defaultStore = defaultStoreConfig ? createStoreForConfig(defaultStoreConfig, runRoot) : new FileBackedStore(`${runRoot}/store.json`);
|
|
115
|
-
this.
|
|
118
|
+
this.resolvedRuntimeAdapterOptions = {
|
|
116
119
|
...runtimeAdapterOptions,
|
|
117
120
|
toolResolver: runtimeAdapterOptions.toolResolver ??
|
|
118
121
|
createResourceToolResolver(workspace, {
|
|
@@ -136,19 +139,51 @@ export class AgentHarness {
|
|
|
136
139
|
((binding) => this.resolveStore(binding)),
|
|
137
140
|
backendResolver: runtimeAdapterOptions.backendResolver ??
|
|
138
141
|
((binding) => createResourceBackendResolver(workspace)(binding)),
|
|
139
|
-
}
|
|
142
|
+
};
|
|
143
|
+
this.runtimeAdapter = new AgentRuntimeAdapter(this.resolvedRuntimeAdapterOptions);
|
|
140
144
|
this.routingSystemPrompt = getRoutingSystemPrompt(workspace.refs);
|
|
141
145
|
this.threadMemorySync = new ThreadMemorySync(this.persistence, this.defaultStore);
|
|
142
146
|
this.unsubscribeThreadMemorySync = this.eventBus.subscribe((event) => {
|
|
143
147
|
void this.threadMemorySync.handleEvent(event);
|
|
144
148
|
});
|
|
149
|
+
const checkpointMaintenanceConfig = readCheckpointMaintenanceConfig(workspace);
|
|
150
|
+
this.checkpointMaintenance = checkpointMaintenanceConfig
|
|
151
|
+
? new CheckpointMaintenanceLoop(discoverCheckpointMaintenanceTargets(workspace), checkpointMaintenanceConfig)
|
|
152
|
+
: null;
|
|
145
153
|
}
|
|
146
154
|
async initialize() {
|
|
147
155
|
await this.persistence.initialize();
|
|
156
|
+
await this.checkpointMaintenance?.start();
|
|
148
157
|
}
|
|
149
158
|
subscribe(listener) {
|
|
150
159
|
return this.eventBus.subscribe(listener);
|
|
151
160
|
}
|
|
161
|
+
getWorkspace() {
|
|
162
|
+
return this.workspace;
|
|
163
|
+
}
|
|
164
|
+
getBinding(agentId) {
|
|
165
|
+
return this.workspace.bindings.get(agentId);
|
|
166
|
+
}
|
|
167
|
+
listAgentTools(agentId) {
|
|
168
|
+
const binding = this.getBinding(agentId);
|
|
169
|
+
if (!binding) {
|
|
170
|
+
throw new Error(`Unknown agent ${agentId}`);
|
|
171
|
+
}
|
|
172
|
+
return binding.langchainAgentParams?.tools ?? binding.deepAgentParams?.tools ?? [];
|
|
173
|
+
}
|
|
174
|
+
resolveAgentTools(agentId) {
|
|
175
|
+
const binding = this.getBinding(agentId);
|
|
176
|
+
if (!binding) {
|
|
177
|
+
throw new Error(`Unknown agent ${agentId}`);
|
|
178
|
+
}
|
|
179
|
+
const compiledTools = this.listAgentTools(agentId);
|
|
180
|
+
const resolver = this.resolvedRuntimeAdapterOptions.toolResolver;
|
|
181
|
+
const resolvedTools = resolver ? resolver(compiledTools.map((tool) => tool.id), binding) : [];
|
|
182
|
+
return compiledTools.map((compiledTool, index) => ({
|
|
183
|
+
compiledTool,
|
|
184
|
+
resolvedTool: resolvedTools[index],
|
|
185
|
+
}));
|
|
186
|
+
}
|
|
152
187
|
async listSessions(filter) {
|
|
153
188
|
const threadSummaries = await this.persistence.listSessions();
|
|
154
189
|
if (!filter?.agentId) {
|
|
@@ -750,6 +785,7 @@ export class AgentHarness {
|
|
|
750
785
|
};
|
|
751
786
|
}
|
|
752
787
|
async close() {
|
|
788
|
+
await this.checkpointMaintenance?.stop();
|
|
753
789
|
this.unsubscribeThreadMemorySync();
|
|
754
790
|
await this.threadMemorySync.close();
|
|
755
791
|
}
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export { AgentRuntimeAdapter, AGENT_INTERRUPT_SENTINEL_PREFIX } from "./agent-runtime-adapter.js";
|
|
2
2
|
export { EventBus } from "./event-bus.js";
|
|
3
3
|
export { FileCheckpointSaver } from "./file-checkpoint-saver.js";
|
|
4
|
+
export { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, maintainSqliteCheckpoints, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
|
|
5
|
+
export { ManagedSqliteSaver } from "./sqlite-maintained-checkpoint-saver.js";
|
|
4
6
|
export { AgentHarness } from "./harness.js";
|
|
5
7
|
export { describeWorkspaceInventory, findAgentBinding, listAgentSkills, listAgentTools, listAvailableAgents, listSpecialists, } from "./inventory.js";
|
|
6
8
|
export * from "./parsing/index.js";
|
package/dist/runtime/index.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export { AgentRuntimeAdapter, AGENT_INTERRUPT_SENTINEL_PREFIX } from "./agent-runtime-adapter.js";
|
|
2
2
|
export { EventBus } from "./event-bus.js";
|
|
3
3
|
export { FileCheckpointSaver } from "./file-checkpoint-saver.js";
|
|
4
|
+
export { CheckpointMaintenanceLoop, discoverCheckpointMaintenanceTargets, maintainSqliteCheckpoints, readCheckpointMaintenanceConfig, } from "./checkpoint-maintenance.js";
|
|
5
|
+
export { ManagedSqliteSaver } from "./sqlite-maintained-checkpoint-saver.js";
|
|
4
6
|
export { AgentHarness } from "./harness.js";
|
|
5
7
|
export { describeWorkspaceInventory, findAgentBinding, listAgentSkills, listAgentTools, listAvailableAgents, listSpecialists, } from "./inventory.js";
|
|
6
8
|
export * from "./parsing/index.js";
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite";
|
|
2
|
+
import type { RunnableConfig } from "@langchain/core/runnables";
|
|
3
|
+
export declare class ManagedSqliteSaver extends SqliteSaver {
|
|
4
|
+
constructor(db: ConstructorParameters<typeof SqliteSaver>[0]);
|
|
5
|
+
prepareMaintenance(): void;
|
|
6
|
+
setup(): void;
|
|
7
|
+
put(config: RunnableConfig, checkpoint: Parameters<SqliteSaver["put"]>[1], metadata: Parameters<SqliteSaver["put"]>[2]): Promise<RunnableConfig<Record<string, any>>>;
|
|
8
|
+
deleteThread(threadId: string): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite";
|
|
2
|
+
export class ManagedSqliteSaver extends SqliteSaver {
|
|
3
|
+
constructor(db) {
|
|
4
|
+
super(db);
|
|
5
|
+
}
|
|
6
|
+
prepareMaintenance() {
|
|
7
|
+
this.setup();
|
|
8
|
+
}
|
|
9
|
+
setup() {
|
|
10
|
+
super.setup();
|
|
11
|
+
this.db.exec(`
|
|
12
|
+
CREATE TABLE IF NOT EXISTS checkpoint_maintenance_meta (
|
|
13
|
+
thread_id TEXT NOT NULL,
|
|
14
|
+
checkpoint_ns TEXT NOT NULL DEFAULT '',
|
|
15
|
+
checkpoint_id TEXT NOT NULL,
|
|
16
|
+
created_at_ms INTEGER NOT NULL,
|
|
17
|
+
PRIMARY KEY (thread_id, checkpoint_ns, checkpoint_id)
|
|
18
|
+
);`);
|
|
19
|
+
}
|
|
20
|
+
async put(config, checkpoint, metadata) {
|
|
21
|
+
const result = await super.put(config, checkpoint, metadata);
|
|
22
|
+
const threadId = result.configurable?.thread_id;
|
|
23
|
+
const checkpointNs = result.configurable?.checkpoint_ns ?? "";
|
|
24
|
+
const checkpointId = result.configurable?.checkpoint_id;
|
|
25
|
+
if (!threadId || !checkpointId) {
|
|
26
|
+
throw new Error("Missing checkpoint identity after SqliteSaver.put");
|
|
27
|
+
}
|
|
28
|
+
this.db
|
|
29
|
+
.prepare(`INSERT OR IGNORE INTO checkpoint_maintenance_meta
|
|
30
|
+
(thread_id, checkpoint_ns, checkpoint_id, created_at_ms)
|
|
31
|
+
VALUES (?, ?, ?, ?)`)
|
|
32
|
+
.run(threadId, checkpointNs, checkpointId, Date.now());
|
|
33
|
+
return result;
|
|
34
|
+
}
|
|
35
|
+
async deleteThread(threadId) {
|
|
36
|
+
await super.deleteThread(threadId);
|
|
37
|
+
this.db.prepare(`DELETE FROM checkpoint_maintenance_meta WHERE thread_id = ?`).run(threadId);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -2,6 +2,7 @@ import path from "node:path";
|
|
|
2
2
|
import { MemorySaver } from "@langchain/langgraph";
|
|
3
3
|
import { SqliteSaver } from "@langchain/langgraph-checkpoint-sqlite";
|
|
4
4
|
import { FileCheckpointSaver } from "../file-checkpoint-saver.js";
|
|
5
|
+
import { ManagedSqliteSaver } from "../sqlite-maintained-checkpoint-saver.js";
|
|
5
6
|
import { createInMemoryStore, FileBackedStore } from "../store.js";
|
|
6
7
|
export function createStoreForConfig(storeConfig, runRoot) {
|
|
7
8
|
const kind = typeof storeConfig.kind === "string" ? storeConfig.kind : "FileStore";
|
|
@@ -27,7 +28,8 @@ export function createCheckpointerForConfig(checkpointerConfig, runRoot) {
|
|
|
27
28
|
return new MemorySaver();
|
|
28
29
|
case "SqliteSaver": {
|
|
29
30
|
const configuredPath = typeof checkpointerConfig.path === "string" ? String(checkpointerConfig.path) : "checkpoints.sqlite";
|
|
30
|
-
|
|
31
|
+
const resolvedPath = path.isAbsolute(configuredPath) ? configuredPath : path.join(runRoot, configuredPath);
|
|
32
|
+
return new ManagedSqliteSaver(SqliteSaver.fromConnString(resolvedPath).db);
|
|
31
33
|
}
|
|
32
34
|
case "FileCheckpointer": {
|
|
33
35
|
const configuredPath = typeof checkpointerConfig.path === "string" ? String(checkpointerConfig.path) : "checkpoints.json";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
type ImportedToolModule = Record<string, unknown>;
|
|
2
|
+
type SchemaLike = {
|
|
3
|
+
parse: (input: unknown) => unknown;
|
|
4
|
+
description?: string;
|
|
5
|
+
shape?: Record<string, unknown>;
|
|
6
|
+
};
|
|
7
|
+
export type LoadedToolModule = {
|
|
8
|
+
implementationName: string;
|
|
9
|
+
invoke: (input: unknown, context?: Record<string, unknown>) => Promise<unknown> | unknown;
|
|
10
|
+
schema: SchemaLike;
|
|
11
|
+
description: string;
|
|
12
|
+
};
|
|
13
|
+
export declare function isSupportedToolModulePath(filePath: string): boolean;
|
|
14
|
+
export declare function discoverAnnotatedFunctionNames(sourceText: string): string[];
|
|
15
|
+
export declare function discoverToolModuleDefinitions(sourceText: string, imported: ImportedToolModule): LoadedToolModule[];
|
|
16
|
+
export declare function loadToolModuleDefinition(imported: ImportedToolModule, implementationName: string): LoadedToolModule;
|
|
17
|
+
export {};
|