@tangle-network/agent-profile-materialize 0.22.3 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp.d.ts +13 -3
- package/dist/mcp.js +23 -5
- package/dist/profile-bundle-cache.d.ts +37 -0
- package/dist/profile-bundle-cache.js +318 -0
- package/dist/profile-bundle-key.d.ts +69 -0
- package/dist/profile-bundle-key.js +62 -0
- package/dist/profile-bundle-lru.d.ts +56 -0
- package/dist/profile-bundle-lru.js +211 -0
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -21,6 +21,7 @@ export type { PrimeProfileErrorKind, PrimeThinkingLevel, } from "./prime-fork.js
|
|
|
21
21
|
export { matchPrimeKernelDeadMarker, PRIME_AGENT_DIR_ENV_VAR, PRIME_AGENT_DIR_ENV_VARS, PRIME_FORK_INSTALL, PRIME_FORK_PIN, PRIME_HARNESS_STATE_ENV_DENYLIST, PRIME_KERNEL_DEAD_MARKERS, PRIME_KERNEL_ENV_PASSTHROUGH, PRIME_MCP_UNSUPPORTED_REASON, PRIME_THINKING_LEVELS, PrimeHarnessStateError, PrimeProfileError, parsePrimeModelsJson, primeAgentDirEnv, primeApiKeyEnvNames, primeThinkingLevel, stripPrimeJsonComments, } from "./prime-fork.js";
|
|
22
22
|
export type { PrimeProfileControls, PrimeProfileSkill, PrimeProfileSubagent, } from "./prime-profile.js";
|
|
23
23
|
export { assertPrimeModelAgreement, hasPrimeProfileMaterial, primeProfileNeedsAgentDir, primeSubagentEntryId, readPrimeProfileControls, } from "./prime-profile.js";
|
|
24
|
+
export * from "./profile-bundle-cache.js";
|
|
24
25
|
export { executionControlConflict, materializeProfile, UNAVAILABLE_AGENT_PROFILE_SECRET_PROVIDER, } from "./profile-materializer.js";
|
|
25
26
|
export type { ResolveAgentProfileResourcesOptions } from "./resource-resolution.js";
|
|
26
27
|
export { normalizeSkillMd, resolveAgentProfileResources, } from "./resource-resolution.js";
|
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ export { BINDABLE_LAUNCH_CONTROLS, EXECUTOR_SYSTEM_PROMPT_BINDINGS, isExecutorSy
|
|
|
14
14
|
export { materializePrimeProfileControls, prunePrimeProfileMaterial, seedPrimeProfileSubagents, writePrimeAgentFile, } from "./prime-agent-dir.js";
|
|
15
15
|
export { matchPrimeKernelDeadMarker, PRIME_AGENT_DIR_ENV_VAR, PRIME_AGENT_DIR_ENV_VARS, PRIME_FORK_INSTALL, PRIME_FORK_PIN, PRIME_HARNESS_STATE_ENV_DENYLIST, PRIME_KERNEL_DEAD_MARKERS, PRIME_KERNEL_ENV_PASSTHROUGH, PRIME_MCP_UNSUPPORTED_REASON, PRIME_THINKING_LEVELS, PrimeHarnessStateError, PrimeProfileError, parsePrimeModelsJson, primeAgentDirEnv, primeApiKeyEnvNames, primeThinkingLevel, stripPrimeJsonComments, } from "./prime-fork.js";
|
|
16
16
|
export { assertPrimeModelAgreement, hasPrimeProfileMaterial, primeProfileNeedsAgentDir, primeSubagentEntryId, readPrimeProfileControls, } from "./prime-profile.js";
|
|
17
|
+
export * from "./profile-bundle-cache.js";
|
|
17
18
|
export { executionControlConflict, materializeProfile, UNAVAILABLE_AGENT_PROFILE_SECRET_PROVIDER, } from "./profile-materializer.js";
|
|
18
19
|
export { normalizeSkillMd, resolveAgentProfileResources, } from "./resource-resolution.js";
|
|
19
20
|
export { prepareSecretConcurrentProfileRevision } from "./secret-concurrent-revision.js";
|
package/dist/mcp.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export declare function materializeMcp(h: HarnessId, mcp: Record<string, ActiveA
|
|
|
3
3
|
* The header a harness sends to a remote MCP server whose tool-call timeout it
|
|
4
4
|
* raised, carrying the timeout it applied in milliseconds. A server may hold a
|
|
5
5
|
* call for nearly that long. A server that does not see the header must assume
|
|
6
|
-
* the harness default, 60 s for Claude Code and
|
|
6
|
+
* the harness default, 60 s for Claude Code, Codex and OpenCode.
|
|
7
7
|
*/
|
|
8
8
|
export declare const MCP_TOOL_TIMEOUT_HEADER = "X-Mcp-Tool-Timeout-Ms";
|
|
9
9
|
/** OpenCode-native MCP configuration used by workspace and startup paths. */
|
|
@@ -12,14 +12,24 @@ export type OpenCodeMcpConfig = Record<string, {
|
|
|
12
12
|
url: string;
|
|
13
13
|
enabled: boolean;
|
|
14
14
|
headers?: Record<string, AgentProfileConfigValue>;
|
|
15
|
+
/** Milliseconds; OpenCode bounds this server's connect, tool listing and tool calls by it. */
|
|
16
|
+
timeout?: number;
|
|
17
|
+
/** `false` turns off OpenCode's OAuth discovery for this server. */
|
|
18
|
+
oauth?: false;
|
|
15
19
|
} | {
|
|
16
20
|
type: "local";
|
|
17
21
|
command: Array<string | AgentProfileConfigValue>;
|
|
18
22
|
enabled: boolean;
|
|
19
23
|
environment?: Record<string, AgentProfileConfigValue>;
|
|
24
|
+
timeout?: number;
|
|
20
25
|
}>;
|
|
21
|
-
/**
|
|
22
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Lower canonical MCP definitions to OpenCode's native startup configuration.
|
|
28
|
+
* `toolTimeouts` holds the validated `metadata.toolTimeoutMs` of each server
|
|
29
|
+
* that declared one; without it OpenCode applies the MCP SDK's 60 s default to
|
|
30
|
+
* every tool call.
|
|
31
|
+
*/
|
|
32
|
+
export declare function openCodeMcpConfig(mcp: Record<string, AgentProfileMcpServer>, toolTimeouts?: ReadonlyMap<string, number>): OpenCodeMcpConfig;
|
|
23
33
|
import type { AgentProfileConfigValue, AgentProfileMcpServer } from "@tangle-network/agent-interface";
|
|
24
34
|
import type { AddPlanFile } from "./plan-json.js";
|
|
25
35
|
import type { HarnessId, Unsupported, WorkspacePlan } from "./plan-types.js";
|
package/dist/mcp.js
CHANGED
|
@@ -58,7 +58,7 @@ export function materializeMcp(h, mcp, plan, add, unsupported) {
|
|
|
58
58
|
// opencode.json "mcp" block (local|remote).
|
|
59
59
|
add(OPENCODE_CONFIG_FILE, serializeWorkspacePlanJson({
|
|
60
60
|
$schema: OPENCODE_CONFIG_SCHEMA,
|
|
61
|
-
mcp: openCodeMcpConfig(mcp),
|
|
61
|
+
mcp: openCodeMcpConfig(mcp, mcpToolTimeouts(mcp, h, unsupported)),
|
|
62
62
|
}));
|
|
63
63
|
break;
|
|
64
64
|
case "gemini": {
|
|
@@ -156,7 +156,7 @@ function tomlInlineStringTable(values) {
|
|
|
156
156
|
* The header a harness sends to a remote MCP server whose tool-call timeout it
|
|
157
157
|
* raised, carrying the timeout it applied in milliseconds. A server may hold a
|
|
158
158
|
* call for nearly that long. A server that does not see the header must assume
|
|
159
|
-
* the harness default, 60 s for Claude Code and
|
|
159
|
+
* the harness default, 60 s for Claude Code, Codex and OpenCode.
|
|
160
160
|
*/
|
|
161
161
|
export const MCP_TOOL_TIMEOUT_HEADER = "X-Mcp-Tool-Timeout-Ms";
|
|
162
162
|
const MIN_MCP_TOOL_TIMEOUT_MS = 1_000;
|
|
@@ -270,21 +270,38 @@ function kimiMcpServer(server) {
|
|
|
270
270
|
...(server.cwd ? { cwd: server.cwd } : {}),
|
|
271
271
|
};
|
|
272
272
|
}
|
|
273
|
-
|
|
274
|
-
|
|
273
|
+
const carriesAuthorization = (headers) => Object.keys(headers ?? {}).some((header) => header.toLowerCase() === "authorization");
|
|
274
|
+
/**
|
|
275
|
+
* Lower canonical MCP definitions to OpenCode's native startup configuration.
|
|
276
|
+
* `toolTimeouts` holds the validated `metadata.toolTimeoutMs` of each server
|
|
277
|
+
* that declared one; without it OpenCode applies the MCP SDK's 60 s default to
|
|
278
|
+
* every tool call.
|
|
279
|
+
*/
|
|
280
|
+
export function openCodeMcpConfig(mcp, toolTimeouts = new Map()) {
|
|
275
281
|
const out = {};
|
|
276
282
|
for (const [name, s] of Object.entries(mcp)) {
|
|
277
283
|
if (s.enabled === false)
|
|
278
284
|
continue;
|
|
285
|
+
// OpenCode also bounds connect and tool listing by this value, so a server
|
|
286
|
+
// that hangs on connect holds startup that long; a refused connection still
|
|
287
|
+
// fails at once.
|
|
288
|
+
const timeout = toolTimeouts.get(name);
|
|
279
289
|
if (s.url) {
|
|
280
290
|
if (s.transport === "sse") {
|
|
281
291
|
throw new Error(`OpenCode MCP server "${name}" uses unsupported SSE transport`);
|
|
282
292
|
}
|
|
293
|
+
const headers = withAppliedToolTimeout(s.headers, timeout);
|
|
283
294
|
out[name] = {
|
|
284
295
|
type: "remote",
|
|
285
296
|
url: s.url,
|
|
286
297
|
enabled: true,
|
|
287
|
-
...(
|
|
298
|
+
...(headers ? { headers } : {}),
|
|
299
|
+
...(timeout !== undefined ? { timeout } : {}),
|
|
300
|
+
// A server that takes its credential from these headers has no OAuth
|
|
301
|
+
// flow. With OAuth on, OpenCode answers its 401 with OAuth discovery at
|
|
302
|
+
// the server's origin, and the agent sees that discovery's failure
|
|
303
|
+
// instead of the server's refusal.
|
|
304
|
+
...(carriesAuthorization(s.headers) ? { oauth: false } : {}),
|
|
288
305
|
};
|
|
289
306
|
continue;
|
|
290
307
|
}
|
|
@@ -299,6 +316,7 @@ export function openCodeMcpConfig(mcp) {
|
|
|
299
316
|
command: [s.command, ...(s.args ?? [])],
|
|
300
317
|
enabled: true,
|
|
301
318
|
...(s.env ? { environment: s.env } : {}),
|
|
319
|
+
...(timeout !== undefined ? { timeout } : {}),
|
|
302
320
|
};
|
|
303
321
|
}
|
|
304
322
|
return out;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type ProfileBundleCache, type ProfileBundleClaim, type ProfileBundleClaimResult, type ProfileBundleManifest } from "./profile-bundle-key.js";
|
|
2
|
+
export * from "./profile-bundle-key.js";
|
|
3
|
+
export { evictProfileBundles, profileBundleManifestPath, sweepProfileBundleCache, } from "./profile-bundle-lru.js";
|
|
4
|
+
/**
|
|
5
|
+
* Admit a just-delivered bundle to the host cache, all-or-nothing. Every
|
|
6
|
+
* object's bytes are verified against its content hash BEFORE anything is
|
|
7
|
+
* renamed into place; a mismatch throws with nothing published. Objects
|
|
8
|
+
* already present are re-verified against the incoming bytes, then the temp
|
|
9
|
+
* copy is dropped — the existing object was verified when it was admitted.
|
|
10
|
+
* The manifest is written LAST as the commit point, followed by lru update
|
|
11
|
+
* and eviction down to the default cap.
|
|
12
|
+
*/
|
|
13
|
+
export declare function admitProfileBundle(root: string, manifest: ProfileBundleManifest, readObject: (sha256: string) => Promise<Uint8Array> | Uint8Array): Promise<void>;
|
|
14
|
+
/**
|
|
15
|
+
* Claim a bundle from a (possibly read-only mounted) cache. Verify-on-read:
|
|
16
|
+
* every object's bytes are re-hashed and compared to BOTH the manifest entry
|
|
17
|
+
* and the claim's expected sha256 before `linkFile` is called — a read-only
|
|
18
|
+
* mount must never serve unverified bytes. `linkFile` returning false sends
|
|
19
|
+
* the file to `missing` (the caller delivers it normally). Status is "hit"
|
|
20
|
+
* only when nothing is missing; a partial bundle is NEVER a hit. The only
|
|
21
|
+
* write is a best-effort lru touch, wrapped so EROFS on a read-only mount is
|
|
22
|
+
* harmless.
|
|
23
|
+
*/
|
|
24
|
+
export declare function claimProfileBundle(root: string, claim: ProfileBundleClaim, linkFile: (objectPath: string, entry: {
|
|
25
|
+
mountPath: string;
|
|
26
|
+
target: string;
|
|
27
|
+
sha256: string;
|
|
28
|
+
mode: number;
|
|
29
|
+
}) => Promise<boolean>): Promise<ProfileBundleClaimResult>;
|
|
30
|
+
/**
|
|
31
|
+
* Open a host-side cache handle over `root`, creating the layout and sweeping
|
|
32
|
+
* stale `incoming/` residue synchronously so the cache is deterministic to
|
|
33
|
+
* use immediately afterwards.
|
|
34
|
+
*/
|
|
35
|
+
export declare function openProfileBundleCache(root: string, options?: {
|
|
36
|
+
maxBytes?: number;
|
|
37
|
+
}): ProfileBundleCache;
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { mkdirSync, readdirSync, rmSync } from "node:fs";
|
|
3
|
+
import { mkdir, open, readFile, rename, rm, stat, writeFile, } from "node:fs/promises";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { PROFILE_BUNDLE_DEFAULT_MAX_BYTES, PROFILE_BUNDLE_MAX_BYTES, PROFILE_BUNDLE_MAX_FILE_BYTES, PROFILE_BUNDLE_MAX_FILES, PROFILE_BUNDLE_SCHEMA, } from "./profile-bundle-key.js";
|
|
6
|
+
import { evictProfileBundles, evictUnlocked, fsyncDir, loadLru, pathExists, profileBundleManifestPath, readManifestFile, saveLru, sweepProfileBundleCache, withRootLock, } from "./profile-bundle-lru.js";
|
|
7
|
+
import { hasControlCharacter, normalizeWorkspaceRelativePath, validateFileMode, } from "./workspace-files.js";
|
|
8
|
+
export * from "./profile-bundle-key.js";
|
|
9
|
+
export { evictProfileBundles, profileBundleManifestPath, sweepProfileBundleCache, } from "./profile-bundle-lru.js";
|
|
10
|
+
const SHA256_HEX_RE = /^[0-9a-f]{64}$/;
|
|
11
|
+
const sha256Hex = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
12
|
+
const nowIso = () => new Date().toISOString();
|
|
13
|
+
/**
|
|
14
|
+
* Content-addressed, per-host cache of delivered profile-file bundles.
|
|
15
|
+
*
|
|
16
|
+
* A production cold sandbox start spends seconds re-delivering the same inline
|
|
17
|
+
* profile file mounts through the box file API on every fresh box. The host
|
|
18
|
+
* owns a cache directory (mounted read-only into boxes) of immutable,
|
|
19
|
+
* content-addressed file objects plus per-bundle manifests. A box "claims" a
|
|
20
|
+
* bundle by verifying every object against its content hash and hardlinking it
|
|
21
|
+
* into the workspace, skipping network delivery; after a normal miss-delivery
|
|
22
|
+
* the box-side publisher "admits" the just-delivered bytes to the host.
|
|
23
|
+
*
|
|
24
|
+
* Durability invariants:
|
|
25
|
+
* - `objects/<sha256>` bytes are immutable and self-certifying: the filename
|
|
26
|
+
* IS the SHA-256 of the bytes. Objects are written to a unique `incoming/`
|
|
27
|
+
* dir and renamed into place only after every byte is verified, so a crash
|
|
28
|
+
* mid-admit leaves no half-published object behind.
|
|
29
|
+
* - `bundles/<key>.json` is the commit point: readers treat a bundle as
|
|
30
|
+
* present only if its manifest exists, and verify every object anyway
|
|
31
|
+
* (verify-on-read), so a read-only or partially corrupt cache mount can
|
|
32
|
+
* never serve unverified bytes.
|
|
33
|
+
* - `lru.json` is advisory accounting only; corruption never breaks admit or
|
|
34
|
+
* claim (see profile-bundle-lru.ts).
|
|
35
|
+
*/
|
|
36
|
+
function validateManifestShape(manifest) {
|
|
37
|
+
if (manifest.schema !== PROFILE_BUNDLE_SCHEMA) {
|
|
38
|
+
throw new Error(`Profile bundle manifest schema mismatch: ${String(manifest.schema)}`);
|
|
39
|
+
}
|
|
40
|
+
if (typeof manifest.bundle !== "string" || manifest.bundle.length === 0) {
|
|
41
|
+
throw new Error("Profile bundle manifest requires a non-empty bundle key");
|
|
42
|
+
}
|
|
43
|
+
if (typeof manifest.createdAt !== "string") {
|
|
44
|
+
throw new Error("Profile bundle manifest requires an ISO createdAt");
|
|
45
|
+
}
|
|
46
|
+
if (!Number.isInteger(manifest.sizeBytes) ||
|
|
47
|
+
manifest.sizeBytes < 0 ||
|
|
48
|
+
manifest.sizeBytes > PROFILE_BUNDLE_MAX_BYTES) {
|
|
49
|
+
throw new Error(`Profile bundle sizeBytes exceeds ${PROFILE_BUNDLE_MAX_BYTES}: ${manifest.sizeBytes}`);
|
|
50
|
+
}
|
|
51
|
+
if (!Array.isArray(manifest.files)) {
|
|
52
|
+
throw new Error("Profile bundle manifest requires a files array");
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Reject anything that could link outside the cache or the workspace:
|
|
57
|
+
* mount paths must already be canonical workspace-relative paths, and targets
|
|
58
|
+
* must not contain traversal segments (targets MAY be absolute — the staged
|
|
59
|
+
* manifest's `path` can be a file-API absolute prefix — but never `..`).
|
|
60
|
+
*/
|
|
61
|
+
function validateTarget(target) {
|
|
62
|
+
if (typeof target !== "string" || target.length === 0) {
|
|
63
|
+
throw new Error("Profile bundle file requires a non-empty target");
|
|
64
|
+
}
|
|
65
|
+
if (hasControlCharacter(target)) {
|
|
66
|
+
throw new Error(`Profile bundle target has control characters: ${target}`);
|
|
67
|
+
}
|
|
68
|
+
const segments = target.replace(/\\/g, "/").split("/");
|
|
69
|
+
if (segments.some((segment) => segment === "..")) {
|
|
70
|
+
throw new Error(`Profile bundle target must not traverse: ${target}`);
|
|
71
|
+
}
|
|
72
|
+
return target;
|
|
73
|
+
}
|
|
74
|
+
function validateManifestFiles(manifest) {
|
|
75
|
+
if (manifest.files.length > PROFILE_BUNDLE_MAX_FILES) {
|
|
76
|
+
throw new Error(`Profile bundle has ${manifest.files.length} files; limit is ${PROFILE_BUNDLE_MAX_FILES}`);
|
|
77
|
+
}
|
|
78
|
+
const mountPaths = new Set();
|
|
79
|
+
const targets = new Set();
|
|
80
|
+
const entries = [];
|
|
81
|
+
for (const file of manifest.files) {
|
|
82
|
+
if (normalizeWorkspaceRelativePath(file.mountPath) !== file.mountPath) {
|
|
83
|
+
throw new Error(`Profile bundle mount path must be canonical: ${file.mountPath}`);
|
|
84
|
+
}
|
|
85
|
+
const target = validateTarget(file.target);
|
|
86
|
+
if (!SHA256_HEX_RE.test(file.sha256)) {
|
|
87
|
+
throw new Error(`Profile bundle sha256 must be lowercase hex without prefix: ${file.sha256}`);
|
|
88
|
+
}
|
|
89
|
+
const mode = validateFileMode(file.mode);
|
|
90
|
+
if (mountPaths.has(file.mountPath)) {
|
|
91
|
+
throw new Error(`Profile bundle has duplicate mount path: ${file.mountPath}`);
|
|
92
|
+
}
|
|
93
|
+
if (targets.has(target)) {
|
|
94
|
+
throw new Error(`Profile bundle has duplicate target: ${target}`);
|
|
95
|
+
}
|
|
96
|
+
mountPaths.add(file.mountPath);
|
|
97
|
+
targets.add(target);
|
|
98
|
+
entries.push({
|
|
99
|
+
mountPath: file.mountPath,
|
|
100
|
+
sha256: file.sha256,
|
|
101
|
+
mode,
|
|
102
|
+
target,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
return entries;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Admit a just-delivered bundle to the host cache, all-or-nothing. Every
|
|
109
|
+
* object's bytes are verified against its content hash BEFORE anything is
|
|
110
|
+
* renamed into place; a mismatch throws with nothing published. Objects
|
|
111
|
+
* already present are re-verified against the incoming bytes, then the temp
|
|
112
|
+
* copy is dropped — the existing object was verified when it was admitted.
|
|
113
|
+
* The manifest is written LAST as the commit point, followed by lru update
|
|
114
|
+
* and eviction down to the default cap.
|
|
115
|
+
*/
|
|
116
|
+
export async function admitProfileBundle(root, manifest, readObject) {
|
|
117
|
+
validateManifestShape(manifest);
|
|
118
|
+
const entries = validateManifestFiles(manifest);
|
|
119
|
+
await sweepProfileBundleCache(root);
|
|
120
|
+
await mkdir(join(root, "objects"), { recursive: true });
|
|
121
|
+
await mkdir(join(root, "bundles"), { recursive: true });
|
|
122
|
+
await mkdir(join(root, "incoming"), { recursive: true });
|
|
123
|
+
await withRootLock(root, async () => {
|
|
124
|
+
const manifestPath = profileBundleManifestPath(root, manifest.bundle);
|
|
125
|
+
const existing = await readManifestFile(manifestPath);
|
|
126
|
+
if (existing && existing.bundle !== manifest.bundle) {
|
|
127
|
+
throw new Error(`Profile bundle key collides with an admitted bundle: ${manifest.bundle}`);
|
|
128
|
+
}
|
|
129
|
+
const incoming = join(root, "incoming", randomUUID());
|
|
130
|
+
await mkdir(incoming, { recursive: true });
|
|
131
|
+
try {
|
|
132
|
+
const staged = [];
|
|
133
|
+
let totalBytes = 0;
|
|
134
|
+
for (const entry of entries) {
|
|
135
|
+
const bytes = await readObject(entry.sha256);
|
|
136
|
+
const buffer = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
|
|
137
|
+
if (buffer.byteLength > PROFILE_BUNDLE_MAX_FILE_BYTES) {
|
|
138
|
+
throw new Error(`Profile bundle file ${entry.mountPath} exceeds ${PROFILE_BUNDLE_MAX_FILE_BYTES} bytes`);
|
|
139
|
+
}
|
|
140
|
+
if (sha256Hex(buffer) !== entry.sha256) {
|
|
141
|
+
throw new Error(`Profile bundle object hash mismatch for ${entry.mountPath}`);
|
|
142
|
+
}
|
|
143
|
+
totalBytes += buffer.byteLength;
|
|
144
|
+
const object = join(root, "objects", entry.sha256);
|
|
145
|
+
if (await pathExists(object)) {
|
|
146
|
+
// Already admitted; the bytes above were re-verified, so drop the temp.
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const temporary = join(incoming, entry.sha256);
|
|
150
|
+
const handle = await open(temporary, "w", (entry.mode & 0o100) !== 0 ? 0o755 : 0o644);
|
|
151
|
+
try {
|
|
152
|
+
await handle.write(buffer);
|
|
153
|
+
await handle.sync();
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
await handle.close();
|
|
157
|
+
}
|
|
158
|
+
staged.push({ temporary, object });
|
|
159
|
+
}
|
|
160
|
+
if (totalBytes > PROFILE_BUNDLE_MAX_BYTES) {
|
|
161
|
+
throw new Error(`Profile bundle exceeds ${PROFILE_BUNDLE_MAX_BYTES} bytes: ${totalBytes}`);
|
|
162
|
+
}
|
|
163
|
+
for (const { temporary, object } of staged) {
|
|
164
|
+
await rename(temporary, object);
|
|
165
|
+
}
|
|
166
|
+
await fsyncDir(join(root, "objects"));
|
|
167
|
+
const stagedManifest = join(incoming, "manifest.json");
|
|
168
|
+
await writeFile(stagedManifest, JSON.stringify(manifest));
|
|
169
|
+
const manifestHandle = await open(stagedManifest, "r");
|
|
170
|
+
try {
|
|
171
|
+
await manifestHandle.sync();
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
await manifestHandle.close();
|
|
175
|
+
}
|
|
176
|
+
await rename(stagedManifest, manifestPath);
|
|
177
|
+
await fsyncDir(join(root, "bundles"));
|
|
178
|
+
const lru = await loadLru(root);
|
|
179
|
+
const used = nowIso();
|
|
180
|
+
for (const entry of entries) {
|
|
181
|
+
lru.objects[entry.sha256] = {
|
|
182
|
+
bytes: (await stat(join(root, "objects", entry.sha256))).size,
|
|
183
|
+
lastUsed: used,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
lru.bundles[manifest.bundle] = {
|
|
187
|
+
bytes: totalBytes,
|
|
188
|
+
lastUsed: used,
|
|
189
|
+
createdAt: manifest.createdAt,
|
|
190
|
+
};
|
|
191
|
+
await saveLru(root, lru);
|
|
192
|
+
}
|
|
193
|
+
finally {
|
|
194
|
+
await rm(incoming, { recursive: true, force: true });
|
|
195
|
+
}
|
|
196
|
+
await evictUnlocked(root, PROFILE_BUNDLE_DEFAULT_MAX_BYTES);
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Claim a bundle from a (possibly read-only mounted) cache. Verify-on-read:
|
|
201
|
+
* every object's bytes are re-hashed and compared to BOTH the manifest entry
|
|
202
|
+
* and the claim's expected sha256 before `linkFile` is called — a read-only
|
|
203
|
+
* mount must never serve unverified bytes. `linkFile` returning false sends
|
|
204
|
+
* the file to `missing` (the caller delivers it normally). Status is "hit"
|
|
205
|
+
* only when nothing is missing; a partial bundle is NEVER a hit. The only
|
|
206
|
+
* write is a best-effort lru touch, wrapped so EROFS on a read-only mount is
|
|
207
|
+
* harmless.
|
|
208
|
+
*/
|
|
209
|
+
export async function claimProfileBundle(root, claim, linkFile) {
|
|
210
|
+
const missAll = () => ({
|
|
211
|
+
status: "miss",
|
|
212
|
+
missing: claim.files.map((file) => file.mountPath),
|
|
213
|
+
});
|
|
214
|
+
const manifest = await readManifestFile(profileBundleManifestPath(root, claim.bundle));
|
|
215
|
+
if (!manifest ||
|
|
216
|
+
manifest.schema !== PROFILE_BUNDLE_SCHEMA ||
|
|
217
|
+
manifest.bundle !== claim.bundle ||
|
|
218
|
+
!Array.isArray(manifest.files)) {
|
|
219
|
+
return missAll();
|
|
220
|
+
}
|
|
221
|
+
let entries;
|
|
222
|
+
try {
|
|
223
|
+
entries = validateManifestFiles(manifest);
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
return missAll();
|
|
227
|
+
}
|
|
228
|
+
const expected = new Map();
|
|
229
|
+
for (const file of claim.files) {
|
|
230
|
+
// Fail closed: a claim with a non-canonical path is rejected outright so a
|
|
231
|
+
// tampered claim can never link outside the workspace.
|
|
232
|
+
if (normalizeWorkspaceRelativePath(file.mountPath) !== file.mountPath) {
|
|
233
|
+
throw new Error(`Profile bundle claim path is not canonical: ${file.mountPath}`);
|
|
234
|
+
}
|
|
235
|
+
if (!SHA256_HEX_RE.test(file.sha256)) {
|
|
236
|
+
throw new Error(`Profile bundle claim sha256 is not hex: ${file.sha256}`);
|
|
237
|
+
}
|
|
238
|
+
if (expected.has(file.mountPath)) {
|
|
239
|
+
throw new Error(`Profile bundle claim has duplicate mount path: ${file.mountPath}`);
|
|
240
|
+
}
|
|
241
|
+
expected.set(file.mountPath, file.sha256);
|
|
242
|
+
}
|
|
243
|
+
const tableMatches = entries.length === expected.size &&
|
|
244
|
+
entries.every((entry) => expected.get(entry.mountPath) === entry.sha256);
|
|
245
|
+
if (!tableMatches) {
|
|
246
|
+
return missAll();
|
|
247
|
+
}
|
|
248
|
+
const linked = [];
|
|
249
|
+
const missing = [];
|
|
250
|
+
for (const entry of entries) {
|
|
251
|
+
const objectPath = join(root, "objects", entry.sha256);
|
|
252
|
+
let bytes;
|
|
253
|
+
try {
|
|
254
|
+
bytes = await readFile(objectPath);
|
|
255
|
+
}
|
|
256
|
+
catch {
|
|
257
|
+
missing.push(entry.mountPath);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
if (sha256Hex(bytes) !== entry.sha256 ||
|
|
261
|
+
entry.sha256 !== expected.get(entry.mountPath)) {
|
|
262
|
+
missing.push(entry.mountPath);
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (await linkFile(objectPath, entry)) {
|
|
266
|
+
linked.push(entry.mountPath);
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
missing.push(entry.mountPath);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (missing.length > 0)
|
|
273
|
+
return { status: "miss", missing };
|
|
274
|
+
try {
|
|
275
|
+
await withRootLock(root, async () => {
|
|
276
|
+
const lru = await loadLru(root);
|
|
277
|
+
const used = nowIso();
|
|
278
|
+
for (const entry of entries) {
|
|
279
|
+
const object = lru.objects[entry.sha256];
|
|
280
|
+
if (object)
|
|
281
|
+
object.lastUsed = used;
|
|
282
|
+
}
|
|
283
|
+
const bundleEntry = lru.bundles[claim.bundle];
|
|
284
|
+
if (bundleEntry)
|
|
285
|
+
bundleEntry.lastUsed = used;
|
|
286
|
+
await saveLru(root, lru);
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// Best-effort accounting on a possibly read-only mount.
|
|
291
|
+
}
|
|
292
|
+
return { status: "hit", linked };
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Open a host-side cache handle over `root`, creating the layout and sweeping
|
|
296
|
+
* stale `incoming/` residue synchronously so the cache is deterministic to
|
|
297
|
+
* use immediately afterwards.
|
|
298
|
+
*/
|
|
299
|
+
export function openProfileBundleCache(root, options) {
|
|
300
|
+
const maxBytes = options?.maxBytes ?? PROFILE_BUNDLE_DEFAULT_MAX_BYTES;
|
|
301
|
+
if (!Number.isFinite(maxBytes) || maxBytes < 0) {
|
|
302
|
+
throw new Error(`Profile bundle cache maxBytes must be >= 0: ${maxBytes}`);
|
|
303
|
+
}
|
|
304
|
+
mkdirSync(join(root, "objects"), { recursive: true });
|
|
305
|
+
mkdirSync(join(root, "bundles"), { recursive: true });
|
|
306
|
+
const incoming = join(root, "incoming");
|
|
307
|
+
mkdirSync(incoming, { recursive: true });
|
|
308
|
+
for (const entry of readdirSync(incoming)) {
|
|
309
|
+
rmSync(join(incoming, entry), { recursive: true, force: true });
|
|
310
|
+
}
|
|
311
|
+
return {
|
|
312
|
+
root,
|
|
313
|
+
maxBytes,
|
|
314
|
+
admit: (manifest, readObject) => admitProfileBundle(root, manifest, readObject),
|
|
315
|
+
evict: () => evictProfileBundles(root, maxBytes),
|
|
316
|
+
sweep: () => sweepProfileBundleCache(root),
|
|
317
|
+
};
|
|
318
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import type { AgentProfileFileMount } from "@tangle-network/agent-interface";
|
|
2
|
+
export declare const PROFILE_BUNDLE_SCHEMA = "agent-profile-bundle/v1";
|
|
3
|
+
export declare const PROFILE_BUNDLE_MAX_FILES = 512;
|
|
4
|
+
export declare const PROFILE_BUNDLE_MAX_BYTES: number;
|
|
5
|
+
export declare const PROFILE_BUNDLE_MAX_FILE_BYTES: number;
|
|
6
|
+
export declare const PROFILE_BUNDLE_DEFAULT_MAX_BYTES: number;
|
|
7
|
+
export interface ProfileBundleFileEntry {
|
|
8
|
+
/** Logical workspace-relative mount path, exactly as declared on the profile. */
|
|
9
|
+
mountPath: string;
|
|
10
|
+
/** SHA-256 of the content bytes as lowercase hex, WITHOUT the `sha256:` prefix. */
|
|
11
|
+
sha256: string;
|
|
12
|
+
/** Full file mode (0o644 default, 0o755 for executable mounts). */
|
|
13
|
+
mode: number;
|
|
14
|
+
}
|
|
15
|
+
export interface ProfileBundleManifest {
|
|
16
|
+
schema: string;
|
|
17
|
+
/** Raw bundle key (e.g. `sha256:<hex>`); also recoverable from the filename. */
|
|
18
|
+
bundle: string;
|
|
19
|
+
createdAt: string;
|
|
20
|
+
sizeBytes: number;
|
|
21
|
+
files: Array<ProfileBundleFileEntry & {
|
|
22
|
+
target: string;
|
|
23
|
+
}>;
|
|
24
|
+
}
|
|
25
|
+
export interface ProfileBundleClaim {
|
|
26
|
+
bundle: string;
|
|
27
|
+
files: Array<{
|
|
28
|
+
mountPath: string;
|
|
29
|
+
sha256: string;
|
|
30
|
+
}>;
|
|
31
|
+
}
|
|
32
|
+
export type ProfileBundleClaimResult = {
|
|
33
|
+
status: "hit";
|
|
34
|
+
linked: string[];
|
|
35
|
+
} | {
|
|
36
|
+
status: "miss";
|
|
37
|
+
missing: string[];
|
|
38
|
+
};
|
|
39
|
+
export interface ProfileBundleCache {
|
|
40
|
+
root: string;
|
|
41
|
+
maxBytes: number;
|
|
42
|
+
admit(manifest: ProfileBundleManifest, readObject: (sha256: string) => Promise<Uint8Array> | Uint8Array): Promise<void>;
|
|
43
|
+
evict(): Promise<void>;
|
|
44
|
+
sweep(): Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Derive the canonical bundle file entry for one inline mount. The sha256 is
|
|
48
|
+
* computed over the exact inline content bytes and the mode mirrors the SDK's
|
|
49
|
+
* `profileFileMode`/`isExecutableProfileFile`: `mount.executable` wins when
|
|
50
|
+
* present, otherwise a path under a `bin/` or `sbin/` directory is executable.
|
|
51
|
+
* A mount whose resource is not inline is a caller bug and throws, matching
|
|
52
|
+
* the fail-closed discipline of {@link resolveInlineRef}.
|
|
53
|
+
*/
|
|
54
|
+
export declare function profileBundleFileEntry(mount: AgentProfileFileMount): ProfileBundleFileEntry;
|
|
55
|
+
/**
|
|
56
|
+
* The canonical file table: one entry per mount, sorted by (mountPath, sha256)
|
|
57
|
+
* so any array order of the same mounts produces the same table.
|
|
58
|
+
*/
|
|
59
|
+
export declare function profileBundleFileTable(mounts: readonly AgentProfileFileMount[]): ProfileBundleFileEntry[];
|
|
60
|
+
/**
|
|
61
|
+
* The content-addressed bundle key. The digest covers EXACTLY the schema
|
|
62
|
+
* version and, per file, the logical mount path, the sha256 of the content
|
|
63
|
+
* bytes, and the full file mode — the same image-content-hash discipline as
|
|
64
|
+
* scripts/ci/image-content-hash.mjs: hash exactly the inputs that determine
|
|
65
|
+
* the cached bytes (and their placement semantics), nothing more. Any profile
|
|
66
|
+
* change — content, path, or mode — produces a new key; anything else
|
|
67
|
+
* (mount order, transport-only target paths, timestamps) does not.
|
|
68
|
+
*/
|
|
69
|
+
export declare function profileFileBundleKey(mounts: readonly AgentProfileFileMount[]): string;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { canonicalCandidateDigest } from "@tangle-network/agent-interface";
|
|
3
|
+
import { resolveInlineRef } from "./resource-resolution.js";
|
|
4
|
+
import { compareStrings } from "./workspace-files.js";
|
|
5
|
+
// The bundle key, wire shapes and limits shared by the SDK, sidecar and
|
|
6
|
+
// host-agent. The cache itself (admit, claim, open) is profile-bundle-cache.ts.
|
|
7
|
+
export const PROFILE_BUNDLE_SCHEMA = "agent-profile-bundle/v1";
|
|
8
|
+
export const PROFILE_BUNDLE_MAX_FILES = 512;
|
|
9
|
+
export const PROFILE_BUNDLE_MAX_BYTES = 256 * 1024 * 1024;
|
|
10
|
+
export const PROFILE_BUNDLE_MAX_FILE_BYTES = 32 * 1024 * 1024;
|
|
11
|
+
export const PROFILE_BUNDLE_DEFAULT_MAX_BYTES = 1024 * 1024 * 1024;
|
|
12
|
+
// Mirrors PROFILE_BIN_DIR_RE in products/sandbox/sdk/src/profile-file-mounts.ts,
|
|
13
|
+
// the canonical definition for staged profile delivery. The SDK consumes this
|
|
14
|
+
// package, so the regex source is mirrored (not imported); tests assert parity
|
|
15
|
+
// with the SDK's profileFileMode semantics for representative paths.
|
|
16
|
+
const PROFILE_BIN_DIR_RE = /(^|\/)(s?bin)\//;
|
|
17
|
+
/**
|
|
18
|
+
* Derive the canonical bundle file entry for one inline mount. The sha256 is
|
|
19
|
+
* computed over the exact inline content bytes and the mode mirrors the SDK's
|
|
20
|
+
* `profileFileMode`/`isExecutableProfileFile`: `mount.executable` wins when
|
|
21
|
+
* present, otherwise a path under a `bin/` or `sbin/` directory is executable.
|
|
22
|
+
* A mount whose resource is not inline is a caller bug and throws, matching
|
|
23
|
+
* the fail-closed discipline of {@link resolveInlineRef}.
|
|
24
|
+
*/
|
|
25
|
+
export function profileBundleFileEntry(mount) {
|
|
26
|
+
const resolved = resolveInlineRef(mount.resource);
|
|
27
|
+
if (resolved === null) {
|
|
28
|
+
throw new Error(`profileBundleFileEntry: ${mount.path} requires an inline resource`);
|
|
29
|
+
}
|
|
30
|
+
const executable = mount.executable ?? PROFILE_BIN_DIR_RE.test(mount.path);
|
|
31
|
+
return {
|
|
32
|
+
mountPath: mount.path,
|
|
33
|
+
sha256: createHash("sha256").update(resolved.content, "utf8").digest("hex"),
|
|
34
|
+
mode: executable ? 0o755 : 0o644,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The canonical file table: one entry per mount, sorted by (mountPath, sha256)
|
|
39
|
+
* so any array order of the same mounts produces the same table.
|
|
40
|
+
*/
|
|
41
|
+
export function profileBundleFileTable(mounts) {
|
|
42
|
+
return mounts
|
|
43
|
+
.map(profileBundleFileEntry)
|
|
44
|
+
.sort((left, right) => left.mountPath !== right.mountPath
|
|
45
|
+
? compareStrings(left.mountPath, right.mountPath)
|
|
46
|
+
: compareStrings(left.sha256, right.sha256));
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The content-addressed bundle key. The digest covers EXACTLY the schema
|
|
50
|
+
* version and, per file, the logical mount path, the sha256 of the content
|
|
51
|
+
* bytes, and the full file mode — the same image-content-hash discipline as
|
|
52
|
+
* scripts/ci/image-content-hash.mjs: hash exactly the inputs that determine
|
|
53
|
+
* the cached bytes (and their placement semantics), nothing more. Any profile
|
|
54
|
+
* change — content, path, or mode — produces a new key; anything else
|
|
55
|
+
* (mount order, transport-only target paths, timestamps) does not.
|
|
56
|
+
*/
|
|
57
|
+
export function profileFileBundleKey(mounts) {
|
|
58
|
+
return canonicalCandidateDigest({
|
|
59
|
+
schema: PROFILE_BUNDLE_SCHEMA,
|
|
60
|
+
files: profileBundleFileTable(mounts),
|
|
61
|
+
});
|
|
62
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Advisory lru accounting for the profile bundle cache (see
|
|
3
|
+
* profile-bundle-cache.ts for the cache layout and durability invariants).
|
|
4
|
+
* `lru.json` is never load-bearing: it is rebuilt from a disk scan whenever it
|
|
5
|
+
* is absent or corrupt, and every write to it is best-effort so a full or
|
|
6
|
+
* read-only cache can never break admit, claim, or eviction.
|
|
7
|
+
*/
|
|
8
|
+
export interface ProfileBundleLru {
|
|
9
|
+
v: 1;
|
|
10
|
+
objects: Record<string, {
|
|
11
|
+
bytes: number;
|
|
12
|
+
lastUsed: string;
|
|
13
|
+
}>;
|
|
14
|
+
bundles: Record<string, {
|
|
15
|
+
bytes: number;
|
|
16
|
+
lastUsed: string;
|
|
17
|
+
createdAt: string;
|
|
18
|
+
}>;
|
|
19
|
+
}
|
|
20
|
+
export declare function pathExists(path: string): Promise<boolean>;
|
|
21
|
+
export declare function fsyncDir(dir: string): Promise<void>;
|
|
22
|
+
export declare function withRootLock<T>(root: string, run: () => Promise<T>): Promise<T>;
|
|
23
|
+
export declare function readManifestFile(path: string): Promise<{
|
|
24
|
+
schema?: unknown;
|
|
25
|
+
bundle?: unknown;
|
|
26
|
+
createdAt?: unknown;
|
|
27
|
+
files?: unknown;
|
|
28
|
+
} | null>;
|
|
29
|
+
/**
|
|
30
|
+
* Bundle keys come from `canonicalCandidateDigest` as `sha256:<hex>`; the `:`
|
|
31
|
+
* is encoded so the key is a safe single path segment on any host filesystem.
|
|
32
|
+
* The raw key is always recoverable from the manifest's `bundle` field.
|
|
33
|
+
*/
|
|
34
|
+
export declare function profileBundleManifestPath(root: string, bundle: string): string;
|
|
35
|
+
/** Rebuild advisory lru accounting from a disk scan (objects are immutable). */
|
|
36
|
+
/**
|
|
37
|
+
* Remove stale `incoming/` admission directories. An admit only ever renames
|
|
38
|
+
* OUT of `incoming/`, so anything found here is residue from an interrupted
|
|
39
|
+
* admission and is safe to delete; objects and manifests are untouched.
|
|
40
|
+
*/
|
|
41
|
+
export declare function sweepProfileBundleCache(root: string): Promise<void>;
|
|
42
|
+
export declare function loadLru(root: string): Promise<ProfileBundleLru>;
|
|
43
|
+
/** Best-effort by contract: lru corruption or a full cache must never break admit/claim. */
|
|
44
|
+
export declare function saveLru(root: string, lru: ProfileBundleLru): Promise<void>;
|
|
45
|
+
/** Eviction body; callers must already hold the per-root lock. */
|
|
46
|
+
export declare function evictUnlocked(root: string, maxBytes: number): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Evict least-recently-used objects until total object bytes are within
|
|
49
|
+
* `maxBytes`. Unreferenced objects go first (they reclaim space with no
|
|
50
|
+
* bundle impact); if the cap still does not hold, referenced objects are
|
|
51
|
+
* evicted oldest-first, which makes their bundles partial — claims then miss
|
|
52
|
+
* and re-deliver, so eviction can never serve wrong bytes. Bundle manifests
|
|
53
|
+
* referencing no existing objects are removed. Mid-admit `incoming/` dirs are
|
|
54
|
+
* never touched.
|
|
55
|
+
*/
|
|
56
|
+
export declare function evictProfileBundles(root: string, maxBytes: number): Promise<void>;
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { open, readdir, readFile, rename, rm, stat, writeFile, } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { compareStrings } from "./workspace-files.js";
|
|
5
|
+
export async function pathExists(path) {
|
|
6
|
+
try {
|
|
7
|
+
await stat(path);
|
|
8
|
+
return true;
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export async function fsyncDir(dir) {
|
|
15
|
+
try {
|
|
16
|
+
const handle = await open(dir, "r");
|
|
17
|
+
try {
|
|
18
|
+
await handle.sync();
|
|
19
|
+
}
|
|
20
|
+
finally {
|
|
21
|
+
await handle.close();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
// Best-effort durability: a cache without directory fsync still verifies.
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** Serialize admits/evictions per cache root so lru read-modify-write stays deterministic. */
|
|
29
|
+
const rootQueues = new Map();
|
|
30
|
+
export function withRootLock(root, run) {
|
|
31
|
+
const previous = rootQueues.get(root) ?? Promise.resolve();
|
|
32
|
+
const next = previous.then(run, run);
|
|
33
|
+
rootQueues.set(root, next.catch(() => undefined));
|
|
34
|
+
return next;
|
|
35
|
+
}
|
|
36
|
+
export async function readManifestFile(path) {
|
|
37
|
+
try {
|
|
38
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Bundle keys come from `canonicalCandidateDigest` as `sha256:<hex>`; the `:`
|
|
46
|
+
* is encoded so the key is a safe single path segment on any host filesystem.
|
|
47
|
+
* The raw key is always recoverable from the manifest's `bundle` field.
|
|
48
|
+
*/
|
|
49
|
+
export function profileBundleManifestPath(root, bundle) {
|
|
50
|
+
const filename = bundle.replace(/[^A-Za-z0-9._-]/g, (char) => `_${char.codePointAt(0)?.toString(16) ?? "3f"}_`);
|
|
51
|
+
return join(root, "bundles", `${filename}.json`);
|
|
52
|
+
}
|
|
53
|
+
/** Rebuild advisory lru accounting from a disk scan (objects are immutable). */
|
|
54
|
+
/**
|
|
55
|
+
* Remove stale `incoming/` admission directories. An admit only ever renames
|
|
56
|
+
* OUT of `incoming/`, so anything found here is residue from an interrupted
|
|
57
|
+
* admission and is safe to delete; objects and manifests are untouched.
|
|
58
|
+
*/
|
|
59
|
+
export async function sweepProfileBundleCache(root) {
|
|
60
|
+
const incoming = join(root, "incoming");
|
|
61
|
+
let entries;
|
|
62
|
+
try {
|
|
63
|
+
entries = await readdir(incoming);
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
for (const entry of entries) {
|
|
69
|
+
await rm(join(incoming, entry), { recursive: true, force: true });
|
|
70
|
+
}
|
|
71
|
+
await fsyncDir(incoming);
|
|
72
|
+
}
|
|
73
|
+
async function rebuildLru(root) {
|
|
74
|
+
const lru = { v: 1, objects: {}, bundles: {} };
|
|
75
|
+
try {
|
|
76
|
+
for (const name of await readdir(join(root, "objects"))) {
|
|
77
|
+
const info = await stat(join(root, "objects", name));
|
|
78
|
+
if (info.isFile()) {
|
|
79
|
+
lru.objects[name] = {
|
|
80
|
+
bytes: info.size,
|
|
81
|
+
lastUsed: info.mtime.toISOString(),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// Missing objects dir: fresh cache.
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
for (const name of await readdir(join(root, "bundles"))) {
|
|
91
|
+
const manifest = await readManifestFile(join(root, "bundles", name));
|
|
92
|
+
if (!manifest || typeof manifest.bundle !== "string")
|
|
93
|
+
continue;
|
|
94
|
+
const info = await stat(join(root, "bundles", name));
|
|
95
|
+
const files = Array.isArray(manifest.files) ? manifest.files : [];
|
|
96
|
+
const referenced = files.map((file) => file.sha256);
|
|
97
|
+
const bytes = referenced.reduce((sum, sha) => sum + (lru.objects[sha ?? ""]?.bytes ?? 0), 0);
|
|
98
|
+
lru.bundles[manifest.bundle] = {
|
|
99
|
+
bytes,
|
|
100
|
+
lastUsed: info.mtime.toISOString(),
|
|
101
|
+
createdAt: typeof manifest.createdAt === "string"
|
|
102
|
+
? manifest.createdAt
|
|
103
|
+
: info.mtime.toISOString(),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
// Missing bundles dir: fresh cache.
|
|
109
|
+
}
|
|
110
|
+
return lru;
|
|
111
|
+
}
|
|
112
|
+
export async function loadLru(root) {
|
|
113
|
+
try {
|
|
114
|
+
const parsed = JSON.parse(await readFile(join(root, "lru.json"), "utf8"));
|
|
115
|
+
if (parsed &&
|
|
116
|
+
parsed.v === 1 &&
|
|
117
|
+
typeof parsed.objects === "object" &&
|
|
118
|
+
typeof parsed.bundles === "object") {
|
|
119
|
+
return parsed;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Corrupt or missing: rebuild below.
|
|
124
|
+
}
|
|
125
|
+
return rebuildLru(root);
|
|
126
|
+
}
|
|
127
|
+
/** Best-effort by contract: lru corruption or a full cache must never break admit/claim. */
|
|
128
|
+
export async function saveLru(root, lru) {
|
|
129
|
+
try {
|
|
130
|
+
const path = join(root, "lru.json");
|
|
131
|
+
const temporary = `${path}.tmp-${randomUUID()}`;
|
|
132
|
+
await writeFile(temporary, JSON.stringify(lru));
|
|
133
|
+
await rename(temporary, path);
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
// Read-only or full cache: eviction still held on disk.
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/** Eviction body; callers must already hold the per-root lock. */
|
|
140
|
+
export async function evictUnlocked(root, maxBytes) {
|
|
141
|
+
const lru = await loadLru(root);
|
|
142
|
+
const objectsDir = join(root, "objects");
|
|
143
|
+
for (const sha of Object.keys(lru.objects)) {
|
|
144
|
+
if (!(await pathExists(join(objectsDir, sha)))) {
|
|
145
|
+
delete lru.objects[sha];
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const manifests = new Map();
|
|
149
|
+
const bundlesDir = join(root, "bundles");
|
|
150
|
+
for (const name of await readdir(bundlesDir)) {
|
|
151
|
+
const manifest = await readManifestFile(join(bundlesDir, name));
|
|
152
|
+
if (manifest && typeof manifest.bundle === "string") {
|
|
153
|
+
manifests.set(manifest.bundle, manifest);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
await rm(join(bundlesDir, name), { force: true });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const referenced = new Set();
|
|
160
|
+
for (const manifest of manifests.values()) {
|
|
161
|
+
const files = Array.isArray(manifest.files) ? manifest.files : [];
|
|
162
|
+
for (const file of files) {
|
|
163
|
+
const sha = file.sha256;
|
|
164
|
+
if (typeof sha === "string")
|
|
165
|
+
referenced.add(sha);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
let total = Object.values(lru.objects).reduce((sum, o) => sum + o.bytes, 0);
|
|
169
|
+
const byLastUsed = (sha) => lru.objects[sha]?.lastUsed ?? "";
|
|
170
|
+
const live = Object.keys(lru.objects);
|
|
171
|
+
const unreferenced = live
|
|
172
|
+
.filter((sha) => !referenced.has(sha))
|
|
173
|
+
.sort((a, b) => compareStrings(byLastUsed(a), byLastUsed(b)));
|
|
174
|
+
const referencedLive = live
|
|
175
|
+
.filter((sha) => referenced.has(sha))
|
|
176
|
+
.sort((a, b) => compareStrings(byLastUsed(a), byLastUsed(b)));
|
|
177
|
+
for (const sha of [...unreferenced, ...referencedLive]) {
|
|
178
|
+
if (total <= maxBytes)
|
|
179
|
+
break;
|
|
180
|
+
await rm(join(objectsDir, sha), { force: true });
|
|
181
|
+
total -= lru.objects[sha]?.bytes ?? 0;
|
|
182
|
+
delete lru.objects[sha];
|
|
183
|
+
}
|
|
184
|
+
// Bundles whose every object is gone can never be claimed; drop their manifests.
|
|
185
|
+
for (const [bundle, manifest] of manifests) {
|
|
186
|
+
const files = Array.isArray(manifest.files) ? manifest.files : [];
|
|
187
|
+
const alive = files.some((file) => Boolean(lru.objects[file.sha256 ?? ""]));
|
|
188
|
+
if (!alive) {
|
|
189
|
+
await rm(profileBundleManifestPath(root, bundle), { force: true });
|
|
190
|
+
delete lru.bundles[bundle];
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
await fsyncDir(objectsDir);
|
|
194
|
+
await fsyncDir(bundlesDir);
|
|
195
|
+
await saveLru(root, lru);
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Evict least-recently-used objects until total object bytes are within
|
|
199
|
+
* `maxBytes`. Unreferenced objects go first (they reclaim space with no
|
|
200
|
+
* bundle impact); if the cap still does not hold, referenced objects are
|
|
201
|
+
* evicted oldest-first, which makes their bundles partial — claims then miss
|
|
202
|
+
* and re-deliver, so eviction can never serve wrong bytes. Bundle manifests
|
|
203
|
+
* referencing no existing objects are removed. Mid-admit `incoming/` dirs are
|
|
204
|
+
* never touched.
|
|
205
|
+
*/
|
|
206
|
+
export async function evictProfileBundles(root, maxBytes) {
|
|
207
|
+
if (!Number.isFinite(maxBytes) || maxBytes < 0) {
|
|
208
|
+
throw new Error(`Profile bundle cache maxBytes must be >= 0: ${maxBytes}`);
|
|
209
|
+
}
|
|
210
|
+
await withRootLock(root, () => evictUnlocked(root, maxBytes));
|
|
211
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-profile-materialize",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.23.0",
|
|
4
4
|
"description": "Materialize an AgentProfile into the native per-harness workspace files (the one shared materializer for cli-bridge, agent-dev-container, and VB).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"repository": {
|