@foldrun/core 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -10
- package/dist/src/approvals.d.ts +104 -1
- package/dist/src/approvals.js +192 -2
- package/dist/src/budget.d.ts +32 -0
- package/dist/src/budget.js +177 -0
- package/dist/src/catalog.js +2 -1
- package/dist/src/confine.js +6 -1
- package/dist/src/egress.d.ts +19 -2
- package/dist/src/egress.js +2 -1
- package/dist/src/evals.d.ts +4 -0
- package/dist/src/evals.js +20 -11
- package/dist/src/flow-lint.d.ts +4 -0
- package/dist/src/flow-lint.js +108 -0
- package/dist/src/host-env.d.ts +8 -0
- package/dist/src/host-env.js +48 -0
- package/dist/src/json-schema.d.ts +12 -0
- package/dist/src/json-schema.js +202 -0
- package/dist/src/library.js +13 -1
- package/dist/src/notify.d.ts +79 -1
- package/dist/src/notify.js +243 -9
- package/dist/src/paths.d.ts +5 -0
- package/dist/src/paths.js +32 -0
- package/dist/src/platform.d.ts +102 -2
- package/dist/src/platform.js +18 -2
- package/dist/src/provider-health.d.ts +55 -0
- package/dist/src/provider-health.js +232 -0
- package/dist/src/run-container.d.ts +57 -4
- package/dist/src/run-container.js +147 -41
- package/dist/src/runner.d.ts +39 -3
- package/dist/src/runner.js +422 -73
- package/dist/src/script-tools.d.ts +12 -0
- package/dist/src/script-tools.js +10 -2
- package/dist/src/secret-health.d.ts +59 -0
- package/dist/src/secret-health.js +155 -0
- package/dist/src/secrets.d.ts +19 -1
- package/dist/src/secrets.js +44 -12
- package/dist/src/starter.js +3 -0
- package/dist/src/step-exec.d.ts +47 -1
- package/dist/src/step-exec.js +155 -60
- package/dist/src/store.d.ts +276 -3
- package/dist/src/store.js +552 -26
- package/dist/src/tar.js +2 -1
- package/dist/src/test-mode.d.ts +111 -0
- package/dist/src/test-mode.js +285 -0
- package/dist/src/tool-test.js +48 -21
- package/dist/src/translator.js +2 -1
- package/dist/src/trigger-log.d.ts +43 -0
- package/dist/src/trigger-log.js +122 -0
- package/dist/src/triggers.d.ts +20 -0
- package/dist/src/triggers.js +91 -10
- package/dist/src/webhook.d.ts +15 -3
- package/dist/src/webhook.js +11 -11
- package/package.json +4 -4
- package/src/approvals.ts +253 -2
- package/src/budget.ts +178 -0
- package/src/catalog.ts +2 -1
- package/src/confine.ts +6 -1
- package/src/egress.ts +20 -4
- package/src/evals.ts +25 -9
- package/src/flow-lint.ts +118 -0
- package/src/host-env.ts +51 -0
- package/src/json-schema.ts +184 -0
- package/src/library.ts +13 -1
- package/src/notify.ts +272 -11
- package/src/paths.ts +29 -0
- package/src/platform.ts +107 -3
- package/src/provider-health.ts +274 -0
- package/src/run-container.ts +175 -36
- package/src/runner.ts +435 -65
- package/src/script-tools.ts +18 -2
- package/src/secret-health.ts +185 -0
- package/src/secrets.ts +44 -10
- package/src/starter.ts +3 -0
- package/src/step-exec.ts +128 -13
- package/src/storage.ts +8 -2
- package/src/store.ts +746 -25
- package/src/tar.ts +2 -1
- package/src/test-mode.ts +311 -0
- package/src/tool-test.ts +49 -20
- package/src/translator.ts +2 -1
- package/src/trigger-log.ts +159 -0
- package/src/triggers.ts +92 -10
- package/src/webhook.ts +17 -11
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { type ProviderSpec } from "./store.ts";
|
|
2
|
+
export type HealthVerdict = "ok" | "credential" | "not-found" | "busy" | "unreachable" | "provider-error";
|
|
3
|
+
export interface ProviderCheck {
|
|
4
|
+
/** How this provider is written in the file: its preset name, or its host. */
|
|
5
|
+
provider: string;
|
|
6
|
+
/** The model id actually asked for — what a run would send. */
|
|
7
|
+
model: string;
|
|
8
|
+
verdict: HealthVerdict;
|
|
9
|
+
ok: boolean;
|
|
10
|
+
status: number | null;
|
|
11
|
+
ms: number;
|
|
12
|
+
/** One sentence a person can act on, in the provider's own words where
|
|
13
|
+
* they said anything useful. */
|
|
14
|
+
detail: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* One request at the provider, in the wire format it speaks, asking for a
|
|
18
|
+
* single token. Never throws: an unreachable host is a verdict, not an
|
|
19
|
+
* exception, because this runs on a tick.
|
|
20
|
+
*/
|
|
21
|
+
export declare function checkProvider(tenant: string, workspace: string | null, spec: ProviderSpec, model: string): Promise<ProviderCheck>;
|
|
22
|
+
/** A provider block in use, and where it is declared — because "which file
|
|
23
|
+
* do I edit" is the next question after "this key is dead". */
|
|
24
|
+
export interface ProviderInUse {
|
|
25
|
+
spec: ProviderSpec;
|
|
26
|
+
/** The workspace whose vault resolves this block's secrets. Null for the
|
|
27
|
+
* account's own block, which resolves against the account vault. */
|
|
28
|
+
workspace: string | null;
|
|
29
|
+
/** Human-readable origin: "account", "rank-desk", "rank-desk/writer". */
|
|
30
|
+
declaredIn: string;
|
|
31
|
+
/** The model id a run would send for the cheapest tier, which is what a
|
|
32
|
+
* health check should spend. */
|
|
33
|
+
model: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Every distinct provider block this account would actually use, read from
|
|
37
|
+
* the same three places a run reads them: the account, each workspace, each
|
|
38
|
+
* agent.
|
|
39
|
+
*
|
|
40
|
+
* Deduplicated by endpoint and token, because ten agents sharing one gateway
|
|
41
|
+
* are one credential and checking it ten times spends ten times as much to
|
|
42
|
+
* learn the same fact. The origin recorded is the first one seen, which is
|
|
43
|
+
* the broadest — an account block covers everything under it.
|
|
44
|
+
*/
|
|
45
|
+
export declare function providersInUse(tenant: string): ProviderInUse[];
|
|
46
|
+
/**
|
|
47
|
+
* Check every provider this account uses, one at a time.
|
|
48
|
+
*
|
|
49
|
+
* Serial on purpose: this is housekeeping on a tick, the whole point is that
|
|
50
|
+
* it is cheap, and firing twelve model requests at once to save eight
|
|
51
|
+
* seconds nobody is waiting for would be a strange trade.
|
|
52
|
+
*/
|
|
53
|
+
export declare function checkAccountProviders(tenant: string): Promise<(ProviderCheck & {
|
|
54
|
+
declaredIn: string;
|
|
55
|
+
})[]>;
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
// Is the key still good?
|
|
2
|
+
//
|
|
3
|
+
// A customer's own model key can be revoked, expire, run out of credit or
|
|
4
|
+
// have its base URL moved, and today the platform finds out the way the
|
|
5
|
+
// customer does: a desk fails at five in the morning. Twenty-seven of the
|
|
6
|
+
// thirty provider presets have never had an agent run through them from
|
|
7
|
+
// this runtime at all, so "it worked when we set it up" is doing a lot of
|
|
8
|
+
// work in that sentence.
|
|
9
|
+
//
|
|
10
|
+
// This is deliberately NOT the tool-loop probe next door. `foldrun probe`
|
|
11
|
+
// asks "can this model drive an agent", which is a question about a model
|
|
12
|
+
// and is asked once. This asks "does this endpoint still take our key",
|
|
13
|
+
// which is a question about a credential and has to be asked repeatedly —
|
|
14
|
+
// so it has to be cheap. One request, one token of output, no SDK, no tool
|
|
15
|
+
// loop, no agent. A fraction of a cent, and it answers the thing that
|
|
16
|
+
// actually breaks.
|
|
17
|
+
//
|
|
18
|
+
// It reads the same provider block a run reads, resolved the same way, so a
|
|
19
|
+
// pass here means a run would reach the same place with the same key.
|
|
20
|
+
import fs from "node:fs";
|
|
21
|
+
import path from "node:path";
|
|
22
|
+
import matter from "gray-matter";
|
|
23
|
+
import { getSecret } from "./secrets.js";
|
|
24
|
+
import { noteSecretUse, healthKey } from "./secret-health.js";
|
|
25
|
+
import { accountDir, workspaceDir, listWorkspaces, listAgents, parseProvider } from "./store.js";
|
|
26
|
+
/** How long to wait. A provider slower than this is not one a step should
|
|
27
|
+
* be waiting on either, and the check must never hold a tick. */
|
|
28
|
+
const TIMEOUT_MS = 15_000;
|
|
29
|
+
/** What each status means for a credential, and what a person does about
|
|
30
|
+
* it. Kept as a table because the mapping IS the feature: "401" tells you
|
|
31
|
+
* nothing you can act on, "the key is no longer accepted" tells you
|
|
32
|
+
* everything. */
|
|
33
|
+
function verdictFor(status) {
|
|
34
|
+
if (status >= 200 && status < 300)
|
|
35
|
+
return { verdict: "ok", detail: "the key was accepted and the model answered" };
|
|
36
|
+
if (status === 401 || status === 403) {
|
|
37
|
+
return { verdict: "credential", detail: "the key is no longer accepted — revoked, rotated, or wrong for this endpoint" };
|
|
38
|
+
}
|
|
39
|
+
if (status === 402)
|
|
40
|
+
return { verdict: "credential", detail: "the account behind this key is out of credit" };
|
|
41
|
+
if (status === 404) {
|
|
42
|
+
return { verdict: "not-found", detail: "the endpoint or the model id does not exist there — check base_url and models:" };
|
|
43
|
+
}
|
|
44
|
+
if (status === 429) {
|
|
45
|
+
// Not a health problem. A busy provider is a provider that is working,
|
|
46
|
+
// and reporting it as broken would train people to ignore the alert.
|
|
47
|
+
return { verdict: "busy", detail: "rate limited right now — the key is fine" };
|
|
48
|
+
}
|
|
49
|
+
if (status >= 500)
|
|
50
|
+
return { verdict: "provider-error", detail: "the provider is failing at its end" };
|
|
51
|
+
return { verdict: "provider-error", detail: `unexpected HTTP ${status}` };
|
|
52
|
+
}
|
|
53
|
+
/** `${SECRET}` in a token or header, filled from this account's vault. */
|
|
54
|
+
function resolve(tenant, workspace, raw) {
|
|
55
|
+
return raw.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => {
|
|
56
|
+
const hit = getSecret(tenant, name, workspace ?? undefined);
|
|
57
|
+
return hit ? hit.value : whole;
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* One request at the provider, in the wire format it speaks, asking for a
|
|
62
|
+
* single token. Never throws: an unreachable host is a verdict, not an
|
|
63
|
+
* exception, because this runs on a tick.
|
|
64
|
+
*/
|
|
65
|
+
export async function checkProvider(tenant, workspace, spec, model) {
|
|
66
|
+
const label = spec.name || (() => {
|
|
67
|
+
try {
|
|
68
|
+
return new URL(spec.baseUrl).host;
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
return spec.baseUrl || "provider";
|
|
72
|
+
}
|
|
73
|
+
})();
|
|
74
|
+
const began = Date.now();
|
|
75
|
+
const fail = (verdict, detail, status = null) => ({
|
|
76
|
+
provider: label,
|
|
77
|
+
model,
|
|
78
|
+
verdict,
|
|
79
|
+
ok: false,
|
|
80
|
+
status,
|
|
81
|
+
ms: Date.now() - began,
|
|
82
|
+
detail,
|
|
83
|
+
});
|
|
84
|
+
const token = resolve(tenant, workspace, spec.token);
|
|
85
|
+
if (!token || token.includes("${")) {
|
|
86
|
+
return fail("credential", `the secret named in provider.token is not in this account's vault (${spec.token})`);
|
|
87
|
+
}
|
|
88
|
+
if (!spec.baseUrl)
|
|
89
|
+
return fail("unreachable", "this provider block has no base_url");
|
|
90
|
+
// The three wire formats, each asking for the smallest possible answer.
|
|
91
|
+
const base = spec.baseUrl.replace(/\/+$/, "");
|
|
92
|
+
const anthropic = spec.format === "anthropic";
|
|
93
|
+
const url = anthropic
|
|
94
|
+
? `${base}/v1/messages`
|
|
95
|
+
: spec.format === "responses"
|
|
96
|
+
? `${base}/responses`
|
|
97
|
+
: `${base}/chat/completions`;
|
|
98
|
+
const body = anthropic
|
|
99
|
+
? { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] }
|
|
100
|
+
: spec.format === "responses"
|
|
101
|
+
? { model, max_output_tokens: 16, input: "hi" }
|
|
102
|
+
: { model, max_tokens: 1, messages: [{ role: "user", content: "hi" }] };
|
|
103
|
+
const headers = { "content-type": "application/json" };
|
|
104
|
+
// An Anthropic-format endpoint takes the key in whichever header its
|
|
105
|
+
// `auth:` says; everything else is a bearer token.
|
|
106
|
+
if (anthropic && spec.auth === "x-api-key") {
|
|
107
|
+
headers["x-api-key"] = token;
|
|
108
|
+
headers["anthropic-version"] = "2023-06-01";
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
headers.authorization = `Bearer ${token}`;
|
|
112
|
+
if (anthropic)
|
|
113
|
+
headers["anthropic-version"] = "2023-06-01";
|
|
114
|
+
}
|
|
115
|
+
for (const [k, v] of Object.entries(spec.headers ?? {}))
|
|
116
|
+
headers[k.toLowerCase()] = resolve(tenant, workspace, v);
|
|
117
|
+
try {
|
|
118
|
+
const res = await fetch(url, {
|
|
119
|
+
method: "POST",
|
|
120
|
+
headers,
|
|
121
|
+
body: JSON.stringify(body),
|
|
122
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
123
|
+
});
|
|
124
|
+
const { verdict, detail } = verdictFor(res.status);
|
|
125
|
+
// The check is itself a use of the credential, and the most informative
|
|
126
|
+
// one there is: it was made for exactly this purpose.
|
|
127
|
+
const named = spec.token.match(/^\$\{([A-Z][A-Z0-9_]*)\}$/)?.[1];
|
|
128
|
+
if (named) {
|
|
129
|
+
let host = spec.baseUrl;
|
|
130
|
+
try {
|
|
131
|
+
host = new URL(spec.baseUrl).host;
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
// an unparseable base_url is its own problem, reported above
|
|
135
|
+
}
|
|
136
|
+
const hit = getSecret(tenant, named, workspace ?? undefined);
|
|
137
|
+
noteSecretUse(tenant, healthKey(named, hit?.scope === "workspace" ? "workspace" : "account", workspace), { host, status: res.status });
|
|
138
|
+
}
|
|
139
|
+
// The provider's own words when it refused. They are the difference
|
|
140
|
+
// between "fix your key" and "fix your model id", and they are only
|
|
141
|
+
// ever in the body.
|
|
142
|
+
let said = "";
|
|
143
|
+
if (!res.ok) {
|
|
144
|
+
said = (await res.text().catch(() => "")).replace(/\s+/g, " ").slice(0, 300);
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
provider: label,
|
|
148
|
+
model,
|
|
149
|
+
verdict,
|
|
150
|
+
// Busy counts as healthy: the key worked, the provider is simply full.
|
|
151
|
+
ok: verdict === "ok" || verdict === "busy",
|
|
152
|
+
status: res.status,
|
|
153
|
+
ms: Date.now() - began,
|
|
154
|
+
detail: said ? `${detail} — ${said}` : detail,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
159
|
+
return fail("unreachable", /timeout|aborted/i.test(message) ? `no answer within ${TIMEOUT_MS / 1000}s` : message);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
/** The cheapest tier's id for a provider block: what it calls `fast`, or
|
|
163
|
+
* our own fast model when it remaps nothing. */
|
|
164
|
+
function cheapestModel(spec) {
|
|
165
|
+
return spec.models.fast || spec.models.default || spec.models.max || "claude-haiku-4-5-20251001";
|
|
166
|
+
}
|
|
167
|
+
function frontmatterOf(file) {
|
|
168
|
+
try {
|
|
169
|
+
return (matter(fs.readFileSync(file, "utf8")).data ?? {});
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
return {};
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Every distinct provider block this account would actually use, read from
|
|
177
|
+
* the same three places a run reads them: the account, each workspace, each
|
|
178
|
+
* agent.
|
|
179
|
+
*
|
|
180
|
+
* Deduplicated by endpoint and token, because ten agents sharing one gateway
|
|
181
|
+
* are one credential and checking it ten times spends ten times as much to
|
|
182
|
+
* learn the same fact. The origin recorded is the first one seen, which is
|
|
183
|
+
* the broadest — an account block covers everything under it.
|
|
184
|
+
*/
|
|
185
|
+
export function providersInUse(tenant) {
|
|
186
|
+
const out = [];
|
|
187
|
+
const seen = new Set();
|
|
188
|
+
const add = (raw, workspace, declaredIn) => {
|
|
189
|
+
const spec = parseProvider(raw);
|
|
190
|
+
if (!spec || !spec.baseUrl)
|
|
191
|
+
return;
|
|
192
|
+
// The token is part of the identity: the same gateway with two different
|
|
193
|
+
// keys is two credentials, and either can die on its own.
|
|
194
|
+
const key = `${workspace ?? ""}|${spec.baseUrl}|${spec.token}|${cheapestModel(spec)}`;
|
|
195
|
+
if (seen.has(key))
|
|
196
|
+
return;
|
|
197
|
+
seen.add(key);
|
|
198
|
+
out.push({ spec, workspace, declaredIn, model: cheapestModel(spec) });
|
|
199
|
+
};
|
|
200
|
+
add(frontmatterOf(path.join(accountDir(tenant), "AGENTS.md")).provider, null, "account");
|
|
201
|
+
for (const ws of safeList(() => listWorkspaces(tenant))) {
|
|
202
|
+
const dir = workspaceDir(tenant, ws.name);
|
|
203
|
+
add(frontmatterOf(path.join(dir, "AGENTS.md")).provider, ws.name, ws.name);
|
|
204
|
+
for (const agent of safeList(() => listAgents(tenant, ws.name))) {
|
|
205
|
+
add(frontmatterOf(path.join(dir, "agents", agent.name, "agent.md")).provider, ws.name, `${ws.name}/${agent.name}`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
|
210
|
+
function safeList(fn) {
|
|
211
|
+
try {
|
|
212
|
+
return fn();
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return [];
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Check every provider this account uses, one at a time.
|
|
220
|
+
*
|
|
221
|
+
* Serial on purpose: this is housekeeping on a tick, the whole point is that
|
|
222
|
+
* it is cheap, and firing twelve model requests at once to save eight
|
|
223
|
+
* seconds nobody is waiting for would be a strange trade.
|
|
224
|
+
*/
|
|
225
|
+
export async function checkAccountProviders(tenant) {
|
|
226
|
+
const results = [];
|
|
227
|
+
for (const use of providersInUse(tenant)) {
|
|
228
|
+
const check = await checkProvider(tenant, use.workspace, use.spec, use.model);
|
|
229
|
+
results.push({ ...check, declaredIn: use.declaredIn });
|
|
230
|
+
}
|
|
231
|
+
return results;
|
|
232
|
+
}
|
|
@@ -33,6 +33,7 @@ export interface ContainerStepInput {
|
|
|
33
33
|
/** The step's spend ceiling and the model's per-token price — see
|
|
34
34
|
* step-exec.ts. Values, so the driver can enforce them in the pod. */
|
|
35
35
|
budgetUsd?: number | null;
|
|
36
|
+
budgetNote?: string;
|
|
36
37
|
price?: {
|
|
37
38
|
input: number;
|
|
38
39
|
output: number;
|
|
@@ -40,6 +41,9 @@ export interface ContainerStepInput {
|
|
|
40
41
|
verify?: string;
|
|
41
42
|
/** `output: json` — checked inside, where the reply is. */
|
|
42
43
|
output?: "json";
|
|
44
|
+
/** `schema:` and `max_turns:`, resolved host-side — see ExecOptions. */
|
|
45
|
+
schema?: Record<string, unknown> | boolean;
|
|
46
|
+
maxTurns?: number;
|
|
43
47
|
/** tools: [search] — the directories to search, as paths INSIDE the
|
|
44
48
|
* container (/workspace/…, /library/…). */
|
|
45
49
|
search?: SearchRoot[];
|
|
@@ -131,7 +135,16 @@ export declare function allowedBack(rel: string): boolean;
|
|
|
131
135
|
* step changed without the staging copy the first driver held in /tmp.
|
|
132
136
|
*/
|
|
133
137
|
export declare function hashTree(dir: string): Record<string, string>;
|
|
134
|
-
|
|
138
|
+
/**
|
|
139
|
+
* Both sides changed a file the step was handed. When each is the handed-in
|
|
140
|
+
* content with lines added on the end — a ledger, history.md, a .jsonl — the
|
|
141
|
+
* answer is the base, then the host's additions, then the step's. Only the
|
|
142
|
+
* base's hash may be known, so the base is found as the longest shared
|
|
143
|
+
* leading run of whole lines whose hash matches. Anything else is null: a
|
|
144
|
+
* real conflict, which the caller must not settle by dropping one side.
|
|
145
|
+
*/
|
|
146
|
+
export declare function mergeAppends(was: Buffer | string | null, host: Buffer, next: Buffer): Buffer | null;
|
|
147
|
+
export declare function applyContainerChanges(hostWs: string, containerWs: string, baseline?: string | Record<string, string>, note?: (message: string) => void, divert?: WriteDivert): string[];
|
|
135
148
|
/**
|
|
136
149
|
* Where the driver builds venvs and npm prefixes, as an absolute path *inside*
|
|
137
150
|
* the runner. Fixed, and it has to be: a venv bakes its own absolute path into
|
|
@@ -165,7 +178,17 @@ export declare function runnerImageTag(): string;
|
|
|
165
178
|
* has not been built before. `npm pack` of the installed core, so the image
|
|
166
179
|
* runs the same bytes the server does.
|
|
167
180
|
*/
|
|
168
|
-
|
|
181
|
+
/**
|
|
182
|
+
* `platform`: build for another architecture — "linux/arm64" on the amd64
|
|
183
|
+
* box, through QEMU — and name the result with the arch appended so it never
|
|
184
|
+
* shadows the native image in the daemon. Only the publisher asks for this;
|
|
185
|
+
* a step always runs the native image. Needs binfmt registered on the host
|
|
186
|
+
* (tonistiigi/binfmt); the publisher does that itself, every time, because
|
|
187
|
+
* the registration does not survive a reboot.
|
|
188
|
+
*/
|
|
189
|
+
export declare function ensureRunnerImage(opts?: {
|
|
190
|
+
platform?: string;
|
|
191
|
+
}): {
|
|
169
192
|
tag: string;
|
|
170
193
|
log: string[];
|
|
171
194
|
};
|
|
@@ -176,12 +199,14 @@ export declare function parseDriverLine(line: string): {
|
|
|
176
199
|
call?: string;
|
|
177
200
|
ms?: number;
|
|
178
201
|
err?: boolean;
|
|
179
|
-
} | {
|
|
202
|
+
} | ({
|
|
180
203
|
e: "done";
|
|
181
|
-
} & ContainerStepOutcome | null;
|
|
204
|
+
} & ContainerStepOutcome) | null;
|
|
182
205
|
export interface RunInContainerArgs {
|
|
183
206
|
workspaceRoot: string;
|
|
184
207
|
libraryRoot: string;
|
|
208
|
+
/** The platform's gallery, staged under /library BEFORE the account library so the account's copies win. */
|
|
209
|
+
galleryRoot?: string;
|
|
185
210
|
/** The account this step belongs to — the key for its dependency cache.
|
|
186
211
|
* Optional so an embedder that has no tenancy simply gets no cache. */
|
|
187
212
|
tenant?: string;
|
|
@@ -208,7 +233,35 @@ export interface RunInContainerArgs {
|
|
|
208
233
|
* (consumed 0), and before each output line it applies. The runner
|
|
209
234
|
* records these on the step, so the next driver knows where to attach. */
|
|
210
235
|
checkpoint?: (ref: string, consumed: number) => void;
|
|
236
|
+
/** A test run: where state/ and storage/ changes go instead of the
|
|
237
|
+
* workspace, and who to tell about each. See test-mode.ts. */
|
|
238
|
+
divert?: WriteDivert;
|
|
239
|
+
/** The run is a test run. An isolated executor labels the sandbox so the
|
|
240
|
+
* cluster can cut its network down to the egress proxy: a script tool
|
|
241
|
+
* calls the network directly, not through the proxy where the test-mode
|
|
242
|
+
* policy lives, so without this a test run's script could still send. */
|
|
243
|
+
test?: boolean;
|
|
211
244
|
}
|
|
245
|
+
/** What a test run does with a write it must not apply. */
|
|
246
|
+
export interface WriteDivert {
|
|
247
|
+
/** The directory the diverted files are written under, by their
|
|
248
|
+
* workspace-relative path: runs/<id>/test-writes/. */
|
|
249
|
+
to: string;
|
|
250
|
+
note: (rel: string, summary: string) => void;
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* What crosses into the container as its environment, and how.
|
|
254
|
+
*
|
|
255
|
+
* A docker --env-file is one `KEY=value` per line and cannot carry a
|
|
256
|
+
* newline. @file secrets are multi-line by nature (a PEM key, a cert), so
|
|
257
|
+
* they are staged as files NOW, into the copied-in workspace, and the env
|
|
258
|
+
* carries the container path — the one place a path is stable across the
|
|
259
|
+
* boundary. A plain secret whose value happens to hold a newline gets the
|
|
260
|
+
* same treatment, and the trace says so: it used to be dropped from the
|
|
261
|
+
* env file without a word, and the step failed later, elsewhere, on a
|
|
262
|
+
* variable that read as unset. Everything else goes through the env file.
|
|
263
|
+
*/
|
|
264
|
+
export declare function stageContainerEnv(env: Record<string, string>, wsIn: string, agentRel: string, emit: (type: "info" | "error", text: string) => void): Record<string, string>;
|
|
212
265
|
/** The limits a size class reserves. Large is the install's configured
|
|
213
266
|
* limits unchanged, so existing installs bill and behave identically;
|
|
214
267
|
* small defaults to a quarter-ish slice and is env-tunable. */
|
|
@@ -23,6 +23,7 @@ import path from "node:path";
|
|
|
23
23
|
import crypto from "node:crypto";
|
|
24
24
|
import { spawn, spawnSync } from "node:child_process";
|
|
25
25
|
import { isPlatformPath } from "./store.js";
|
|
26
|
+
import { divertedSummary, isDivertedPath } from "./test-mode.js";
|
|
26
27
|
import { isFileValue, fileContent } from "./secrets.js";
|
|
27
28
|
import { safeTenantSegment } from "./runtime.js";
|
|
28
29
|
import { dataRoot } from "./paths.js";
|
|
@@ -123,7 +124,41 @@ export function hashTree(dir) {
|
|
|
123
124
|
walk(dir);
|
|
124
125
|
return out;
|
|
125
126
|
}
|
|
126
|
-
|
|
127
|
+
const sha = (b) => crypto.createHash("sha256").update(b).digest("hex");
|
|
128
|
+
/**
|
|
129
|
+
* Both sides changed a file the step was handed. When each is the handed-in
|
|
130
|
+
* content with lines added on the end — a ledger, history.md, a .jsonl — the
|
|
131
|
+
* answer is the base, then the host's additions, then the step's. Only the
|
|
132
|
+
* base's hash may be known, so the base is found as the longest shared
|
|
133
|
+
* leading run of whole lines whose hash matches. Anything else is null: a
|
|
134
|
+
* real conflict, which the caller must not settle by dropping one side.
|
|
135
|
+
*/
|
|
136
|
+
export function mergeAppends(was, host, next) {
|
|
137
|
+
// Both sides created the same new file: no shared base to append to, and
|
|
138
|
+
// joining two JSON documents is corruption, not a merge.
|
|
139
|
+
if (was === null)
|
|
140
|
+
return null;
|
|
141
|
+
const h = host.toString("utf8");
|
|
142
|
+
const n = next.toString("utf8");
|
|
143
|
+
let common = 0;
|
|
144
|
+
const max = Math.min(h.length, n.length);
|
|
145
|
+
while (common < max && h.charCodeAt(common) === n.charCodeAt(common))
|
|
146
|
+
common++;
|
|
147
|
+
for (let end = common; end >= 0; end--) {
|
|
148
|
+
if (end > 0 && h[end - 1] !== "\n")
|
|
149
|
+
continue;
|
|
150
|
+
const prefix = h.slice(0, end);
|
|
151
|
+
const matches = typeof was === "string" ? sha(Buffer.from(prefix, "utf8")) === was : Buffer.from(prefix, "utf8").equals(was);
|
|
152
|
+
if (!matches)
|
|
153
|
+
continue;
|
|
154
|
+
const hostAdded = h.slice(end);
|
|
155
|
+
const stepAdded = n.slice(end);
|
|
156
|
+
const joined = hostAdded && !hostAdded.endsWith("\n") ? hostAdded + "\n" : hostAdded;
|
|
157
|
+
return Buffer.from(prefix + joined + stepAdded, "utf8");
|
|
158
|
+
}
|
|
159
|
+
return null;
|
|
160
|
+
}
|
|
161
|
+
export function applyContainerChanges(hostWs, containerWs, baseline, note, divert) {
|
|
127
162
|
const applied = [];
|
|
128
163
|
const walk = (dir) => {
|
|
129
164
|
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
@@ -144,22 +179,61 @@ export function applyContainerChanges(hostWs, containerWs, baseline) {
|
|
|
144
179
|
}
|
|
145
180
|
if (!allowedBack(rel))
|
|
146
181
|
continue;
|
|
147
|
-
|
|
148
|
-
//
|
|
149
|
-
//
|
|
182
|
+
let next = fs.readFileSync(abs);
|
|
183
|
+
// What the step was handed: its bytes (a staging dir) or its hash.
|
|
184
|
+
// null when the file did not exist then; undefined when no baseline.
|
|
185
|
+
let was;
|
|
150
186
|
if (typeof baseline === "string") {
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
continue;
|
|
187
|
+
const p = path.join(baseline, rel);
|
|
188
|
+
was = fs.existsSync(p) ? fs.readFileSync(p) : null;
|
|
154
189
|
}
|
|
155
190
|
else if (baseline) {
|
|
156
|
-
|
|
157
|
-
if (was && was === crypto.createHash("sha256").update(next).digest("hex"))
|
|
158
|
-
continue;
|
|
191
|
+
was = baseline[rel.replaceAll("\\", "/")] ?? null;
|
|
159
192
|
}
|
|
193
|
+
const same = (b) => was === null || was === undefined ? false : typeof was === "string" ? sha(b) === was : b.equals(was);
|
|
194
|
+
// Untouched by this step: whatever the host says now is more current
|
|
195
|
+
// than what we handed in, including a change made while it ran.
|
|
196
|
+
if (same(next))
|
|
197
|
+
continue;
|
|
160
198
|
const target = path.join(hostWs, rel);
|
|
161
|
-
|
|
199
|
+
const hostNow = fs.existsSync(target) ? fs.readFileSync(target) : null;
|
|
200
|
+
if (hostNow && hostNow.equals(next))
|
|
201
|
+
continue;
|
|
202
|
+
// The host moved too since the step was handed its copy — another run
|
|
203
|
+
// or a person wrote it. Writing the step's copy over it erased the
|
|
204
|
+
// other run's ledger rows on 2026-09-14. Merge appends; keep both
|
|
205
|
+
// sides of anything else.
|
|
206
|
+
const hostMoved = was !== undefined && hostNow !== null && !same(hostNow);
|
|
207
|
+
if (hostMoved) {
|
|
208
|
+
const merged = mergeAppends(was ?? null, hostNow, next);
|
|
209
|
+
if (merged) {
|
|
210
|
+
next = merged;
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
const ext = path.extname(rel);
|
|
214
|
+
const stem = rel.slice(0, rel.length - ext.length);
|
|
215
|
+
const conflictRel = `${stem}.conflict-${Date.now().toString(36)}${ext}`;
|
|
216
|
+
const conflictAbs = path.join(hostWs, conflictRel);
|
|
217
|
+
fs.mkdirSync(path.dirname(conflictAbs), { recursive: true });
|
|
218
|
+
fs.writeFileSync(conflictAbs, next);
|
|
219
|
+
applied.push(conflictRel);
|
|
220
|
+
note?.(`write-back conflict on ${rel.replaceAll("\\", "/")}: it changed on the workspace while this step ran, ` +
|
|
221
|
+
`and not only by appending — kept the workspace's version; this step's version is ${conflictRel.replaceAll("\\", "/")}`);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// A test run's state/ and storage/ writes land under the run, not on
|
|
226
|
+
// the workspace: the step ran against the real files and its output
|
|
227
|
+
// is kept where a person can read it, but the next real run starts
|
|
228
|
+
// from state a test never touched. Decided after the merge so the
|
|
229
|
+
// diverted copy is what WOULD have been written, conflicts included.
|
|
230
|
+
if (divert && isDivertedPath(rel)) {
|
|
231
|
+
const aside = path.join(divert.to, rel);
|
|
232
|
+
fs.mkdirSync(path.dirname(aside), { recursive: true });
|
|
233
|
+
fs.writeFileSync(aside, next);
|
|
234
|
+
divert.note(rel.replaceAll("\\", "/"), divertedSummary(rel.replaceAll("\\", "/"), hostNow, next));
|
|
162
235
|
continue;
|
|
236
|
+
}
|
|
163
237
|
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
164
238
|
fs.writeFileSync(target, next);
|
|
165
239
|
applied.push(rel);
|
|
@@ -286,9 +360,9 @@ try {
|
|
|
286
360
|
|
|
287
361
|
// API and script tools, rebuilt in here from their specs. Secrets were
|
|
288
362
|
// substituted into API headers before the input crossed the boundary, so
|
|
289
|
-
// buildApiTools gets an already-resolved environment.
|
|
290
|
-
//
|
|
291
|
-
// the
|
|
363
|
+
// buildApiTools gets an already-resolved environment. verify: gets the
|
|
364
|
+
// container's own environment, whole: it is the boundary, assembled on
|
|
365
|
+
// the host from an allowlist, and the declared secrets are in it.
|
|
292
366
|
const api = buildApiTools("", input.apis, undefined, { env, missing: [] });
|
|
293
367
|
const script = buildScriptTools(
|
|
294
368
|
agentDir,
|
|
@@ -333,10 +407,13 @@ try {
|
|
|
333
407
|
env,
|
|
334
408
|
timeoutSec: input.timeoutSec,
|
|
335
409
|
budgetUsd: input.budgetUsd,
|
|
410
|
+
budgetNote: input.budgetNote,
|
|
336
411
|
price: input.price,
|
|
337
412
|
verify: input.verify,
|
|
338
|
-
verifyEnv:
|
|
413
|
+
verifyEnv: env,
|
|
339
414
|
output: input.output,
|
|
415
|
+
schema: input.schema,
|
|
416
|
+
maxTurns: input.maxTurns,
|
|
340
417
|
// The container is the boundary; the SDK's own bash sandbox here would
|
|
341
418
|
// only block declared network use (SSH, curl) for no added safety.
|
|
342
419
|
sandboxBash: false,
|
|
@@ -442,8 +519,17 @@ export function runnerImageTag() {
|
|
|
442
519
|
* has not been built before. `npm pack` of the installed core, so the image
|
|
443
520
|
* runs the same bytes the server does.
|
|
444
521
|
*/
|
|
445
|
-
|
|
446
|
-
|
|
522
|
+
/**
|
|
523
|
+
* `platform`: build for another architecture — "linux/arm64" on the amd64
|
|
524
|
+
* box, through QEMU — and name the result with the arch appended so it never
|
|
525
|
+
* shadows the native image in the daemon. Only the publisher asks for this;
|
|
526
|
+
* a step always runs the native image. Needs binfmt registered on the host
|
|
527
|
+
* (tonistiigi/binfmt); the publisher does that itself, every time, because
|
|
528
|
+
* the registration does not survive a reboot.
|
|
529
|
+
*/
|
|
530
|
+
export function ensureRunnerImage(opts = {}) {
|
|
531
|
+
const arch = opts.platform ? `-${opts.platform.split("/").pop()}` : "";
|
|
532
|
+
const tag = process.env.FOLDRUN_RUNNER_IMAGE ?? runnerImageTag() + arch;
|
|
447
533
|
const log = [];
|
|
448
534
|
if (process.env.FOLDRUN_RUNNER_IMAGE)
|
|
449
535
|
return { tag, log };
|
|
@@ -463,7 +549,8 @@ export function ensureRunnerImage() {
|
|
|
463
549
|
fs.writeFileSync(path.join(build, "entry.sh"), ENTRY);
|
|
464
550
|
fs.writeFileSync(path.join(build, "Dockerfile"), DOCKERFILE);
|
|
465
551
|
log.push(`building runner image ${tag} (first run only)`);
|
|
466
|
-
const
|
|
552
|
+
const args = ["build", ...(opts.platform ? ["--platform", opts.platform] : []), "-t", tag, build];
|
|
553
|
+
const out = spawnSync(cli(), args, { encoding: "utf8" });
|
|
467
554
|
if (out.status !== 0) {
|
|
468
555
|
throw new Error(`runner image build failed:\n${(out.stderr || out.stdout).slice(-2000)}`);
|
|
469
556
|
}
|
|
@@ -538,6 +625,44 @@ function ensureRunNetwork() {
|
|
|
538
625
|
}
|
|
539
626
|
return RUN_NETWORK;
|
|
540
627
|
}
|
|
628
|
+
/**
|
|
629
|
+
* What crosses into the container as its environment, and how.
|
|
630
|
+
*
|
|
631
|
+
* A docker --env-file is one `KEY=value` per line and cannot carry a
|
|
632
|
+
* newline. @file secrets are multi-line by nature (a PEM key, a cert), so
|
|
633
|
+
* they are staged as files NOW, into the copied-in workspace, and the env
|
|
634
|
+
* carries the container path — the one place a path is stable across the
|
|
635
|
+
* boundary. A plain secret whose value happens to hold a newline gets the
|
|
636
|
+
* same treatment, and the trace says so: it used to be dropped from the
|
|
637
|
+
* env file without a word, and the step failed later, elsewhere, on a
|
|
638
|
+
* variable that read as unset. Everything else goes through the env file.
|
|
639
|
+
*/
|
|
640
|
+
export function stageContainerEnv(env, wsIn, agentRel, emit) {
|
|
641
|
+
const containerEnv = {};
|
|
642
|
+
const stage = (k, content) => {
|
|
643
|
+
const rel = path.join(agentRel, ".secret-files", k.toLowerCase());
|
|
644
|
+
const abs = path.join(wsIn, rel);
|
|
645
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
|
|
646
|
+
fs.writeFileSync(abs, content, { mode: 0o600 });
|
|
647
|
+
return `/workspace/${rel.replaceAll("\\", "/")}`;
|
|
648
|
+
};
|
|
649
|
+
for (const [k, v] of Object.entries(env)) {
|
|
650
|
+
if (typeof v !== "string")
|
|
651
|
+
continue;
|
|
652
|
+
if (isFileValue(v)) {
|
|
653
|
+
containerEnv[k] = stage(k, fileContent(v));
|
|
654
|
+
}
|
|
655
|
+
else if (/[\r\n]/.test(v)) {
|
|
656
|
+
containerEnv[k] = stage(k, v);
|
|
657
|
+
emit("info", `${k} holds a line break and cannot cross as an environment variable — ` +
|
|
658
|
+
`passed as a file instead; $${k} is its path in the sandbox`);
|
|
659
|
+
}
|
|
660
|
+
else {
|
|
661
|
+
containerEnv[k] = v;
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
return containerEnv;
|
|
665
|
+
}
|
|
541
666
|
/** The limits a size class reserves. Large is the install's configured
|
|
542
667
|
* limits unchanged, so existing installs bill and behave identically;
|
|
543
668
|
* small defaults to a quarter-ish slice and is env-tunable. */
|
|
@@ -628,6 +753,8 @@ export async function runStepInContainer(args) {
|
|
|
628
753
|
filter: (src) => !isPlatformPath(path.relative(args.workspaceRoot, src)),
|
|
629
754
|
});
|
|
630
755
|
const libIn = path.join(staging, "library");
|
|
756
|
+
if (args.galleryRoot && fs.existsSync(args.galleryRoot))
|
|
757
|
+
fs.cpSync(args.galleryRoot, libIn, { recursive: true });
|
|
631
758
|
if (fs.existsSync(args.libraryRoot))
|
|
632
759
|
fs.cpSync(args.libraryRoot, libIn, { recursive: true });
|
|
633
760
|
else
|
|
@@ -635,28 +762,7 @@ export async function runStepInContainer(args) {
|
|
|
635
762
|
const jobIn = path.join(staging, "job");
|
|
636
763
|
fs.mkdirSync(jobIn);
|
|
637
764
|
fs.writeFileSync(path.join(jobIn, "input.json"), JSON.stringify(args.input, null, 2));
|
|
638
|
-
|
|
639
|
-
// --env-file cannot carry a newline. So they are staged as files *now*,
|
|
640
|
-
// into the copied-in workspace, and the env carries the container path —
|
|
641
|
-
// the one place a path is stable across the boundary. Everything else
|
|
642
|
-
// goes through the env file. (Without this the key silently vanished:
|
|
643
|
-
// the old filter dropped any value with a newline, and ssh got an empty
|
|
644
|
-
// -i path.)
|
|
645
|
-
const containerEnv = {};
|
|
646
|
-
for (const [k, v] of Object.entries(args.env)) {
|
|
647
|
-
if (typeof v !== "string")
|
|
648
|
-
continue;
|
|
649
|
-
if (isFileValue(v)) {
|
|
650
|
-
const rel = path.join(args.input.agentRel, ".secret-files", k.toLowerCase());
|
|
651
|
-
const abs = path.join(wsIn, rel);
|
|
652
|
-
fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
|
|
653
|
-
fs.writeFileSync(abs, fileContent(v), { mode: 0o600 });
|
|
654
|
-
containerEnv[k] = `/workspace/${rel.replaceAll("\\", "/")}`;
|
|
655
|
-
}
|
|
656
|
-
else if (!v.includes("\n")) {
|
|
657
|
-
containerEnv[k] = v;
|
|
658
|
-
}
|
|
659
|
-
}
|
|
765
|
+
const containerEnv = stageContainerEnv(args.env, wsIn, args.input.agentRel, args.emit);
|
|
660
766
|
const envFile = path.join(staging, "env");
|
|
661
767
|
fs.writeFileSync(envFile, Object.entries(containerEnv).map(([k, v]) => `${k}=${v}`).join("\n"), { mode: 0o600 });
|
|
662
768
|
const flags = [
|
|
@@ -770,7 +876,7 @@ export async function runStepInContainer(args) {
|
|
|
770
876
|
fs.mkdirSync(wsOut);
|
|
771
877
|
const back = spawnSync(cli(), ["cp", `${containerId}:/workspace/.`, wsOut], { encoding: "utf8" });
|
|
772
878
|
if (back.status === 0) {
|
|
773
|
-
applyContainerChanges(args.workspaceRoot, wsOut, wsIn);
|
|
879
|
+
applyContainerChanges(args.workspaceRoot, wsOut, wsIn, (m) => args.emit("error", m), args.divert);
|
|
774
880
|
}
|
|
775
881
|
else {
|
|
776
882
|
args.emit("error", `copy-out failed — the step's file changes were lost:\n${back.stderr.slice(0, 500)}`);
|