@librechat/agents 3.2.42 → 3.2.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/agents/AgentContext.cjs +4 -3
- package/dist/cjs/agents/AgentContext.cjs.map +1 -1
- package/dist/cjs/graphs/Graph.cjs +1 -1
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/hooks/createWorkspacePolicyHook.cjs +1 -1
- package/dist/cjs/llm/bedrock/index.cjs +1 -1
- package/dist/cjs/llm/bedrock/index.cjs.map +1 -1
- package/dist/cjs/main.cjs +2 -0
- package/dist/cjs/messages/cache.cjs +26 -4
- package/dist/cjs/messages/cache.cjs.map +1 -1
- package/dist/cjs/messages/reducer.cjs +15 -2
- package/dist/cjs/messages/reducer.cjs.map +1 -1
- package/dist/cjs/session/JsonlSessionStore.cjs +1 -1
- package/dist/cjs/tools/cloudflare/CloudflareSandboxExecutionEngine.cjs +32 -16
- package/dist/cjs/tools/cloudflare/CloudflareSandboxExecutionEngine.cjs.map +1 -1
- package/dist/cjs/tools/local/CompileCheckTool.cjs +11 -3
- package/dist/cjs/tools/local/CompileCheckTool.cjs.map +1 -1
- package/dist/cjs/tools/local/FileCheckpointer.cjs +8 -3
- package/dist/cjs/tools/local/FileCheckpointer.cjs.map +1 -1
- package/dist/cjs/tools/local/LocalCodingTools.cjs +26 -7
- package/dist/cjs/tools/local/LocalCodingTools.cjs.map +1 -1
- package/dist/cjs/tools/local/LocalExecutionEngine.cjs +2 -2
- package/dist/cjs/tools/local/syntaxCheck.cjs +6 -2
- package/dist/cjs/tools/local/syntaxCheck.cjs.map +1 -1
- package/dist/cjs/tools/local/workspaceFS.cjs +20 -0
- package/dist/cjs/tools/local/workspaceFS.cjs.map +1 -1
- package/dist/esm/agents/AgentContext.mjs +5 -4
- package/dist/esm/agents/AgentContext.mjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +2 -2
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/hooks/createWorkspacePolicyHook.mjs +1 -1
- package/dist/esm/llm/bedrock/index.mjs +2 -2
- package/dist/esm/llm/bedrock/index.mjs.map +1 -1
- package/dist/esm/main.mjs +3 -3
- package/dist/esm/messages/cache.mjs +26 -5
- package/dist/esm/messages/cache.mjs.map +1 -1
- package/dist/esm/messages/reducer.mjs +15 -2
- package/dist/esm/messages/reducer.mjs.map +1 -1
- package/dist/esm/session/JsonlSessionStore.mjs +1 -1
- package/dist/esm/tools/cloudflare/CloudflareSandboxExecutionEngine.mjs +32 -17
- package/dist/esm/tools/cloudflare/CloudflareSandboxExecutionEngine.mjs.map +1 -1
- package/dist/esm/tools/local/CompileCheckTool.mjs +11 -3
- package/dist/esm/tools/local/CompileCheckTool.mjs.map +1 -1
- package/dist/esm/tools/local/FileCheckpointer.mjs +9 -4
- package/dist/esm/tools/local/FileCheckpointer.mjs.map +1 -1
- package/dist/esm/tools/local/LocalCodingTools.mjs +26 -7
- package/dist/esm/tools/local/LocalCodingTools.mjs.map +1 -1
- package/dist/esm/tools/local/LocalExecutionEngine.mjs +2 -2
- package/dist/esm/tools/local/syntaxCheck.mjs +6 -2
- package/dist/esm/tools/local/syntaxCheck.mjs.map +1 -1
- package/dist/esm/tools/local/workspaceFS.mjs +19 -1
- package/dist/esm/tools/local/workspaceFS.mjs.map +1 -1
- package/dist/types/agents/AgentContext.d.ts +3 -2
- package/dist/types/messages/cache.d.ts +12 -2
- package/dist/types/tools/cloudflare/CloudflareSandboxExecutionEngine.d.ts +9 -0
- package/dist/types/tools/local/workspaceFS.d.ts +13 -0
- package/package.json +1 -1
- package/src/agents/AgentContext.ts +8 -3
- package/src/graphs/Graph.ts +7 -1
- package/src/llm/bedrock/index.ts +2 -2
- package/src/messages/cache.test.ts +32 -0
- package/src/messages/cache.ts +31 -5
- package/src/messages/reducer.spec.ts +32 -0
- package/src/messages/reducer.ts +23 -7
- package/src/specs/vllm-reasoning-toolcalls.test.ts +340 -0
- package/src/tools/__tests__/CloudflareSandboxExecution.test.ts +192 -0
- package/src/tools/__tests__/LocalExecutionTools.test.ts +54 -0
- package/src/tools/cloudflare/CloudflareSandboxExecutionEngine.ts +106 -30
- package/src/tools/local/CompileCheckTool.ts +19 -3
- package/src/tools/local/FileCheckpointer.ts +20 -4
- package/src/tools/local/LocalCodingTools.ts +61 -8
- package/src/tools/local/__tests__/FileCheckpointer.test.ts +42 -0
- package/src/tools/local/syntaxCheck.ts +14 -2
- package/src/tools/local/workspaceFS.ts +27 -0
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
createCloudflareBashProgrammaticToolCallingTool,
|
|
11
11
|
createCloudflareProgrammaticToolCallingTool,
|
|
12
12
|
} from '../cloudflare/CloudflareProgrammaticToolCalling';
|
|
13
|
+
import { isWorkspaceClientTimeoutError } from '../local/workspaceFS';
|
|
13
14
|
import { createCloudflareBridgeRuntime } from '../cloudflare/CloudflareBridgeRuntime';
|
|
14
15
|
import { resolveLocalToolsForBinding } from '../local/resolveLocalExecutionTools';
|
|
15
16
|
import { spawnLocalProcess } from '../local/LocalExecutionEngine';
|
|
@@ -388,6 +389,197 @@ describe('Cloudflare sandbox execution backend', () => {
|
|
|
388
389
|
expect(received).not.toHaveProperty('signal');
|
|
389
390
|
});
|
|
390
391
|
|
|
392
|
+
it('rejects with a client-side timeout when sandbox readFile stalls', async () => {
|
|
393
|
+
// The native-DO file-IO RPCs (readFile/writeFile/listFiles/...) have the same
|
|
394
|
+
// stall hazard exec() does: no signal, no enforced timeout. A cold/unresponsive
|
|
395
|
+
// container otherwise hangs the host await until the run-level abort, burning
|
|
396
|
+
// the whole budget on one read (observed live: a `read_file` stalled ~552s).
|
|
397
|
+
jest.useFakeTimers();
|
|
398
|
+
try {
|
|
399
|
+
let readCalls = 0;
|
|
400
|
+
const fs = createCloudflareWorkspaceFS({
|
|
401
|
+
workspaceRoot: '/workspace',
|
|
402
|
+
timeoutMs: 1000,
|
|
403
|
+
sandbox: createRuntime({
|
|
404
|
+
readFile: () => {
|
|
405
|
+
readCalls += 1;
|
|
406
|
+
return new Promise<string>(() => undefined);
|
|
407
|
+
},
|
|
408
|
+
}),
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
const error = (fs.readFile as (p: string) => Promise<unknown>)(
|
|
412
|
+
'/workspace/a.txt'
|
|
413
|
+
).catch((e: unknown) => e);
|
|
414
|
+
// Client backstop = clientFsTimeoutMs(1000) = 6000ms; advance past it.
|
|
415
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
416
|
+
const settled = await error;
|
|
417
|
+
// Must be the DISTINGUISHABLE timeout error so ENOENT-only callers rethrow
|
|
418
|
+
// it instead of mistaking a stalled read for a missing file.
|
|
419
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
420
|
+
expect((settled as Error).message).toMatch(/client-side timeout/);
|
|
421
|
+
expect(readCalls).toBe(1);
|
|
422
|
+
} finally {
|
|
423
|
+
jest.useRealTimers();
|
|
424
|
+
}
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
it('keeps the backstop active while draining a streamed file read', async () => {
|
|
428
|
+
// sandbox.readFile resolves to { content: ReadableStream } whose stream never
|
|
429
|
+
// ends. The race must cover the drain (normalizeReadFileContent), not just the
|
|
430
|
+
// initial RPC, or read_file/open/stat still hang to the run-level abort.
|
|
431
|
+
jest.useFakeTimers();
|
|
432
|
+
try {
|
|
433
|
+
const fs = createCloudflareWorkspaceFS({
|
|
434
|
+
workspaceRoot: '/workspace',
|
|
435
|
+
timeoutMs: 1000,
|
|
436
|
+
sandbox: createRuntime({
|
|
437
|
+
readFile: async () => ({
|
|
438
|
+
content: new ReadableStream<Uint8Array>({
|
|
439
|
+
// start() never enqueues or closes -> the drain stalls forever.
|
|
440
|
+
start() {},
|
|
441
|
+
}),
|
|
442
|
+
}),
|
|
443
|
+
}),
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
const error = (fs.readFile as (p: string) => Promise<unknown>)(
|
|
447
|
+
'/workspace/a.txt'
|
|
448
|
+
).catch((e: unknown) => e);
|
|
449
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
450
|
+
const settled = await error;
|
|
451
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
452
|
+
} finally {
|
|
453
|
+
jest.useRealTimers();
|
|
454
|
+
}
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
it('rethrows a stat directory-probe timeout instead of falling through to readFile', async () => {
|
|
458
|
+
// findChildInfo returns nothing -> the directory probe (listFiles) runs; if it
|
|
459
|
+
// STALLS it must surface, not fall through to the readFile branch (which would
|
|
460
|
+
// burn a SECOND full backstop, ~2x the timeout, before the caller sees it).
|
|
461
|
+
jest.useFakeTimers();
|
|
462
|
+
try {
|
|
463
|
+
let readFileCalls = 0;
|
|
464
|
+
const fs = createCloudflareWorkspaceFS({
|
|
465
|
+
workspaceRoot: '/workspace',
|
|
466
|
+
timeoutMs: 1000,
|
|
467
|
+
sandbox: createRuntime({
|
|
468
|
+
listFiles: (dir) =>
|
|
469
|
+
dir === '/workspace/probe-me'
|
|
470
|
+
? new Promise(() => undefined) // the probe stalls
|
|
471
|
+
: Promise.resolve([]), // findChildInfo's parent listing returns fast
|
|
472
|
+
readFile: () => {
|
|
473
|
+
readFileCalls += 1;
|
|
474
|
+
return Promise.resolve('');
|
|
475
|
+
},
|
|
476
|
+
}),
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
const error = fs.stat('/workspace/probe-me').catch((e: unknown) => e);
|
|
480
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
481
|
+
const settled = await error;
|
|
482
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
483
|
+
// Must NOT have fallen through to the readFile probe.
|
|
484
|
+
expect(readFileCalls).toBe(0);
|
|
485
|
+
} finally {
|
|
486
|
+
jest.useRealTimers();
|
|
487
|
+
}
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it('bounds execute_code temp-dir setup RPCs (mkdir/writeFile) that stall', async () => {
|
|
491
|
+
jest.useFakeTimers();
|
|
492
|
+
try {
|
|
493
|
+
const sandbox = createRuntime({
|
|
494
|
+
// exec would resolve fine; the stall is in the pre-exec mkdir setup.
|
|
495
|
+
mkdir: () => new Promise<{ ok: true }>(() => undefined),
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
const promise = executeCloudflareCode(
|
|
499
|
+
{ lang: 'py', code: 'print("hi")' },
|
|
500
|
+
{ sandbox, workspaceRoot: '/workspace', timeoutMs: 1000 }
|
|
501
|
+
);
|
|
502
|
+
const assertion = expect(promise).rejects.toThrow(/client-side timeout/);
|
|
503
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
504
|
+
await assertion;
|
|
505
|
+
} finally {
|
|
506
|
+
jest.useRealTimers();
|
|
507
|
+
}
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it('still cleans up the temp dir when execute_code setup (writeFile) times out', async () => {
|
|
511
|
+
// The setup RPCs are inside the try, so a stalled writeFile still triggers
|
|
512
|
+
// the finally cleanup — otherwise the late (uncancellable) write leaves an
|
|
513
|
+
// orphaned .lc-exec/<uuid> dir behind on every cold-container failure.
|
|
514
|
+
jest.useFakeTimers();
|
|
515
|
+
try {
|
|
516
|
+
const execCommands: string[] = [];
|
|
517
|
+
const sandbox = createRuntime({
|
|
518
|
+
mkdir: async () => ({ ok: true }),
|
|
519
|
+
writeFile: () => new Promise<{ ok: true }>(() => undefined), // stalls
|
|
520
|
+
exec: async (command) => {
|
|
521
|
+
execCommands.push(command);
|
|
522
|
+
return { exitCode: 0, stdout: '', stderr: '' };
|
|
523
|
+
},
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
const promise = executeCloudflareCode(
|
|
527
|
+
{ lang: 'py', code: 'print("hi")' },
|
|
528
|
+
{ sandbox, workspaceRoot: '/workspace', timeoutMs: 1000 }
|
|
529
|
+
).catch((e: unknown) => e);
|
|
530
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
531
|
+
const settled = await promise;
|
|
532
|
+
expect(isWorkspaceClientTimeoutError(settled)).toBe(true);
|
|
533
|
+
// Cleanup must have been issued despite the setup failure.
|
|
534
|
+
expect(execCommands.some((c) => c.startsWith('rm -rf'))).toBe(true);
|
|
535
|
+
} finally {
|
|
536
|
+
jest.useRealTimers();
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
it('rejects with a client-side timeout when sandbox listFiles stalls', async () => {
|
|
541
|
+
jest.useFakeTimers();
|
|
542
|
+
try {
|
|
543
|
+
const fs = createCloudflareWorkspaceFS({
|
|
544
|
+
workspaceRoot: '/workspace',
|
|
545
|
+
timeoutMs: 1000,
|
|
546
|
+
sandbox: createRuntime({
|
|
547
|
+
listFiles: () =>
|
|
548
|
+
new Promise<t.CloudflareSandboxFileInfo[]>(() => undefined),
|
|
549
|
+
}),
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
const promise = (fs.readdir as (p: string) => Promise<unknown>)(
|
|
553
|
+
'/workspace/sub'
|
|
554
|
+
);
|
|
555
|
+
const assertion = expect(promise).rejects.toThrow(/client-side timeout/);
|
|
556
|
+
await jest.advanceTimersByTimeAsync(6500);
|
|
557
|
+
await assertion;
|
|
558
|
+
} finally {
|
|
559
|
+
jest.useRealTimers();
|
|
560
|
+
}
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
it('does not time out a native FS RPC that returns in time', async () => {
|
|
564
|
+
jest.useFakeTimers();
|
|
565
|
+
try {
|
|
566
|
+
const fs = createCloudflareWorkspaceFS({
|
|
567
|
+
workspaceRoot: '/workspace',
|
|
568
|
+
timeoutMs: 1000,
|
|
569
|
+
sandbox: createRuntime({ readFile: async () => 'contents' }),
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
const result = await (
|
|
573
|
+
fs.readFile as (p: string, e: 'utf8') => Promise<string>
|
|
574
|
+
)('/workspace/a.txt', 'utf8');
|
|
575
|
+
expect(result).toBe('contents');
|
|
576
|
+
// The backstop timer must have been cleared, not left dangling.
|
|
577
|
+
expect(jest.getTimerCount()).toBe(0);
|
|
578
|
+
} finally {
|
|
579
|
+
jest.useRealTimers();
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
|
|
391
583
|
it('passes call-specific timeouts to the Cloudflare spawn wrapper', async () => {
|
|
392
584
|
let execCommand = '';
|
|
393
585
|
let execTimeout: number | undefined;
|
|
@@ -37,6 +37,8 @@ import {
|
|
|
37
37
|
_resetSyntaxCheckProbeCacheForTests,
|
|
38
38
|
} from '../local/syntaxCheck';
|
|
39
39
|
import { resolveLocalToolsForBinding } from '../local/resolveLocalExecutionTools';
|
|
40
|
+
import { WorkspaceClientTimeoutError } from '../local/workspaceFS';
|
|
41
|
+
import type { WorkspaceFS } from '../local/workspaceFS';
|
|
40
42
|
import { LocalFileCheckpointerImpl } from '../local/FileCheckpointer';
|
|
41
43
|
import { createCompileCheckTool } from '../local/CompileCheckTool';
|
|
42
44
|
import { runBashAstChecks } from '../local/bashAst';
|
|
@@ -2317,6 +2319,58 @@ describe('comprehensive review (round 14) — Codex P1 #37 + P2 #38/#40/#41', ()
|
|
|
2317
2319
|
).rejects.toThrow(/syntax check failed.*reverted/i);
|
|
2318
2320
|
expect(await fsp.readFile(file, 'utf8')).toBe(original);
|
|
2319
2321
|
});
|
|
2322
|
+
|
|
2323
|
+
it('write_file: still reverts when the strict validation READ times out', async () => {
|
|
2324
|
+
// A stalled validation read must not leave the just-written bytes on disk —
|
|
2325
|
+
// strict mode's revert contract still applies when validation is
|
|
2326
|
+
// inconclusive due to a timeout (here a brand-new file -> unlink).
|
|
2327
|
+
const cwd = await createTempDir();
|
|
2328
|
+
const file = join(cwd, 'config.json');
|
|
2329
|
+
const unlinked: string[] = [];
|
|
2330
|
+
let written = false;
|
|
2331
|
+
const fakeFs = {
|
|
2332
|
+
readFile: async () => {
|
|
2333
|
+
if (!written) {
|
|
2334
|
+
// write_file pre-read: the file does not exist yet.
|
|
2335
|
+
throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' });
|
|
2336
|
+
}
|
|
2337
|
+
// jsonCheck's post-write validation read stalls.
|
|
2338
|
+
throw new WorkspaceClientTimeoutError(
|
|
2339
|
+
'cloudflare sandbox readFile exceeded 6000ms client-side timeout'
|
|
2340
|
+
);
|
|
2341
|
+
},
|
|
2342
|
+
writeFile: async () => {
|
|
2343
|
+
written = true;
|
|
2344
|
+
},
|
|
2345
|
+
mkdir: async () => undefined,
|
|
2346
|
+
unlink: async (p: string) => {
|
|
2347
|
+
unlinked.push(p);
|
|
2348
|
+
},
|
|
2349
|
+
stat: async () => ({ isFile: () => true, size: 1 }),
|
|
2350
|
+
readdir: async () => [],
|
|
2351
|
+
realpath: async (p: string) => p,
|
|
2352
|
+
} as unknown as WorkspaceFS;
|
|
2353
|
+
|
|
2354
|
+
const bundle = createLocalCodingToolBundle({
|
|
2355
|
+
cwd,
|
|
2356
|
+
postEditSyntaxCheck: 'strict',
|
|
2357
|
+
exec: { fs: fakeFs },
|
|
2358
|
+
});
|
|
2359
|
+
const writeTool = bundle.tools.find(
|
|
2360
|
+
(tt) => tt.name === Constants.WRITE_FILE
|
|
2361
|
+
);
|
|
2362
|
+
await expect(
|
|
2363
|
+
writeTool!.invoke({
|
|
2364
|
+
id: 'wf-strict-read-timeout',
|
|
2365
|
+
name: Constants.WRITE_FILE,
|
|
2366
|
+
args: { path: file, content: '{"ok": true}\n' },
|
|
2367
|
+
type: 'tool_call',
|
|
2368
|
+
})
|
|
2369
|
+
).rejects.toThrow(/client-side timeout/);
|
|
2370
|
+
// strict mode attempted the revert (brand-new file -> unlink), rather than
|
|
2371
|
+
// leaving the unvalidated write on disk.
|
|
2372
|
+
expect(unlinked.length).toBeGreaterThan(0);
|
|
2373
|
+
});
|
|
2320
2374
|
});
|
|
2321
2375
|
|
|
2322
2376
|
describe('fallbackGrep skip sentinels do not count as matches (Codex P2 [43])', () => {
|
|
@@ -5,6 +5,10 @@ import type { WriteFileOptions, MakeDirectoryOptions, Stats } from 'fs';
|
|
|
5
5
|
import type { ChildProcessWithoutNullStreams } from 'child_process';
|
|
6
6
|
import type { FileHandle } from 'fs/promises';
|
|
7
7
|
import type { WorkspaceFS, ReaddirEntry } from '@/tools/local/workspaceFS';
|
|
8
|
+
import {
|
|
9
|
+
WorkspaceClientTimeoutError,
|
|
10
|
+
isWorkspaceClientTimeoutError,
|
|
11
|
+
} from '@/tools/local/workspaceFS';
|
|
8
12
|
import type * as t from '@/types';
|
|
9
13
|
import {
|
|
10
14
|
LOCAL_SPAWN_TIMEOUT_MS,
|
|
@@ -140,6 +144,18 @@ export function clientExecTimeoutMs(timeoutMs: number): number {
|
|
|
140
144
|
return outerTimeoutMs(timeoutMs) + 5000;
|
|
141
145
|
}
|
|
142
146
|
|
|
147
|
+
/**
|
|
148
|
+
* Client-side backstop timeout for a native-DO sandbox FILE-IO RPC
|
|
149
|
+
* (`readFile`/`writeFile`/`listFiles`/`mkdir`/`deleteFile`). Unlike `exec()`
|
|
150
|
+
* there is no in-sandbox `timeout(1)` layer for these to honor, so this is just a
|
|
151
|
+
* few seconds of headroom over the configured tool timeout — enough that a normal
|
|
152
|
+
* (even large, byte-capped) read completes, while a stalled/cold container can't
|
|
153
|
+
* outlast it. See `withClientTimeout` for why the native DO RPC needs this.
|
|
154
|
+
*/
|
|
155
|
+
export function clientFsTimeoutMs(timeoutMs: number): number {
|
|
156
|
+
return timeoutMs + 5000;
|
|
157
|
+
}
|
|
158
|
+
|
|
143
159
|
/**
|
|
144
160
|
* Bound a `sandbox.exec()` await with a CLIENT-SIDE timeout.
|
|
145
161
|
*
|
|
@@ -188,8 +204,8 @@ export async function withClientTimeout<T>(
|
|
|
188
204
|
// only then abort a signal-aware exec, whose resulting AbortError must
|
|
189
205
|
// not surface to the caller instead of the timeout.
|
|
190
206
|
reject(
|
|
191
|
-
new
|
|
192
|
-
`${label} exceeded ${timeoutMs}ms client-side timeout (sandbox
|
|
207
|
+
new WorkspaceClientTimeoutError(
|
|
208
|
+
`${label} exceeded ${timeoutMs}ms client-side timeout (sandbox RPC did not return)`
|
|
193
209
|
)
|
|
194
210
|
);
|
|
195
211
|
options.onTimeout?.();
|
|
@@ -412,12 +428,17 @@ function createDirent(info: t.CloudflareSandboxFileInfo): ReaddirEntry {
|
|
|
412
428
|
|
|
413
429
|
async function findChildInfo(
|
|
414
430
|
sandbox: t.CloudflareSandboxRuntime,
|
|
415
|
-
filePath: string
|
|
431
|
+
filePath: string,
|
|
432
|
+
timeoutMs: number
|
|
416
433
|
): Promise<t.CloudflareSandboxFileInfo | undefined> {
|
|
417
434
|
const parent = path.dirname(filePath);
|
|
418
435
|
const basename = path.basename(filePath);
|
|
419
436
|
const entries = normalizeFileList(
|
|
420
|
-
await
|
|
437
|
+
await withClientTimeout(
|
|
438
|
+
sandbox.listFiles(parent, { includeHidden: true }),
|
|
439
|
+
timeoutMs,
|
|
440
|
+
'cloudflare sandbox listFiles'
|
|
441
|
+
)
|
|
421
442
|
);
|
|
422
443
|
return entries.find((entry) => {
|
|
423
444
|
const absolute = entryAbsolutePath(entry, parent);
|
|
@@ -429,13 +450,29 @@ export function createCloudflareWorkspaceFS(
|
|
|
429
450
|
config: t.CloudflareSandboxExecutionConfig
|
|
430
451
|
): WorkspaceFS {
|
|
431
452
|
const workspaceRoot = getCloudflareWorkspaceRoot(config);
|
|
453
|
+
// Native-DO file-IO RPCs have the SAME stall hazard as exec() (PR #252): no
|
|
454
|
+
// `signal`, no reliably-enforced timeout, so a cold/unresponsive container
|
|
455
|
+
// hangs the host await until the run-level abort — burning the whole budget on
|
|
456
|
+
// one read (observed: a `read_file` that stalled ~552s before the wall-clock
|
|
457
|
+
// budget killed it). Bound every native FS RPC with the same client-side
|
|
458
|
+
// backstop the exec sites use.
|
|
459
|
+
const fsTimeoutMs = clientFsTimeoutMs(config.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
460
|
+
const bound = <T>(op: Promise<T>, label: string): Promise<T> =>
|
|
461
|
+
withClientTimeout(op, fsTimeoutMs, `cloudflare sandbox ${label}`);
|
|
432
462
|
|
|
433
463
|
const fs: WorkspaceFS = {
|
|
434
464
|
readFile: (async (filePath: string, encoding?: 'utf8') => {
|
|
435
465
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
436
466
|
const resolved = toSandboxPath(filePath, workspaceRoot);
|
|
437
|
-
|
|
438
|
-
|
|
467
|
+
// Wrap the stream drain (normalizeReadFileContent) inside the backstop too:
|
|
468
|
+
// a sandbox.readFile that resolves to a { content: ReadableStream } can
|
|
469
|
+
// still stall mid-drain after the RPC promise settled.
|
|
470
|
+
const buffer = await bound(
|
|
471
|
+
(async (): Promise<Buffer> =>
|
|
472
|
+
normalizeReadFileContent(
|
|
473
|
+
await sandbox.readFile(resolved, encoding ? { encoding } : undefined)
|
|
474
|
+
))(),
|
|
475
|
+
'readFile'
|
|
439
476
|
);
|
|
440
477
|
return encoding != null ? buffer.toString(encoding) : buffer;
|
|
441
478
|
}) as WorkspaceFS['readFile'],
|
|
@@ -447,29 +484,46 @@ export function createCloudflareWorkspaceFS(
|
|
|
447
484
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
448
485
|
const resolved = toSandboxPath(filePath, workspaceRoot);
|
|
449
486
|
const normalized = normalizeWriteFileContent(content);
|
|
450
|
-
await
|
|
487
|
+
await bound(
|
|
488
|
+
sandbox.writeFile(resolved, normalized.content, normalized.options),
|
|
489
|
+
'writeFile'
|
|
490
|
+
);
|
|
451
491
|
},
|
|
452
492
|
stat: async (filePath: string) => {
|
|
453
493
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
454
494
|
const resolved = toSandboxPath(filePath, workspaceRoot);
|
|
455
495
|
if (resolved === workspaceRoot) {
|
|
456
496
|
const entries = normalizeFileList(
|
|
457
|
-
await
|
|
497
|
+
await bound(
|
|
498
|
+
sandbox.listFiles(resolved, { includeHidden: true }),
|
|
499
|
+
'listFiles'
|
|
500
|
+
)
|
|
458
501
|
);
|
|
459
502
|
return createStats({ size: entries.length, type: 'directory' });
|
|
460
503
|
}
|
|
461
|
-
const info = await findChildInfo(sandbox, resolved);
|
|
504
|
+
const info = await findChildInfo(sandbox, resolved, fsTimeoutMs);
|
|
462
505
|
if (info != null) {
|
|
463
506
|
return createStats({ size: info.size, type: info.type });
|
|
464
507
|
}
|
|
465
508
|
try {
|
|
466
509
|
const entries = normalizeFileList(
|
|
467
|
-
await
|
|
510
|
+
await bound(
|
|
511
|
+
sandbox.listFiles(resolved, { includeHidden: true }),
|
|
512
|
+
'listFiles'
|
|
513
|
+
)
|
|
468
514
|
);
|
|
469
515
|
return createStats({ size: entries.length, type: 'directory' });
|
|
470
|
-
} catch {
|
|
471
|
-
|
|
472
|
-
|
|
516
|
+
} catch (error) {
|
|
517
|
+
// A directory-probe timeout is a stalled container, not "not a directory".
|
|
518
|
+
// Don't fall through to the readFile branch — that would wait through a
|
|
519
|
+
// SECOND full backstop (~2x the timeout) before surfacing.
|
|
520
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
521
|
+
throw error;
|
|
522
|
+
}
|
|
523
|
+
const buffer = await bound(
|
|
524
|
+
(async (): Promise<Buffer> =>
|
|
525
|
+
normalizeReadFileContent(await sandbox.readFile(resolved)))(),
|
|
526
|
+
'readFile'
|
|
473
527
|
);
|
|
474
528
|
return createStats({ size: buffer.length, type: 'file' });
|
|
475
529
|
}
|
|
@@ -478,7 +532,10 @@ export function createCloudflareWorkspaceFS(
|
|
|
478
532
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
479
533
|
const resolved = toSandboxPath(filePath, workspaceRoot);
|
|
480
534
|
const entries = normalizeFileList(
|
|
481
|
-
await
|
|
535
|
+
await bound(
|
|
536
|
+
sandbox.listFiles(resolved, { includeHidden: true }),
|
|
537
|
+
'listFiles'
|
|
538
|
+
)
|
|
482
539
|
);
|
|
483
540
|
if (options?.withFileTypes === true) {
|
|
484
541
|
return entries.map(createDirent);
|
|
@@ -487,21 +544,29 @@ export function createCloudflareWorkspaceFS(
|
|
|
487
544
|
}) as WorkspaceFS['readdir'],
|
|
488
545
|
mkdir: async (filePath: string, options?: MakeDirectoryOptions) => {
|
|
489
546
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
490
|
-
await
|
|
491
|
-
|
|
492
|
-
|
|
547
|
+
await bound(
|
|
548
|
+
sandbox.mkdir(toSandboxPath(filePath, workspaceRoot), {
|
|
549
|
+
recursive: options?.recursive,
|
|
550
|
+
}),
|
|
551
|
+
'mkdir'
|
|
552
|
+
);
|
|
493
553
|
},
|
|
494
554
|
realpath: async (filePath: string) =>
|
|
495
555
|
toSandboxPath(filePath, workspaceRoot),
|
|
496
556
|
unlink: async (filePath: string) => {
|
|
497
557
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
498
|
-
await
|
|
558
|
+
await bound(
|
|
559
|
+
sandbox.deleteFile(toSandboxPath(filePath, workspaceRoot)),
|
|
560
|
+
'deleteFile'
|
|
561
|
+
);
|
|
499
562
|
},
|
|
500
563
|
open: async (filePath: string, _flags: 'r') => {
|
|
501
564
|
const sandbox = await resolveCloudflareSandbox(config);
|
|
502
565
|
const resolved = toSandboxPath(filePath, workspaceRoot);
|
|
503
|
-
const buffer = await
|
|
504
|
-
|
|
566
|
+
const buffer = await bound(
|
|
567
|
+
(async (): Promise<Buffer> =>
|
|
568
|
+
normalizeReadFileContent(await sandbox.readFile(resolved)))(),
|
|
569
|
+
'readFile'
|
|
505
570
|
);
|
|
506
571
|
return {
|
|
507
572
|
read: async (
|
|
@@ -832,18 +897,29 @@ export async function executeCloudflareCode(
|
|
|
832
897
|
input.args,
|
|
833
898
|
ctx.shell
|
|
834
899
|
);
|
|
835
|
-
await ctx.sandbox.mkdir(tempDir, { recursive: true });
|
|
836
|
-
if (runtime.source != null) {
|
|
837
|
-
await ctx.sandbox.writeFile(
|
|
838
|
-
path.join(tempDir, runtime.fileName),
|
|
839
|
-
runtime.source,
|
|
840
|
-
{
|
|
841
|
-
encoding: 'utf8',
|
|
842
|
-
}
|
|
843
|
-
);
|
|
844
|
-
}
|
|
845
900
|
let execSucceeded = false;
|
|
846
901
|
try {
|
|
902
|
+
// Bound the temp-dir setup RPCs (they run BEFORE the bounded exec): a
|
|
903
|
+
// native-DO stall here would hang the host on a single mkdir/writeFile and
|
|
904
|
+
// burn the run budget. Keep them INSIDE the try so the finally cleanup still
|
|
905
|
+
// removes .lc-exec/<uuid> if setup throws — the uncancellable write can land
|
|
906
|
+
// late on a cold container, so an orphaned dir would otherwise accumulate.
|
|
907
|
+
await withClientTimeout(
|
|
908
|
+
ctx.sandbox.mkdir(tempDir, { recursive: true }),
|
|
909
|
+
clientFsTimeoutMs(ctx.timeoutMs),
|
|
910
|
+
'cloudflare sandbox mkdir'
|
|
911
|
+
);
|
|
912
|
+
if (runtime.source != null) {
|
|
913
|
+
await withClientTimeout(
|
|
914
|
+
ctx.sandbox.writeFile(
|
|
915
|
+
path.join(tempDir, runtime.fileName),
|
|
916
|
+
runtime.source,
|
|
917
|
+
{ encoding: 'utf8' }
|
|
918
|
+
),
|
|
919
|
+
clientFsTimeoutMs(ctx.timeoutMs),
|
|
920
|
+
'cloudflare sandbox writeFile'
|
|
921
|
+
);
|
|
922
|
+
}
|
|
847
923
|
const result = await execWithClientTimeout(
|
|
848
924
|
ctx.sandbox,
|
|
849
925
|
withInSandboxTimeout(runtime.command, ctx.timeoutMs),
|
|
@@ -27,6 +27,7 @@ import { resolve } from 'path';
|
|
|
27
27
|
import { tool } from '@langchain/core/tools';
|
|
28
28
|
import type { DynamicStructuredTool } from '@langchain/core/tools';
|
|
29
29
|
import type { WorkspaceFS } from './workspaceFS';
|
|
30
|
+
import { isWorkspaceClientTimeoutError } from './workspaceFS';
|
|
30
31
|
import type * as t from '@/types';
|
|
31
32
|
import {
|
|
32
33
|
getLocalCwd,
|
|
@@ -74,7 +75,12 @@ async function pathExists(fs: WorkspaceFS, p: string): Promise<boolean> {
|
|
|
74
75
|
try {
|
|
75
76
|
await fs.stat(p);
|
|
76
77
|
return true;
|
|
77
|
-
} catch {
|
|
78
|
+
} catch (error) {
|
|
79
|
+
// A stalled-RPC timeout is not "absent" — don't let it pick a weaker/wrong
|
|
80
|
+
// toolchain after waiting through the backstop; surface it.
|
|
81
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
78
84
|
return false;
|
|
79
85
|
}
|
|
80
86
|
}
|
|
@@ -94,7 +100,12 @@ async function detect(cwd: string, fs: WorkspaceFS): Promise<Detection> {
|
|
|
94
100
|
if (await pathExists(fs, resolve(cwd, 'package.json'))) {
|
|
95
101
|
const pkgRaw = await fs
|
|
96
102
|
.readFile(resolve(cwd, 'package.json'), 'utf8')
|
|
97
|
-
.catch(() =>
|
|
103
|
+
.catch((error) => {
|
|
104
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
105
|
+
throw error;
|
|
106
|
+
}
|
|
107
|
+
return '';
|
|
108
|
+
});
|
|
98
109
|
if (pkgRaw.includes('"typescript"')) {
|
|
99
110
|
return {
|
|
100
111
|
kind: 'typescript',
|
|
@@ -124,7 +135,12 @@ async function detect(cwd: string, fs: WorkspaceFS): Promise<Detection> {
|
|
|
124
135
|
) {
|
|
125
136
|
const pyToml = await fs
|
|
126
137
|
.readFile(resolve(cwd, 'pyproject.toml'), 'utf8')
|
|
127
|
-
.catch(() =>
|
|
138
|
+
.catch((error) => {
|
|
139
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
140
|
+
throw error;
|
|
141
|
+
}
|
|
142
|
+
return '';
|
|
143
|
+
});
|
|
128
144
|
if (pyToml.includes('mypy')) {
|
|
129
145
|
return {
|
|
130
146
|
kind: 'python-mypy',
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { dirname } from 'path';
|
|
2
2
|
import type { WorkspaceFS } from './workspaceFS';
|
|
3
3
|
import type * as t from '@/types';
|
|
4
|
-
import { nodeWorkspaceFS } from './workspaceFS';
|
|
4
|
+
import { isWorkspaceClientTimeoutError, nodeWorkspaceFS } from './workspaceFS';
|
|
5
5
|
|
|
6
6
|
type Snapshot = { kind: 'absent' } | { kind: 'present'; content: Buffer };
|
|
7
7
|
|
|
@@ -41,7 +41,12 @@ export class LocalFileCheckpointerImpl implements t.LocalFileCheckpointer {
|
|
|
41
41
|
let info;
|
|
42
42
|
try {
|
|
43
43
|
info = await this.fs.stat(absolutePath);
|
|
44
|
-
} catch {
|
|
44
|
+
} catch (error) {
|
|
45
|
+
// A stalled-RPC timeout is NOT "file absent" — snapshotting it as absent
|
|
46
|
+
// would delete an existing file on revert. Surface it instead.
|
|
47
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
45
50
|
this.snapshots.set(absolutePath, { kind: 'absent' });
|
|
46
51
|
return;
|
|
47
52
|
}
|
|
@@ -60,7 +65,14 @@ export class LocalFileCheckpointerImpl implements t.LocalFileCheckpointer {
|
|
|
60
65
|
let restored = 0;
|
|
61
66
|
for (const [path, snapshot] of this.snapshots.entries()) {
|
|
62
67
|
if (snapshot.kind === 'absent') {
|
|
63
|
-
await this.fs.unlink(path).catch(() =>
|
|
68
|
+
await this.fs.unlink(path).catch((error) => {
|
|
69
|
+
// A timed-out delete did NOT happen — surface it rather than counting
|
|
70
|
+
// the path as restored (which would falsely claim the workspace is
|
|
71
|
+
// back to its pre-write state).
|
|
72
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
});
|
|
64
76
|
restored++;
|
|
65
77
|
continue;
|
|
66
78
|
}
|
|
@@ -68,7 +80,11 @@ export class LocalFileCheckpointerImpl implements t.LocalFileCheckpointer {
|
|
|
68
80
|
await this.fs.mkdir(dirname(path), { recursive: true });
|
|
69
81
|
await this.fs.writeFile(path, snapshot.content);
|
|
70
82
|
restored++;
|
|
71
|
-
} catch {
|
|
83
|
+
} catch (error) {
|
|
84
|
+
// A timed-out restore left the bad write in place — surface it.
|
|
85
|
+
if (isWorkspaceClientTimeoutError(error)) {
|
|
86
|
+
throw error;
|
|
87
|
+
}
|
|
72
88
|
// Best-effort: ignore individual restore failures so the rest
|
|
73
89
|
// of the rewind continues.
|
|
74
90
|
}
|