@paperclipai/plugin-daytona 2026.1001.0 → 2026.1002.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/plugin.js CHANGED
@@ -1,8 +1,9 @@
1
1
  import path from "node:path";
2
2
  import { createHash, randomUUID } from "node:crypto";
3
3
  import { Daytona, DaytonaNotFoundError, DaytonaTimeoutError } from "@daytonaio/sdk";
4
- import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER } from "@paperclipai/plugin-sdk";
4
+ import { decodeChannelBytes, definePlugin, NOOP_PLUGIN_TRACER, PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
5
5
  import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js";
6
+ import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js";
6
7
  // The Claude `setup-token` login pseudo-terminal (PTY) session for this provider.
7
8
  // The session runs the login command on a real pseudo-terminal, streams the
8
9
  // terminal output, and delivers the delayed browser code plus the Enter byte. A
@@ -140,7 +141,7 @@ function parseOptionalNumber(value) {
140
141
  return Number.isFinite(parsed) ? parsed : null;
141
142
  }
142
143
  function parseDriverConfig(raw) {
143
- const timeoutMs = Number(raw.timeoutMs ?? 300_000);
144
+ const timeoutMs = Number(raw.timeoutMs ?? DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS);
144
145
  const livenessTimeoutMs = Number(raw.livenessTimeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS);
145
146
  return {
146
147
  apiKey: parseOptionalString(raw.apiKey),
@@ -149,7 +150,7 @@ function parseDriverConfig(raw) {
149
150
  snapshot: parseOptionalString(raw.snapshot),
150
151
  image: parseOptionalString(raw.image),
151
152
  language: parseOptionalString(raw.language),
152
- timeoutMs: Number.isFinite(timeoutMs) ? Math.trunc(timeoutMs) : 300_000,
153
+ timeoutMs: Number.isFinite(timeoutMs) ? Math.trunc(timeoutMs) : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS,
153
154
  livenessTimeoutMs: Number.isFinite(livenessTimeoutMs) ? Math.trunc(livenessTimeoutMs) : DEFAULT_LIVENESS_TIMEOUT_MS,
154
155
  cpu: parseOptionalNumber(raw.cpu),
155
156
  memory: parseOptionalNumber(raw.memory),
@@ -323,7 +324,7 @@ async function drainSandboxBeforeTermination(sandbox, scope) {
323
324
  }
324
325
  }
325
326
  async function terminateAtProvider(scope, operation, action) {
326
- const timeoutMs = scope.config.timeoutMs > 0 ? scope.config.timeoutMs : 300_000;
327
+ const timeoutMs = scope.config.timeoutMs > 0 ? scope.config.timeoutMs : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS;
327
328
  return withLivenessTimeout(operation, timeoutMs + LIVENESS_START_TIMEOUT_MARGIN_MS, action);
328
329
  }
329
330
  async function ensureSandboxStarted(sandbox, timeoutSeconds) {
@@ -342,12 +343,21 @@ async function ensureSandboxStarted(sandbox, timeoutSeconds) {
342
343
  }
343
344
  await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds));
344
345
  }
345
- async function resolveSandboxWorkingDirectory(sandbox) {
346
+ function hasMissingSandboxContainer(sandbox) {
347
+ if (sandbox.state !== "error" || sandbox.recoverable !== false || typeof sandbox.errorReason !== "string")
348
+ return false;
349
+ if (typeof sandbox.id !== "string" || !sandbox.id)
350
+ return false;
351
+ return sandbox.errorReason ===
352
+ `not found: failed to inspect sandbox container ${sandbox.id}: Error response from daemon: No such container: ${sandbox.id}`;
353
+ }
354
+ async function resolveSandboxWorkingDirectory(sandbox, create = true) {
346
355
  const root = (await sandbox.getWorkDir())?.trim()
347
356
  || (await sandbox.getUserHomeDir())?.trim()
348
357
  || "/home/daytona";
349
358
  const remoteCwd = path.posix.join(root, "paperclip-workspace");
350
- await sandbox.fs.createFolder(remoteCwd, "755");
359
+ if (create)
360
+ await sandbox.fs.createFolder(remoteCwd, "755");
351
361
  return remoteCwd;
352
362
  }
353
363
  async function detectSandboxShellCommand(sandbox, timeoutSeconds) {
@@ -368,12 +378,13 @@ function parseProbeInteger(value) {
368
378
  return Number.isInteger(parsed) ? parsed : null;
369
379
  }
370
380
  function workspaceSentinelToken(input) {
371
- if (!input.config.reuseLease || !input.params.agentId || (!input.params.executionWorkspaceId && !input.params.issueId)) {
381
+ if ((!input.config.reuseLease && !input.params.runId) || !input.params.agentId || (!input.params.executionWorkspaceId && !input.params.issueId)) {
372
382
  return null;
373
383
  }
374
384
  return createHash("sha256")
375
385
  .update(stableStringify({
376
386
  provider: "daytona",
387
+ ...(!input.config.reuseLease ? { ephemeralRunId: input.params.runId, ephemeralProviderLeaseId: input.providerLeaseId } : {}),
377
388
  companyId: input.params.companyId,
378
389
  environmentId: input.params.environmentId,
379
390
  agentId: input.params.agentId,
@@ -398,7 +409,7 @@ function workspaceSentinelPath(remoteCwd) {
398
409
  }
399
410
  async function writeWorkspaceSentinel(input) {
400
411
  const sentinelPath = workspaceSentinelPath(input.remoteCwd);
401
- const token = workspaceSentinelToken({ params: input.params, config: input.config });
412
+ const token = workspaceSentinelToken({ params: input.params, config: input.config, providerLeaseId: input.sandbox.id });
402
413
  if (!token) {
403
414
  return { path: sentinelPath, token: null, result: "skipped" };
404
415
  }
@@ -415,7 +426,8 @@ async function writeWorkspaceSentinel(input) {
415
426
  provider: "daytona",
416
427
  writtenAt: new Date().toISOString(),
417
428
  }, null, 2), "utf8"), sentinelPath, input.timeoutSeconds);
418
- return { path: sentinelPath, token, result: "written" };
429
+ return { path: sentinelPath, token, result: "written",
430
+ ...(!input.config.reuseLease ? { runId: input.params.runId, providerLeaseId: input.sandbox.id } : {}) };
419
431
  }
420
432
  async function verifyWorkspaceSentinel(input) {
421
433
  const metadataSentinel = isRecord(input.leaseMetadata?.workspaceSentinel)
@@ -424,6 +436,8 @@ async function verifyWorkspaceSentinel(input) {
424
436
  const sentinelPath = typeof metadataSentinel?.path === "string"
425
437
  ? metadataSentinel.path
426
438
  : workspaceSentinelPath(input.remoteCwd);
439
+ const binding = typeof metadataSentinel?.runId === "string" && typeof metadataSentinel?.providerLeaseId === "string"
440
+ ? { runId: metadataSentinel.runId, providerLeaseId: metadataSentinel.providerLeaseId } : {};
427
441
  const expectedToken = typeof metadataSentinel?.token === "string" ? metadataSentinel.token : null;
428
442
  if (!expectedToken) {
429
443
  return { path: sentinelPath, token: null, result: "missing" };
@@ -438,6 +452,7 @@ async function verifyWorkspaceSentinel(input) {
438
452
  return {
439
453
  path: sentinelPath,
440
454
  token: expectedToken,
455
+ ...binding,
441
456
  result: actualToken === expectedToken ? "matched" : "mismatch",
442
457
  };
443
458
  }
@@ -693,18 +708,83 @@ async function createSandbox(params, config, options = {}) {
693
708
  throw new Error(resourceRequestError);
694
709
  }
695
710
  const client = createDaytonaClient(config);
696
- const createParams = buildCreateParams(config, buildSandboxLabels({
697
- companyId: params.companyId,
698
- environmentId: params.environmentId,
699
- runId: "runId" in params ? params.runId : undefined,
700
- setupSessionId: "sessionId" in params ? params.sessionId : undefined,
701
- purpose: options.purpose,
702
- reuseLease: config.reuseLease,
703
- }));
704
- const sandbox = await client.create(createParams, {
705
- timeout: toTimeoutSeconds(config.timeoutMs),
706
- });
707
- return sandbox;
711
+ // The SDK can create a sandbox and then throw while waiting for it to start.
712
+ // Keep an attempt-specific provider name so that failure cannot lose ownership.
713
+ const attemptId = randomUUID();
714
+ const name = `paperclip-create-${attemptId}`;
715
+ const labels = { ...buildSandboxLabels({
716
+ companyId: params.companyId,
717
+ environmentId: params.environmentId,
718
+ runId: "runId" in params ? params.runId : undefined,
719
+ setupSessionId: "sessionId" in params ? params.sessionId : undefined,
720
+ purpose: options.purpose,
721
+ reuseLease: config.reuseLease,
722
+ }), "paperclip-create-attempt": attemptId };
723
+ // The SDK mutates params.labels (for example, code-toolbox-language).
724
+ // Preserve our immutable ownership snapshot for validation and retry.
725
+ const createParams = { ...buildCreateParams(config, { ...labels }), name };
726
+ const cleanup = {
727
+ providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId,
728
+ ...("runId" in params ? { runId: params.runId } : {}),
729
+ attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config),
730
+ };
731
+ options.onCreateAttempt?.(cleanup);
732
+ try {
733
+ return await client.create(createParams, {
734
+ timeout: toTimeoutSeconds(config.timeoutMs),
735
+ });
736
+ }
737
+ catch (createError) {
738
+ try {
739
+ // A not-found lookup after an uncertain create is not a deletion receipt:
740
+ // the provider may still materialize the request. Keep the name in the
741
+ // error so cleanup can be reconciled rather than declaring success.
742
+ await destroyFailedCreation(config, cleanup);
743
+ }
744
+ catch (cleanupError) {
745
+ throw new PluginEnvironmentCreationCleanupError([createError, cleanupError], `Daytona sandbox creation failed; cleanup could not be confirmed for ${name}`, cleanup);
746
+ }
747
+ throw createError;
748
+ }
749
+ }
750
+ // Resolve by the immutable creation name, not through the admitted-lease cache:
751
+ // the SDK's returned sandbox ID differs from this provisional name. Persist a
752
+ // verified provider ID before host-driven deletion. Its later absence is then
753
+ // conclusive, even if the deletion receipt or database update was lost.
754
+ async function destroyFailedCreation(config, cleanup, requireDurableObservation = false) {
755
+ if (cleanup.providerLeaseId !== `paperclip-create-${cleanup.attemptId}` ||
756
+ cleanup.accountFingerprint !== sandboxAccountDiscriminator(config) ||
757
+ cleanup.labels["paperclip-provider"] !== "daytona" ||
758
+ cleanup.labels["paperclip-create-attempt"] !== cleanup.attemptId ||
759
+ cleanup.labels["paperclip-company-id"] !== cleanup.companyId ||
760
+ cleanup.labels["paperclip-environment-id"] !== cleanup.environmentId ||
761
+ (cleanup.runId !== undefined && cleanup.labels["paperclip-run-id"] !== cleanup.runId)) {
762
+ throw new Error("Failed-create sandbox ownership does not match");
763
+ }
764
+ const client = createDaytonaClient(config);
765
+ let orphan;
766
+ try {
767
+ orphan = await withLivenessTimeout("sandbox.failedCreateLookup", 10_000, () => client.get(cleanup.observedProviderLeaseId ?? cleanup.providerLeaseId));
768
+ }
769
+ catch (error) {
770
+ // A name that has never been observed may still materialize. An ID already
771
+ // observed with exact ownership cannot materialize as a new allocation.
772
+ if (cleanup.observedProviderLeaseId && error instanceof DaytonaNotFoundError)
773
+ return;
774
+ throw error;
775
+ }
776
+ if ((cleanup.observedProviderLeaseId && orphan.id !== cleanup.observedProviderLeaseId) ||
777
+ orphan.name !== cleanup.providerLeaseId || Object.entries(cleanup.labels).some(([key, value]) => orphan.labels?.[key] !== value)) {
778
+ throw new Error("Failed-create sandbox ownership does not match");
779
+ }
780
+ if (!cleanup.observedProviderLeaseId) {
781
+ cleanup.observedProviderLeaseId = orphan.id;
782
+ if (requireDurableObservation) {
783
+ // No destructive call yet: the host journals the observation and retries.
784
+ throw new PluginEnvironmentCreationCleanupError([], "Persist observed sandbox identity before cleanup", cleanup);
785
+ }
786
+ }
787
+ await withLivenessTimeout("sandbox.failedCreateDelete", 15_000, () => orphan.delete(10, true));
708
788
  }
709
789
  // Non-secret provider/account fingerprint. Uses the *resolved* key (config or
710
790
  // DAYTONA_API_KEY env fallback) so an env-provided credential is still scoped,
@@ -1307,10 +1387,9 @@ async function executeOneShot(sandbox, params, config) {
1307
1387
  }
1308
1388
  }
1309
1389
  }
1310
- // Poll interval for a session command's exit code. The live spike measured a
1311
- // session command resolving in about 260-300 ms, so a short interval keeps the
1312
- // poll responsive without a busy loop.
1313
- const SESSION_POLL_INTERVAL_MS = 50;
1390
+ // Fallback reads full log snapshots as well as status. Limit the heavier
1391
+ // polling path while keeping output available to interactive commands.
1392
+ const SESSION_LOG_POLL_INTERVAL_MS = 1000;
1314
1393
  function sleep(ms) {
1315
1394
  return new Promise((resolve) => setTimeout(resolve, ms));
1316
1395
  }
@@ -1319,10 +1398,10 @@ function sleep(ms) {
1319
1398
  // first read almost always holds the code. These delays cover the rare case
1320
1399
  // where the first read has no code yet.
1321
1400
  const SESSION_EXIT_CODE_RETRY_DELAYS_MS = [50, 100, 200];
1322
- // A bounded reconnect for the log stream. A disconnect settles the stream
1323
- // promise as a rejection while the command still runs on the server. One
1324
- // reconnect replays the log from byte 0; the stream buffer drops the replayed
1325
- // prefix by byte offset. After this many reconnects the dispatch falls back to
1401
+ // A bounded reconnect for the log stream. The SDK resolves on WebSocket close,
1402
+ // which does not prove that the command exited. A reconnect replays the log
1403
+ // from byte 0; the stream buffer drops the replayed prefix by byte offset.
1404
+ // After this many reconnects the dispatch falls back to
1326
1405
  // the poll path.
1327
1406
  const MAX_SESSION_STREAM_RECONNECTS = 1;
1328
1407
  // Buffers the stdout and stderr of one session command from the callback log
@@ -1341,14 +1420,25 @@ const MAX_SESSION_STREAM_RECONNECTS = 1;
1341
1420
  // time at read. It does not copy the earlier output on each append, so total
1342
1421
  // buffering work stays linear in the output size, not quadratic.
1343
1422
  function createSessionStreamBuffer(onNewTail) {
1423
+ const createStream = () => ({
1424
+ chunks: [], length: 0, connectionBytes: 0, pendingReplacement: "", previousReplacement: null,
1425
+ });
1344
1426
  const streams = {
1345
- stdout: { chunks: [], length: 0, connectionBytes: 0 },
1346
- stderr: { chunks: [], length: 0, connectionBytes: 0 },
1427
+ stdout: createStream(),
1428
+ stderr: createStream(),
1347
1429
  };
1348
- function append(streamName, stream, chunk) {
1349
- const buf = Buffer.from(chunk, "utf8");
1430
+ function append(streamName, stream, chunk, final = false, publish = true) {
1431
+ // The SDK flushes an incomplete UTF-8 character as U+FFFD on socket close.
1432
+ // Do not publish that suffix or count its replacement bytes until a replay
1433
+ // confirms it. A real final U+FFFD is flushed at confirmed command exit.
1434
+ const text = stream.pendingReplacement + chunk;
1435
+ stream.pendingReplacement = final ? "" : (text.match(/\uFFFD+$/u)?.[0] ?? "");
1436
+ const buf = Buffer.from(text.slice(0, text.length - stream.pendingReplacement.length), "utf8");
1350
1437
  const start = stream.connectionBytes;
1351
1438
  stream.connectionBytes = start + buf.length;
1439
+ if (stream.previousReplacement && stream.connectionBytes > stream.previousReplacement.offset) {
1440
+ stream.previousReplacement = null;
1441
+ }
1352
1442
  // The whole chunk falls before the delivered byte count, so it is a replay.
1353
1443
  if (start + buf.length <= stream.length) {
1354
1444
  return;
@@ -1361,7 +1451,7 @@ function createSessionStreamBuffer(onNewTail) {
1361
1451
  stream.length += tail.length;
1362
1452
  // Deliver only the genuinely new tail to the live sink, so a replayed
1363
1453
  // prefix on a reconnect never reaches the host twice.
1364
- if (onNewTail && tail.length > 0) {
1454
+ if (publish && onNewTail && tail.length > 0) {
1365
1455
  onNewTail(streamName, tail.toString("utf8"));
1366
1456
  }
1367
1457
  }
@@ -1371,8 +1461,34 @@ function createSessionStreamBuffer(onNewTail) {
1371
1461
  // Reset the per-connection read cursors after a reconnect, so the replayed
1372
1462
  // prefix drops against the already-delivered byte count.
1373
1463
  resetConnectionCursors() {
1374
- streams.stdout.connectionBytes = 0;
1375
- streams.stderr.connectionBytes = 0;
1464
+ for (const stream of Object.values(streams)) {
1465
+ if (stream.pendingReplacement &&
1466
+ (!stream.previousReplacement || stream.connectionBytes > stream.previousReplacement.offset ||
1467
+ (stream.connectionBytes === stream.previousReplacement.offset &&
1468
+ stream.pendingReplacement.length > stream.previousReplacement.text.length))) {
1469
+ stream.previousReplacement = { offset: stream.connectionBytes, text: stream.pendingReplacement };
1470
+ }
1471
+ stream.pendingReplacement = "";
1472
+ stream.connectionBytes = 0;
1473
+ }
1474
+ },
1475
+ finish(publish = true) {
1476
+ for (const name of ["stdout", "stderr"]) {
1477
+ const stream = streams[name];
1478
+ if (stream.previousReplacement?.offset === stream.connectionBytes &&
1479
+ stream.previousReplacement.text.length > stream.pendingReplacement.length) {
1480
+ stream.pendingReplacement = stream.previousReplacement.text;
1481
+ }
1482
+ append(name, stream, "", true, publish);
1483
+ // A short terminal snapshot must not discard a genuine trailing
1484
+ // replacement character already received from a longer stream.
1485
+ const previous = stream.previousReplacement;
1486
+ if (previous && previous.offset >= stream.length) {
1487
+ stream.connectionBytes = previous.offset;
1488
+ append(name, stream, previous.text, true, publish);
1489
+ }
1490
+ stream.previousReplacement = null;
1491
+ }
1376
1492
  },
1377
1493
  get stdout() {
1378
1494
  return Buffer.concat(streams.stdout.chunks).toString("utf8");
@@ -1382,40 +1498,90 @@ function createSessionStreamBuffer(onNewTail) {
1382
1498
  },
1383
1499
  };
1384
1500
  }
1501
+ class SessionCommandDeadlineError extends Error {
1502
+ }
1503
+ class SessionObservationTimeoutError extends Error {
1504
+ }
1505
+ async function beforeSessionDeadline(deadlineMs, action) {
1506
+ const remainingMs = deadlineMs - Date.now();
1507
+ if (remainingMs <= 0)
1508
+ throw new SessionCommandDeadlineError();
1509
+ let timer;
1510
+ try {
1511
+ return await Promise.race([
1512
+ action(),
1513
+ new Promise((_resolve, reject) => {
1514
+ timer = setTimeout(() => reject(new SessionCommandDeadlineError()), remainingMs);
1515
+ }),
1516
+ ]);
1517
+ }
1518
+ finally {
1519
+ if (timer !== undefined)
1520
+ clearTimeout(timer);
1521
+ }
1522
+ }
1523
+ // A provider operation timeout bounds one observation, not a healthy stream's
1524
+ // lifetime. The caller's RPC/run guard and session teardown own that lifetime.
1525
+ async function observeSessionCommand(timeoutMs, action) {
1526
+ try {
1527
+ return await beforeSessionDeadline(Date.now() + timeoutMs, action);
1528
+ }
1529
+ catch (error) {
1530
+ if (error instanceof SessionCommandDeadlineError || error instanceof DaytonaTimeoutError) {
1531
+ throw new SessionObservationTimeoutError();
1532
+ }
1533
+ throw error;
1534
+ }
1535
+ }
1385
1536
  // Stream stdout and stderr of one session command from the callback log form.
1386
- // The stream buffer drops a replayed prefix by byte offset on a reconnect. A
1387
- // disconnect rejects the stream promise; the dispatch reconnects a bounded
1388
- // number of times, then reports failure so the caller falls back to the poll
1389
- // path.
1390
- async function runSessionLogStream(sandbox, sessionId, commandId, onNewTail) {
1391
- const buffer = createSessionStreamBuffer(onNewTail);
1537
+ // The stream buffer drops a replayed prefix by byte offset on a reconnect.
1538
+ // Both a rejected stream and a clean close without a confirmed process exit
1539
+ // use bounded reconnects, then polling.
1540
+ async function runSessionLogStream(sandbox, sessionId, commandId, timeoutMs, buffer) {
1392
1541
  let reconnects = 0;
1542
+ let closedStream = false;
1393
1543
  while (true) {
1544
+ let streamClosed = false;
1545
+ let acceptingOutput = true;
1394
1546
  try {
1395
- await sandbox.process.getSessionCommandLogs(sessionId, commandId, buffer.onStdout, buffer.onStderr);
1396
- return { ok: true, stdout: buffer.stdout, stderr: buffer.stderr };
1547
+ await sandbox.process.getSessionCommandLogs(sessionId, commandId, (chunk) => { if (acceptingOutput)
1548
+ buffer.onStdout(chunk); }, (chunk) => { if (acceptingOutput)
1549
+ buffer.onStderr(chunk); });
1550
+ streamClosed = true;
1551
+ closedStream = true;
1397
1552
  }
1398
1553
  catch {
1399
- if (reconnects >= MAX_SESSION_STREAM_RECONNECTS) {
1400
- return { ok: false };
1401
- }
1402
- reconnects += 1;
1403
- buffer.resetConnectionCursors();
1554
+ // The command can still be running after a log transport failure.
1404
1555
  }
1556
+ finally {
1557
+ // The SDK exposes no cancellation handle for this socket. Session
1558
+ // teardown owns closing it; late data cannot change a settled execution.
1559
+ acceptingOutput = false;
1560
+ }
1561
+ if (streamClosed) {
1562
+ const exitCode = await readSessionExitCode(sandbox, sessionId, commandId, timeoutMs);
1563
+ if (exitCode !== null)
1564
+ return { exitCode, closedStream, buffer };
1565
+ }
1566
+ if (reconnects >= MAX_SESSION_STREAM_RECONNECTS) {
1567
+ return { exitCode: null, closedStream, buffer };
1568
+ }
1569
+ reconnects += 1;
1570
+ buffer.resetConnectionCursors();
1405
1571
  }
1406
1572
  }
1407
1573
  // Read the exit code one time after the log stream ends. The exit code is
1408
1574
  // available within one poll, so the first read almost always holds it. Add a
1409
1575
  // small bounded retry with backoff only for the rare case where the first read
1410
1576
  // has no code yet. Return null when no read holds a numeric code.
1411
- async function readSessionExitCode(sandbox, sessionId, commandId) {
1412
- const first = await sandbox.process.getSessionCommand(sessionId, commandId);
1577
+ async function readSessionExitCode(sandbox, sessionId, commandId, timeoutMs) {
1578
+ const first = await observeSessionCommand(timeoutMs, () => sandbox.process.getSessionCommand(sessionId, commandId));
1413
1579
  if (typeof first.exitCode === "number") {
1414
1580
  return first.exitCode;
1415
1581
  }
1416
1582
  for (const delayMs of SESSION_EXIT_CODE_RETRY_DELAYS_MS) {
1417
1583
  await sleep(delayMs);
1418
- const status = await sandbox.process.getSessionCommand(sessionId, commandId);
1584
+ const status = await observeSessionCommand(timeoutMs, () => sandbox.process.getSessionCommand(sessionId, commandId));
1419
1585
  if (typeof status.exitCode === "number") {
1420
1586
  return status.exitCode;
1421
1587
  }
@@ -1450,6 +1616,8 @@ async function executeInSession(sandbox, sessionId, params, config) {
1450
1616
  // the exec wall-time spent before the abort, so a slow command is still
1451
1617
  // attributed to the provider boundary.
1452
1618
  let execStart = null;
1619
+ let confirmedExitCode = null;
1620
+ const streamBuffer = createSessionStreamBuffer((stream, text) => pluginContext?.execution.log(stream, text));
1453
1621
  try {
1454
1622
  if (stdinPath) {
1455
1623
  await sandbox.fs.uploadFile(Buffer.from(params.stdin ?? "", "utf8"), stdinPath, timeoutSeconds);
@@ -1469,76 +1637,109 @@ async function executeInSession(sandbox, sessionId, params, config) {
1469
1637
  const commandId = dispatched.cmdId;
1470
1638
  // Log-stream path. A session command always tries the stream first: it
1471
1639
  // streams stdout and stderr from the callback log form, then reads the exit
1472
- // code one time. On a stream failure, fall through to the poll path below,
1473
- // because the command still runs to its exit on the server.
1640
+ // code. A log socket closing without a recorded exit leaves the command
1641
+ // running, so reconnect and then poll the same command without replaying it.
1474
1642
  //
1475
1643
  // Emit each genuinely new output chunk to the host during the active execute
1476
1644
  // call. The host routes it to the runner log sink by the host-issued
1477
1645
  // invocation id. This is a no-op when no plugin context is set (a direct
1478
1646
  // test call) or when the host has no active execute route.
1479
- const streamResult = await runSessionLogStream(sandbox, sessionId, commandId, (stream, text) => pluginContext?.execution.log(stream, text));
1480
- if (streamResult.ok) {
1481
- const exitCode = await readSessionExitCode(sandbox, sessionId, commandId);
1647
+ const streamResult = await runSessionLogStream(sandbox, sessionId, commandId, effectiveTimeoutMs, streamBuffer);
1648
+ if (streamResult.exitCode !== null) {
1649
+ confirmedExitCode = streamResult.exitCode;
1650
+ // A log socket can close before the process emits its final bytes. Even
1651
+ // a now-visible exit code does not prove the streamed prefix is complete.
1652
+ const logs = await observeSessionCommand(effectiveTimeoutMs, () => sandbox.process.getSessionCommandLogs(sessionId, commandId));
1653
+ streamBuffer.resetConnectionCursors();
1654
+ streamBuffer.onStdout(logs?.stdout ?? "");
1655
+ streamBuffer.onStderr(logs?.stderr ?? "");
1656
+ streamBuffer.finish();
1482
1657
  const durationMs = timingNow() - execStart;
1483
1658
  return {
1484
- exitCode,
1659
+ exitCode: streamResult.exitCode,
1485
1660
  timedOut: false,
1486
- stdout: streamResult.stdout,
1487
- stderr: streamResult.stderr,
1661
+ stdout: streamResult.buffer.stdout,
1662
+ stderr: streamResult.buffer.stderr,
1488
1663
  metadata: { durationMs },
1489
1664
  };
1490
1665
  }
1491
- // Poll for the exit code; the SDK has no wait method. The poll deadline uses
1492
- // the wall clock, separate from the injected timing clock that measures the
1493
- // reported `durationMs`. The poll path is the fallback when the log stream
1494
- // fails.
1666
+ // Keep forwarding output while waiting for the actual command exit. ACP
1667
+ // peers may need a tool response before they can finish the command. Waiting
1668
+ // for exit before reading logs would strand those peers after a stream loss.
1669
+ // An EOF after a live stream does not end the command's lifetime. Bound
1670
+ // each recovery read independently; the caller still owns stop/teardown.
1671
+ // Preserve the legacy budget starting at fallback entry when both stream
1672
+ // attempts fail, including a stream that fails after running for hours.
1495
1673
  const deadlineMs = Date.now() + effectiveTimeoutMs;
1496
- let exitCode = null;
1497
- while (true) {
1498
- const status = await sandbox.process.getSessionCommand(sessionId, commandId);
1499
- if (typeof status.exitCode === "number") {
1500
- exitCode = status.exitCode;
1501
- break;
1674
+ const observe = async (action) => {
1675
+ try {
1676
+ return await (streamResult.closedStream
1677
+ ? observeSessionCommand(effectiveTimeoutMs, action)
1678
+ : beforeSessionDeadline(deadlineMs, action));
1502
1679
  }
1503
- if (Date.now() >= deadlineMs) {
1680
+ catch (error) {
1681
+ if (error instanceof DaytonaTimeoutError)
1682
+ throw new SessionObservationTimeoutError();
1683
+ throw error;
1684
+ }
1685
+ };
1686
+ while (true) {
1687
+ const status = await observe(() => sandbox.process.getSessionCommand(sessionId, commandId));
1688
+ if (typeof status.exitCode === "number")
1689
+ confirmedExitCode = status.exitCode;
1690
+ const logs = await observe(() => sandbox.process.getSessionCommandLogs(sessionId, commandId));
1691
+ streamResult.buffer.resetConnectionCursors();
1692
+ streamResult.buffer.onStdout(logs.stdout ?? "");
1693
+ streamResult.buffer.onStderr(logs.stderr ?? "");
1694
+ if (confirmedExitCode !== null) {
1695
+ streamResult.buffer.finish();
1504
1696
  const durationMs = timingNow() - execStart;
1505
- const timeoutMessage = gitNet
1506
- ? `Git network operation timed out after ${Math.round(effectiveTimeoutMs / 1000)} s — the remote may be unreachable or noninteractive credentials are not configured.`
1507
- : `Command timed out after ${Math.round(effectiveTimeoutMs / 1000)} s.`;
1508
1697
  return {
1509
- exitCode: null,
1510
- timedOut: true,
1511
- stdout: "",
1512
- stderr: `${timeoutMessage}\n`,
1698
+ exitCode: confirmedExitCode,
1699
+ timedOut: false,
1700
+ stdout: streamResult.buffer.stdout,
1701
+ stderr: streamResult.buffer.stderr,
1513
1702
  metadata: { durationMs },
1514
1703
  };
1515
1704
  }
1516
- await sleep(SESSION_POLL_INTERVAL_MS);
1705
+ // Full log snapshots are heavier than a status read. Limit fallback
1706
+ // traffic to one snapshot per second. Failed-stream fallback retains
1707
+ // its original deadline, including the time spent waiting between reads.
1708
+ await sleep(streamResult.closedStream ? SESSION_LOG_POLL_INTERVAL_MS
1709
+ : Math.max(0, Math.min(SESSION_LOG_POLL_INTERVAL_MS, deadlineMs - Date.now())));
1517
1710
  }
1518
- // Read true, separated stdout and stderr from the logs endpoint. The
1519
- // synchronous dispatch response fields are optional, so the logs endpoint is
1520
- // the source of truth.
1521
- const logs = await sandbox.process.getSessionCommandLogs(sessionId, commandId);
1522
- const durationMs = timingNow() - execStart;
1523
- return {
1524
- exitCode,
1525
- timedOut: false,
1526
- stdout: logs.stdout ?? "",
1527
- stderr: logs.stderr ?? "",
1528
- metadata: { durationMs },
1529
- };
1530
1711
  }
1531
1712
  catch (error) {
1532
- if (error instanceof DaytonaTimeoutError) {
1713
+ if (error instanceof SessionObservationTimeoutError) {
1714
+ // Preserve undecided decoder suffixes in the partial result, without
1715
+ // injecting a possibly incomplete character into the live protocol.
1716
+ streamBuffer.finish(false);
1717
+ return {
1718
+ exitCode: null,
1719
+ timedOut: true,
1720
+ stdout: streamBuffer.stdout,
1721
+ stderr: `${streamBuffer.stderr}Session log observation timed out after ${Math.round(effectiveTimeoutMs / 1000)} s; ${confirmedExitCode === null ? "command exit remains unconfirmed" : "final output may be incomplete"}.\n`,
1722
+ metadata: {
1723
+ ...(execStart != null ? { durationMs: timingNow() - execStart } : {}),
1724
+ timeoutScope: "session_log_observation",
1725
+ commandExitConfirmed: confirmedExitCode !== null,
1726
+ ...(confirmedExitCode !== null ? { observedExitCode: confirmedExitCode } : {}),
1727
+ },
1728
+ };
1729
+ }
1730
+ if (error instanceof DaytonaTimeoutError || error instanceof SessionCommandDeadlineError) {
1731
+ streamBuffer.finish(false);
1533
1732
  const timeoutMessage = gitNet
1534
1733
  ? `Git network operation timed out after ${Math.round(effectiveTimeoutMs / 1000)} s — the remote may be unreachable or noninteractive credentials are not configured.`
1535
- : error.message.trim();
1734
+ : error instanceof SessionCommandDeadlineError
1735
+ ? `Command timed out after ${Math.round(effectiveTimeoutMs / 1000)} s.`
1736
+ : error.message.trim();
1536
1737
  const durationMs = execStart != null ? timingNow() - execStart : undefined;
1537
1738
  return {
1538
1739
  exitCode: null,
1539
1740
  timedOut: true,
1540
- stdout: "",
1541
- stderr: `${timeoutMessage}\n`,
1741
+ stdout: streamBuffer.stdout,
1742
+ stderr: `${streamBuffer.stderr}${timeoutMessage}\n`,
1542
1743
  ...(durationMs != null ? { metadata: { durationMs } } : {}),
1543
1744
  };
1544
1745
  }
@@ -1685,62 +1886,131 @@ const plugin = definePlugin({
1685
1886
  },
1686
1887
  async onEnvironmentAcquireLease(params) {
1687
1888
  const config = parseDriverConfig(params.config);
1688
- const sandbox = await createSandbox(params, config);
1689
- try {
1690
- const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
1691
- const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs));
1692
- // Configure a provider-side destroy time at or before a caller deadline, so
1693
- // an abandoned sandbox self-destroys even if Paperclip is down. The lease
1694
- // carries the real provider expiry (or none) as evidence of the bound.
1695
- const expiresAt = await configureSandboxExpiry({
1696
- sandbox,
1697
- requestedExpiresAt: params.requestedExpiresAt,
1698
- nowMs: Date.now(),
1699
- });
1700
- const workspaceSentinel = await writeWorkspaceSentinel({
1701
- sandbox,
1702
- remoteCwd,
1703
- params,
1704
- config,
1705
- timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
1706
- });
1707
- sandboxHandleLeaseAdmissionStates.open({
1708
- driverKey: params.driverKey,
1709
- companyId: params.companyId,
1710
- environmentId: params.environmentId,
1711
- providerLeaseId: sandbox.id,
1712
- config,
1889
+ // One budget covers creation, setup, and inline cleanup. The host leaves
1890
+ // 30 seconds beyond this deadline to receive/journal the cleanup record.
1891
+ const budgetMs = config.timeoutMs > 0 ? config.timeoutMs : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS;
1892
+ const deadline = Date.now() + budgetMs;
1893
+ let expired = false;
1894
+ let phase = "create";
1895
+ let cleanup;
1896
+ const timeoutFailure = () => {
1897
+ expired = true;
1898
+ const cause = new Error(`Daytona lease acquisition exceeded ${budgetMs} ms during ${phase}`);
1899
+ return cleanup
1900
+ ? new PluginEnvironmentCreationCleanupError([cause], "Daytona lease acquisition timed out; allocation cleanup is pending", { ...cleanup, labels: { ...cleanup.labels } })
1901
+ : cause;
1902
+ };
1903
+ const assertActive = () => {
1904
+ if (expired || Date.now() >= deadline)
1905
+ throw timeoutFailure();
1906
+ };
1907
+ const acquire = async () => {
1908
+ const sandbox = await createSandbox(params, config, {
1909
+ onCreateAttempt: (attempt) => { cleanup = attempt; },
1713
1910
  });
1714
- // Seed the handle cache with the fresh handle under the exact scope that
1715
- // `onEnvironmentRealizeWorkspace` reads (providerLeaseId === sandbox.id).
1716
- // Realize then reuses this handle instead of paying a real `client.get`.
1717
- sandboxHandleCache.seed({
1718
- driverKey: params.driverKey,
1719
- companyId: params.companyId,
1720
- environmentId: params.environmentId,
1721
- providerLeaseId: sandbox.id,
1722
- config,
1723
- }, sandbox);
1724
- return {
1725
- providerLeaseId: sandbox.id,
1726
- expiresAt,
1727
- metadata: leaseMetadata({
1728
- config,
1911
+ try {
1912
+ assertActive();
1913
+ if (cleanup)
1914
+ cleanup.observedProviderLeaseId = sandbox.id;
1915
+ phase = "workspace";
1916
+ const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
1917
+ assertActive();
1918
+ phase = "shell";
1919
+ const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now())));
1920
+ assertActive();
1921
+ // Configure a provider-side destroy time at or before a caller deadline, so
1922
+ // an abandoned sandbox self-destroys even if Paperclip is down. The lease
1923
+ // carries the real provider expiry (or none) as evidence of the bound.
1924
+ phase = "expiry";
1925
+ const expiresAt = await configureSandboxExpiry({
1926
+ sandbox,
1927
+ requestedExpiresAt: params.requestedExpiresAt,
1928
+ nowMs: Date.now(),
1929
+ });
1930
+ assertActive();
1931
+ phase = "sentinel";
1932
+ const workspaceSentinel = await writeWorkspaceSentinel({
1729
1933
  sandbox,
1730
- shellCommand,
1731
1934
  remoteCwd,
1732
- resumedLease: false,
1733
- workspaceSentinel,
1935
+ params,
1936
+ config,
1937
+ timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())),
1938
+ });
1939
+ assertActive();
1940
+ sandboxHandleLeaseAdmissionStates.open({
1941
+ driverKey: params.driverKey,
1942
+ companyId: params.companyId,
1943
+ environmentId: params.environmentId,
1944
+ providerLeaseId: sandbox.id,
1945
+ config,
1946
+ });
1947
+ // Seed the handle cache with the fresh handle under the exact scope that
1948
+ // `onEnvironmentRealizeWorkspace` reads (providerLeaseId === sandbox.id).
1949
+ // Realize then reuses this handle instead of paying a real `client.get`.
1950
+ sandboxHandleCache.seed({
1951
+ driverKey: params.driverKey,
1952
+ companyId: params.companyId,
1953
+ environmentId: params.environmentId,
1954
+ providerLeaseId: sandbox.id,
1955
+ config,
1956
+ }, sandbox);
1957
+ return {
1958
+ providerLeaseId: sandbox.id,
1959
+ expiresAt,
1960
+ metadata: leaseMetadata({
1961
+ config,
1962
+ sandbox,
1963
+ shellCommand,
1964
+ remoteCwd,
1965
+ resumedLease: false,
1966
+ workspaceSentinel,
1967
+ }),
1968
+ };
1969
+ }
1970
+ catch (error) {
1971
+ // After timeout the host owns the durable cleanup record. A late SDK
1972
+ // completion must not admit this lease or start another setup phase.
1973
+ if (!expired) {
1974
+ phase = "cleanup";
1975
+ try {
1976
+ await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())));
1977
+ }
1978
+ catch (cleanupError) {
1979
+ if (cleanup) {
1980
+ throw new PluginEnvironmentCreationCleanupError([error, cleanupError], "Daytona lease setup failed; allocation cleanup is pending", { ...cleanup, labels: { ...cleanup.labels } });
1981
+ }
1982
+ throw error;
1983
+ }
1984
+ }
1985
+ throw error;
1986
+ }
1987
+ };
1988
+ let timer;
1989
+ try {
1990
+ return await Promise.race([
1991
+ acquire(),
1992
+ new Promise((_resolve, reject) => {
1993
+ timer = setTimeout(() => reject(timeoutFailure()), budgetMs);
1734
1994
  }),
1735
- };
1995
+ ]);
1736
1996
  }
1737
- catch (error) {
1738
- await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined);
1739
- throw error;
1997
+ finally {
1998
+ clearTimeout(timer);
1740
1999
  }
1741
2000
  },
1742
2001
  async onEnvironmentResumeLease(params) {
1743
2002
  const config = parseDriverConfig(params.config);
2003
+ if (params.leaseMetadata?.reuseLease === false) {
2004
+ const sentinel = isRecord(params.leaseMetadata.workspaceSentinel) ? params.leaseMetadata.workspaceSentinel : null;
2005
+ const intent = isRecord(params.leaseMetadata.nativeWorkspaceExportResume) ? params.leaseMetadata.nativeWorkspaceExportResume : null;
2006
+ // Ephemeral recovery is bound at acquisition. Never mint proof during
2007
+ // resume, accept a copied sentinel for a replacement, or waive legacy proof.
2008
+ if (typeof sentinel?.token !== "string" || !sentinel.token || typeof sentinel.runId !== "string" || !sentinel.runId
2009
+ || sentinel.providerLeaseId !== params.providerLeaseId || params.leaseMetadata.sandboxId !== params.providerLeaseId
2010
+ || (intent && intent.runId !== sentinel.runId)) {
2011
+ return { providerLeaseId: null, metadata: { expired: true, workspaceSentinel: { result: "mismatch" } } };
2012
+ }
2013
+ }
1744
2014
  const scope = {
1745
2015
  driverKey: params.driverKey,
1746
2016
  companyId: params.companyId,
@@ -1758,6 +2028,26 @@ const plugin = definePlugin({
1758
2028
  if (!sandbox) {
1759
2029
  return { providerLeaseId: null, metadata: { expired: true } };
1760
2030
  }
2031
+ // Daytona can retain the API record after losing its container. Confirm
2032
+ // the exact provider report again before allowing the host's existing
2033
+ // backup-guarded replacement path. Unknown errors preserve the lease.
2034
+ if (hasMissingSandboxContainer(sandbox)) {
2035
+ try {
2036
+ await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData());
2037
+ assertHandleMatchesLease(sandbox, params.providerLeaseId);
2038
+ }
2039
+ catch (error) {
2040
+ evictSandboxHandle(scope);
2041
+ if (error instanceof DaytonaNotFoundError) {
2042
+ return { providerLeaseId: null, metadata: { expired: true } };
2043
+ }
2044
+ throw error;
2045
+ }
2046
+ if (hasMissingSandboxContainer(sandbox)) {
2047
+ evictSandboxHandle(scope);
2048
+ return { providerLeaseId: null, metadata: { expired: true } };
2049
+ }
2050
+ }
1761
2051
  // A stopped sandbox loses its session shell, so the stored session id is
1762
2052
  // stale after a real restart. Clear the id only when the sandbox is not
1763
2053
  // already running, and clear it before the restart. A stopped sandbox has
@@ -1776,7 +2066,7 @@ const plugin = definePlugin({
1776
2066
  const resumedFromState = sandbox.state ?? null;
1777
2067
  await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
1778
2068
  try {
1779
- const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
2069
+ const remoteCwd = await resolveSandboxWorkingDirectory(sandbox, false);
1780
2070
  // C3: a resumed lease must clear the workspace sentinel before it is
1781
2071
  // trusted, even when the handle came from the cache. On any non-match we
1782
2072
  // evict the cached handle and expire the lease so a stale/foreign sandbox
@@ -1821,9 +2111,44 @@ const plugin = definePlugin({
1821
2111
  }
1822
2112
  }, { allowClosed: true });
1823
2113
  },
2114
+ async onEnvironmentStopLease(params) {
2115
+ if (!params.providerLeaseId)
2116
+ throw new Error("Daytona stop requires an exact sandbox identity");
2117
+ const config = parseDriverConfig(params.config);
2118
+ const scope = { driverKey: params.driverKey, companyId: params.companyId,
2119
+ environmentId: params.environmentId, providerLeaseId: params.providerLeaseId, config };
2120
+ const teardownGate = sandboxHandleTeardownGates.begin(scope);
2121
+ sandboxHandleLeaseAdmissionStates.close(scope);
2122
+ try {
2123
+ const sandbox = await getSandboxOrNull(scope, { bypassTeardownGate: true });
2124
+ if (!sandbox)
2125
+ throw new Error("Daytona retained sandbox is unavailable");
2126
+ evictSandboxHandle(scope);
2127
+ // Provider auto-delete can destroy a stopped sandbox without an explicit
2128
+ // delete call. Confirm the provider disabled it before stopping saved work;
2129
+ // the SDK setter also updates a local field, which is not sufficient proof.
2130
+ const deadline = Math.min(config.livenessTimeoutMs, 30_000);
2131
+ await withLivenessTimeout("sandbox.setAutoDeleteInterval", deadline, () => sandbox.setAutoDeleteInterval(-1));
2132
+ await withLivenessTimeout("sandbox.refreshData", deadline, () => sandbox.refreshData());
2133
+ if (sandbox.autoDeleteInterval !== -1)
2134
+ throw new Error("Daytona retention policy was not confirmed");
2135
+ if (sandbox.state !== "stopped") {
2136
+ await terminateAtProvider(scope, "sandbox.stop", () => sandbox.stop(Math.min(toTimeoutSeconds(config.timeoutMs), 30)));
2137
+ }
2138
+ sandboxHandleSessionStore.clear(scope);
2139
+ await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
2140
+ return { providerLeaseId: params.providerLeaseId, state: "stopped" };
2141
+ }
2142
+ finally {
2143
+ sandboxHandleTeardownGates.end(scope, teardownGate);
2144
+ evictSandboxHandle(scope);
2145
+ }
2146
+ },
1824
2147
  async onEnvironmentReleaseLease(params) {
1825
2148
  if (!params.providerLeaseId)
1826
2149
  return;
2150
+ if (params.resourceDisposition === "stop_and_retain")
2151
+ return plugin.definition.onEnvironmentStopLease(params);
1827
2152
  const config = parseDriverConfig(params.config);
1828
2153
  const scope = {
1829
2154
  driverKey: params.driverKey,
@@ -1895,6 +2220,17 @@ const plugin = definePlugin({
1895
2220
  if (!params.providerLeaseId)
1896
2221
  return;
1897
2222
  const config = parseDriverConfig(params.config);
2223
+ if (params.leaseMetadata?.failedCreateCleanup !== undefined) {
2224
+ const cleanup = readEnvironmentCreationCleanupError({ data: {
2225
+ schema: "paperclip/environment-creation-cleanup/v1", cleanup: params.leaseMetadata.failedCreateCleanup,
2226
+ } });
2227
+ if (!cleanup || cleanup.companyId !== params.companyId || cleanup.environmentId !== params.environmentId ||
2228
+ cleanup.providerLeaseId !== params.providerLeaseId) {
2229
+ throw new Error("Failed-create sandbox cleanup scope does not match");
2230
+ }
2231
+ await destroyFailedCreation(config, cleanup, true);
2232
+ return { providerLeaseId: params.providerLeaseId, state: "destroyed" };
2233
+ }
1898
2234
  const scope = {
1899
2235
  driverKey: params.driverKey,
1900
2236
  companyId: params.companyId,
@@ -2361,6 +2697,7 @@ const plugin = definePlugin({
2361
2697
  operations: params.operations,
2362
2698
  remoteDir,
2363
2699
  timeoutSeconds,
2700
+ onArchiveRecovery: () => pluginContext?.logger.info("Workspace export omitted unsafe links; retrying with confined entries."),
2364
2701
  });
2365
2702
  sandboxHandleCache.markFresh(scope);
2366
2703
  return result;