deepline 0.3.213 → 0.3.214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +122 -32
  2. package/dist/bundling-sources/sdk/src/compat.ts +2 -0
  3. package/dist/bundling-sources/sdk/src/deployment-protection.ts +19 -0
  4. package/dist/bundling-sources/sdk/src/http.ts +3 -7
  5. package/dist/bundling-sources/sdk/src/index.ts +7 -0
  6. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  7. package/dist/bundling-sources/sdk/src/types.ts +25 -0
  8. package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
  9. package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
  10. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +33 -7
  11. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +11 -1
  12. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +3 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +53 -1
  14. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +187 -6
  15. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +12 -3
  16. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -2
  17. package/dist/bundling-sources/shared_libs/plays/settings-contract.ts +176 -0
  18. package/dist/cli/index.js +560 -92
  19. package/dist/cli/index.mjs +560 -92
  20. package/dist/index.d.mts +165 -3
  21. package/dist/index.d.ts +165 -3
  22. package/dist/index.js +67 -35
  23. package/dist/index.mjs +67 -35
  24. package/dist/install-integrity.json +2 -0
  25. package/dist/release.d.mts +1 -1
  26. package/dist/release.d.ts +1 -1
  27. package/dist/release.js +1 -1
  28. package/dist/release.mjs +1 -1
  29. package/package.json +3 -2
@@ -1,3 +1,8 @@
1
+ import type {
2
+ PlaySettingsResult,
3
+ PlaySettingsPatch,
4
+ PlaySettingsPathsPage,
5
+ } from '../../shared_libs/plays/settings-contract';
1
6
  import { iterateCursorPages } from './pagination.js';
2
7
  import type { ToolResultBilling } from '../../shared_libs/plays/tool-result-types.js';
3
8
  /**
@@ -825,6 +830,36 @@ export type RunsGetOptions = {
825
830
  failedLogLimit?: number;
826
831
  };
827
832
 
833
+ export type RunGetStatus =
834
+ | {
835
+ available: false;
836
+ reason: 'not_projected' | 'read_failed';
837
+ }
838
+ | {
839
+ available: true;
840
+ observedAt: number;
841
+ source: {
842
+ ledgerNextSeq: number | null;
843
+ runSnapshotLedgerNextSeq: number | null;
844
+ aligned: boolean | null;
845
+ };
846
+ calls: {
847
+ logicalCalls: { value: number | null; confirmed: boolean };
848
+ toolDispatches: { value: number | null; confirmed: boolean };
849
+ provider429s: { value: number | null; confirmed: boolean };
850
+ };
851
+ focusStep: {
852
+ id: string;
853
+ label: string | null;
854
+ status: string | null;
855
+ rows: {
856
+ succeeded: number | null;
857
+ failed: number | null;
858
+ total: number | null;
859
+ };
860
+ } | null;
861
+ };
862
+
828
863
  const RUNS_FAILED_LOG_LIMIT = 20;
829
864
 
830
865
  /** Streaming options for `client.runs.tail(...)`. */
@@ -963,7 +998,10 @@ export type PlaySecretCheck = {
963
998
  export type RunsNamespace = {
964
999
  /** Get current run status by public run id. */
965
1000
  get: (runId: string, options?: RunsGetOptions) => Promise<PlayStatus>;
966
- /** Read bounded scalar lifecycle and confirmed work counts only. */
1001
+ /**
1002
+ * @deprecated Use `client.runs.get(runId)` for status, progress, logs, and results.
1003
+ * This scalar-only reader remains for compatibility with existing callers.
1004
+ */
967
1005
  status: (runId: string) => Promise<RunLifecycleStatus>;
968
1006
  /** Explicitly read the retained original input (may include customer data). */
969
1007
  input: (runId: string) => Promise<{
@@ -2494,10 +2532,25 @@ function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
2494
2532
  : typeof raw.workflowId === 'string'
2495
2533
  ? raw.workflowId
2496
2534
  : (packageRun?.id ?? '');
2535
+ const legacyOverview = runPackage?.overview ?? raw.overview;
2536
+ const runStatus = runPackage?.runStatus ?? raw.runStatus;
2497
2537
  return {
2498
2538
  ...(raw as unknown as Omit<PlayStatus, 'runId' | 'status'>),
2499
2539
  runId,
2500
- ...(runPackage ? { package: runPackage, outputs: runPackage.outputs } : {}),
2540
+ ...(isRecord(legacyOverview)
2541
+ ? {
2542
+ overview: legacyOverview as unknown as RunLifecycleStatus,
2543
+ }
2544
+ : {}),
2545
+ ...(isRecord(runStatus)
2546
+ ? { runStatus: runStatus as unknown as RunGetStatus }
2547
+ : {}),
2548
+ ...(runPackage
2549
+ ? {
2550
+ package: runPackage,
2551
+ outputs: runPackage.outputs,
2552
+ }
2553
+ : {}),
2501
2554
  status,
2502
2555
  };
2503
2556
  }
@@ -2850,19 +2903,6 @@ function playRunResultFromStatus(
2850
2903
  };
2851
2904
  }
2852
2905
 
2853
- function playRunStatusFromState(state: PlayLiveStatusState): PlayStatus {
2854
- return {
2855
- runId: state.runId,
2856
- status: state.status,
2857
- progress: {
2858
- status: state.status,
2859
- logs: state.logs,
2860
- ...(state.error ? { error: state.error } : {}),
2861
- },
2862
- ...('result' in state ? { result: state.result } : {}),
2863
- };
2864
- }
2865
-
2866
2906
  /**
2867
2907
  * Low-level typed REST client with authentication, retries, and localhost failover.
2868
2908
  *
@@ -3347,6 +3387,12 @@ export class DeeplineClient {
3347
3387
  options.includeSearchDebug ? 'true' : 'false',
3348
3388
  );
3349
3389
  params.set('search_mode', options.searchMode ?? 'v2');
3390
+ if (options.limit !== undefined) {
3391
+ params.set('limit', String(options.limit));
3392
+ }
3393
+ if (options.offset !== undefined) {
3394
+ params.set('offset', String(options.offset));
3395
+ }
3350
3396
  if (options.categories?.trim()) {
3351
3397
  params.set('categories', options.categories.trim());
3352
3398
  }
@@ -4807,8 +4853,25 @@ export class DeeplineClient {
4807
4853
  const query = params.size > 0 ? `?${params.toString()}` : '';
4808
4854
  const response = await this.http.get<Record<string, unknown>>(
4809
4855
  `/api/v2/runs/${encodeURIComponent(runId)}${query}`,
4856
+ { headers: { 'x-deepline-run-status': '1' } },
4810
4857
  );
4858
+ const responseRunStatus = isPlayRunPackage(response)
4859
+ ? response.runStatus
4860
+ : isPlayRunPackage(response.package)
4861
+ ? response.package.runStatus
4862
+ : response.runStatus;
4863
+ const responseOverview = isPlayRunPackage(response)
4864
+ ? response.overview
4865
+ : isPlayRunPackage(response.package)
4866
+ ? response.package.overview
4867
+ : response.overview;
4811
4868
  const status = normalizePlayStatus(response);
4869
+ if (isRecord(responseRunStatus)) {
4870
+ status.runStatus = responseRunStatus as unknown as RunGetStatus;
4871
+ }
4872
+ if (isRecord(responseOverview)) {
4873
+ status.overview = responseOverview as unknown as RunLifecycleStatus;
4874
+ }
4812
4875
  if (options?.failedLogs !== true || status.status !== 'failed') {
4813
4876
  return status;
4814
4877
  }
@@ -4859,7 +4922,10 @@ export class DeeplineClient {
4859
4922
  };
4860
4923
  }
4861
4924
 
4862
- /** Read only the bounded scalar projection; does not hydrate the run package. */
4925
+ /** Fetch only the legacy scalar lifecycle status for a run.
4926
+ *
4927
+ * @deprecated Use `client.runs.get(runId)` for the canonical run read.
4928
+ */
4863
4929
  async getRunLifecycleStatus(runId: string): Promise<RunLifecycleStatus> {
4864
4930
  const response = await this.http.get<RunLifecycleStatus>(
4865
4931
  `/api/v2/runs/${encodeURIComponent(runId)}/status`,
@@ -5067,9 +5133,7 @@ export class DeeplineClient {
5067
5133
  if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
5068
5134
  continue;
5069
5135
  }
5070
- return await this.getRunStatus(status.runId || runId).catch(
5071
- () => state.latest ?? playRunStatusFromState(state),
5072
- );
5136
+ return await this.getRunStatus(status.runId || runId);
5073
5137
  }
5074
5138
  if (options?.signal?.aborted) {
5075
5139
  throw new DeeplineError('Run observation aborted.', undefined, 'ABORTED');
@@ -5125,6 +5189,7 @@ export class DeeplineClient {
5125
5189
  for (;;) {
5126
5190
  const connectedAt = Date.now();
5127
5191
  let sawEvent = false;
5192
+ let terminalRunId: string | null = null;
5128
5193
  let endedReason = 'stream window ended before a terminal event';
5129
5194
  try {
5130
5195
  for await (const event of this.streamPlayRunEvents(runId, {
@@ -5137,9 +5202,8 @@ export class DeeplineClient {
5137
5202
  if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
5138
5203
  continue;
5139
5204
  }
5140
- return await this.getRunStatus(status.runId || runId).catch(
5141
- () => state.latest ?? playRunStatusFromState(state),
5142
- );
5205
+ terminalRunId = status.runId || runId;
5206
+ break;
5143
5207
  }
5144
5208
  } catch (error) {
5145
5209
  if (options?.signal?.aborted || !isTransientPlayStreamError(error)) {
@@ -5148,18 +5212,16 @@ export class DeeplineClient {
5148
5212
  endedReason = error instanceof Error ? error.message : String(error);
5149
5213
  }
5150
5214
 
5215
+ if (terminalRunId) {
5216
+ return await this.getRunStatus(terminalRunId);
5217
+ }
5218
+
5151
5219
  // Window ended without a terminal event. The run may have finished
5152
5220
  // during the gap — re-check durable status once before reconnecting.
5153
- // Non-transient status failures (e.g. 404 = run gone) fail loudly.
5154
- let refreshed: PlayStatus | null = null;
5155
- try {
5156
- refreshed = await this.getRunStatus(runId);
5157
- } catch (error) {
5158
- if (!isTransientPlayStreamError(error)) {
5159
- throw error;
5160
- }
5161
- }
5162
- if (refreshed && TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
5221
+ // `runs.get` owns the durable snapshot; its optional status evidence is
5222
+ // additive and must not prevent old servers from ending this stream.
5223
+ const refreshed = await this.getRunStatus(runId);
5224
+ if (TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
5163
5225
  return refreshed;
5164
5226
  }
5165
5227
 
@@ -5551,6 +5613,34 @@ export class DeeplineClient {
5551
5613
  };
5552
5614
  }
5553
5615
 
5616
+ /** Read definition-level settings, independently of code revisions. */
5617
+ async getPlaySettings(playName: string): Promise<PlaySettingsResult> {
5618
+ return this.http.get(
5619
+ `/api/v2/plays/${encodeURIComponent(playName)}/settings`,
5620
+ );
5621
+ }
5622
+
5623
+ /** Retry-safe field patch; omitted fields retain their current value. */
5624
+ async updatePlaySettings(
5625
+ playName: string,
5626
+ patch: PlaySettingsPatch,
5627
+ ): Promise<PlaySettingsResult> {
5628
+ return this.http.patch(
5629
+ `/api/v2/plays/${encodeURIComponent(playName)}/settings`,
5630
+ patch,
5631
+ );
5632
+ }
5633
+
5634
+ /** List one bounded page of existing Play locations; continue with cursor. */
5635
+ async listPlaySettingsPaths(
5636
+ options: { root?: 'person' | 'team'; limit?: number; cursor?: string } = {},
5637
+ ): Promise<PlaySettingsPathsPage> {
5638
+ const query = new URLSearchParams();
5639
+ for (const [key, value] of Object.entries(options))
5640
+ if (value !== undefined) query.set(key, String(value));
5641
+ return this.http.get(`/api/v2/plays/settings/paths?${query}`);
5642
+ }
5643
+
5554
5644
  /** Set whether an org-owned Play sorts before unpinned Plays. */
5555
5645
  async setPlayPinned(
5556
5646
  playName: string,
@@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
3
3
  import { homedir } from 'node:os';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { sdkCliStateDirPath } from './config.js';
6
+ import { deploymentProtectionHeaders } from './deployment-protection.js';
6
7
 
7
8
  export type SdkCompatibilityStatus =
8
9
  | 'current'
@@ -334,6 +335,7 @@ export async function checkSdkCompatibility(
334
335
  const response = await fetch(url, {
335
336
  method: 'GET',
336
337
  headers: {
338
+ ...deploymentProtectionHeaders(),
337
339
  'User-Agent': `deepline-ts-sdk/${SDK_VERSION}`,
338
340
  'X-Deepline-SDK-Version': SDK_VERSION,
339
341
  'X-Deepline-API-Major': String(SDK_API_MAJOR),
@@ -0,0 +1,19 @@
1
+ /** Headers shared by SDK requests and the CLI's directory deployment body. */
2
+ export function deploymentProtectionHeaders(): Record<string, string> {
3
+ const token =
4
+ typeof process !== 'undefined'
5
+ ? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
6
+ : undefined;
7
+ return token ? { 'x-vercel-protection-bypass': token } : {};
8
+ }
9
+
10
+ export const fetchWithDeploymentProtection: typeof fetch = (input, init) => {
11
+ const headers = new Headers(
12
+ input instanceof Request ? input.headers : undefined,
13
+ );
14
+ new Headers(init?.headers).forEach((value, name) => headers.set(name, value));
15
+ for (const [name, value] of Object.entries(deploymentProtectionHeaders())) {
16
+ headers.set(name, value);
17
+ }
18
+ return fetch(input, { ...init, headers });
19
+ };
@@ -35,6 +35,7 @@ import { SDK_API_CONTRACT, SDK_VERSION } from './version.js';
35
35
  import type { LiveEventEnvelope } from './types.js';
36
36
  import { detectAgentRuntime, isCoworkLikeSandbox } from './agent-runtime.js';
37
37
  import { readSdkSkillsLocalVersion } from './skills-version.js';
38
+ import { deploymentProtectionHeaders } from './deployment-protection.js';
38
39
  import {
39
40
  ABSURD_RELEASE_OVERRIDE_HEADER,
40
41
  COORDINATOR_INTERNAL_TOKEN_HEADER,
@@ -266,18 +267,12 @@ export class HttpClient {
266
267
  'X-Deepline-API-Contract': SDK_API_CONTRACT,
267
268
  'X-Deepline-Run-Result-Shape': 'canonical',
268
269
  ...extra,
270
+ ...deploymentProtectionHeaders(),
269
271
  };
270
272
  const skillsVersion = this.readSkillsVersionHeader();
271
273
  if (skillsVersion) {
272
274
  headers['X-Deepline-Skills-Version'] = skillsVersion;
273
275
  }
274
- const bypassToken =
275
- typeof process !== 'undefined'
276
- ? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
277
- : undefined;
278
- if (bypassToken) {
279
- headers['x-vercel-protection-bypass'] = bypassToken;
280
- }
281
276
  const playArtifactR2Prefix =
282
277
  typeof process !== 'undefined'
283
278
  ? process.env?.DEEPLINE_PLAY_ARTIFACT_R2_PREFIX
@@ -706,6 +701,7 @@ export class HttpClient {
706
701
  | 'onResponse'
707
702
  | 'allowNotFound'
708
703
  | 'idempotencyKey'
704
+ | 'headers'
709
705
  >,
710
706
  ): Promise<T> {
711
707
  return this.request<T>(path, {
@@ -1,3 +1,10 @@
1
+ export type {
2
+ PlaySettings,
3
+ PlayPath,
4
+ PlaySettingsPatch,
5
+ PlaySettingsResult,
6
+ PlaySettingsPathsPage,
7
+ } from '../../shared_libs/plays/settings-contract';
1
8
  /**
2
9
  * Deepline SDK — TypeScript client for the Deepline enrichment platform.
3
10
  *
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
200
200
  // getters keep their established compatibility behavior.
201
201
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
202
202
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
203
- version: '0.3.213',
203
+ version: '0.3.214',
204
204
  updateSummary:
205
205
  'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
206
206
  packageCapabilities: {
@@ -1,4 +1,6 @@
1
+ import type { PlaySettings } from '../../shared_libs/plays/settings-contract';
1
2
  import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest';
3
+ import type { RunGetStatus, RunLifecycleStatus } from './client.js';
2
4
  import type { PlayGrepMatch } from '../../shared_libs/plays/play-grep-search';
3
5
  import type { PlayRuntimeSelection } from '../../shared_libs/play-runtime/runtime-environment';
4
6
  import type {
@@ -466,6 +468,10 @@ export type InferenceQuote = {
466
468
  export interface ToolSearchOptions {
467
469
  /** Free-text search query. */
468
470
  query?: string;
471
+ /** Maximum number of ranked tools to return (1-50; defaults to 20). */
472
+ limit?: number;
473
+ /** Zero-based offset into the ranked result set. */
474
+ offset?: number;
469
475
  /** Comma-separated category filter such as `company_search` or `email_finder`. */
470
476
  categories?: string;
471
477
  /** Optional explicit search terms used by agent/CLI callers. */
@@ -488,6 +494,12 @@ export interface ToolSearchResult {
488
494
  count?: number;
489
495
  /** Matching count within the selected retrieval set. Check total_is_exact. */
490
496
  total?: number;
497
+ /** Zero-based offset used for this page. */
498
+ offset?: number;
499
+ /** Maximum number of ranked tools requested for this page. */
500
+ limit?: number;
501
+ /** Offset to pass for the next page, when more results are in this result set. */
502
+ next_offset?: number;
491
503
  /** False when indexed retrieval uses a bounded candidate set. */
492
504
  total_is_exact?: boolean;
493
505
  /** True when indexed retrieval may have additional matching tools. */
@@ -729,6 +741,12 @@ export interface PlayRunPackage {
729
741
  schemaVersion: 1;
730
742
  /** Package discriminator. */
731
743
  kind: 'play_run';
744
+ /** Ledger position of the detailed snapshot, when provided by this server. */
745
+ source?: { ledgerNextSeq: number | null };
746
+ /** Legacy status evidence returned by older servers. */
747
+ overview?: RunLifecycleStatus;
748
+ /** Optional bounded live evidence from the existing run status projection. */
749
+ runStatus?: RunGetStatus;
732
750
  /** Run identity, status, timing, and dashboard metadata. */
733
751
  run: {
734
752
  id: string;
@@ -834,6 +852,10 @@ export type PlayRunOutcome =
834
852
  export interface PlayStatus {
835
853
  /** Public play-run identifier. */
836
854
  runId: string;
855
+ /** Legacy status evidence returned by older servers. */
856
+ overview?: RunLifecycleStatus;
857
+ /** Additive bounded evidence when returned by the current server. */
858
+ runStatus?: RunGetStatus;
837
859
  /** Public Deepline play-run API version. */
838
860
  apiVersion?: number;
839
861
  /** Saved play name for this run, when available. */
@@ -1310,6 +1332,8 @@ export interface PlaySheetSummary {
1310
1332
  * Returned by {@link DeeplineClient.getPlay}.
1311
1333
  */
1312
1334
  export interface PlayDefinitionDetail {
1335
+ /** Definition settings; never attached to a revision. */
1336
+ settings?: PlaySettings;
1313
1337
  /** Convex document ID. */
1314
1338
  _id: string;
1315
1339
  /** Stable registry key. */
@@ -1415,6 +1439,7 @@ export type PlayCostEstimate =
1415
1439
  };
1416
1440
 
1417
1441
  export interface PlayListItem {
1442
+ settings?: PlaySettings;
1418
1443
  playKey?: string;
1419
1444
  reference?: string;
1420
1445
  name: string;
@@ -939,6 +939,24 @@
939
939
  "failureAlert": "convex_action_wrapper",
940
940
  "domainReportRequired": true
941
941
  },
942
+ {
943
+ "id": "convex.refresh-tool-usage-popularity",
944
+ "name": "Refresh tool usage popularity",
945
+ "scheduler": "convex_cron",
946
+ "cadence": "37 0 * * *",
947
+ "timezone": "UTC",
948
+ "source": "packages/convex/crons.ts",
949
+ "locator": "refresh tool usage popularity snapshot",
950
+ "owner": "integrations",
951
+ "criticality": "operational",
952
+ "purpose": "Sum distinct user-days per tool over the last 28 complete UTC days into the snapshot tools search uses as a ranking prior, then delete user-day rows past retention.",
953
+ "successDefinition": "A new snapshot row is written and expired user-day rows are drained; an empty snapshot is valid before any usage is folded.",
954
+ "failureDefinition": "Reading daily counts, writing the snapshot, or draining expired rows fails, leaving search on the previous snapshot.",
955
+ "startGraceMs": 120000,
956
+ "runTimeoutMs": 1800000,
957
+ "failureAlert": "convex_action_wrapper",
958
+ "domainReportRequired": true
959
+ },
942
960
  {
943
961
  "id": "convex.salesforce-oauth-refresh-maintenance",
944
962
  "name": "Maintain Salesforce OAuth refresh tokens",
@@ -14,6 +14,7 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
14
14
  'convex.cleanup-expired-api-key-claims',
15
15
  'convex.cleanup-expired-analytics-ip2company-cache',
16
16
  'convex.cleanup-expired-domain-health-cache',
17
+ 'convex.refresh-tool-usage-popularity',
17
18
  'convex.salesforce-oauth-refresh-maintenance',
18
19
  'convex.monitor-fleet-convergence-tick',
19
20
  'convex.monitor-provider-repair-recovery-tick',
@@ -2,6 +2,8 @@ import { spawnSync } from 'node:child_process';
2
2
 
3
3
  const FLY_REGISTRY_COMMAND_TIMEOUT_MS = 30_000;
4
4
  const RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS = 10_000;
5
+ export const FLY_REGISTRY_MANIFEST_ATTEMPTS = 3;
6
+ const FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS = 1_500;
5
7
 
6
8
  type FlyRegistryCommandResult = Readonly<{
7
9
  error?: Error & { code?: string };
@@ -59,15 +61,39 @@ export function runtimeImageManifestExists(
59
61
  options: {
60
62
  timeoutMs?: number;
61
63
  spawnCommand?: FlyRegistryCommandSpawn;
64
+ sleep?: (ms: number) => void;
62
65
  } = {},
63
66
  ): boolean {
64
- return run(
65
- 'docker',
66
- ['manifest', 'inspect', image],
67
- true,
68
- options.timeoutMs ?? RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS,
69
- options.spawnCommand,
70
- );
67
+ const timeoutMs = options.timeoutMs ?? RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS;
68
+ const sleep = options.sleep ?? sleepSync;
69
+ for (
70
+ let attempt = 1;
71
+ attempt <= FLY_REGISTRY_MANIFEST_ATTEMPTS;
72
+ attempt += 1
73
+ ) {
74
+ try {
75
+ return run(
76
+ 'docker',
77
+ ['manifest', 'inspect', image],
78
+ true,
79
+ timeoutMs,
80
+ options.spawnCommand,
81
+ );
82
+ } catch (error) {
83
+ if (
84
+ !(error instanceof Error) ||
85
+ (error as NodeJS.ErrnoException).code !== 'ETIMEDOUT' ||
86
+ attempt === FLY_REGISTRY_MANIFEST_ATTEMPTS
87
+ ) {
88
+ throw error;
89
+ }
90
+ console.warn(
91
+ `${error.message} Retrying manifest probe (${attempt}/${FLY_REGISTRY_MANIFEST_ATTEMPTS}).`,
92
+ );
93
+ sleep(FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS * attempt);
94
+ }
95
+ }
96
+ throw new Error('Fly registry manifest probe exhausted without a result.');
71
97
  }
72
98
 
73
99
  export const FLY_REGISTRY_AUTH_ATTEMPTS = 3;
@@ -3614,8 +3614,15 @@ async function main() {
3614
3614
  process.exit(1);
3615
3615
  },
3616
3616
  });
3617
+ // The Daytona watchdog is a separate process and can keep runner liveness
3618
+ // advancing while synchronous customer JavaScript occupies this process's
3619
+ // event loop. Hand off only after the existing durable `waiting` fence and
3620
+ // immediately before the marker that starts customer execution.
3621
+ const lastHeartbeatAtMs = heartbeat?.lastSuccessAtMs();
3622
+ heartbeat?.stop();
3617
3623
  writeRunnerRuntimeStartedMarker(
3618
3624
  process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH,
3625
+ lastHeartbeatAtMs === undefined ? {} : { lastHeartbeatAtMs },
3619
3626
  );
3620
3627
  let result: PlayRunnerResult | null = null;
3621
3628
  let executionFailed = false;
@@ -3791,7 +3798,7 @@ export async function materializeTransportedArtifactWhileRunnerParks(
3791
3798
 
3792
3799
  export function writeRunnerRuntimeStartedMarker(
3793
3800
  path: string | null | undefined,
3794
- detail: { pid?: number; at?: number } = {},
3801
+ detail: { pid?: number; at?: number; lastHeartbeatAtMs?: number } = {},
3795
3802
  ): void {
3796
3803
  const runtimeStartedPath = path?.trim();
3797
3804
  if (!runtimeStartedPath) return;
@@ -3801,6 +3808,9 @@ export function writeRunnerRuntimeStartedMarker(
3801
3808
  started: true,
3802
3809
  pid: detail.pid ?? process.pid,
3803
3810
  at: detail.at ?? Date.now(),
3811
+ ...(typeof detail.lastHeartbeatAtMs === 'number'
3812
+ ? { lastHeartbeatAtMs: detail.lastHeartbeatAtMs }
3813
+ : {}),
3804
3814
  }),
3805
3815
  'utf-8',
3806
3816
  );
@@ -100,6 +100,8 @@ export interface RunnerHeartbeatLoopHandle {
100
100
  executionAcknowledged: Promise<void>;
101
101
  /** Trigger an immediate heartbeat while waiting for the park acknowledgement. */
102
102
  requestHeartbeat: () => void;
103
+ /** Last durable heartbeat timestamp, used for an external liveness handoff. */
104
+ lastSuccessAtMs: () => number;
103
105
  }
104
106
 
105
107
  const APP_RUNTIME_COMPAT_PATH = '/api/v2/plays/runtime';
@@ -420,5 +422,6 @@ export function startRunnerHeartbeatLoop(
420
422
  firstHeartbeat,
421
423
  executionAcknowledged,
422
424
  requestHeartbeat: tick,
425
+ lastSuccessAtMs: () => lastSuccessAt,
423
426
  };
424
427
  }
@@ -27,6 +27,7 @@ import {
27
27
  RUNTIME_SANDBOX_MEMORY_ABORT_CODE,
28
28
  RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE,
29
29
  RUNTIME_SANDBOX_OOM_MESSAGE,
30
+ RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE,
30
31
  } from '../../run-failure';
31
32
  import {
32
33
  buildDaytonaPayloadMaterializerSource,
@@ -293,6 +294,41 @@ function watchdogMemoryAbortDiagnostic() {
293
294
  return null;
294
295
  }
295
296
 
297
+ function watchdogRunnerHeartbeatUnavailableDiagnostic() {
298
+ try {
299
+ if (!terminationDiagnosticPath) return null;
300
+ const diagnostic = JSON.parse(
301
+ fs.readFileSync(terminationDiagnosticPath, 'utf8'),
302
+ );
303
+ if (
304
+ !diagnostic ||
305
+ diagnostic.schemaVersion !== 1 ||
306
+ diagnostic.reason !== 'runner_heartbeat_unreachable'
307
+ ) {
308
+ return null;
309
+ }
310
+ return {
311
+ heartbeatAttempt: Number.isSafeInteger(diagnostic.heartbeatAttempt)
312
+ ? diagnostic.heartbeatAttempt
313
+ : null,
314
+ heartbeatAgeMs: Number.isFinite(diagnostic.heartbeatAgeMs)
315
+ ? Math.max(0, diagnostic.heartbeatAgeMs)
316
+ : null,
317
+ leaseMs: Number.isFinite(diagnostic.leaseMs)
318
+ ? Math.max(0, diagnostic.leaseMs)
319
+ : null,
320
+ lastErrorCode:
321
+ typeof diagnostic.lastErrorCode === 'string' &&
322
+ /^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+|TimeoutError|redirect_[0-9]{3}|http_[0-9]{3}|protocol_error|unknown)$/.test(
323
+ diagnostic.lastErrorCode,
324
+ )
325
+ ? diagnostic.lastErrorCode
326
+ : 'unknown',
327
+ };
328
+ } catch {}
329
+ return null;
330
+ }
331
+
296
332
  function memoryAbortCause(diagnostic) {
297
333
  const parts = [
298
334
  ${JSON.stringify(`${RUNTIME_SANDBOX_MEMORY_ABORT_CODE}: ${RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE}`)},
@@ -373,6 +409,7 @@ async function main() {
373
409
  // SIGKILL does not collapse into the wrapper's generic exit code 1.
374
410
  const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
375
411
  const memoryAbort = watchdogMemoryAbortDiagnostic();
412
+ const heartbeatUnavailable = watchdogRunnerHeartbeatUnavailableDiagnostic();
376
413
  const sandboxOom =
377
414
  !memoryAbort &&
378
415
  sandboxKilled &&
@@ -383,6 +420,10 @@ async function main() {
383
420
  ? memoryAbortCause(memoryAbort)
384
421
  : sandboxOom
385
422
  ? 'RUNTIME_SANDBOX_OOM: ' + ${JSON.stringify(RUNTIME_SANDBOX_OOM_MESSAGE)}
423
+ : heartbeatUnavailable
424
+ ? 'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' +
425
+ ${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)} +
426
+ ' heartbeat=' + JSON.stringify(heartbeatUnavailable)
386
427
  : sandboxKilled
387
428
  ? 'RUNTIME_SANDBOX_KILLED: ' + ${JSON.stringify(RUNTIME_SANDBOX_KILLED_MESSAGE)}
388
429
  :
@@ -408,6 +449,15 @@ async function main() {
408
449
  message: ${JSON.stringify(RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE)},
409
450
  retryable: false,
410
451
  cause: synthesizedError,
452
+ }] }
453
+ : {}),
454
+ ...(heartbeatUnavailable && !runtimeLimitExceeded && !memoryAbort && !sandboxOom
455
+ ? { errors: [{
456
+ code: 'RUNTIME_CONTROL_PLANE_UNAVAILABLE',
457
+ phase: 'infrastructure',
458
+ message: ${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)},
459
+ retryable: false,
460
+ cause: synthesizedError,
411
461
  }] }
412
462
  : {}),
413
463
  ...(sandboxOom
@@ -428,7 +478,9 @@ async function main() {
428
478
  cause: synthesizedError,
429
479
  }] }
430
480
  : {}),
431
- logs: [],
481
+ logs: heartbeatUnavailable
482
+ ? ['[runtime.failure_diagnostic] ' + JSON.stringify({ event: 'runner_heartbeat_unavailable', ...heartbeatUnavailable })]
483
+ : [],
432
484
  stats: {},
433
485
  steps: [],
434
486
  checkpointDisposition: '${RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT}',