deepline 0.3.213 → 0.3.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +122 -32
- package/dist/bundling-sources/sdk/src/compat.ts +2 -0
- package/dist/bundling-sources/sdk/src/deployment-protection.ts +19 -0
- package/dist/bundling-sources/sdk/src/http.ts +3 -7
- package/dist/bundling-sources/sdk/src/index.ts +7 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +25 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +33 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +53 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +187 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +12 -3
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -2
- package/dist/bundling-sources/shared_libs/plays/settings-contract.ts +176 -0
- package/dist/cli/index.js +560 -92
- package/dist/cli/index.mjs +560 -92
- package/dist/index.d.mts +165 -3
- package/dist/index.d.ts +165 -3
- package/dist/index.js +67 -35
- package/dist/index.mjs +67 -35
- package/dist/install-integrity.json +2 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +3 -2
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlaySettingsResult,
|
|
3
|
+
PlaySettingsPatch,
|
|
4
|
+
PlaySettingsPathsPage,
|
|
5
|
+
} from '../../shared_libs/plays/settings-contract';
|
|
1
6
|
import { iterateCursorPages } from './pagination.js';
|
|
2
7
|
import type { ToolResultBilling } from '../../shared_libs/plays/tool-result-types.js';
|
|
3
8
|
/**
|
|
@@ -825,6 +830,36 @@ export type RunsGetOptions = {
|
|
|
825
830
|
failedLogLimit?: number;
|
|
826
831
|
};
|
|
827
832
|
|
|
833
|
+
export type RunGetStatus =
|
|
834
|
+
| {
|
|
835
|
+
available: false;
|
|
836
|
+
reason: 'not_projected' | 'read_failed';
|
|
837
|
+
}
|
|
838
|
+
| {
|
|
839
|
+
available: true;
|
|
840
|
+
observedAt: number;
|
|
841
|
+
source: {
|
|
842
|
+
ledgerNextSeq: number | null;
|
|
843
|
+
runSnapshotLedgerNextSeq: number | null;
|
|
844
|
+
aligned: boolean | null;
|
|
845
|
+
};
|
|
846
|
+
calls: {
|
|
847
|
+
logicalCalls: { value: number | null; confirmed: boolean };
|
|
848
|
+
toolDispatches: { value: number | null; confirmed: boolean };
|
|
849
|
+
provider429s: { value: number | null; confirmed: boolean };
|
|
850
|
+
};
|
|
851
|
+
focusStep: {
|
|
852
|
+
id: string;
|
|
853
|
+
label: string | null;
|
|
854
|
+
status: string | null;
|
|
855
|
+
rows: {
|
|
856
|
+
succeeded: number | null;
|
|
857
|
+
failed: number | null;
|
|
858
|
+
total: number | null;
|
|
859
|
+
};
|
|
860
|
+
} | null;
|
|
861
|
+
};
|
|
862
|
+
|
|
828
863
|
const RUNS_FAILED_LOG_LIMIT = 20;
|
|
829
864
|
|
|
830
865
|
/** Streaming options for `client.runs.tail(...)`. */
|
|
@@ -963,7 +998,10 @@ export type PlaySecretCheck = {
|
|
|
963
998
|
export type RunsNamespace = {
|
|
964
999
|
/** Get current run status by public run id. */
|
|
965
1000
|
get: (runId: string, options?: RunsGetOptions) => Promise<PlayStatus>;
|
|
966
|
-
/**
|
|
1001
|
+
/**
|
|
1002
|
+
* @deprecated Use `client.runs.get(runId)` for status, progress, logs, and results.
|
|
1003
|
+
* This scalar-only reader remains for compatibility with existing callers.
|
|
1004
|
+
*/
|
|
967
1005
|
status: (runId: string) => Promise<RunLifecycleStatus>;
|
|
968
1006
|
/** Explicitly read the retained original input (may include customer data). */
|
|
969
1007
|
input: (runId: string) => Promise<{
|
|
@@ -2494,10 +2532,25 @@ function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
|
|
|
2494
2532
|
: typeof raw.workflowId === 'string'
|
|
2495
2533
|
? raw.workflowId
|
|
2496
2534
|
: (packageRun?.id ?? '');
|
|
2535
|
+
const legacyOverview = runPackage?.overview ?? raw.overview;
|
|
2536
|
+
const runStatus = runPackage?.runStatus ?? raw.runStatus;
|
|
2497
2537
|
return {
|
|
2498
2538
|
...(raw as unknown as Omit<PlayStatus, 'runId' | 'status'>),
|
|
2499
2539
|
runId,
|
|
2500
|
-
...(
|
|
2540
|
+
...(isRecord(legacyOverview)
|
|
2541
|
+
? {
|
|
2542
|
+
overview: legacyOverview as unknown as RunLifecycleStatus,
|
|
2543
|
+
}
|
|
2544
|
+
: {}),
|
|
2545
|
+
...(isRecord(runStatus)
|
|
2546
|
+
? { runStatus: runStatus as unknown as RunGetStatus }
|
|
2547
|
+
: {}),
|
|
2548
|
+
...(runPackage
|
|
2549
|
+
? {
|
|
2550
|
+
package: runPackage,
|
|
2551
|
+
outputs: runPackage.outputs,
|
|
2552
|
+
}
|
|
2553
|
+
: {}),
|
|
2501
2554
|
status,
|
|
2502
2555
|
};
|
|
2503
2556
|
}
|
|
@@ -2850,19 +2903,6 @@ function playRunResultFromStatus(
|
|
|
2850
2903
|
};
|
|
2851
2904
|
}
|
|
2852
2905
|
|
|
2853
|
-
function playRunStatusFromState(state: PlayLiveStatusState): PlayStatus {
|
|
2854
|
-
return {
|
|
2855
|
-
runId: state.runId,
|
|
2856
|
-
status: state.status,
|
|
2857
|
-
progress: {
|
|
2858
|
-
status: state.status,
|
|
2859
|
-
logs: state.logs,
|
|
2860
|
-
...(state.error ? { error: state.error } : {}),
|
|
2861
|
-
},
|
|
2862
|
-
...('result' in state ? { result: state.result } : {}),
|
|
2863
|
-
};
|
|
2864
|
-
}
|
|
2865
|
-
|
|
2866
2906
|
/**
|
|
2867
2907
|
* Low-level typed REST client with authentication, retries, and localhost failover.
|
|
2868
2908
|
*
|
|
@@ -3347,6 +3387,12 @@ export class DeeplineClient {
|
|
|
3347
3387
|
options.includeSearchDebug ? 'true' : 'false',
|
|
3348
3388
|
);
|
|
3349
3389
|
params.set('search_mode', options.searchMode ?? 'v2');
|
|
3390
|
+
if (options.limit !== undefined) {
|
|
3391
|
+
params.set('limit', String(options.limit));
|
|
3392
|
+
}
|
|
3393
|
+
if (options.offset !== undefined) {
|
|
3394
|
+
params.set('offset', String(options.offset));
|
|
3395
|
+
}
|
|
3350
3396
|
if (options.categories?.trim()) {
|
|
3351
3397
|
params.set('categories', options.categories.trim());
|
|
3352
3398
|
}
|
|
@@ -4807,8 +4853,25 @@ export class DeeplineClient {
|
|
|
4807
4853
|
const query = params.size > 0 ? `?${params.toString()}` : '';
|
|
4808
4854
|
const response = await this.http.get<Record<string, unknown>>(
|
|
4809
4855
|
`/api/v2/runs/${encodeURIComponent(runId)}${query}`,
|
|
4856
|
+
{ headers: { 'x-deepline-run-status': '1' } },
|
|
4810
4857
|
);
|
|
4858
|
+
const responseRunStatus = isPlayRunPackage(response)
|
|
4859
|
+
? response.runStatus
|
|
4860
|
+
: isPlayRunPackage(response.package)
|
|
4861
|
+
? response.package.runStatus
|
|
4862
|
+
: response.runStatus;
|
|
4863
|
+
const responseOverview = isPlayRunPackage(response)
|
|
4864
|
+
? response.overview
|
|
4865
|
+
: isPlayRunPackage(response.package)
|
|
4866
|
+
? response.package.overview
|
|
4867
|
+
: response.overview;
|
|
4811
4868
|
const status = normalizePlayStatus(response);
|
|
4869
|
+
if (isRecord(responseRunStatus)) {
|
|
4870
|
+
status.runStatus = responseRunStatus as unknown as RunGetStatus;
|
|
4871
|
+
}
|
|
4872
|
+
if (isRecord(responseOverview)) {
|
|
4873
|
+
status.overview = responseOverview as unknown as RunLifecycleStatus;
|
|
4874
|
+
}
|
|
4812
4875
|
if (options?.failedLogs !== true || status.status !== 'failed') {
|
|
4813
4876
|
return status;
|
|
4814
4877
|
}
|
|
@@ -4859,7 +4922,10 @@ export class DeeplineClient {
|
|
|
4859
4922
|
};
|
|
4860
4923
|
}
|
|
4861
4924
|
|
|
4862
|
-
/**
|
|
4925
|
+
/** Fetch only the legacy scalar lifecycle status for a run.
|
|
4926
|
+
*
|
|
4927
|
+
* @deprecated Use `client.runs.get(runId)` for the canonical run read.
|
|
4928
|
+
*/
|
|
4863
4929
|
async getRunLifecycleStatus(runId: string): Promise<RunLifecycleStatus> {
|
|
4864
4930
|
const response = await this.http.get<RunLifecycleStatus>(
|
|
4865
4931
|
`/api/v2/runs/${encodeURIComponent(runId)}/status`,
|
|
@@ -5067,9 +5133,7 @@ export class DeeplineClient {
|
|
|
5067
5133
|
if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
5068
5134
|
continue;
|
|
5069
5135
|
}
|
|
5070
|
-
return await this.getRunStatus(status.runId || runId)
|
|
5071
|
-
() => state.latest ?? playRunStatusFromState(state),
|
|
5072
|
-
);
|
|
5136
|
+
return await this.getRunStatus(status.runId || runId);
|
|
5073
5137
|
}
|
|
5074
5138
|
if (options?.signal?.aborted) {
|
|
5075
5139
|
throw new DeeplineError('Run observation aborted.', undefined, 'ABORTED');
|
|
@@ -5125,6 +5189,7 @@ export class DeeplineClient {
|
|
|
5125
5189
|
for (;;) {
|
|
5126
5190
|
const connectedAt = Date.now();
|
|
5127
5191
|
let sawEvent = false;
|
|
5192
|
+
let terminalRunId: string | null = null;
|
|
5128
5193
|
let endedReason = 'stream window ended before a terminal event';
|
|
5129
5194
|
try {
|
|
5130
5195
|
for await (const event of this.streamPlayRunEvents(runId, {
|
|
@@ -5137,9 +5202,8 @@ export class DeeplineClient {
|
|
|
5137
5202
|
if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
5138
5203
|
continue;
|
|
5139
5204
|
}
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
);
|
|
5205
|
+
terminalRunId = status.runId || runId;
|
|
5206
|
+
break;
|
|
5143
5207
|
}
|
|
5144
5208
|
} catch (error) {
|
|
5145
5209
|
if (options?.signal?.aborted || !isTransientPlayStreamError(error)) {
|
|
@@ -5148,18 +5212,16 @@ export class DeeplineClient {
|
|
|
5148
5212
|
endedReason = error instanceof Error ? error.message : String(error);
|
|
5149
5213
|
}
|
|
5150
5214
|
|
|
5215
|
+
if (terminalRunId) {
|
|
5216
|
+
return await this.getRunStatus(terminalRunId);
|
|
5217
|
+
}
|
|
5218
|
+
|
|
5151
5219
|
// Window ended without a terminal event. The run may have finished
|
|
5152
5220
|
// during the gap — re-check durable status once before reconnecting.
|
|
5153
|
-
//
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
} catch (error) {
|
|
5158
|
-
if (!isTransientPlayStreamError(error)) {
|
|
5159
|
-
throw error;
|
|
5160
|
-
}
|
|
5161
|
-
}
|
|
5162
|
-
if (refreshed && TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
|
|
5221
|
+
// `runs.get` owns the durable snapshot; its optional status evidence is
|
|
5222
|
+
// additive and must not prevent old servers from ending this stream.
|
|
5223
|
+
const refreshed = await this.getRunStatus(runId);
|
|
5224
|
+
if (TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
|
|
5163
5225
|
return refreshed;
|
|
5164
5226
|
}
|
|
5165
5227
|
|
|
@@ -5551,6 +5613,34 @@ export class DeeplineClient {
|
|
|
5551
5613
|
};
|
|
5552
5614
|
}
|
|
5553
5615
|
|
|
5616
|
+
/** Read definition-level settings, independently of code revisions. */
|
|
5617
|
+
async getPlaySettings(playName: string): Promise<PlaySettingsResult> {
|
|
5618
|
+
return this.http.get(
|
|
5619
|
+
`/api/v2/plays/${encodeURIComponent(playName)}/settings`,
|
|
5620
|
+
);
|
|
5621
|
+
}
|
|
5622
|
+
|
|
5623
|
+
/** Retry-safe field patch; omitted fields retain their current value. */
|
|
5624
|
+
async updatePlaySettings(
|
|
5625
|
+
playName: string,
|
|
5626
|
+
patch: PlaySettingsPatch,
|
|
5627
|
+
): Promise<PlaySettingsResult> {
|
|
5628
|
+
return this.http.patch(
|
|
5629
|
+
`/api/v2/plays/${encodeURIComponent(playName)}/settings`,
|
|
5630
|
+
patch,
|
|
5631
|
+
);
|
|
5632
|
+
}
|
|
5633
|
+
|
|
5634
|
+
/** List one bounded page of existing Play locations; continue with cursor. */
|
|
5635
|
+
async listPlaySettingsPaths(
|
|
5636
|
+
options: { root?: 'person' | 'team'; limit?: number; cursor?: string } = {},
|
|
5637
|
+
): Promise<PlaySettingsPathsPage> {
|
|
5638
|
+
const query = new URLSearchParams();
|
|
5639
|
+
for (const [key, value] of Object.entries(options))
|
|
5640
|
+
if (value !== undefined) query.set(key, String(value));
|
|
5641
|
+
return this.http.get(`/api/v2/plays/settings/paths?${query}`);
|
|
5642
|
+
}
|
|
5643
|
+
|
|
5554
5644
|
/** Set whether an org-owned Play sorts before unpinned Plays. */
|
|
5555
5645
|
async setPlayPinned(
|
|
5556
5646
|
playName: string,
|
|
@@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
3
3
|
import { homedir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { sdkCliStateDirPath } from './config.js';
|
|
6
|
+
import { deploymentProtectionHeaders } from './deployment-protection.js';
|
|
6
7
|
|
|
7
8
|
export type SdkCompatibilityStatus =
|
|
8
9
|
| 'current'
|
|
@@ -334,6 +335,7 @@ export async function checkSdkCompatibility(
|
|
|
334
335
|
const response = await fetch(url, {
|
|
335
336
|
method: 'GET',
|
|
336
337
|
headers: {
|
|
338
|
+
...deploymentProtectionHeaders(),
|
|
337
339
|
'User-Agent': `deepline-ts-sdk/${SDK_VERSION}`,
|
|
338
340
|
'X-Deepline-SDK-Version': SDK_VERSION,
|
|
339
341
|
'X-Deepline-API-Major': String(SDK_API_MAJOR),
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Headers shared by SDK requests and the CLI's directory deployment body. */
|
|
2
|
+
export function deploymentProtectionHeaders(): Record<string, string> {
|
|
3
|
+
const token =
|
|
4
|
+
typeof process !== 'undefined'
|
|
5
|
+
? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
|
|
6
|
+
: undefined;
|
|
7
|
+
return token ? { 'x-vercel-protection-bypass': token } : {};
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const fetchWithDeploymentProtection: typeof fetch = (input, init) => {
|
|
11
|
+
const headers = new Headers(
|
|
12
|
+
input instanceof Request ? input.headers : undefined,
|
|
13
|
+
);
|
|
14
|
+
new Headers(init?.headers).forEach((value, name) => headers.set(name, value));
|
|
15
|
+
for (const [name, value] of Object.entries(deploymentProtectionHeaders())) {
|
|
16
|
+
headers.set(name, value);
|
|
17
|
+
}
|
|
18
|
+
return fetch(input, { ...init, headers });
|
|
19
|
+
};
|
|
@@ -35,6 +35,7 @@ import { SDK_API_CONTRACT, SDK_VERSION } from './version.js';
|
|
|
35
35
|
import type { LiveEventEnvelope } from './types.js';
|
|
36
36
|
import { detectAgentRuntime, isCoworkLikeSandbox } from './agent-runtime.js';
|
|
37
37
|
import { readSdkSkillsLocalVersion } from './skills-version.js';
|
|
38
|
+
import { deploymentProtectionHeaders } from './deployment-protection.js';
|
|
38
39
|
import {
|
|
39
40
|
ABSURD_RELEASE_OVERRIDE_HEADER,
|
|
40
41
|
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
@@ -266,18 +267,12 @@ export class HttpClient {
|
|
|
266
267
|
'X-Deepline-API-Contract': SDK_API_CONTRACT,
|
|
267
268
|
'X-Deepline-Run-Result-Shape': 'canonical',
|
|
268
269
|
...extra,
|
|
270
|
+
...deploymentProtectionHeaders(),
|
|
269
271
|
};
|
|
270
272
|
const skillsVersion = this.readSkillsVersionHeader();
|
|
271
273
|
if (skillsVersion) {
|
|
272
274
|
headers['X-Deepline-Skills-Version'] = skillsVersion;
|
|
273
275
|
}
|
|
274
|
-
const bypassToken =
|
|
275
|
-
typeof process !== 'undefined'
|
|
276
|
-
? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
|
|
277
|
-
: undefined;
|
|
278
|
-
if (bypassToken) {
|
|
279
|
-
headers['x-vercel-protection-bypass'] = bypassToken;
|
|
280
|
-
}
|
|
281
276
|
const playArtifactR2Prefix =
|
|
282
277
|
typeof process !== 'undefined'
|
|
283
278
|
? process.env?.DEEPLINE_PLAY_ARTIFACT_R2_PREFIX
|
|
@@ -706,6 +701,7 @@ export class HttpClient {
|
|
|
706
701
|
| 'onResponse'
|
|
707
702
|
| 'allowNotFound'
|
|
708
703
|
| 'idempotencyKey'
|
|
704
|
+
| 'headers'
|
|
709
705
|
>,
|
|
710
706
|
): Promise<T> {
|
|
711
707
|
return this.request<T>(path, {
|
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.214',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import type { PlaySettings } from '../../shared_libs/plays/settings-contract';
|
|
1
2
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest';
|
|
3
|
+
import type { RunGetStatus, RunLifecycleStatus } from './client.js';
|
|
2
4
|
import type { PlayGrepMatch } from '../../shared_libs/plays/play-grep-search';
|
|
3
5
|
import type { PlayRuntimeSelection } from '../../shared_libs/play-runtime/runtime-environment';
|
|
4
6
|
import type {
|
|
@@ -466,6 +468,10 @@ export type InferenceQuote = {
|
|
|
466
468
|
export interface ToolSearchOptions {
|
|
467
469
|
/** Free-text search query. */
|
|
468
470
|
query?: string;
|
|
471
|
+
/** Maximum number of ranked tools to return (1-50; defaults to 20). */
|
|
472
|
+
limit?: number;
|
|
473
|
+
/** Zero-based offset into the ranked result set. */
|
|
474
|
+
offset?: number;
|
|
469
475
|
/** Comma-separated category filter such as `company_search` or `email_finder`. */
|
|
470
476
|
categories?: string;
|
|
471
477
|
/** Optional explicit search terms used by agent/CLI callers. */
|
|
@@ -488,6 +494,12 @@ export interface ToolSearchResult {
|
|
|
488
494
|
count?: number;
|
|
489
495
|
/** Matching count within the selected retrieval set. Check total_is_exact. */
|
|
490
496
|
total?: number;
|
|
497
|
+
/** Zero-based offset used for this page. */
|
|
498
|
+
offset?: number;
|
|
499
|
+
/** Maximum number of ranked tools requested for this page. */
|
|
500
|
+
limit?: number;
|
|
501
|
+
/** Offset to pass for the next page, when more results are in this result set. */
|
|
502
|
+
next_offset?: number;
|
|
491
503
|
/** False when indexed retrieval uses a bounded candidate set. */
|
|
492
504
|
total_is_exact?: boolean;
|
|
493
505
|
/** True when indexed retrieval may have additional matching tools. */
|
|
@@ -729,6 +741,12 @@ export interface PlayRunPackage {
|
|
|
729
741
|
schemaVersion: 1;
|
|
730
742
|
/** Package discriminator. */
|
|
731
743
|
kind: 'play_run';
|
|
744
|
+
/** Ledger position of the detailed snapshot, when provided by this server. */
|
|
745
|
+
source?: { ledgerNextSeq: number | null };
|
|
746
|
+
/** Legacy status evidence returned by older servers. */
|
|
747
|
+
overview?: RunLifecycleStatus;
|
|
748
|
+
/** Optional bounded live evidence from the existing run status projection. */
|
|
749
|
+
runStatus?: RunGetStatus;
|
|
732
750
|
/** Run identity, status, timing, and dashboard metadata. */
|
|
733
751
|
run: {
|
|
734
752
|
id: string;
|
|
@@ -834,6 +852,10 @@ export type PlayRunOutcome =
|
|
|
834
852
|
export interface PlayStatus {
|
|
835
853
|
/** Public play-run identifier. */
|
|
836
854
|
runId: string;
|
|
855
|
+
/** Legacy status evidence returned by older servers. */
|
|
856
|
+
overview?: RunLifecycleStatus;
|
|
857
|
+
/** Additive bounded evidence when returned by the current server. */
|
|
858
|
+
runStatus?: RunGetStatus;
|
|
837
859
|
/** Public Deepline play-run API version. */
|
|
838
860
|
apiVersion?: number;
|
|
839
861
|
/** Saved play name for this run, when available. */
|
|
@@ -1310,6 +1332,8 @@ export interface PlaySheetSummary {
|
|
|
1310
1332
|
* Returned by {@link DeeplineClient.getPlay}.
|
|
1311
1333
|
*/
|
|
1312
1334
|
export interface PlayDefinitionDetail {
|
|
1335
|
+
/** Definition settings; never attached to a revision. */
|
|
1336
|
+
settings?: PlaySettings;
|
|
1313
1337
|
/** Convex document ID. */
|
|
1314
1338
|
_id: string;
|
|
1315
1339
|
/** Stable registry key. */
|
|
@@ -1415,6 +1439,7 @@ export type PlayCostEstimate =
|
|
|
1415
1439
|
};
|
|
1416
1440
|
|
|
1417
1441
|
export interface PlayListItem {
|
|
1442
|
+
settings?: PlaySettings;
|
|
1418
1443
|
playKey?: string;
|
|
1419
1444
|
reference?: string;
|
|
1420
1445
|
name: string;
|
|
@@ -939,6 +939,24 @@
|
|
|
939
939
|
"failureAlert": "convex_action_wrapper",
|
|
940
940
|
"domainReportRequired": true
|
|
941
941
|
},
|
|
942
|
+
{
|
|
943
|
+
"id": "convex.refresh-tool-usage-popularity",
|
|
944
|
+
"name": "Refresh tool usage popularity",
|
|
945
|
+
"scheduler": "convex_cron",
|
|
946
|
+
"cadence": "37 0 * * *",
|
|
947
|
+
"timezone": "UTC",
|
|
948
|
+
"source": "packages/convex/crons.ts",
|
|
949
|
+
"locator": "refresh tool usage popularity snapshot",
|
|
950
|
+
"owner": "integrations",
|
|
951
|
+
"criticality": "operational",
|
|
952
|
+
"purpose": "Sum distinct user-days per tool over the last 28 complete UTC days into the snapshot tools search uses as a ranking prior, then delete user-day rows past retention.",
|
|
953
|
+
"successDefinition": "A new snapshot row is written and expired user-day rows are drained; an empty snapshot is valid before any usage is folded.",
|
|
954
|
+
"failureDefinition": "Reading daily counts, writing the snapshot, or draining expired rows fails, leaving search on the previous snapshot.",
|
|
955
|
+
"startGraceMs": 120000,
|
|
956
|
+
"runTimeoutMs": 1800000,
|
|
957
|
+
"failureAlert": "convex_action_wrapper",
|
|
958
|
+
"domainReportRequired": true
|
|
959
|
+
},
|
|
942
960
|
{
|
|
943
961
|
"id": "convex.salesforce-oauth-refresh-maintenance",
|
|
944
962
|
"name": "Maintain Salesforce OAuth refresh tokens",
|
|
@@ -14,6 +14,7 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
|
|
|
14
14
|
'convex.cleanup-expired-api-key-claims',
|
|
15
15
|
'convex.cleanup-expired-analytics-ip2company-cache',
|
|
16
16
|
'convex.cleanup-expired-domain-health-cache',
|
|
17
|
+
'convex.refresh-tool-usage-popularity',
|
|
17
18
|
'convex.salesforce-oauth-refresh-maintenance',
|
|
18
19
|
'convex.monitor-fleet-convergence-tick',
|
|
19
20
|
'convex.monitor-provider-repair-recovery-tick',
|
|
@@ -2,6 +2,8 @@ import { spawnSync } from 'node:child_process';
|
|
|
2
2
|
|
|
3
3
|
const FLY_REGISTRY_COMMAND_TIMEOUT_MS = 30_000;
|
|
4
4
|
const RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS = 10_000;
|
|
5
|
+
export const FLY_REGISTRY_MANIFEST_ATTEMPTS = 3;
|
|
6
|
+
const FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS = 1_500;
|
|
5
7
|
|
|
6
8
|
type FlyRegistryCommandResult = Readonly<{
|
|
7
9
|
error?: Error & { code?: string };
|
|
@@ -59,15 +61,39 @@ export function runtimeImageManifestExists(
|
|
|
59
61
|
options: {
|
|
60
62
|
timeoutMs?: number;
|
|
61
63
|
spawnCommand?: FlyRegistryCommandSpawn;
|
|
64
|
+
sleep?: (ms: number) => void;
|
|
62
65
|
} = {},
|
|
63
66
|
): boolean {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
)
|
|
67
|
+
const timeoutMs = options.timeoutMs ?? RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS;
|
|
68
|
+
const sleep = options.sleep ?? sleepSync;
|
|
69
|
+
for (
|
|
70
|
+
let attempt = 1;
|
|
71
|
+
attempt <= FLY_REGISTRY_MANIFEST_ATTEMPTS;
|
|
72
|
+
attempt += 1
|
|
73
|
+
) {
|
|
74
|
+
try {
|
|
75
|
+
return run(
|
|
76
|
+
'docker',
|
|
77
|
+
['manifest', 'inspect', image],
|
|
78
|
+
true,
|
|
79
|
+
timeoutMs,
|
|
80
|
+
options.spawnCommand,
|
|
81
|
+
);
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (
|
|
84
|
+
!(error instanceof Error) ||
|
|
85
|
+
(error as NodeJS.ErrnoException).code !== 'ETIMEDOUT' ||
|
|
86
|
+
attempt === FLY_REGISTRY_MANIFEST_ATTEMPTS
|
|
87
|
+
) {
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
console.warn(
|
|
91
|
+
`${error.message} Retrying manifest probe (${attempt}/${FLY_REGISTRY_MANIFEST_ATTEMPTS}).`,
|
|
92
|
+
);
|
|
93
|
+
sleep(FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS * attempt);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
throw new Error('Fly registry manifest probe exhausted without a result.');
|
|
71
97
|
}
|
|
72
98
|
|
|
73
99
|
export const FLY_REGISTRY_AUTH_ATTEMPTS = 3;
|
|
@@ -3614,8 +3614,15 @@ async function main() {
|
|
|
3614
3614
|
process.exit(1);
|
|
3615
3615
|
},
|
|
3616
3616
|
});
|
|
3617
|
+
// The Daytona watchdog is a separate process and can keep runner liveness
|
|
3618
|
+
// advancing while synchronous customer JavaScript occupies this process's
|
|
3619
|
+
// event loop. Hand off only after the existing durable `waiting` fence and
|
|
3620
|
+
// immediately before the marker that starts customer execution.
|
|
3621
|
+
const lastHeartbeatAtMs = heartbeat?.lastSuccessAtMs();
|
|
3622
|
+
heartbeat?.stop();
|
|
3617
3623
|
writeRunnerRuntimeStartedMarker(
|
|
3618
3624
|
process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH,
|
|
3625
|
+
lastHeartbeatAtMs === undefined ? {} : { lastHeartbeatAtMs },
|
|
3619
3626
|
);
|
|
3620
3627
|
let result: PlayRunnerResult | null = null;
|
|
3621
3628
|
let executionFailed = false;
|
|
@@ -3791,7 +3798,7 @@ export async function materializeTransportedArtifactWhileRunnerParks(
|
|
|
3791
3798
|
|
|
3792
3799
|
export function writeRunnerRuntimeStartedMarker(
|
|
3793
3800
|
path: string | null | undefined,
|
|
3794
|
-
detail: { pid?: number; at?: number } = {},
|
|
3801
|
+
detail: { pid?: number; at?: number; lastHeartbeatAtMs?: number } = {},
|
|
3795
3802
|
): void {
|
|
3796
3803
|
const runtimeStartedPath = path?.trim();
|
|
3797
3804
|
if (!runtimeStartedPath) return;
|
|
@@ -3801,6 +3808,9 @@ export function writeRunnerRuntimeStartedMarker(
|
|
|
3801
3808
|
started: true,
|
|
3802
3809
|
pid: detail.pid ?? process.pid,
|
|
3803
3810
|
at: detail.at ?? Date.now(),
|
|
3811
|
+
...(typeof detail.lastHeartbeatAtMs === 'number'
|
|
3812
|
+
? { lastHeartbeatAtMs: detail.lastHeartbeatAtMs }
|
|
3813
|
+
: {}),
|
|
3804
3814
|
}),
|
|
3805
3815
|
'utf-8',
|
|
3806
3816
|
);
|
|
@@ -100,6 +100,8 @@ export interface RunnerHeartbeatLoopHandle {
|
|
|
100
100
|
executionAcknowledged: Promise<void>;
|
|
101
101
|
/** Trigger an immediate heartbeat while waiting for the park acknowledgement. */
|
|
102
102
|
requestHeartbeat: () => void;
|
|
103
|
+
/** Last durable heartbeat timestamp, used for an external liveness handoff. */
|
|
104
|
+
lastSuccessAtMs: () => number;
|
|
103
105
|
}
|
|
104
106
|
|
|
105
107
|
const APP_RUNTIME_COMPAT_PATH = '/api/v2/plays/runtime';
|
|
@@ -420,5 +422,6 @@ export function startRunnerHeartbeatLoop(
|
|
|
420
422
|
firstHeartbeat,
|
|
421
423
|
executionAcknowledged,
|
|
422
424
|
requestHeartbeat: tick,
|
|
425
|
+
lastSuccessAtMs: () => lastSuccessAt,
|
|
423
426
|
};
|
|
424
427
|
}
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
RUNTIME_SANDBOX_MEMORY_ABORT_CODE,
|
|
28
28
|
RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE,
|
|
29
29
|
RUNTIME_SANDBOX_OOM_MESSAGE,
|
|
30
|
+
RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE,
|
|
30
31
|
} from '../../run-failure';
|
|
31
32
|
import {
|
|
32
33
|
buildDaytonaPayloadMaterializerSource,
|
|
@@ -293,6 +294,41 @@ function watchdogMemoryAbortDiagnostic() {
|
|
|
293
294
|
return null;
|
|
294
295
|
}
|
|
295
296
|
|
|
297
|
+
function watchdogRunnerHeartbeatUnavailableDiagnostic() {
|
|
298
|
+
try {
|
|
299
|
+
if (!terminationDiagnosticPath) return null;
|
|
300
|
+
const diagnostic = JSON.parse(
|
|
301
|
+
fs.readFileSync(terminationDiagnosticPath, 'utf8'),
|
|
302
|
+
);
|
|
303
|
+
if (
|
|
304
|
+
!diagnostic ||
|
|
305
|
+
diagnostic.schemaVersion !== 1 ||
|
|
306
|
+
diagnostic.reason !== 'runner_heartbeat_unreachable'
|
|
307
|
+
) {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
return {
|
|
311
|
+
heartbeatAttempt: Number.isSafeInteger(diagnostic.heartbeatAttempt)
|
|
312
|
+
? diagnostic.heartbeatAttempt
|
|
313
|
+
: null,
|
|
314
|
+
heartbeatAgeMs: Number.isFinite(diagnostic.heartbeatAgeMs)
|
|
315
|
+
? Math.max(0, diagnostic.heartbeatAgeMs)
|
|
316
|
+
: null,
|
|
317
|
+
leaseMs: Number.isFinite(diagnostic.leaseMs)
|
|
318
|
+
? Math.max(0, diagnostic.leaseMs)
|
|
319
|
+
: null,
|
|
320
|
+
lastErrorCode:
|
|
321
|
+
typeof diagnostic.lastErrorCode === 'string' &&
|
|
322
|
+
/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+|TimeoutError|redirect_[0-9]{3}|http_[0-9]{3}|protocol_error|unknown)$/.test(
|
|
323
|
+
diagnostic.lastErrorCode,
|
|
324
|
+
)
|
|
325
|
+
? diagnostic.lastErrorCode
|
|
326
|
+
: 'unknown',
|
|
327
|
+
};
|
|
328
|
+
} catch {}
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
|
|
296
332
|
function memoryAbortCause(diagnostic) {
|
|
297
333
|
const parts = [
|
|
298
334
|
${JSON.stringify(`${RUNTIME_SANDBOX_MEMORY_ABORT_CODE}: ${RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE}`)},
|
|
@@ -373,6 +409,7 @@ async function main() {
|
|
|
373
409
|
// SIGKILL does not collapse into the wrapper's generic exit code 1.
|
|
374
410
|
const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
|
|
375
411
|
const memoryAbort = watchdogMemoryAbortDiagnostic();
|
|
412
|
+
const heartbeatUnavailable = watchdogRunnerHeartbeatUnavailableDiagnostic();
|
|
376
413
|
const sandboxOom =
|
|
377
414
|
!memoryAbort &&
|
|
378
415
|
sandboxKilled &&
|
|
@@ -383,6 +420,10 @@ async function main() {
|
|
|
383
420
|
? memoryAbortCause(memoryAbort)
|
|
384
421
|
: sandboxOom
|
|
385
422
|
? 'RUNTIME_SANDBOX_OOM: ' + ${JSON.stringify(RUNTIME_SANDBOX_OOM_MESSAGE)}
|
|
423
|
+
: heartbeatUnavailable
|
|
424
|
+
? 'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' +
|
|
425
|
+
${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)} +
|
|
426
|
+
' heartbeat=' + JSON.stringify(heartbeatUnavailable)
|
|
386
427
|
: sandboxKilled
|
|
387
428
|
? 'RUNTIME_SANDBOX_KILLED: ' + ${JSON.stringify(RUNTIME_SANDBOX_KILLED_MESSAGE)}
|
|
388
429
|
:
|
|
@@ -408,6 +449,15 @@ async function main() {
|
|
|
408
449
|
message: ${JSON.stringify(RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE)},
|
|
409
450
|
retryable: false,
|
|
410
451
|
cause: synthesizedError,
|
|
452
|
+
}] }
|
|
453
|
+
: {}),
|
|
454
|
+
...(heartbeatUnavailable && !runtimeLimitExceeded && !memoryAbort && !sandboxOom
|
|
455
|
+
? { errors: [{
|
|
456
|
+
code: 'RUNTIME_CONTROL_PLANE_UNAVAILABLE',
|
|
457
|
+
phase: 'infrastructure',
|
|
458
|
+
message: ${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)},
|
|
459
|
+
retryable: false,
|
|
460
|
+
cause: synthesizedError,
|
|
411
461
|
}] }
|
|
412
462
|
: {}),
|
|
413
463
|
...(sandboxOom
|
|
@@ -428,7 +478,9 @@ async function main() {
|
|
|
428
478
|
cause: synthesizedError,
|
|
429
479
|
}] }
|
|
430
480
|
: {}),
|
|
431
|
-
logs:
|
|
481
|
+
logs: heartbeatUnavailable
|
|
482
|
+
? ['[runtime.failure_diagnostic] ' + JSON.stringify({ event: 'runner_heartbeat_unavailable', ...heartbeatUnavailable })]
|
|
483
|
+
: [],
|
|
432
484
|
stats: {},
|
|
433
485
|
steps: [],
|
|
434
486
|
checkpointDisposition: '${RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT}',
|