deepline 0.3.212 → 0.3.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +137 -32
- package/dist/bundling-sources/sdk/src/compat.ts +2 -0
- package/dist/bundling-sources/sdk/src/deployment-protection.ts +19 -0
- package/dist/bundling-sources/sdk/src/http.ts +3 -7
- package/dist/bundling-sources/sdk/src/index.ts +7 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +25 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +18 -0
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +33 -7
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-heartbeat.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +53 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +187 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +67 -37
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +12 -3
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +5 -2
- package/dist/bundling-sources/shared_libs/plays/settings-contract.ts +176 -0
- package/dist/cli/index.js +572 -92
- package/dist/cli/index.mjs +572 -92
- package/dist/index.d.mts +191 -3
- package/dist/index.d.ts +191 -3
- package/dist/index.js +67 -35
- package/dist/index.mjs +67 -35
- package/dist/install-integrity.json +2 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +3 -2
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
PlaySettingsResult,
|
|
3
|
+
PlaySettingsPatch,
|
|
4
|
+
PlaySettingsPathsPage,
|
|
5
|
+
} from '../../shared_libs/plays/settings-contract';
|
|
1
6
|
import { iterateCursorPages } from './pagination.js';
|
|
2
7
|
import type { ToolResultBilling } from '../../shared_libs/plays/tool-result-types.js';
|
|
3
8
|
/**
|
|
@@ -825,6 +830,36 @@ export type RunsGetOptions = {
|
|
|
825
830
|
failedLogLimit?: number;
|
|
826
831
|
};
|
|
827
832
|
|
|
833
|
+
export type RunGetStatus =
|
|
834
|
+
| {
|
|
835
|
+
available: false;
|
|
836
|
+
reason: 'not_projected' | 'read_failed';
|
|
837
|
+
}
|
|
838
|
+
| {
|
|
839
|
+
available: true;
|
|
840
|
+
observedAt: number;
|
|
841
|
+
source: {
|
|
842
|
+
ledgerNextSeq: number | null;
|
|
843
|
+
runSnapshotLedgerNextSeq: number | null;
|
|
844
|
+
aligned: boolean | null;
|
|
845
|
+
};
|
|
846
|
+
calls: {
|
|
847
|
+
logicalCalls: { value: number | null; confirmed: boolean };
|
|
848
|
+
toolDispatches: { value: number | null; confirmed: boolean };
|
|
849
|
+
provider429s: { value: number | null; confirmed: boolean };
|
|
850
|
+
};
|
|
851
|
+
focusStep: {
|
|
852
|
+
id: string;
|
|
853
|
+
label: string | null;
|
|
854
|
+
status: string | null;
|
|
855
|
+
rows: {
|
|
856
|
+
succeeded: number | null;
|
|
857
|
+
failed: number | null;
|
|
858
|
+
total: number | null;
|
|
859
|
+
};
|
|
860
|
+
} | null;
|
|
861
|
+
};
|
|
862
|
+
|
|
828
863
|
const RUNS_FAILED_LOG_LIMIT = 20;
|
|
829
864
|
|
|
830
865
|
/** Streaming options for `client.runs.tail(...)`. */
|
|
@@ -963,7 +998,10 @@ export type PlaySecretCheck = {
|
|
|
963
998
|
export type RunsNamespace = {
|
|
964
999
|
/** Get current run status by public run id. */
|
|
965
1000
|
get: (runId: string, options?: RunsGetOptions) => Promise<PlayStatus>;
|
|
966
|
-
/**
|
|
1001
|
+
/**
|
|
1002
|
+
* @deprecated Use `client.runs.get(runId)` for status, progress, logs, and results.
|
|
1003
|
+
* This scalar-only reader remains for compatibility with existing callers.
|
|
1004
|
+
*/
|
|
967
1005
|
status: (runId: string) => Promise<RunLifecycleStatus>;
|
|
968
1006
|
/** Explicitly read the retained original input (may include customer data). */
|
|
969
1007
|
input: (runId: string) => Promise<{
|
|
@@ -1026,6 +1064,21 @@ export type RunLifecycleStatus =
|
|
|
1026
1064
|
startedAt: number | null;
|
|
1027
1065
|
finishedAt: number | null;
|
|
1028
1066
|
};
|
|
1067
|
+
/** Null for scheduler-only admission or a pre-expansion projection row. */
|
|
1068
|
+
source?: { ledgerNextSeq: number | null };
|
|
1069
|
+
/** One durable current activity, not proof that every row is blocked. */
|
|
1070
|
+
activity?: {
|
|
1071
|
+
kind: string;
|
|
1072
|
+
reason: string | null;
|
|
1073
|
+
targetKind: string | null;
|
|
1074
|
+
observedAt: number | null;
|
|
1075
|
+
nextAt: number | null;
|
|
1076
|
+
} | null;
|
|
1077
|
+
calls?: {
|
|
1078
|
+
logicalCalls: { value: number | null; confirmed: boolean };
|
|
1079
|
+
providerRequests: { value: number | null; confirmed: boolean };
|
|
1080
|
+
provider429s: { value: number | null; confirmed: boolean };
|
|
1081
|
+
};
|
|
1029
1082
|
steps: {
|
|
1030
1083
|
observed: number;
|
|
1031
1084
|
running: number;
|
|
@@ -2479,10 +2532,25 @@ function normalizePlayStatus(raw: Record<string, unknown>): PlayStatus {
|
|
|
2479
2532
|
: typeof raw.workflowId === 'string'
|
|
2480
2533
|
? raw.workflowId
|
|
2481
2534
|
: (packageRun?.id ?? '');
|
|
2535
|
+
const legacyOverview = runPackage?.overview ?? raw.overview;
|
|
2536
|
+
const runStatus = runPackage?.runStatus ?? raw.runStatus;
|
|
2482
2537
|
return {
|
|
2483
2538
|
...(raw as unknown as Omit<PlayStatus, 'runId' | 'status'>),
|
|
2484
2539
|
runId,
|
|
2485
|
-
...(
|
|
2540
|
+
...(isRecord(legacyOverview)
|
|
2541
|
+
? {
|
|
2542
|
+
overview: legacyOverview as unknown as RunLifecycleStatus,
|
|
2543
|
+
}
|
|
2544
|
+
: {}),
|
|
2545
|
+
...(isRecord(runStatus)
|
|
2546
|
+
? { runStatus: runStatus as unknown as RunGetStatus }
|
|
2547
|
+
: {}),
|
|
2548
|
+
...(runPackage
|
|
2549
|
+
? {
|
|
2550
|
+
package: runPackage,
|
|
2551
|
+
outputs: runPackage.outputs,
|
|
2552
|
+
}
|
|
2553
|
+
: {}),
|
|
2486
2554
|
status,
|
|
2487
2555
|
};
|
|
2488
2556
|
}
|
|
@@ -2835,19 +2903,6 @@ function playRunResultFromStatus(
|
|
|
2835
2903
|
};
|
|
2836
2904
|
}
|
|
2837
2905
|
|
|
2838
|
-
function playRunStatusFromState(state: PlayLiveStatusState): PlayStatus {
|
|
2839
|
-
return {
|
|
2840
|
-
runId: state.runId,
|
|
2841
|
-
status: state.status,
|
|
2842
|
-
progress: {
|
|
2843
|
-
status: state.status,
|
|
2844
|
-
logs: state.logs,
|
|
2845
|
-
...(state.error ? { error: state.error } : {}),
|
|
2846
|
-
},
|
|
2847
|
-
...('result' in state ? { result: state.result } : {}),
|
|
2848
|
-
};
|
|
2849
|
-
}
|
|
2850
|
-
|
|
2851
2906
|
/**
|
|
2852
2907
|
* Low-level typed REST client with authentication, retries, and localhost failover.
|
|
2853
2908
|
*
|
|
@@ -3332,6 +3387,12 @@ export class DeeplineClient {
|
|
|
3332
3387
|
options.includeSearchDebug ? 'true' : 'false',
|
|
3333
3388
|
);
|
|
3334
3389
|
params.set('search_mode', options.searchMode ?? 'v2');
|
|
3390
|
+
if (options.limit !== undefined) {
|
|
3391
|
+
params.set('limit', String(options.limit));
|
|
3392
|
+
}
|
|
3393
|
+
if (options.offset !== undefined) {
|
|
3394
|
+
params.set('offset', String(options.offset));
|
|
3395
|
+
}
|
|
3335
3396
|
if (options.categories?.trim()) {
|
|
3336
3397
|
params.set('categories', options.categories.trim());
|
|
3337
3398
|
}
|
|
@@ -4792,8 +4853,25 @@ export class DeeplineClient {
|
|
|
4792
4853
|
const query = params.size > 0 ? `?${params.toString()}` : '';
|
|
4793
4854
|
const response = await this.http.get<Record<string, unknown>>(
|
|
4794
4855
|
`/api/v2/runs/${encodeURIComponent(runId)}${query}`,
|
|
4856
|
+
{ headers: { 'x-deepline-run-status': '1' } },
|
|
4795
4857
|
);
|
|
4858
|
+
const responseRunStatus = isPlayRunPackage(response)
|
|
4859
|
+
? response.runStatus
|
|
4860
|
+
: isPlayRunPackage(response.package)
|
|
4861
|
+
? response.package.runStatus
|
|
4862
|
+
: response.runStatus;
|
|
4863
|
+
const responseOverview = isPlayRunPackage(response)
|
|
4864
|
+
? response.overview
|
|
4865
|
+
: isPlayRunPackage(response.package)
|
|
4866
|
+
? response.package.overview
|
|
4867
|
+
: response.overview;
|
|
4796
4868
|
const status = normalizePlayStatus(response);
|
|
4869
|
+
if (isRecord(responseRunStatus)) {
|
|
4870
|
+
status.runStatus = responseRunStatus as unknown as RunGetStatus;
|
|
4871
|
+
}
|
|
4872
|
+
if (isRecord(responseOverview)) {
|
|
4873
|
+
status.overview = responseOverview as unknown as RunLifecycleStatus;
|
|
4874
|
+
}
|
|
4797
4875
|
if (options?.failedLogs !== true || status.status !== 'failed') {
|
|
4798
4876
|
return status;
|
|
4799
4877
|
}
|
|
@@ -4844,7 +4922,10 @@ export class DeeplineClient {
|
|
|
4844
4922
|
};
|
|
4845
4923
|
}
|
|
4846
4924
|
|
|
4847
|
-
/**
|
|
4925
|
+
/** Fetch only the legacy scalar lifecycle status for a run.
|
|
4926
|
+
*
|
|
4927
|
+
* @deprecated Use `client.runs.get(runId)` for the canonical run read.
|
|
4928
|
+
*/
|
|
4848
4929
|
async getRunLifecycleStatus(runId: string): Promise<RunLifecycleStatus> {
|
|
4849
4930
|
const response = await this.http.get<RunLifecycleStatus>(
|
|
4850
4931
|
`/api/v2/runs/${encodeURIComponent(runId)}/status`,
|
|
@@ -5052,9 +5133,7 @@ export class DeeplineClient {
|
|
|
5052
5133
|
if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
5053
5134
|
continue;
|
|
5054
5135
|
}
|
|
5055
|
-
return await this.getRunStatus(status.runId || runId)
|
|
5056
|
-
() => state.latest ?? playRunStatusFromState(state),
|
|
5057
|
-
);
|
|
5136
|
+
return await this.getRunStatus(status.runId || runId);
|
|
5058
5137
|
}
|
|
5059
5138
|
if (options?.signal?.aborted) {
|
|
5060
5139
|
throw new DeeplineError('Run observation aborted.', undefined, 'ABORTED');
|
|
@@ -5110,6 +5189,7 @@ export class DeeplineClient {
|
|
|
5110
5189
|
for (;;) {
|
|
5111
5190
|
const connectedAt = Date.now();
|
|
5112
5191
|
let sawEvent = false;
|
|
5192
|
+
let terminalRunId: string | null = null;
|
|
5113
5193
|
let endedReason = 'stream window ended before a terminal event';
|
|
5114
5194
|
try {
|
|
5115
5195
|
for await (const event of this.streamPlayRunEvents(runId, {
|
|
@@ -5122,9 +5202,8 @@ export class DeeplineClient {
|
|
|
5122
5202
|
if (!status || !TERMINAL_PLAY_STATUSES.has(status.status)) {
|
|
5123
5203
|
continue;
|
|
5124
5204
|
}
|
|
5125
|
-
|
|
5126
|
-
|
|
5127
|
-
);
|
|
5205
|
+
terminalRunId = status.runId || runId;
|
|
5206
|
+
break;
|
|
5128
5207
|
}
|
|
5129
5208
|
} catch (error) {
|
|
5130
5209
|
if (options?.signal?.aborted || !isTransientPlayStreamError(error)) {
|
|
@@ -5133,18 +5212,16 @@ export class DeeplineClient {
|
|
|
5133
5212
|
endedReason = error instanceof Error ? error.message : String(error);
|
|
5134
5213
|
}
|
|
5135
5214
|
|
|
5215
|
+
if (terminalRunId) {
|
|
5216
|
+
return await this.getRunStatus(terminalRunId);
|
|
5217
|
+
}
|
|
5218
|
+
|
|
5136
5219
|
// Window ended without a terminal event. The run may have finished
|
|
5137
5220
|
// during the gap — re-check durable status once before reconnecting.
|
|
5138
|
-
//
|
|
5139
|
-
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
} catch (error) {
|
|
5143
|
-
if (!isTransientPlayStreamError(error)) {
|
|
5144
|
-
throw error;
|
|
5145
|
-
}
|
|
5146
|
-
}
|
|
5147
|
-
if (refreshed && TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
|
|
5221
|
+
// `runs.get` owns the durable snapshot; its optional status evidence is
|
|
5222
|
+
// additive and must not prevent old servers from ending this stream.
|
|
5223
|
+
const refreshed = await this.getRunStatus(runId);
|
|
5224
|
+
if (TERMINAL_PLAY_STATUSES.has(refreshed.status)) {
|
|
5148
5225
|
return refreshed;
|
|
5149
5226
|
}
|
|
5150
5227
|
|
|
@@ -5536,6 +5613,34 @@ export class DeeplineClient {
|
|
|
5536
5613
|
};
|
|
5537
5614
|
}
|
|
5538
5615
|
|
|
5616
|
+
/** Read definition-level settings, independently of code revisions. */
|
|
5617
|
+
async getPlaySettings(playName: string): Promise<PlaySettingsResult> {
|
|
5618
|
+
return this.http.get(
|
|
5619
|
+
`/api/v2/plays/${encodeURIComponent(playName)}/settings`,
|
|
5620
|
+
);
|
|
5621
|
+
}
|
|
5622
|
+
|
|
5623
|
+
/** Retry-safe field patch; omitted fields retain their current value. */
|
|
5624
|
+
async updatePlaySettings(
|
|
5625
|
+
playName: string,
|
|
5626
|
+
patch: PlaySettingsPatch,
|
|
5627
|
+
): Promise<PlaySettingsResult> {
|
|
5628
|
+
return this.http.patch(
|
|
5629
|
+
`/api/v2/plays/${encodeURIComponent(playName)}/settings`,
|
|
5630
|
+
patch,
|
|
5631
|
+
);
|
|
5632
|
+
}
|
|
5633
|
+
|
|
5634
|
+
/** List one bounded page of existing Play locations; continue with cursor. */
|
|
5635
|
+
async listPlaySettingsPaths(
|
|
5636
|
+
options: { root?: 'person' | 'team'; limit?: number; cursor?: string } = {},
|
|
5637
|
+
): Promise<PlaySettingsPathsPage> {
|
|
5638
|
+
const query = new URLSearchParams();
|
|
5639
|
+
for (const [key, value] of Object.entries(options))
|
|
5640
|
+
if (value !== undefined) query.set(key, String(value));
|
|
5641
|
+
return this.http.get(`/api/v2/plays/settings/paths?${query}`);
|
|
5642
|
+
}
|
|
5643
|
+
|
|
5539
5644
|
/** Set whether an org-owned Play sorts before unpinned Plays. */
|
|
5540
5645
|
async setPlayPinned(
|
|
5541
5646
|
playName: string,
|
|
@@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
3
3
|
import { homedir } from 'node:os';
|
|
4
4
|
import { dirname, join } from 'node:path';
|
|
5
5
|
import { sdkCliStateDirPath } from './config.js';
|
|
6
|
+
import { deploymentProtectionHeaders } from './deployment-protection.js';
|
|
6
7
|
|
|
7
8
|
export type SdkCompatibilityStatus =
|
|
8
9
|
| 'current'
|
|
@@ -334,6 +335,7 @@ export async function checkSdkCompatibility(
|
|
|
334
335
|
const response = await fetch(url, {
|
|
335
336
|
method: 'GET',
|
|
336
337
|
headers: {
|
|
338
|
+
...deploymentProtectionHeaders(),
|
|
337
339
|
'User-Agent': `deepline-ts-sdk/${SDK_VERSION}`,
|
|
338
340
|
'X-Deepline-SDK-Version': SDK_VERSION,
|
|
339
341
|
'X-Deepline-API-Major': String(SDK_API_MAJOR),
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Headers shared by SDK requests and the CLI's directory deployment body. */
|
|
2
|
+
export function deploymentProtectionHeaders(): Record<string, string> {
|
|
3
|
+
const token =
|
|
4
|
+
typeof process !== 'undefined'
|
|
5
|
+
? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
|
|
6
|
+
: undefined;
|
|
7
|
+
return token ? { 'x-vercel-protection-bypass': token } : {};
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export const fetchWithDeploymentProtection: typeof fetch = (input, init) => {
|
|
11
|
+
const headers = new Headers(
|
|
12
|
+
input instanceof Request ? input.headers : undefined,
|
|
13
|
+
);
|
|
14
|
+
new Headers(init?.headers).forEach((value, name) => headers.set(name, value));
|
|
15
|
+
for (const [name, value] of Object.entries(deploymentProtectionHeaders())) {
|
|
16
|
+
headers.set(name, value);
|
|
17
|
+
}
|
|
18
|
+
return fetch(input, { ...init, headers });
|
|
19
|
+
};
|
|
@@ -35,6 +35,7 @@ import { SDK_API_CONTRACT, SDK_VERSION } from './version.js';
|
|
|
35
35
|
import type { LiveEventEnvelope } from './types.js';
|
|
36
36
|
import { detectAgentRuntime, isCoworkLikeSandbox } from './agent-runtime.js';
|
|
37
37
|
import { readSdkSkillsLocalVersion } from './skills-version.js';
|
|
38
|
+
import { deploymentProtectionHeaders } from './deployment-protection.js';
|
|
38
39
|
import {
|
|
39
40
|
ABSURD_RELEASE_OVERRIDE_HEADER,
|
|
40
41
|
COORDINATOR_INTERNAL_TOKEN_HEADER,
|
|
@@ -266,18 +267,12 @@ export class HttpClient {
|
|
|
266
267
|
'X-Deepline-API-Contract': SDK_API_CONTRACT,
|
|
267
268
|
'X-Deepline-Run-Result-Shape': 'canonical',
|
|
268
269
|
...extra,
|
|
270
|
+
...deploymentProtectionHeaders(),
|
|
269
271
|
};
|
|
270
272
|
const skillsVersion = this.readSkillsVersionHeader();
|
|
271
273
|
if (skillsVersion) {
|
|
272
274
|
headers['X-Deepline-Skills-Version'] = skillsVersion;
|
|
273
275
|
}
|
|
274
|
-
const bypassToken =
|
|
275
|
-
typeof process !== 'undefined'
|
|
276
|
-
? process.env?.VERCEL_PROTECTION_BYPASS_TOKEN
|
|
277
|
-
: undefined;
|
|
278
|
-
if (bypassToken) {
|
|
279
|
-
headers['x-vercel-protection-bypass'] = bypassToken;
|
|
280
|
-
}
|
|
281
276
|
const playArtifactR2Prefix =
|
|
282
277
|
typeof process !== 'undefined'
|
|
283
278
|
? process.env?.DEEPLINE_PLAY_ARTIFACT_R2_PREFIX
|
|
@@ -706,6 +701,7 @@ export class HttpClient {
|
|
|
706
701
|
| 'onResponse'
|
|
707
702
|
| 'allowNotFound'
|
|
708
703
|
| 'idempotencyKey'
|
|
704
|
+
| 'headers'
|
|
709
705
|
>,
|
|
710
706
|
): Promise<T> {
|
|
711
707
|
return this.request<T>(path, {
|
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.214',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import type { PlaySettings } from '../../shared_libs/plays/settings-contract';
|
|
1
2
|
import type { PlayCompilerManifest } from '../../shared_libs/plays/compiler-manifest';
|
|
3
|
+
import type { RunGetStatus, RunLifecycleStatus } from './client.js';
|
|
2
4
|
import type { PlayGrepMatch } from '../../shared_libs/plays/play-grep-search';
|
|
3
5
|
import type { PlayRuntimeSelection } from '../../shared_libs/play-runtime/runtime-environment';
|
|
4
6
|
import type {
|
|
@@ -466,6 +468,10 @@ export type InferenceQuote = {
|
|
|
466
468
|
export interface ToolSearchOptions {
|
|
467
469
|
/** Free-text search query. */
|
|
468
470
|
query?: string;
|
|
471
|
+
/** Maximum number of ranked tools to return (1-50; defaults to 20). */
|
|
472
|
+
limit?: number;
|
|
473
|
+
/** Zero-based offset into the ranked result set. */
|
|
474
|
+
offset?: number;
|
|
469
475
|
/** Comma-separated category filter such as `company_search` or `email_finder`. */
|
|
470
476
|
categories?: string;
|
|
471
477
|
/** Optional explicit search terms used by agent/CLI callers. */
|
|
@@ -488,6 +494,12 @@ export interface ToolSearchResult {
|
|
|
488
494
|
count?: number;
|
|
489
495
|
/** Matching count within the selected retrieval set. Check total_is_exact. */
|
|
490
496
|
total?: number;
|
|
497
|
+
/** Zero-based offset used for this page. */
|
|
498
|
+
offset?: number;
|
|
499
|
+
/** Maximum number of ranked tools requested for this page. */
|
|
500
|
+
limit?: number;
|
|
501
|
+
/** Offset to pass for the next page, when more results are in this result set. */
|
|
502
|
+
next_offset?: number;
|
|
491
503
|
/** False when indexed retrieval uses a bounded candidate set. */
|
|
492
504
|
total_is_exact?: boolean;
|
|
493
505
|
/** True when indexed retrieval may have additional matching tools. */
|
|
@@ -729,6 +741,12 @@ export interface PlayRunPackage {
|
|
|
729
741
|
schemaVersion: 1;
|
|
730
742
|
/** Package discriminator. */
|
|
731
743
|
kind: 'play_run';
|
|
744
|
+
/** Ledger position of the detailed snapshot, when provided by this server. */
|
|
745
|
+
source?: { ledgerNextSeq: number | null };
|
|
746
|
+
/** Legacy status evidence returned by older servers. */
|
|
747
|
+
overview?: RunLifecycleStatus;
|
|
748
|
+
/** Optional bounded live evidence from the existing run status projection. */
|
|
749
|
+
runStatus?: RunGetStatus;
|
|
732
750
|
/** Run identity, status, timing, and dashboard metadata. */
|
|
733
751
|
run: {
|
|
734
752
|
id: string;
|
|
@@ -834,6 +852,10 @@ export type PlayRunOutcome =
|
|
|
834
852
|
export interface PlayStatus {
|
|
835
853
|
/** Public play-run identifier. */
|
|
836
854
|
runId: string;
|
|
855
|
+
/** Legacy status evidence returned by older servers. */
|
|
856
|
+
overview?: RunLifecycleStatus;
|
|
857
|
+
/** Additive bounded evidence when returned by the current server. */
|
|
858
|
+
runStatus?: RunGetStatus;
|
|
837
859
|
/** Public Deepline play-run API version. */
|
|
838
860
|
apiVersion?: number;
|
|
839
861
|
/** Saved play name for this run, when available. */
|
|
@@ -1310,6 +1332,8 @@ export interface PlaySheetSummary {
|
|
|
1310
1332
|
* Returned by {@link DeeplineClient.getPlay}.
|
|
1311
1333
|
*/
|
|
1312
1334
|
export interface PlayDefinitionDetail {
|
|
1335
|
+
/** Definition settings; never attached to a revision. */
|
|
1336
|
+
settings?: PlaySettings;
|
|
1313
1337
|
/** Convex document ID. */
|
|
1314
1338
|
_id: string;
|
|
1315
1339
|
/** Stable registry key. */
|
|
@@ -1415,6 +1439,7 @@ export type PlayCostEstimate =
|
|
|
1415
1439
|
};
|
|
1416
1440
|
|
|
1417
1441
|
export interface PlayListItem {
|
|
1442
|
+
settings?: PlaySettings;
|
|
1418
1443
|
playKey?: string;
|
|
1419
1444
|
reference?: string;
|
|
1420
1445
|
name: string;
|
|
@@ -939,6 +939,24 @@
|
|
|
939
939
|
"failureAlert": "convex_action_wrapper",
|
|
940
940
|
"domainReportRequired": true
|
|
941
941
|
},
|
|
942
|
+
{
|
|
943
|
+
"id": "convex.refresh-tool-usage-popularity",
|
|
944
|
+
"name": "Refresh tool usage popularity",
|
|
945
|
+
"scheduler": "convex_cron",
|
|
946
|
+
"cadence": "37 0 * * *",
|
|
947
|
+
"timezone": "UTC",
|
|
948
|
+
"source": "packages/convex/crons.ts",
|
|
949
|
+
"locator": "refresh tool usage popularity snapshot",
|
|
950
|
+
"owner": "integrations",
|
|
951
|
+
"criticality": "operational",
|
|
952
|
+
"purpose": "Sum distinct user-days per tool over the last 28 complete UTC days into the snapshot tools search uses as a ranking prior, then delete user-day rows past retention.",
|
|
953
|
+
"successDefinition": "A new snapshot row is written and expired user-day rows are drained; an empty snapshot is valid before any usage is folded.",
|
|
954
|
+
"failureDefinition": "Reading daily counts, writing the snapshot, or draining expired rows fails, leaving search on the previous snapshot.",
|
|
955
|
+
"startGraceMs": 120000,
|
|
956
|
+
"runTimeoutMs": 1800000,
|
|
957
|
+
"failureAlert": "convex_action_wrapper",
|
|
958
|
+
"domainReportRequired": true
|
|
959
|
+
},
|
|
942
960
|
{
|
|
943
961
|
"id": "convex.salesforce-oauth-refresh-maintenance",
|
|
944
962
|
"name": "Maintain Salesforce OAuth refresh tokens",
|
|
@@ -14,6 +14,7 @@ export const MANUALLY_RUNNABLE_CONVEX_CRON_JOB_IDS = [
|
|
|
14
14
|
'convex.cleanup-expired-api-key-claims',
|
|
15
15
|
'convex.cleanup-expired-analytics-ip2company-cache',
|
|
16
16
|
'convex.cleanup-expired-domain-health-cache',
|
|
17
|
+
'convex.refresh-tool-usage-popularity',
|
|
17
18
|
'convex.salesforce-oauth-refresh-maintenance',
|
|
18
19
|
'convex.monitor-fleet-convergence-tick',
|
|
19
20
|
'convex.monitor-provider-repair-recovery-tick',
|
|
@@ -2,6 +2,8 @@ import { spawnSync } from 'node:child_process';
|
|
|
2
2
|
|
|
3
3
|
const FLY_REGISTRY_COMMAND_TIMEOUT_MS = 30_000;
|
|
4
4
|
const RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS = 10_000;
|
|
5
|
+
export const FLY_REGISTRY_MANIFEST_ATTEMPTS = 3;
|
|
6
|
+
const FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS = 1_500;
|
|
5
7
|
|
|
6
8
|
type FlyRegistryCommandResult = Readonly<{
|
|
7
9
|
error?: Error & { code?: string };
|
|
@@ -59,15 +61,39 @@ export function runtimeImageManifestExists(
|
|
|
59
61
|
options: {
|
|
60
62
|
timeoutMs?: number;
|
|
61
63
|
spawnCommand?: FlyRegistryCommandSpawn;
|
|
64
|
+
sleep?: (ms: number) => void;
|
|
62
65
|
} = {},
|
|
63
66
|
): boolean {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
)
|
|
67
|
+
const timeoutMs = options.timeoutMs ?? RUNTIME_IMAGE_MANIFEST_TIMEOUT_MS;
|
|
68
|
+
const sleep = options.sleep ?? sleepSync;
|
|
69
|
+
for (
|
|
70
|
+
let attempt = 1;
|
|
71
|
+
attempt <= FLY_REGISTRY_MANIFEST_ATTEMPTS;
|
|
72
|
+
attempt += 1
|
|
73
|
+
) {
|
|
74
|
+
try {
|
|
75
|
+
return run(
|
|
76
|
+
'docker',
|
|
77
|
+
['manifest', 'inspect', image],
|
|
78
|
+
true,
|
|
79
|
+
timeoutMs,
|
|
80
|
+
options.spawnCommand,
|
|
81
|
+
);
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (
|
|
84
|
+
!(error instanceof Error) ||
|
|
85
|
+
(error as NodeJS.ErrnoException).code !== 'ETIMEDOUT' ||
|
|
86
|
+
attempt === FLY_REGISTRY_MANIFEST_ATTEMPTS
|
|
87
|
+
) {
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
console.warn(
|
|
91
|
+
`${error.message} Retrying manifest probe (${attempt}/${FLY_REGISTRY_MANIFEST_ATTEMPTS}).`,
|
|
92
|
+
);
|
|
93
|
+
sleep(FLY_REGISTRY_MANIFEST_RETRY_DELAY_MS * attempt);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
throw new Error('Fly registry manifest probe exhausted without a result.');
|
|
71
97
|
}
|
|
72
98
|
|
|
73
99
|
export const FLY_REGISTRY_AUTH_ATTEMPTS = 3;
|
|
@@ -3614,8 +3614,15 @@ async function main() {
|
|
|
3614
3614
|
process.exit(1);
|
|
3615
3615
|
},
|
|
3616
3616
|
});
|
|
3617
|
+
// The Daytona watchdog is a separate process and can keep runner liveness
|
|
3618
|
+
// advancing while synchronous customer JavaScript occupies this process's
|
|
3619
|
+
// event loop. Hand off only after the existing durable `waiting` fence and
|
|
3620
|
+
// immediately before the marker that starts customer execution.
|
|
3621
|
+
const lastHeartbeatAtMs = heartbeat?.lastSuccessAtMs();
|
|
3622
|
+
heartbeat?.stop();
|
|
3617
3623
|
writeRunnerRuntimeStartedMarker(
|
|
3618
3624
|
process.env.DEEPLINE_PLAY_RUNNER_RUNTIME_STARTED_PATH,
|
|
3625
|
+
lastHeartbeatAtMs === undefined ? {} : { lastHeartbeatAtMs },
|
|
3619
3626
|
);
|
|
3620
3627
|
let result: PlayRunnerResult | null = null;
|
|
3621
3628
|
let executionFailed = false;
|
|
@@ -3791,7 +3798,7 @@ export async function materializeTransportedArtifactWhileRunnerParks(
|
|
|
3791
3798
|
|
|
3792
3799
|
export function writeRunnerRuntimeStartedMarker(
|
|
3793
3800
|
path: string | null | undefined,
|
|
3794
|
-
detail: { pid?: number; at?: number } = {},
|
|
3801
|
+
detail: { pid?: number; at?: number; lastHeartbeatAtMs?: number } = {},
|
|
3795
3802
|
): void {
|
|
3796
3803
|
const runtimeStartedPath = path?.trim();
|
|
3797
3804
|
if (!runtimeStartedPath) return;
|
|
@@ -3801,6 +3808,9 @@ export function writeRunnerRuntimeStartedMarker(
|
|
|
3801
3808
|
started: true,
|
|
3802
3809
|
pid: detail.pid ?? process.pid,
|
|
3803
3810
|
at: detail.at ?? Date.now(),
|
|
3811
|
+
...(typeof detail.lastHeartbeatAtMs === 'number'
|
|
3812
|
+
? { lastHeartbeatAtMs: detail.lastHeartbeatAtMs }
|
|
3813
|
+
: {}),
|
|
3804
3814
|
}),
|
|
3805
3815
|
'utf-8',
|
|
3806
3816
|
);
|
|
@@ -100,6 +100,8 @@ export interface RunnerHeartbeatLoopHandle {
|
|
|
100
100
|
executionAcknowledged: Promise<void>;
|
|
101
101
|
/** Trigger an immediate heartbeat while waiting for the park acknowledgement. */
|
|
102
102
|
requestHeartbeat: () => void;
|
|
103
|
+
/** Last durable heartbeat timestamp, used for an external liveness handoff. */
|
|
104
|
+
lastSuccessAtMs: () => number;
|
|
103
105
|
}
|
|
104
106
|
|
|
105
107
|
const APP_RUNTIME_COMPAT_PATH = '/api/v2/plays/runtime';
|
|
@@ -420,5 +422,6 @@ export function startRunnerHeartbeatLoop(
|
|
|
420
422
|
firstHeartbeat,
|
|
421
423
|
executionAcknowledged,
|
|
422
424
|
requestHeartbeat: tick,
|
|
425
|
+
lastSuccessAtMs: () => lastSuccessAt,
|
|
423
426
|
};
|
|
424
427
|
}
|
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
RUNTIME_SANDBOX_MEMORY_ABORT_CODE,
|
|
28
28
|
RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE,
|
|
29
29
|
RUNTIME_SANDBOX_OOM_MESSAGE,
|
|
30
|
+
RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE,
|
|
30
31
|
} from '../../run-failure';
|
|
31
32
|
import {
|
|
32
33
|
buildDaytonaPayloadMaterializerSource,
|
|
@@ -293,6 +294,41 @@ function watchdogMemoryAbortDiagnostic() {
|
|
|
293
294
|
return null;
|
|
294
295
|
}
|
|
295
296
|
|
|
297
|
+
function watchdogRunnerHeartbeatUnavailableDiagnostic() {
|
|
298
|
+
try {
|
|
299
|
+
if (!terminationDiagnosticPath) return null;
|
|
300
|
+
const diagnostic = JSON.parse(
|
|
301
|
+
fs.readFileSync(terminationDiagnosticPath, 'utf8'),
|
|
302
|
+
);
|
|
303
|
+
if (
|
|
304
|
+
!diagnostic ||
|
|
305
|
+
diagnostic.schemaVersion !== 1 ||
|
|
306
|
+
diagnostic.reason !== 'runner_heartbeat_unreachable'
|
|
307
|
+
) {
|
|
308
|
+
return null;
|
|
309
|
+
}
|
|
310
|
+
return {
|
|
311
|
+
heartbeatAttempt: Number.isSafeInteger(diagnostic.heartbeatAttempt)
|
|
312
|
+
? diagnostic.heartbeatAttempt
|
|
313
|
+
: null,
|
|
314
|
+
heartbeatAgeMs: Number.isFinite(diagnostic.heartbeatAgeMs)
|
|
315
|
+
? Math.max(0, diagnostic.heartbeatAgeMs)
|
|
316
|
+
: null,
|
|
317
|
+
leaseMs: Number.isFinite(diagnostic.leaseMs)
|
|
318
|
+
? Math.max(0, diagnostic.leaseMs)
|
|
319
|
+
: null,
|
|
320
|
+
lastErrorCode:
|
|
321
|
+
typeof diagnostic.lastErrorCode === 'string' &&
|
|
322
|
+
/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+|TimeoutError|redirect_[0-9]{3}|http_[0-9]{3}|protocol_error|unknown)$/.test(
|
|
323
|
+
diagnostic.lastErrorCode,
|
|
324
|
+
)
|
|
325
|
+
? diagnostic.lastErrorCode
|
|
326
|
+
: 'unknown',
|
|
327
|
+
};
|
|
328
|
+
} catch {}
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
|
|
296
332
|
function memoryAbortCause(diagnostic) {
|
|
297
333
|
const parts = [
|
|
298
334
|
${JSON.stringify(`${RUNTIME_SANDBOX_MEMORY_ABORT_CODE}: ${RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE}`)},
|
|
@@ -373,6 +409,7 @@ async function main() {
|
|
|
373
409
|
// SIGKILL does not collapse into the wrapper's generic exit code 1.
|
|
374
410
|
const sandboxKilled = exitCode === 137 || watchdogObservedChildSigkill();
|
|
375
411
|
const memoryAbort = watchdogMemoryAbortDiagnostic();
|
|
412
|
+
const heartbeatUnavailable = watchdogRunnerHeartbeatUnavailableDiagnostic();
|
|
376
413
|
const sandboxOom =
|
|
377
414
|
!memoryAbort &&
|
|
378
415
|
sandboxKilled &&
|
|
@@ -383,6 +420,10 @@ async function main() {
|
|
|
383
420
|
? memoryAbortCause(memoryAbort)
|
|
384
421
|
: sandboxOom
|
|
385
422
|
? 'RUNTIME_SANDBOX_OOM: ' + ${JSON.stringify(RUNTIME_SANDBOX_OOM_MESSAGE)}
|
|
423
|
+
: heartbeatUnavailable
|
|
424
|
+
? 'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' +
|
|
425
|
+
${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)} +
|
|
426
|
+
' heartbeat=' + JSON.stringify(heartbeatUnavailable)
|
|
386
427
|
: sandboxKilled
|
|
387
428
|
? 'RUNTIME_SANDBOX_KILLED: ' + ${JSON.stringify(RUNTIME_SANDBOX_KILLED_MESSAGE)}
|
|
388
429
|
:
|
|
@@ -408,6 +449,15 @@ async function main() {
|
|
|
408
449
|
message: ${JSON.stringify(RUNTIME_SANDBOX_MEMORY_ABORT_MESSAGE)},
|
|
409
450
|
retryable: false,
|
|
410
451
|
cause: synthesizedError,
|
|
452
|
+
}] }
|
|
453
|
+
: {}),
|
|
454
|
+
...(heartbeatUnavailable && !runtimeLimitExceeded && !memoryAbort && !sandboxOom
|
|
455
|
+
? { errors: [{
|
|
456
|
+
code: 'RUNTIME_CONTROL_PLANE_UNAVAILABLE',
|
|
457
|
+
phase: 'infrastructure',
|
|
458
|
+
message: ${JSON.stringify(RUNTIME_CONTROL_PLANE_UNAVAILABLE_MESSAGE)},
|
|
459
|
+
retryable: false,
|
|
460
|
+
cause: synthesizedError,
|
|
411
461
|
}] }
|
|
412
462
|
: {}),
|
|
413
463
|
...(sandboxOom
|
|
@@ -428,7 +478,9 @@ async function main() {
|
|
|
428
478
|
cause: synthesizedError,
|
|
429
479
|
}] }
|
|
430
480
|
: {}),
|
|
431
|
-
logs:
|
|
481
|
+
logs: heartbeatUnavailable
|
|
482
|
+
? ['[runtime.failure_diagnostic] ' + JSON.stringify({ event: 'runner_heartbeat_unavailable', ...heartbeatUnavailable })]
|
|
483
|
+
: [],
|
|
432
484
|
stats: {},
|
|
433
485
|
steps: [],
|
|
434
486
|
checkpointDisposition: '${RUNNER_TERMINAL_CHECKPOINT_DISPOSITION.ABSENT}',
|