deepline 0.1.198 → 0.1.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +5 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +51 -17
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/README.md +1 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +16 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +40 -9
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +108 -39
- package/dist/bundling-sources/sdk/src/http.ts +5 -2
- package/dist/bundling-sources/sdk/src/play.ts +1 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +7 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/worker-play-entry.ts +11 -55
- package/dist/bundling-sources/shared_libs/play-runtime/adaptive-tool-dispatcher.ts +355 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +172 -73
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1794 -377
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +67 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +161 -39
- package/dist/bundling-sources/shared_libs/play-runtime/governor/adaptive-admission.ts +218 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +115 -25
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +17 -17
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/live-events.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +265 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +135 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-input.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +0 -3
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +10 -1
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +9 -15
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +262 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-heartbeat-supervisor.ts +64 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +76 -8
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +59 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +269 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +433 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +227 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +882 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +573 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/bundle.ts +264 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/index.ts +33 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runner-events.ts +35 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +53 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +0 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +459 -103
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-infra-mode.ts +23 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +23 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +16 -24
- package/dist/bundling-sources/shared_libs/play-runtime/serial-task-queue.ts +26 -0
- package/dist/bundling-sources/shared_libs/play-runtime/submit-limits.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/vercel-protection.ts +81 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +54 -9
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +17 -3
- package/dist/bundling-sources/shared_libs/play-runtime/worker-api-types.ts +0 -20
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +105 -4
- package/dist/bundling-sources/shared_libs/plays/row-identity.ts +4 -11
- package/dist/cli/index.js +441 -101
- package/dist/cli/index.mjs +441 -101
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +8 -8
- package/dist/index.mjs +8 -8
- package/dist/plays/bundle-play-file.mjs +15 -20
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-diagnosis.ts +0 -321
- package/dist/bundling-sources/shared_libs/play-runtime/hatchet-cold-execution-target.ts +0 -158
- package/dist/bundling-sources/shared_libs/temporal/constants.ts +0 -39
- package/dist/bundling-sources/shared_libs/temporal/preview-config.ts +0 -150
package/dist/index.d.mts
CHANGED
|
@@ -3640,7 +3640,7 @@ interface DeeplinePlayRuntimeContext {
|
|
|
3640
3640
|
*
|
|
3641
3641
|
* // Poll status
|
|
3642
3642
|
* const status = await job.status();
|
|
3643
|
-
* console.log(status.
|
|
3643
|
+
* console.log(status.status); // 'running'
|
|
3644
3644
|
*
|
|
3645
3645
|
* // Stream logs until completion
|
|
3646
3646
|
* const finalStatus = await job.tail({
|
package/dist/index.d.ts
CHANGED
|
@@ -3640,7 +3640,7 @@ interface DeeplinePlayRuntimeContext {
|
|
|
3640
3640
|
*
|
|
3641
3641
|
* // Poll status
|
|
3642
3642
|
* const status = await job.status();
|
|
3643
|
-
* console.log(status.
|
|
3643
|
+
* console.log(status.status); // 'running'
|
|
3644
3644
|
*
|
|
3645
3645
|
* // Stream logs until completion
|
|
3646
3646
|
* const finalStatus = await job.tail({
|
package/dist/index.js
CHANGED
|
@@ -422,10 +422,10 @@ var SDK_RELEASE = {
|
|
|
422
422
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
423
423
|
// fields shipped in 0.1.153.
|
|
424
424
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
425
|
-
version: "0.1.
|
|
425
|
+
version: "0.1.200",
|
|
426
426
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
427
427
|
supportPolicy: {
|
|
428
|
-
latest: "0.1.
|
|
428
|
+
latest: "0.1.200",
|
|
429
429
|
minimumSupported: "0.1.53",
|
|
430
430
|
deprecatedBelow: "0.1.53",
|
|
431
431
|
commandMinimumSupported: [
|
|
@@ -866,7 +866,10 @@ var HttpClient = class {
|
|
|
866
866
|
* @param path - API path (e.g. `"/api/v2/tools"`)
|
|
867
867
|
*/
|
|
868
868
|
async get(path) {
|
|
869
|
-
return this.request(path, {
|
|
869
|
+
return this.request(path, {
|
|
870
|
+
method: "GET",
|
|
871
|
+
retryApiErrors: true
|
|
872
|
+
});
|
|
870
873
|
}
|
|
871
874
|
async *streamSse(path, options) {
|
|
872
875
|
const url = `${this.config.baseUrl}${path}`;
|
|
@@ -1140,7 +1143,6 @@ function resolveTimingWindow(input) {
|
|
|
1140
1143
|
// ../shared_libs/play-runtime/run-terminal-source.ts
|
|
1141
1144
|
var PLAY_RUN_LEDGER_EVENT_SOURCES = [
|
|
1142
1145
|
"worker",
|
|
1143
|
-
"temporal",
|
|
1144
1146
|
"convex",
|
|
1145
1147
|
"coordinator",
|
|
1146
1148
|
"system"
|
|
@@ -4772,13 +4774,11 @@ function sanitizeIdentifierPart(value) {
|
|
|
4772
4774
|
function validateIdentifierPart(rawValue, label, maxLength) {
|
|
4773
4775
|
const sanitized = sanitizeIdentifierPart(rawValue);
|
|
4774
4776
|
if (!sanitized) {
|
|
4775
|
-
throw new Error(
|
|
4776
|
-
`${label} must contain at least one letter or number after normalization. Use only letters, numbers, underscores, or hyphens.`
|
|
4777
|
-
);
|
|
4777
|
+
throw new Error(`${label} must contain a letter or number.`);
|
|
4778
4778
|
}
|
|
4779
4779
|
if (sanitized.length > maxLength) {
|
|
4780
4780
|
throw new Error(
|
|
4781
|
-
`${label} is too long after normalization (${sanitized.length}/${maxLength})
|
|
4781
|
+
`${label} is too long after normalization (${sanitized.length}/${maxLength}): "${sanitized}".`
|
|
4782
4782
|
);
|
|
4783
4783
|
}
|
|
4784
4784
|
return sanitized;
|
package/dist/index.mjs
CHANGED
|
@@ -352,10 +352,10 @@ var SDK_RELEASE = {
|
|
|
352
352
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
353
353
|
// fields shipped in 0.1.153.
|
|
354
354
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
355
|
-
version: "0.1.
|
|
355
|
+
version: "0.1.200",
|
|
356
356
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
357
357
|
supportPolicy: {
|
|
358
|
-
latest: "0.1.
|
|
358
|
+
latest: "0.1.200",
|
|
359
359
|
minimumSupported: "0.1.53",
|
|
360
360
|
deprecatedBelow: "0.1.53",
|
|
361
361
|
commandMinimumSupported: [
|
|
@@ -796,7 +796,10 @@ var HttpClient = class {
|
|
|
796
796
|
* @param path - API path (e.g. `"/api/v2/tools"`)
|
|
797
797
|
*/
|
|
798
798
|
async get(path) {
|
|
799
|
-
return this.request(path, {
|
|
799
|
+
return this.request(path, {
|
|
800
|
+
method: "GET",
|
|
801
|
+
retryApiErrors: true
|
|
802
|
+
});
|
|
800
803
|
}
|
|
801
804
|
async *streamSse(path, options) {
|
|
802
805
|
const url = `${this.config.baseUrl}${path}`;
|
|
@@ -1070,7 +1073,6 @@ function resolveTimingWindow(input) {
|
|
|
1070
1073
|
// ../shared_libs/play-runtime/run-terminal-source.ts
|
|
1071
1074
|
var PLAY_RUN_LEDGER_EVENT_SOURCES = [
|
|
1072
1075
|
"worker",
|
|
1073
|
-
"temporal",
|
|
1074
1076
|
"convex",
|
|
1075
1077
|
"coordinator",
|
|
1076
1078
|
"system"
|
|
@@ -4702,13 +4704,11 @@ function sanitizeIdentifierPart(value) {
|
|
|
4702
4704
|
function validateIdentifierPart(rawValue, label, maxLength) {
|
|
4703
4705
|
const sanitized = sanitizeIdentifierPart(rawValue);
|
|
4704
4706
|
if (!sanitized) {
|
|
4705
|
-
throw new Error(
|
|
4706
|
-
`${label} must contain at least one letter or number after normalization. Use only letters, numbers, underscores, or hyphens.`
|
|
4707
|
-
);
|
|
4707
|
+
throw new Error(`${label} must contain a letter or number.`);
|
|
4708
4708
|
}
|
|
4709
4709
|
if (sanitized.length > maxLength) {
|
|
4710
4710
|
throw new Error(
|
|
4711
|
-
`${label} is too long after normalization (${sanitized.length}/${maxLength})
|
|
4711
|
+
`${label} is too long after normalization (${sanitized.length}/${maxLength}): "${sanitized}".`
|
|
4712
4712
|
);
|
|
4713
4713
|
}
|
|
4714
4714
|
return sanitized;
|
|
@@ -1733,22 +1733,22 @@ var PLAY_DEDUP_BACKENDS = {
|
|
|
1733
1733
|
|
|
1734
1734
|
// ../shared_libs/play-runtime/scheduler-backend.ts
|
|
1735
1735
|
var PLAY_SCHEDULER_BACKENDS = {
|
|
1736
|
-
temporal: "temporal",
|
|
1737
1736
|
cfWorkflows: "cf-workflows",
|
|
1737
|
+
/** Private legacy id retained only so old persisted rows can be interpreted. */
|
|
1738
|
+
postgres: "postgres",
|
|
1738
1739
|
/**
|
|
1739
|
-
*
|
|
1740
|
-
*
|
|
1740
|
+
* Postgres-native durable-execution scheduler (vendored Absurd engine). Runs
|
|
1741
|
+
* on the battle-tested run-state store without an external workflow engine.
|
|
1742
|
+
* Selected explicitly by the `absurd` profile.
|
|
1741
1743
|
*/
|
|
1742
|
-
|
|
1743
|
-
hatchet: "hatchet",
|
|
1744
|
+
absurd: "absurd",
|
|
1744
1745
|
inProcess: "in-process"
|
|
1745
1746
|
};
|
|
1746
1747
|
|
|
1747
1748
|
// ../shared_libs/play-runtime/providers.ts
|
|
1748
1749
|
var PLAY_RUNTIME_PROVIDER_IDS = {
|
|
1749
1750
|
workersEdge: "workers_edge",
|
|
1750
|
-
|
|
1751
|
-
local: "local"
|
|
1751
|
+
absurd: "absurd"
|
|
1752
1752
|
};
|
|
1753
1753
|
var PLAY_RUNTIME_PROVIDERS = {
|
|
1754
1754
|
workers_edge: {
|
|
@@ -1759,21 +1759,16 @@ var PLAY_RUNTIME_PROVIDERS = {
|
|
|
1759
1759
|
artifactKind: PLAY_ARTIFACT_KINDS.esmWorkers,
|
|
1760
1760
|
label: "Cloudflare Dynamic Workflows + Dynamic Workers + DO dedup"
|
|
1761
1761
|
},
|
|
1762
|
-
|
|
1763
|
-
id: PLAY_RUNTIME_PROVIDER_IDS.
|
|
1764
|
-
scheduler: PLAY_SCHEDULER_BACKENDS.
|
|
1762
|
+
absurd: {
|
|
1763
|
+
id: PLAY_RUNTIME_PROVIDER_IDS.absurd,
|
|
1764
|
+
scheduler: PLAY_SCHEDULER_BACKENDS.absurd,
|
|
1765
|
+
// Server runs execute on Daytona; the local-dev/test shape swaps the runner
|
|
1766
|
+
// axis to local_process (same as the local profile) via an explicit
|
|
1767
|
+
// runtimeBackend override at submit while tests drive local_process.
|
|
1765
1768
|
runner: PLAY_RUNTIME_BACKENDS.daytona,
|
|
1766
1769
|
dedup: PLAY_DEDUP_BACKENDS.inMemory,
|
|
1767
1770
|
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
1768
|
-
label: "
|
|
1769
|
-
},
|
|
1770
|
-
local: {
|
|
1771
|
-
id: PLAY_RUNTIME_PROVIDER_IDS.local,
|
|
1772
|
-
scheduler: PLAY_SCHEDULER_BACKENDS.temporal,
|
|
1773
|
-
runner: PLAY_RUNTIME_BACKENDS.localProcess,
|
|
1774
|
-
dedup: PLAY_DEDUP_BACKENDS.inMemory,
|
|
1775
|
-
artifactKind: PLAY_ARTIFACT_KINDS.cjsNode20,
|
|
1776
|
-
label: "Local Temporal scheduler + local subprocess runner (tests)"
|
|
1771
|
+
label: "Absurd (Postgres-native durable) scheduler + one-shot Daytona runner + receipt-backed in-process dedup"
|
|
1777
1772
|
}
|
|
1778
1773
|
};
|
|
1779
1774
|
function defaultPlayRuntimeProvider() {
|
|
@@ -1786,7 +1781,7 @@ function resolvePlayRuntimeProvider(override) {
|
|
|
1786
1781
|
return PLAY_RUNTIME_PROVIDERS[id];
|
|
1787
1782
|
}
|
|
1788
1783
|
throw new Error(
|
|
1789
|
-
`
|
|
1784
|
+
`Unsupported play runtime provider "${id}". Expected one of: ${Object.keys(
|
|
1790
1785
|
PLAY_RUNTIME_PROVIDERS
|
|
1791
1786
|
).join(", ")}.`
|
|
1792
1787
|
);
|
package/package.json
CHANGED
|
@@ -1,321 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
HATCHET_COLD_EXECUTION_PHASES,
|
|
3
|
-
HATCHET_COLD_EXECUTION_TARGET,
|
|
4
|
-
type HatchetColdExecutionPhaseId,
|
|
5
|
-
type HatchetColdExecutionPhaseSummary,
|
|
6
|
-
} from './hatchet-cold-execution-target';
|
|
7
|
-
|
|
8
|
-
export type HatchetColdExecutionPhaseSeverity =
|
|
9
|
-
| 'missing'
|
|
10
|
-
| 'ok'
|
|
11
|
-
| 'near_budget'
|
|
12
|
-
| 'over_budget';
|
|
13
|
-
|
|
14
|
-
export type HatchetColdExecutionPhaseDiagnosis = {
|
|
15
|
-
id: HatchetColdExecutionPhaseId;
|
|
16
|
-
phase: string;
|
|
17
|
-
budgetMs: number;
|
|
18
|
-
p95: number | null;
|
|
19
|
-
overBudgetMs: number | null;
|
|
20
|
-
severity: HatchetColdExecutionPhaseSeverity;
|
|
21
|
-
ownerModule: string;
|
|
22
|
-
ownerFiles: readonly string[];
|
|
23
|
-
interpretation: string;
|
|
24
|
-
nextAction: string;
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
export type HatchetColdExecutionDiagnosis = {
|
|
28
|
-
ok: boolean;
|
|
29
|
-
targetName: string;
|
|
30
|
-
thresholdMs: number;
|
|
31
|
-
overallP95: number | null;
|
|
32
|
-
overallOverThresholdMs: number | null;
|
|
33
|
-
bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
|
|
34
|
-
phases: HatchetColdExecutionPhaseDiagnosis[];
|
|
35
|
-
observations: string[];
|
|
36
|
-
nextActions: string[];
|
|
37
|
-
summary: string;
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
type DiagnoseInput = {
|
|
41
|
-
overallP95: number | null;
|
|
42
|
-
thresholdMs?: number;
|
|
43
|
-
phaseSummaries: readonly HatchetColdExecutionPhaseSummary[];
|
|
44
|
-
};
|
|
45
|
-
|
|
46
|
-
type PhaseOwnership = {
|
|
47
|
-
ownerModule: string;
|
|
48
|
-
ownerFiles: readonly string[];
|
|
49
|
-
interpretation: string;
|
|
50
|
-
nextAction: string;
|
|
51
|
-
};
|
|
52
|
-
|
|
53
|
-
const PHASE_OWNERSHIP: Record<HatchetColdExecutionPhaseId, PhaseOwnership> = {
|
|
54
|
-
submit: {
|
|
55
|
-
ownerModule: 'Play Run Launch Plan',
|
|
56
|
-
ownerFiles: [
|
|
57
|
-
'src/lib/plays/start-run.ts',
|
|
58
|
-
'src/lib/plays/launch/',
|
|
59
|
-
'scripts/check-hatchet-cold-execution-latency.ts',
|
|
60
|
-
],
|
|
61
|
-
interpretation:
|
|
62
|
-
'Submit time is being spent before the worker can claim the run: app validation, authority minting, start-row persistence, scheduler-row creation, or Hatchet dispatch.',
|
|
63
|
-
nextAction:
|
|
64
|
-
'Inspect routeTimings plus schema_ready / scheduler_row_submitted / run_no_wait_done deltas, then move non-launch work out of the hot path or overlap insert and dispatch.',
|
|
65
|
-
},
|
|
66
|
-
hatchet_dispatch_claim: {
|
|
67
|
-
ownerModule: 'Play Scheduler Adapter',
|
|
68
|
-
ownerFiles: [
|
|
69
|
-
'src/lib/plays/scheduler-backends/hatchet-dispatch.ts',
|
|
70
|
-
'src/lib/plays/scheduler-backends/hatchet-runtime-worker.ts',
|
|
71
|
-
'deploy/fly/hatchet/',
|
|
72
|
-
],
|
|
73
|
-
interpretation:
|
|
74
|
-
'Hatchet assignment or worker claim is slower than budget, so the run is waiting in orchestration before Daytona work begins.',
|
|
75
|
-
nextAction:
|
|
76
|
-
'Check Hatchet engine placement, worker replica health, worker slots, and dispatch mode before changing Daytona or payload transport.',
|
|
77
|
-
},
|
|
78
|
-
daytona_create: {
|
|
79
|
-
ownerModule: 'One-Shot Sandbox Runner',
|
|
80
|
-
ownerFiles: [
|
|
81
|
-
'apps/temporal-worker/src/runner-backends/backends/daytona.ts',
|
|
82
|
-
'apps/temporal-worker/src/runner-backends/backends/daytona-lifecycle.ts',
|
|
83
|
-
],
|
|
84
|
-
interpretation:
|
|
85
|
-
'Fresh Daytona sandbox creation is consuming the cold-execution budget before runner files upload.',
|
|
86
|
-
nextAction:
|
|
87
|
-
'Compare default image versus configured snapshot startup, then verify create timing in Daytona lifecycle events before touching artifact upload.',
|
|
88
|
-
},
|
|
89
|
-
daytona_upload: {
|
|
90
|
-
ownerModule: 'Play Artifact Target',
|
|
91
|
-
ownerFiles: [
|
|
92
|
-
'apps/temporal-worker/src/runner-backends/backends/daytona-payload-transport.ts',
|
|
93
|
-
'shared_libs/plays/artifact-transport.ts',
|
|
94
|
-
'apps/play-runner/src/entry.ts',
|
|
95
|
-
],
|
|
96
|
-
interpretation:
|
|
97
|
-
'Runner bundle, play artifact, config, or materialized-file staging is now the dominant transfer cost into the one-shot sandbox.',
|
|
98
|
-
nextAction:
|
|
99
|
-
'Measure uploaded bytes by label, then reduce runner/artifact/config bytes or move immutable runner material into the snapshot without changing the one-shot sandbox Interface.',
|
|
100
|
-
},
|
|
101
|
-
daytona_execute: {
|
|
102
|
-
ownerModule: 'One-Shot Sandbox Runner',
|
|
103
|
-
ownerFiles: [
|
|
104
|
-
'apps/temporal-worker/src/runner-backends/backends/daytona.ts',
|
|
105
|
-
'apps/play-runner/src/entry.ts',
|
|
106
|
-
'shared_libs/play-runtime/protocol.ts',
|
|
107
|
-
],
|
|
108
|
-
interpretation:
|
|
109
|
-
'The sandbox command is slow after files are present, usually runner boot, artifact load/decode, module startup, or the hello-world execution path.',
|
|
110
|
-
nextAction:
|
|
111
|
-
'Inspect the captured runner output and runtimeTiming; shrink startup imports or shift decode/load work into upload/snapshot prep instead of adding pools.',
|
|
112
|
-
},
|
|
113
|
-
terminal_observation: {
|
|
114
|
-
ownerModule: 'Run Snapshot',
|
|
115
|
-
ownerFiles: [
|
|
116
|
-
'src/lib/plays/scheduler-backends/hatchet-completion-publisher.ts',
|
|
117
|
-
'src/lib/plays/scheduler-backends/postgres-progress.ts',
|
|
118
|
-
'convex/model/runLogStream.ts',
|
|
119
|
-
],
|
|
120
|
-
interpretation:
|
|
121
|
-
'The worker has reached terminal state, but the customer-visible observer is seeing it late.',
|
|
122
|
-
nextAction:
|
|
123
|
-
'Check terminal write ordering, NOTIFY/direct observer connection, and polling fallback before changing execution or scheduler code.',
|
|
124
|
-
},
|
|
125
|
-
};
|
|
126
|
-
|
|
127
|
-
export function diagnoseHatchetColdExecution(
|
|
128
|
-
input: DiagnoseInput,
|
|
129
|
-
): HatchetColdExecutionDiagnosis {
|
|
130
|
-
const thresholdMs =
|
|
131
|
-
input.thresholdMs ?? HATCHET_COLD_EXECUTION_TARGET.thresholdMs;
|
|
132
|
-
const summariesById = new Map(
|
|
133
|
-
input.phaseSummaries.map((summary) => [summary.id, summary]),
|
|
134
|
-
);
|
|
135
|
-
const phases = HATCHET_COLD_EXECUTION_PHASES.map((definition) => {
|
|
136
|
-
const summary = summariesById.get(definition.id);
|
|
137
|
-
const p95 = summary?.p95 ?? null;
|
|
138
|
-
const overBudgetMs =
|
|
139
|
-
p95 === null ? null : Math.max(0, p95 - definition.budgetMs);
|
|
140
|
-
const severity = phaseSeverity(p95, definition.budgetMs);
|
|
141
|
-
return {
|
|
142
|
-
id: definition.id,
|
|
143
|
-
phase: definition.phase,
|
|
144
|
-
budgetMs: definition.budgetMs,
|
|
145
|
-
p95,
|
|
146
|
-
overBudgetMs,
|
|
147
|
-
severity,
|
|
148
|
-
...PHASE_OWNERSHIP[definition.id],
|
|
149
|
-
};
|
|
150
|
-
});
|
|
151
|
-
|
|
152
|
-
const overallOverThresholdMs =
|
|
153
|
-
input.overallP95 === null
|
|
154
|
-
? null
|
|
155
|
-
: Math.max(0, input.overallP95 - thresholdMs);
|
|
156
|
-
const ok = input.overallP95 !== null && input.overallP95 <= thresholdMs;
|
|
157
|
-
const bottleneck = chooseBottleneck(phases);
|
|
158
|
-
const observations = buildObservations({
|
|
159
|
-
ok,
|
|
160
|
-
thresholdMs,
|
|
161
|
-
overallP95: input.overallP95,
|
|
162
|
-
overallOverThresholdMs,
|
|
163
|
-
phases,
|
|
164
|
-
bottleneck,
|
|
165
|
-
});
|
|
166
|
-
const nextActions = buildNextActions(bottleneck, phases);
|
|
167
|
-
|
|
168
|
-
return {
|
|
169
|
-
ok,
|
|
170
|
-
targetName: HATCHET_COLD_EXECUTION_TARGET.name,
|
|
171
|
-
thresholdMs,
|
|
172
|
-
overallP95: input.overallP95,
|
|
173
|
-
overallOverThresholdMs,
|
|
174
|
-
bottleneck,
|
|
175
|
-
phases,
|
|
176
|
-
observations,
|
|
177
|
-
nextActions,
|
|
178
|
-
summary: buildSummary({ ok, input, thresholdMs, bottleneck }),
|
|
179
|
-
};
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
export function formatHatchetColdExecutionDiagnosis(
|
|
183
|
-
diagnosis: HatchetColdExecutionDiagnosis,
|
|
184
|
-
): string[] {
|
|
185
|
-
const lines = [
|
|
186
|
-
`[hatchet-cold-diagnosis] ${diagnosis.summary}`,
|
|
187
|
-
...diagnosis.observations.map(
|
|
188
|
-
(observation) => `[hatchet-cold-diagnosis] ${observation}`,
|
|
189
|
-
),
|
|
190
|
-
];
|
|
191
|
-
if (diagnosis.bottleneck) {
|
|
192
|
-
lines.push(
|
|
193
|
-
`[hatchet-cold-diagnosis] owner=${diagnosis.bottleneck.ownerModule}`,
|
|
194
|
-
`[hatchet-cold-diagnosis] next=${diagnosis.bottleneck.nextAction}`,
|
|
195
|
-
);
|
|
196
|
-
}
|
|
197
|
-
return lines;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
function phaseSeverity(
|
|
201
|
-
p95: number | null,
|
|
202
|
-
budgetMs: number,
|
|
203
|
-
): HatchetColdExecutionPhaseSeverity {
|
|
204
|
-
if (p95 === null) return 'missing';
|
|
205
|
-
if (p95 > budgetMs) return 'over_budget';
|
|
206
|
-
if (p95 >= Math.floor(budgetMs * 0.85)) return 'near_budget';
|
|
207
|
-
return 'ok';
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
function chooseBottleneck(
|
|
211
|
-
phases: readonly HatchetColdExecutionPhaseDiagnosis[],
|
|
212
|
-
): HatchetColdExecutionPhaseDiagnosis | null {
|
|
213
|
-
const overBudget = phases
|
|
214
|
-
.filter((phase) => phase.severity === 'over_budget')
|
|
215
|
-
.sort(
|
|
216
|
-
(left, right) =>
|
|
217
|
-
(right.overBudgetMs ?? 0) - (left.overBudgetMs ?? 0) ||
|
|
218
|
-
(right.p95 ?? 0) - (left.p95 ?? 0),
|
|
219
|
-
);
|
|
220
|
-
if (overBudget[0]) return overBudget[0];
|
|
221
|
-
|
|
222
|
-
const nearBudget = phases
|
|
223
|
-
.filter((phase) => phase.severity === 'near_budget')
|
|
224
|
-
.sort((left, right) => (right.p95 ?? 0) - (left.p95 ?? 0));
|
|
225
|
-
if (nearBudget[0]) return nearBudget[0];
|
|
226
|
-
|
|
227
|
-
const missing = phases.find((phase) => phase.severity === 'missing');
|
|
228
|
-
return missing ?? null;
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
function buildObservations(input: {
|
|
232
|
-
ok: boolean;
|
|
233
|
-
thresholdMs: number;
|
|
234
|
-
overallP95: number | null;
|
|
235
|
-
overallOverThresholdMs: number | null;
|
|
236
|
-
phases: readonly HatchetColdExecutionPhaseDiagnosis[];
|
|
237
|
-
bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
|
|
238
|
-
}): string[] {
|
|
239
|
-
const observations: string[] = [];
|
|
240
|
-
if (input.overallP95 === null) {
|
|
241
|
-
observations.push(
|
|
242
|
-
'No overall p95 was observed; latency target cannot be evaluated.',
|
|
243
|
-
);
|
|
244
|
-
} else if (input.ok) {
|
|
245
|
-
observations.push(
|
|
246
|
-
`Overall p95=${input.overallP95}ms is within threshold=${input.thresholdMs}ms.`,
|
|
247
|
-
);
|
|
248
|
-
} else {
|
|
249
|
-
observations.push(
|
|
250
|
-
`Overall p95=${input.overallP95}ms exceeds threshold=${input.thresholdMs}ms by ${input.overallOverThresholdMs}ms.`,
|
|
251
|
-
);
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
const overBudget = input.phases.filter(
|
|
255
|
-
(phase) => phase.severity === 'over_budget',
|
|
256
|
-
);
|
|
257
|
-
if (overBudget.length > 0) {
|
|
258
|
-
observations.push(
|
|
259
|
-
`Over-budget phases: ${overBudget
|
|
260
|
-
.map(
|
|
261
|
-
(phase) =>
|
|
262
|
-
`${phase.id} p95=${phase.p95}ms budget=${phase.budgetMs}ms (+${phase.overBudgetMs}ms)`,
|
|
263
|
-
)
|
|
264
|
-
.join(', ')}.`,
|
|
265
|
-
);
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
const missing = input.phases.filter((phase) => phase.severity === 'missing');
|
|
269
|
-
if (missing.length > 0) {
|
|
270
|
-
observations.push(
|
|
271
|
-
`Missing phase samples: ${missing.map((phase) => phase.id).join(', ')}.`,
|
|
272
|
-
);
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
if (input.bottleneck) {
|
|
276
|
-
observations.push(
|
|
277
|
-
`Primary bottleneck: ${input.bottleneck.id} owned by ${input.bottleneck.ownerModule}.`,
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
return observations;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
function buildNextActions(
|
|
284
|
-
bottleneck: HatchetColdExecutionPhaseDiagnosis | null,
|
|
285
|
-
phases: readonly HatchetColdExecutionPhaseDiagnosis[],
|
|
286
|
-
): string[] {
|
|
287
|
-
const actions: string[] = [];
|
|
288
|
-
if (bottleneck) {
|
|
289
|
-
actions.push(bottleneck.nextAction);
|
|
290
|
-
}
|
|
291
|
-
const missing = phases.find((phase) => phase.severity === 'missing');
|
|
292
|
-
if (missing) {
|
|
293
|
-
actions.push(
|
|
294
|
-
'Fix missing runtimeTiming/phase sample emission before trusting the latency diagnosis.',
|
|
295
|
-
);
|
|
296
|
-
}
|
|
297
|
-
if (actions.length === 0) {
|
|
298
|
-
actions.push(
|
|
299
|
-
'Keep the current phase budgets and rerun the same gate after the next runtime change.',
|
|
300
|
-
);
|
|
301
|
-
}
|
|
302
|
-
return [...new Set(actions)];
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
function buildSummary(input: {
|
|
306
|
-
ok: boolean;
|
|
307
|
-
input: DiagnoseInput;
|
|
308
|
-
thresholdMs: number;
|
|
309
|
-
bottleneck: HatchetColdExecutionPhaseDiagnosis | null;
|
|
310
|
-
}): string {
|
|
311
|
-
if (input.input.overallP95 === null) {
|
|
312
|
-
return `${HATCHET_COLD_EXECUTION_TARGET.name}: inconclusive; no overall p95 sample.`;
|
|
313
|
-
}
|
|
314
|
-
if (input.ok) {
|
|
315
|
-
return `${HATCHET_COLD_EXECUTION_TARGET.name}: pass at p95=${input.input.overallP95}ms.`;
|
|
316
|
-
}
|
|
317
|
-
const owner = input.bottleneck
|
|
318
|
-
? `; primary owner ${input.bottleneck.ownerModule}`
|
|
319
|
-
: '';
|
|
320
|
-
return `${HATCHET_COLD_EXECUTION_TARGET.name}: fail at p95=${input.input.overallP95}ms over threshold=${input.thresholdMs}ms${owner}.`;
|
|
321
|
-
}
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
export type HatchetColdExecutionPhaseId =
|
|
2
|
-
| 'submit'
|
|
3
|
-
| 'hatchet_dispatch_claim'
|
|
4
|
-
| 'daytona_create'
|
|
5
|
-
| 'daytona_upload'
|
|
6
|
-
| 'daytona_execute'
|
|
7
|
-
| 'terminal_observation';
|
|
8
|
-
|
|
9
|
-
export type HatchetColdExecutionPhaseDefinition = {
|
|
10
|
-
id: HatchetColdExecutionPhaseId;
|
|
11
|
-
header: string;
|
|
12
|
-
phase: string;
|
|
13
|
-
budgetMs: number;
|
|
14
|
-
description: string;
|
|
15
|
-
};
|
|
16
|
-
|
|
17
|
-
export type HatchetColdExecutionSample = Partial<
|
|
18
|
-
Record<HatchetColdExecutionPhaseId, number | null>
|
|
19
|
-
>;
|
|
20
|
-
|
|
21
|
-
export type HatchetColdExecutionPhaseSummary = {
|
|
22
|
-
phase: string;
|
|
23
|
-
id: HatchetColdExecutionPhaseId;
|
|
24
|
-
budgetMs: number;
|
|
25
|
-
p50: number | null;
|
|
26
|
-
p95: number | null;
|
|
27
|
-
max: number | null;
|
|
28
|
-
observed: number;
|
|
29
|
-
ok: boolean | null;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export const HATCHET_COLD_EXECUTION_TARGET = {
|
|
33
|
-
name: 'hatchet-cold-execution-under-2s',
|
|
34
|
-
description:
|
|
35
|
-
'Customer-observed cold execution from submit through terminal observation on the real Hatchet + Daytona path.',
|
|
36
|
-
thresholdMs: 2_000,
|
|
37
|
-
percentile: 95,
|
|
38
|
-
} as const;
|
|
39
|
-
|
|
40
|
-
export const HATCHET_COLD_EXECUTION_PHASES = [
|
|
41
|
-
{
|
|
42
|
-
id: 'submit',
|
|
43
|
-
header: 'hc_submit',
|
|
44
|
-
phase: 'hatchet_cold.submit',
|
|
45
|
-
budgetMs: 300,
|
|
46
|
-
description:
|
|
47
|
-
'Customer submit route response, including app validation, authority minting, scheduler row creation, and Hatchet dispatch.',
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
id: 'hatchet_dispatch_claim',
|
|
51
|
-
header: 'hc_hatchet',
|
|
52
|
-
phase: 'hatchet_cold.hatchet_dispatch_claim',
|
|
53
|
-
budgetMs: 250,
|
|
54
|
-
description:
|
|
55
|
-
'Hatchet dispatch and worker claim time from persisted scheduler row to claimed execution.',
|
|
56
|
-
},
|
|
57
|
-
{
|
|
58
|
-
id: 'daytona_create',
|
|
59
|
-
header: 'hc_create',
|
|
60
|
-
phase: 'hatchet_cold.daytona_create',
|
|
61
|
-
budgetMs: 300,
|
|
62
|
-
description: 'Fresh Daytona sandbox creation for the claimed run.',
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
id: 'daytona_upload',
|
|
66
|
-
header: 'hc_upload',
|
|
67
|
-
phase: 'hatchet_cold.daytona_upload',
|
|
68
|
-
budgetMs: 350,
|
|
69
|
-
description:
|
|
70
|
-
'Runner bundle, play artifact, config, and staged-file upload into the Daytona sandbox.',
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
id: 'daytona_execute',
|
|
74
|
-
header: 'hc_execute',
|
|
75
|
-
phase: 'hatchet_cold.daytona_execute',
|
|
76
|
-
budgetMs: 650,
|
|
77
|
-
description:
|
|
78
|
-
'Daytona command execution until the play runner returns its terminal result.',
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
id: 'terminal_observation',
|
|
82
|
-
header: 'hc_observe',
|
|
83
|
-
phase: 'hatchet_cold.terminal_observation',
|
|
84
|
-
budgetMs: 150,
|
|
85
|
-
description:
|
|
86
|
-
'Lag between scheduler terminal persistence and the customer-visible observer noticing terminal state.',
|
|
87
|
-
},
|
|
88
|
-
] as const satisfies readonly HatchetColdExecutionPhaseDefinition[];
|
|
89
|
-
|
|
90
|
-
export const HATCHET_COLD_EXECUTION_PHASE_DEFINITIONS =
|
|
91
|
-
HATCHET_COLD_EXECUTION_PHASES.map((definition) => ({
|
|
92
|
-
header: definition.header,
|
|
93
|
-
phase: definition.phase,
|
|
94
|
-
source: 'hatchet_cold',
|
|
95
|
-
description: `${definition.description} Budget: ${definition.budgetMs}ms.`,
|
|
96
|
-
}));
|
|
97
|
-
|
|
98
|
-
export function validateHatchetColdExecutionTarget(): string[] {
|
|
99
|
-
const errors: string[] = [];
|
|
100
|
-
const phaseIds = new Set<string>();
|
|
101
|
-
const phaseNames = new Set<string>();
|
|
102
|
-
let budgetTotal = 0;
|
|
103
|
-
for (const phase of HATCHET_COLD_EXECUTION_PHASES) {
|
|
104
|
-
if (phaseIds.has(phase.id)) {
|
|
105
|
-
errors.push(`Duplicate phase id: ${phase.id}`);
|
|
106
|
-
}
|
|
107
|
-
phaseIds.add(phase.id);
|
|
108
|
-
if (phaseNames.has(phase.phase)) {
|
|
109
|
-
errors.push(`Duplicate phase name: ${phase.phase}`);
|
|
110
|
-
}
|
|
111
|
-
phaseNames.add(phase.phase);
|
|
112
|
-
if (!Number.isFinite(phase.budgetMs) || phase.budgetMs <= 0) {
|
|
113
|
-
errors.push(`Phase ${phase.id} must have a positive budget.`);
|
|
114
|
-
}
|
|
115
|
-
budgetTotal += phase.budgetMs;
|
|
116
|
-
}
|
|
117
|
-
if (budgetTotal !== HATCHET_COLD_EXECUTION_TARGET.thresholdMs) {
|
|
118
|
-
errors.push(
|
|
119
|
-
`Phase budgets sum to ${budgetTotal}ms, expected ${HATCHET_COLD_EXECUTION_TARGET.thresholdMs}ms.`,
|
|
120
|
-
);
|
|
121
|
-
}
|
|
122
|
-
return errors;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export function summarizeHatchetColdExecutionPhases(
|
|
126
|
-
samples: readonly HatchetColdExecutionSample[],
|
|
127
|
-
): HatchetColdExecutionPhaseSummary[] {
|
|
128
|
-
return HATCHET_COLD_EXECUTION_PHASES.map((definition) => {
|
|
129
|
-
const values = samples
|
|
130
|
-
.map((sample) => sample[definition.id])
|
|
131
|
-
.filter(
|
|
132
|
-
(value): value is number =>
|
|
133
|
-
typeof value === 'number' && Number.isFinite(value),
|
|
134
|
-
)
|
|
135
|
-
.sort((left, right) => left - right);
|
|
136
|
-
const p50 = percentile(values, 50);
|
|
137
|
-
const p95 = percentile(values, HATCHET_COLD_EXECUTION_TARGET.percentile);
|
|
138
|
-
return {
|
|
139
|
-
phase: definition.phase,
|
|
140
|
-
id: definition.id,
|
|
141
|
-
budgetMs: definition.budgetMs,
|
|
142
|
-
p50,
|
|
143
|
-
p95,
|
|
144
|
-
max: values.at(-1) ?? null,
|
|
145
|
-
observed: values.length,
|
|
146
|
-
ok: p95 === null ? null : p95 <= definition.budgetMs,
|
|
147
|
-
};
|
|
148
|
-
});
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function percentile(sortedValues: readonly number[], p: number): number | null {
|
|
152
|
-
if (sortedValues.length === 0) return null;
|
|
153
|
-
const index = Math.min(
|
|
154
|
-
sortedValues.length - 1,
|
|
155
|
-
Math.max(0, Math.ceil((p / 100) * sortedValues.length) - 1),
|
|
156
|
-
);
|
|
157
|
-
return sortedValues[index] ?? null;
|
|
158
|
-
}
|