deepline 0.3.246 → 0.3.248
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +177 -13
- package/dist/bundling-sources/sdk/src/index.ts +8 -0
- package/dist/bundling-sources/sdk/src/plays/input-example.ts +40 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/runs/overview.ts +457 -0
- package/dist/bundling-sources/sdk/src/runs/receipts.ts +23 -0
- package/dist/bundling-sources/sdk/src/tool-calls.ts +21 -1
- package/dist/bundling-sources/sdk/src/types.ts +16 -1
- package/dist/bundling-sources/shared_libs/observability/scheduled-jobs.json +37 -1
- package/dist/bundling-sources/shared_libs/observability/scheduled-work-manual.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-data-plane/column-names.ts +17 -3
- package/dist/bundling-sources/shared_libs/play-data-plane/sheet-contract.ts +40 -8
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +26 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +150 -19
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +2 -1
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +6 -3
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-janitor.ts +26 -21
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-sandbox-reaper.ts +23 -21
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +17 -0
- package/dist/bundling-sources/shared_libs/play-runtime/log-context.ts +105 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +40 -4
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +50 -5
- package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +14 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +56 -16
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-control-transport.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-launch-transport.ts +327 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +87 -30
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +24 -13
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-start-admission.ts +23 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +76 -33
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +90 -35
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +62 -21
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +368 -66
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +22 -4
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-ledger-outbox-recovery.ts +106 -81
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-event-search.ts +1347 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-queue-health.ts +16 -19
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +14 -6
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +16 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +3 -2
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +5 -2
- package/dist/bundling-sources/shared_libs/plays/dataset-summary.ts +11 -7
- package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +5 -2
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +11 -2
- package/dist/cli/index.js +3504 -2368
- package/dist/cli/index.mjs +3188 -2052
- package/dist/{compiler-manifest-n8ZtPv2h.d.mts → compiler-manifest-DavmLa8y.d.mts} +1 -0
- package/dist/{compiler-manifest-n8ZtPv2h.d.ts → compiler-manifest-DavmLa8y.d.ts} +1 -0
- package/dist/index.d.mts +97 -5
- package/dist/index.d.ts +97 -5
- package/dist/index.js +135 -15
- package/dist/index.mjs +135 -15
- package/dist/install-integrity.json +9 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -25,6 +25,7 @@ import type {
|
|
|
25
25
|
PlayRunnerRuntimeTiming,
|
|
26
26
|
} from '../protocol';
|
|
27
27
|
import { PROD_DEFAULT_RUNTIME_SCHEDULER_SCHEMA } from '../runtime-scheduler-topology';
|
|
28
|
+
import { RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER } from './runtime-event-search';
|
|
28
29
|
import type { PlayRunnerRuntimeResource } from '../runner-backends/types';
|
|
29
30
|
import {
|
|
30
31
|
RuntimeSandboxOperationState,
|
|
@@ -76,6 +77,10 @@ import {
|
|
|
76
77
|
type DocflowObservationUpsert,
|
|
77
78
|
} from '../docflow-observation';
|
|
78
79
|
import { normalizePlayDocflowNodeIoState } from '../docflow-node-io';
|
|
80
|
+
import {
|
|
81
|
+
normalizePlayLogContext,
|
|
82
|
+
type PlayLogContext,
|
|
83
|
+
} from '../log-context';
|
|
79
84
|
import {
|
|
80
85
|
workReceiptFailureKindCodeForWrite,
|
|
81
86
|
workReceiptFailureKindFromCode,
|
|
@@ -299,7 +304,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
299
304
|
// v46 adds a bounded provider-scope Retry-After hold with shared adaptive-rate
|
|
300
305
|
// state. The scope owns learning and dispatch across runs and tool rules;
|
|
301
306
|
// existing deadline rows expand additively and initialize on their first seed.
|
|
302
|
-
|
|
307
|
+
// v47 attaches the append-only runtime event projection to durable outbox and
|
|
308
|
+
// receipt transitions; no existing runtime wire field changes.
|
|
309
|
+
// v48 expands optional compact-log columns and readers before writer enablement.
|
|
310
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 48;
|
|
303
311
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
304
312
|
'scheduler_schema_migrations';
|
|
305
313
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
@@ -885,6 +893,8 @@ const OUTBOX_RETRY_MAX_MS = 15 * 60 * 1_000;
|
|
|
885
893
|
// diagnostic window, then delete in small, independently committed batches.
|
|
886
894
|
export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_MS =
|
|
887
895
|
30 * 24 * 60 * 60 * 1_000;
|
|
896
|
+
export const POSTGRES_SCHEDULER_OUTBOX_LOG_BODY_RETENTION_MS =
|
|
897
|
+
7 * 24 * 60 * 60 * 1_000;
|
|
888
898
|
export const POSTGRES_SCHEDULER_OUTBOX_DELIVERED_RETENTION_BATCH_LIMIT = 1_000;
|
|
889
899
|
const RUN_LEDGER_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
|
|
890
900
|
const COMPUTE_BILLING_DISPATCH_TOKEN_TIMEOUT_MS = 15_000;
|
|
@@ -1101,6 +1111,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
|
|
|
1101
1111
|
rateRequesterFairShare?: boolean;
|
|
1102
1112
|
/** v45 added durable Retry-After holds shared across rules and runs. */
|
|
1103
1113
|
rateScopeRetryAfter?: boolean;
|
|
1114
|
+
/** v48 expands compact log storage; prior readers ignore these columns. */
|
|
1115
|
+
runtimeLogStorage?: boolean;
|
|
1104
1116
|
}>;
|
|
1105
1117
|
|
|
1106
1118
|
export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
@@ -1718,6 +1730,22 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1718
1730
|
AND index_row.indisvalid
|
|
1719
1731
|
)
|
|
1720
1732
|
)
|
|
1733
|
+
AND (
|
|
1734
|
+
$38::boolean = false
|
|
1735
|
+
OR (
|
|
1736
|
+
EXISTS (
|
|
1737
|
+
SELECT 1 FROM information_schema.columns
|
|
1738
|
+
WHERE table_schema = $1 AND table_name = 'runs'
|
|
1739
|
+
AND column_name = 'run_log_store' AND is_nullable = 'YES'
|
|
1740
|
+
)
|
|
1741
|
+
AND (
|
|
1742
|
+
SELECT count(*) FROM information_schema.columns
|
|
1743
|
+
WHERE table_schema = $1 AND table_name = 'attempts'
|
|
1744
|
+
AND column_name IN ('runner_result_digest', 'runner_log_payload_json')
|
|
1745
|
+
AND is_nullable = 'YES'
|
|
1746
|
+
) = 2
|
|
1747
|
+
)
|
|
1748
|
+
)
|
|
1721
1749
|
AS ready
|
|
1722
1750
|
`,
|
|
1723
1751
|
[
|
|
@@ -1758,6 +1786,7 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1758
1786
|
`${quotedSchema}.${quoteIdent('rate_requesters')}`,
|
|
1759
1787
|
requirements.rateScopeRetryAfter !== false,
|
|
1760
1788
|
`${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
|
|
1789
|
+
requirements.runtimeLogStorage !== false,
|
|
1761
1790
|
],
|
|
1762
1791
|
);
|
|
1763
1792
|
if (currentShape.rows[0]?.ready !== true) return false;
|
|
@@ -1980,6 +2009,19 @@ export async function ensurePostgresSchedulerOnlineAdditiveExpansion(
|
|
|
1980
2009
|
options,
|
|
1981
2010
|
)} (state, updated_at)
|
|
1982
2011
|
`);
|
|
2012
|
+
await ensurePostgresSchedulerRunLogStorageExpansion(client, options);
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
/** Deployment-only expansion; absent values keep legacy storage authoritative. */
|
|
2016
|
+
async function ensurePostgresSchedulerRunLogStorageExpansion(
|
|
2017
|
+
client: PostgresSchedulerQueryClient,
|
|
2018
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
2019
|
+
): Promise<void> {
|
|
2020
|
+
await client.query(`ALTER TABLE ${tableName('runs', options)}
|
|
2021
|
+
ADD COLUMN IF NOT EXISTS run_log_store text`);
|
|
2022
|
+
await client.query(`ALTER TABLE ${tableName('attempts', options)}
|
|
2023
|
+
ADD COLUMN IF NOT EXISTS runner_result_digest text,
|
|
2024
|
+
ADD COLUMN IF NOT EXISTS runner_log_payload_json jsonb`);
|
|
1983
2025
|
}
|
|
1984
2026
|
|
|
1985
2027
|
/** Read the highest deployment-owned scheduler schema marker. */
|
|
@@ -2546,6 +2588,7 @@ export async function ensurePostgresSchedulerSchema(
|
|
|
2546
2588
|
ALTER TABLE ${tableName('attempts', options)}
|
|
2547
2589
|
ADD COLUMN IF NOT EXISTS runner_terminal_received_at timestamptz
|
|
2548
2590
|
`);
|
|
2591
|
+
await ensurePostgresSchedulerRunLogStorageExpansion(client, options);
|
|
2549
2592
|
// Authoritative provenance for terminal reporting. Do not infer this from
|
|
2550
2593
|
// the completed/cleanup state shape: direct ingress and compatibility wake
|
|
2551
2594
|
// can both produce those rows.
|
|
@@ -4949,7 +4992,14 @@ async function submitPostgresSchedulerRunTransaction(
|
|
|
4949
4992
|
'graphHash', $5::text,
|
|
4950
4993
|
'runtimeBackend', $6::text,
|
|
4951
4994
|
'schedulerBackend', launch_payload.launch_json ->> 'schedulerBackend',
|
|
4952
|
-
'schedulerSchema', launch_payload.launch_json ->> 'schedulerSchema',
|
|
4995
|
+
'schedulerSchema', COALESCE(launch_payload.launch_json ->> 'schedulerSchema', '${schemaName(options)}'),
|
|
4996
|
+
'runLogStore', CASE WHEN EXISTS (
|
|
4997
|
+
SELECT 1 FROM pg_proc AS proc
|
|
4998
|
+
JOIN pg_namespace AS namespace ON namespace.oid = proc.pronamespace
|
|
4999
|
+
WHERE namespace.nspname = '${schemaName(options)}'
|
|
5000
|
+
AND proc.proname = 'search_outbox_event'
|
|
5001
|
+
AND obj_description(proc.oid, 'pg_proc') = '${RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER}'
|
|
5002
|
+
) THEN 'planet_scale' END,
|
|
4953
5003
|
'executionProfile', launch_payload.launch_json ->> 'executionProfile',
|
|
4954
5004
|
'runtimeReleaseId', launch_payload.launch_json ->> 'runtimeReleaseId',
|
|
4955
5005
|
'runtimeDeployVersion', launch_payload.launch_json ->> 'runtimeDeployVersion',
|
|
@@ -6023,7 +6073,14 @@ export async function firePostgresSchedulerCronSchedule(
|
|
|
6023
6073
|
'graphHash', launch_payload.launch_json ->> 'graphHash',
|
|
6024
6074
|
'runtimeBackend', launch_payload.launch_json ->> 'runtimeBackend',
|
|
6025
6075
|
'schedulerBackend', launch_payload.launch_json ->> 'schedulerBackend',
|
|
6026
|
-
'schedulerSchema', launch_payload.launch_json ->> 'schedulerSchema',
|
|
6076
|
+
'schedulerSchema', COALESCE(launch_payload.launch_json ->> 'schedulerSchema', '${schemaName(options)}'),
|
|
6077
|
+
'runLogStore', CASE WHEN EXISTS (
|
|
6078
|
+
SELECT 1 FROM pg_proc AS proc
|
|
6079
|
+
JOIN pg_namespace AS namespace ON namespace.oid = proc.pronamespace
|
|
6080
|
+
WHERE namespace.nspname = '${schemaName(options)}'
|
|
6081
|
+
AND proc.proname = 'search_outbox_event'
|
|
6082
|
+
AND obj_description(proc.oid, 'pg_proc') = '${RUNTIME_EVENT_SEARCH_OUTBOX_PROJECTION_MARKER}'
|
|
6083
|
+
) THEN 'planet_scale' END,
|
|
6027
6084
|
'executionProfile', launch_payload.launch_json ->> 'executionProfile',
|
|
6028
6085
|
'runtimeReleaseId', launch_payload.launch_json ->> 'runtimeReleaseId',
|
|
6029
6086
|
'runtimeDeployVersion', launch_payload.launch_json ->> 'runtimeDeployVersion',
|
|
@@ -8472,6 +8529,36 @@ export type PostgresSchedulerAttemptAuthority = Pick<
|
|
|
8472
8529
|
(typeof POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS)[number]
|
|
8473
8530
|
>;
|
|
8474
8531
|
|
|
8532
|
+
type PostgresSchedulerAuthorityRow = PostgresSchedulerAttemptAuthority & {
|
|
8533
|
+
launch_required_fields_valid: boolean;
|
|
8534
|
+
};
|
|
8535
|
+
|
|
8536
|
+
// Keep JSON types intact: text columns would coerce malformed launch scalars.
|
|
8537
|
+
const POSTGRES_SCHEDULER_AUTHORITY_COLUMNS = [
|
|
8538
|
+
...POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS,
|
|
8539
|
+
'playId',
|
|
8540
|
+
'executorToken',
|
|
8541
|
+
]
|
|
8542
|
+
.map((field) => `${quoteIdent(field)} jsonb`)
|
|
8543
|
+
.join(', ');
|
|
8544
|
+
const POSTGRES_SCHEDULER_AUTHORITY_SELECT = `${POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS.map((field) => `launch.${quoteIdent(field)}`).join(', ')},
|
|
8545
|
+
${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
|
|
8546
|
+
.map((field) => `jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`)
|
|
8547
|
+
.join(' AND ')} AS launch_required_fields_valid`;
|
|
8548
|
+
|
|
8549
|
+
function parsePostgresSchedulerAuthorityRow(
|
|
8550
|
+
row: PostgresSchedulerAuthorityRow | undefined,
|
|
8551
|
+
): PostgresSchedulerAttemptAuthority | null {
|
|
8552
|
+
if (!row) return null;
|
|
8553
|
+
const { launch_required_fields_valid, ...authority } = row;
|
|
8554
|
+
if (launch_required_fields_valid !== true) {
|
|
8555
|
+
throw new Error(
|
|
8556
|
+
'Postgres scheduler launch_json is missing required fields.',
|
|
8557
|
+
);
|
|
8558
|
+
}
|
|
8559
|
+
return authority;
|
|
8560
|
+
}
|
|
8561
|
+
|
|
8475
8562
|
/**
|
|
8476
8563
|
* Prove the same attempt lease as the full execution reader, returning only
|
|
8477
8564
|
* the token issuer's authority. Input, graph, and artifact bytes never cross
|
|
@@ -8484,29 +8571,13 @@ export async function readPostgresSchedulerAttemptAuthorityForLease(
|
|
|
8484
8571
|
input: { runId: string; attempt: number; leaseToken: string },
|
|
8485
8572
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
8486
8573
|
): Promise<PostgresSchedulerAttemptAuthority | null> {
|
|
8487
|
-
const
|
|
8488
|
-
// Keep JSON types intact. A text record column would coerce malformed launch
|
|
8489
|
-
// scalars into strings, bypassing the full reader's required-field checks.
|
|
8490
|
-
const columns = [...fields, 'playId', 'executorToken']
|
|
8491
|
-
.map((field) => `${quoteIdent(field)} jsonb`)
|
|
8492
|
-
.join(', ');
|
|
8493
|
-
const result = await client.query<
|
|
8494
|
-
PostgresSchedulerAttemptAuthority & {
|
|
8495
|
-
launch_required_fields_valid: boolean;
|
|
8496
|
-
}
|
|
8497
|
-
>(
|
|
8574
|
+
const result = await client.query<PostgresSchedulerAuthorityRow>(
|
|
8498
8575
|
`
|
|
8499
|
-
SELECT ${
|
|
8500
|
-
${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
|
|
8501
|
-
.map(
|
|
8502
|
-
(field) =>
|
|
8503
|
-
`jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`,
|
|
8504
|
-
)
|
|
8505
|
-
.join(' AND ')} AS launch_required_fields_valid
|
|
8576
|
+
SELECT ${POSTGRES_SCHEDULER_AUTHORITY_SELECT}
|
|
8506
8577
|
FROM ${tableName('runs', options)} AS runs
|
|
8507
8578
|
JOIN ${tableName('attempts', options)} AS attempts
|
|
8508
8579
|
ON attempts.run_id = runs.run_id
|
|
8509
|
-
CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${
|
|
8580
|
+
CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${POSTGRES_SCHEDULER_AUTHORITY_COLUMNS})
|
|
8510
8581
|
WHERE runs.run_id = $1
|
|
8511
8582
|
AND runs.status IN ('queued', 'running', 'waiting')
|
|
8512
8583
|
AND attempts.attempt = $2
|
|
@@ -8516,15 +8587,7 @@ export async function readPostgresSchedulerAttemptAuthorityForLease(
|
|
|
8516
8587
|
`,
|
|
8517
8588
|
[input.runId, input.attempt, input.leaseToken],
|
|
8518
8589
|
);
|
|
8519
|
-
|
|
8520
|
-
if (!row) return null;
|
|
8521
|
-
const { launch_required_fields_valid, ...authority } = row;
|
|
8522
|
-
if (launch_required_fields_valid !== true) {
|
|
8523
|
-
throw new Error(
|
|
8524
|
-
'Postgres scheduler launch_json is missing required fields.',
|
|
8525
|
-
);
|
|
8526
|
-
}
|
|
8527
|
-
return authority;
|
|
8590
|
+
return parsePostgresSchedulerAuthorityRow(result.rows[0]);
|
|
8528
8591
|
}
|
|
8529
8592
|
|
|
8530
8593
|
/**
|
|
@@ -9817,10 +9880,11 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
9817
9880
|
const ledgerTerminalResult = terminalRunResultForLedger(terminalResult, {
|
|
9818
9881
|
content: 'full',
|
|
9819
9882
|
});
|
|
9820
|
-
const
|
|
9883
|
+
const terminalLogProjection = terminalLogProjectionForOutbox(
|
|
9821
9884
|
input.result,
|
|
9822
9885
|
input.terminalLogOutbox,
|
|
9823
9886
|
);
|
|
9887
|
+
const terminalLogLines = terminalLogProjection.lines;
|
|
9824
9888
|
const runtimeTiming = extractRunnerRuntimeTiming(input.result);
|
|
9825
9889
|
const terminalWorkCounterSegment = terminalWorkCounterSegmentFromRunnerResult(
|
|
9826
9890
|
input.result,
|
|
@@ -9910,7 +9974,13 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
9910
9974
|
AND target.attempt = $2
|
|
9911
9975
|
AND target.status IN ('running', 'waiting')
|
|
9912
9976
|
AND target.lease_token = $11
|
|
9913
|
-
AND (
|
|
9977
|
+
AND (
|
|
9978
|
+
target.runner_result_json IS NULL
|
|
9979
|
+
OR target.runner_result_json = $3::jsonb
|
|
9980
|
+
OR target.runner_result_digest = encode(
|
|
9981
|
+
sha256(convert_to(($3::jsonb)::text, 'UTF8')), 'hex'
|
|
9982
|
+
)
|
|
9983
|
+
)
|
|
9914
9984
|
AND run.run_id = target.run_id
|
|
9915
9985
|
AND NOT EXISTS (
|
|
9916
9986
|
SELECT 1
|
|
@@ -10070,9 +10140,10 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10070
10140
|
jsonb_build_object(
|
|
10071
10141
|
'output', $6::jsonb,
|
|
10072
10142
|
'terminalLogLines', $7::jsonb,
|
|
10143
|
+
'terminalLogContexts', $17::jsonb,
|
|
10073
10144
|
'terminalProgressEvents', $8::jsonb,
|
|
10074
10145
|
'resultSummary', $16::jsonb,
|
|
10075
|
-
'terminalWorkCounterSegment', $
|
|
10146
|
+
'terminalWorkCounterSegment', $18::jsonb
|
|
10076
10147
|
),
|
|
10077
10148
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
10078
10149
|
FROM run_terminal
|
|
@@ -10107,6 +10178,7 @@ export async function directCompletePostgresSchedulerRunnerTerminal(
|
|
|
10107
10178
|
input.engineWake.wakeChannel,
|
|
10108
10179
|
input.allowCappedCompletion === true,
|
|
10109
10180
|
stringifyPostgresJson(resultSummary),
|
|
10181
|
+
JSON.stringify(terminalLogProjection.lineContexts),
|
|
10110
10182
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
10111
10183
|
],
|
|
10112
10184
|
);
|
|
@@ -10618,6 +10690,36 @@ export async function claimPostgresSchedulerComputeBillingJobs(
|
|
|
10618
10690
|
}));
|
|
10619
10691
|
}
|
|
10620
10692
|
|
|
10693
|
+
/**
|
|
10694
|
+
* Issue terminal compute-billing authority under the same settlement lease as
|
|
10695
|
+
* the full worker reader, without hydrating execution input or retained graphs.
|
|
10696
|
+
* Historical launches keep their absent-descriptor fallback in the issuer.
|
|
10697
|
+
*/
|
|
10698
|
+
export async function readComputeBillingDispatchAuthorityForLease(
|
|
10699
|
+
client: PostgresSchedulerQueryClient,
|
|
10700
|
+
input: { runId: string; leaseOwner: string; attemptCount?: number },
|
|
10701
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
10702
|
+
): Promise<PostgresSchedulerAttemptAuthority | null> {
|
|
10703
|
+
const result = await client.query<PostgresSchedulerAuthorityRow>(
|
|
10704
|
+
`
|
|
10705
|
+
SELECT ${POSTGRES_SCHEDULER_AUTHORITY_SELECT}
|
|
10706
|
+
FROM ${tableName('compute_billing_jobs_v2', options)} AS job
|
|
10707
|
+
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = job.run_id
|
|
10708
|
+
CROSS JOIN LATERAL jsonb_to_record(runs.launch_json)
|
|
10709
|
+
AS launch(${POSTGRES_SCHEDULER_AUTHORITY_COLUMNS})
|
|
10710
|
+
WHERE job.run_id = $1
|
|
10711
|
+
AND job.status = 'settling'
|
|
10712
|
+
AND job.lease_owner = $2
|
|
10713
|
+
AND job.lease_expires_at > now()
|
|
10714
|
+
AND ($3::integer IS NULL OR job.attempt_count = $3)
|
|
10715
|
+
AND runs.status IN ('completed', 'failed', 'cancelled')
|
|
10716
|
+
LIMIT 1
|
|
10717
|
+
`,
|
|
10718
|
+
[input.runId, input.leaseOwner, input.attemptCount ?? null],
|
|
10719
|
+
);
|
|
10720
|
+
return parsePostgresSchedulerAuthorityRow(result.rows[0]);
|
|
10721
|
+
}
|
|
10722
|
+
|
|
10621
10723
|
export async function readComputeBillingDispatchLaunchForLease(
|
|
10622
10724
|
client: PostgresSchedulerQueryClient,
|
|
10623
10725
|
input: { runId: string; leaseOwner: string; attemptCount?: number },
|
|
@@ -12075,7 +12177,8 @@ export async function isPostgresSchedulerRunnerProgressAttemptCurrent(
|
|
|
12075
12177
|
* Bumps `heartbeat_at` on the runner's attempt row (the same liveness column
|
|
12076
12178
|
* the worker's own state transitions maintain — no new schema) and returns the
|
|
12077
12179
|
* play-level run status so the gateway can put cancellation on the heartbeat
|
|
12078
|
-
* channel. Returns null when the run
|
|
12180
|
+
* channel. Returns null when the run is missing or this attempt is no longer
|
|
12181
|
+
* current while the run remains active.
|
|
12079
12182
|
*/
|
|
12080
12183
|
export async function heartbeatPostgresSchedulerDetachedRunner(
|
|
12081
12184
|
client: PostgresSchedulerQueryClient,
|
|
@@ -12125,7 +12228,26 @@ export async function heartbeatPostgresSchedulerDetachedRunner(
|
|
|
12125
12228
|
`,
|
|
12126
12229
|
[input.runId, input.attempt],
|
|
12127
12230
|
);
|
|
12128
|
-
|
|
12231
|
+
const status = heartbeat.rows[0]?.status;
|
|
12232
|
+
if (status) return status;
|
|
12233
|
+
|
|
12234
|
+
// A terminal writer can commit while the UPDATE waits for its attempt lock.
|
|
12235
|
+
// PostgreSQL then rechecks the changed attempt and skips it, but the UNION
|
|
12236
|
+
// still reads the statement's earlier snapshot. Read terminal state with a
|
|
12237
|
+
// fresh snapshot after that wait; never acknowledge a replacement's active
|
|
12238
|
+
// state without renewing this exact attempt through the fence above.
|
|
12239
|
+
const terminal = await client.query<{
|
|
12240
|
+
status: 'completed' | 'failed' | 'cancelled';
|
|
12241
|
+
}>(
|
|
12242
|
+
`
|
|
12243
|
+
SELECT status
|
|
12244
|
+
FROM ${tableName('runs', options)}
|
|
12245
|
+
WHERE run_id = $1
|
|
12246
|
+
AND status IN ('completed', 'failed', 'cancelled')
|
|
12247
|
+
`,
|
|
12248
|
+
[input.runId],
|
|
12249
|
+
);
|
|
12250
|
+
return terminal.rows[0]?.status ?? null;
|
|
12129
12251
|
}
|
|
12130
12252
|
|
|
12131
12253
|
export async function claimPostgresSchedulerSignal(
|
|
@@ -12214,6 +12336,7 @@ export async function completePostgresSchedulerAttempt(
|
|
|
12214
12336
|
rowOutcomes?: unknown;
|
|
12215
12337
|
runtimeTiming?: PlayRunnerRuntimeTiming | null;
|
|
12216
12338
|
terminalLogLines?: readonly string[] | null;
|
|
12339
|
+
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
12217
12340
|
terminalWorkCounterSegment?: PlayRunLedgerTerminalWorkCounterSegment;
|
|
12218
12341
|
},
|
|
12219
12342
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
@@ -12223,10 +12346,25 @@ export async function completePostgresSchedulerAttempt(
|
|
|
12223
12346
|
content: 'full',
|
|
12224
12347
|
});
|
|
12225
12348
|
const resultSummary = terminalRowOutcomesSummary(input.rowOutcomes);
|
|
12226
|
-
|
|
12227
|
-
.
|
|
12228
|
-
.
|
|
12349
|
+
if (
|
|
12350
|
+
input.terminalLogContexts != null &&
|
|
12351
|
+
input.terminalLogContexts.length !== (input.terminalLogLines ?? []).length
|
|
12352
|
+
) {
|
|
12353
|
+
throw new TypeError(
|
|
12354
|
+
'Terminal log contexts must align with terminal log lines.',
|
|
12355
|
+
);
|
|
12356
|
+
}
|
|
12357
|
+
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
12358
|
+
.map((line, index) => ({
|
|
12359
|
+
line: line.trim(),
|
|
12360
|
+
context: input.terminalLogContexts?.[index] ?? null,
|
|
12361
|
+
}))
|
|
12362
|
+
.filter((entry) => entry.line.length > 0)
|
|
12229
12363
|
.slice(-LOG_TAIL_LIMIT);
|
|
12364
|
+
const terminalLogLines = terminalLogEntries.map((entry) => entry.line);
|
|
12365
|
+
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
12366
|
+
normalizePlayLogContext(entry.context),
|
|
12367
|
+
);
|
|
12230
12368
|
const completed = await client.query<{ run_id: string }>(
|
|
12231
12369
|
`
|
|
12232
12370
|
WITH completed AS (
|
|
@@ -12402,8 +12540,9 @@ export async function completePostgresSchedulerAttempt(
|
|
|
12402
12540
|
jsonb_build_object(
|
|
12403
12541
|
'output', $9::jsonb,
|
|
12404
12542
|
'terminalLogLines', $8::jsonb,
|
|
12543
|
+
'terminalLogContexts', $11::jsonb,
|
|
12405
12544
|
'resultSummary', $10::jsonb,
|
|
12406
|
-
'terminalWorkCounterSegment', $
|
|
12545
|
+
'terminalWorkCounterSegment', $12::jsonb
|
|
12407
12546
|
),
|
|
12408
12547
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
12409
12548
|
FROM run_terminal
|
|
@@ -12425,6 +12564,7 @@ export async function completePostgresSchedulerAttempt(
|
|
|
12425
12564
|
JSON.stringify(terminalLogLines),
|
|
12426
12565
|
stringifyPostgresJson(ledgerTerminalResult ?? null),
|
|
12427
12566
|
stringifyPostgresJson(resultSummary),
|
|
12567
|
+
JSON.stringify(terminalLogContexts),
|
|
12428
12568
|
stringifyPostgresJson(input.terminalWorkCounterSegment ?? null),
|
|
12429
12569
|
],
|
|
12430
12570
|
);
|
|
@@ -12443,6 +12583,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
12443
12583
|
error: string;
|
|
12444
12584
|
result?: unknown;
|
|
12445
12585
|
terminalLogLines?: readonly string[] | null;
|
|
12586
|
+
terminalLogContexts?: readonly (PlayLogContext | null)[] | null;
|
|
12446
12587
|
/** Refuse to overwrite an authoritative detached runner terminal that won
|
|
12447
12588
|
* after claim. Resumable `suspended` receipts may be superseded by the
|
|
12448
12589
|
* hard ceiling in the same fenced statement. */
|
|
@@ -12478,10 +12619,25 @@ export async function failPostgresSchedulerAttempt(
|
|
|
12478
12619
|
);
|
|
12479
12620
|
// Keep the recovery tail separate from result: large failures are represented
|
|
12480
12621
|
// by a result reference, but their customer guidance must still be logged.
|
|
12481
|
-
|
|
12482
|
-
.
|
|
12483
|
-
.
|
|
12622
|
+
if (
|
|
12623
|
+
input.terminalLogContexts != null &&
|
|
12624
|
+
input.terminalLogContexts.length !== (input.terminalLogLines ?? []).length
|
|
12625
|
+
) {
|
|
12626
|
+
throw new TypeError(
|
|
12627
|
+
'Terminal log contexts must align with terminal log lines.',
|
|
12628
|
+
);
|
|
12629
|
+
}
|
|
12630
|
+
const terminalLogEntries = (input.terminalLogLines ?? [])
|
|
12631
|
+
.map((line, index) => ({
|
|
12632
|
+
line: line.trim(),
|
|
12633
|
+
context: input.terminalLogContexts?.[index] ?? null,
|
|
12634
|
+
}))
|
|
12635
|
+
.filter((entry) => entry.line.length > 0)
|
|
12484
12636
|
.slice(-LOG_TAIL_LIMIT);
|
|
12637
|
+
const terminalLogLines = terminalLogEntries.map((entry) => entry.line);
|
|
12638
|
+
const terminalLogContexts = terminalLogEntries.map((entry) =>
|
|
12639
|
+
normalizePlayLogContext(entry.context),
|
|
12640
|
+
);
|
|
12485
12641
|
const failed = await client.query<{ run_id: string }>(
|
|
12486
12642
|
`
|
|
12487
12643
|
WITH failed AS (
|
|
@@ -12717,8 +12873,9 @@ export async function failPostgresSchedulerAttempt(
|
|
|
12717
12873
|
'error', $4::jsonb ->> 'message',
|
|
12718
12874
|
'result', $8::jsonb,
|
|
12719
12875
|
'terminalLogLines', $12::jsonb,
|
|
12876
|
+
'terminalLogContexts', $14::jsonb,
|
|
12720
12877
|
'resultSummary', $13::jsonb,
|
|
12721
|
-
'terminalWorkCounterSegment', $
|
|
12878
|
+
'terminalWorkCounterSegment', $15::jsonb
|
|
12722
12879
|
),
|
|
12723
12880
|
${causalOutboxCreatedAt('run_terminal.run_id', options)}
|
|
12724
12881
|
FROM run_terminal
|
|
@@ -12743,6 +12900,7 @@ export async function failPostgresSchedulerAttempt(
|
|
|
12743
12900
|
input.preserveRunnerTerminal === true,
|
|
12744
12901
|
JSON.stringify(terminalLogLines),
|
|
12745
12902
|
stringifyPostgresJson(resultSummary),
|
|
12903
|
+
JSON.stringify(terminalLogContexts),
|
|
12746
12904
|
stringifyPostgresJson(terminalWorkCounterSegment ?? null),
|
|
12747
12905
|
],
|
|
12748
12906
|
);
|
|
@@ -13431,37 +13589,62 @@ function boundedRunLogTailMetadata(logs: string[]): {
|
|
|
13431
13589
|
return tail.length > 0 ? { tail, totalLogCount: logs.length } : null;
|
|
13432
13590
|
}
|
|
13433
13591
|
|
|
13434
|
-
function
|
|
13592
|
+
function terminalLogProjectionForOutbox(
|
|
13435
13593
|
runnerResult: unknown,
|
|
13436
13594
|
mode: PostgresSchedulerTerminalLogOutbox | undefined,
|
|
13437
|
-
): string[] {
|
|
13595
|
+
): { lines: string[]; lineContexts: Array<PlayLogContext | null> } {
|
|
13438
13596
|
if (
|
|
13439
13597
|
mode !== 'ctx_log_tail' ||
|
|
13440
13598
|
!runnerResult ||
|
|
13441
13599
|
typeof runnerResult !== 'object'
|
|
13442
13600
|
) {
|
|
13443
|
-
return [];
|
|
13601
|
+
return { lines: [], lineContexts: [] };
|
|
13444
13602
|
}
|
|
13445
|
-
const
|
|
13603
|
+
const result = runnerResult as {
|
|
13604
|
+
logs?: unknown;
|
|
13605
|
+
logContexts?: unknown;
|
|
13606
|
+
};
|
|
13607
|
+
const logs = result.logs;
|
|
13446
13608
|
if (!Array.isArray(logs)) {
|
|
13447
|
-
return [];
|
|
13609
|
+
return { lines: [], lineContexts: [] };
|
|
13610
|
+
}
|
|
13611
|
+
if (
|
|
13612
|
+
result.logContexts !== undefined &&
|
|
13613
|
+
(!Array.isArray(result.logContexts) ||
|
|
13614
|
+
result.logContexts.length !== logs.length)
|
|
13615
|
+
) {
|
|
13616
|
+
throw new TypeError('Terminal log contexts must align with runner logs.');
|
|
13448
13617
|
}
|
|
13449
|
-
|
|
13450
|
-
.
|
|
13451
|
-
|
|
13618
|
+
const logContexts = Array.isArray(result.logContexts)
|
|
13619
|
+
? result.logContexts
|
|
13620
|
+
: undefined;
|
|
13621
|
+
const entries = logs
|
|
13622
|
+
.map((line, index) => {
|
|
13623
|
+
if (typeof line !== 'string') return null;
|
|
13452
13624
|
const normalized = readProvenanceTag(line).line.trim();
|
|
13453
13625
|
// Retain bounded terminal measurement summaries alongside timestamped
|
|
13454
13626
|
// customer/recovery lines after a detached runner has exited.
|
|
13455
|
-
|
|
13627
|
+
const retained =
|
|
13456
13628
|
CTX_LOG_TIMESTAMP_PREFIX_RE.test(normalized) ||
|
|
13457
13629
|
normalized.startsWith('[runtime.failure_diagnostic] ') ||
|
|
13458
13630
|
normalized.startsWith('[runtime.phase_summary] ') ||
|
|
13459
13631
|
normalized.startsWith('[runtime.logs] ') ||
|
|
13460
|
-
normalized.startsWith('[runtime.log_transport] ')
|
|
13461
|
-
);
|
|
13632
|
+
normalized.startsWith('[runtime.log_transport] ');
|
|
13633
|
+
if (!retained) return null;
|
|
13634
|
+
return {
|
|
13635
|
+
line: line.trim(),
|
|
13636
|
+
context: normalizePlayLogContext(logContexts?.[index]),
|
|
13637
|
+
};
|
|
13462
13638
|
})
|
|
13463
|
-
.
|
|
13639
|
+
.filter(
|
|
13640
|
+
(entry): entry is { line: string; context: PlayLogContext | null } =>
|
|
13641
|
+
entry !== null,
|
|
13642
|
+
)
|
|
13464
13643
|
.slice(-LOG_TAIL_LIMIT);
|
|
13644
|
+
return {
|
|
13645
|
+
lines: entries.map((entry) => entry.line),
|
|
13646
|
+
lineContexts: entries.map((entry) => entry.context),
|
|
13647
|
+
};
|
|
13465
13648
|
}
|
|
13466
13649
|
|
|
13467
13650
|
export function attachRunnerWorkProgressMetadata(
|
|
@@ -13694,6 +13877,10 @@ export async function executePostgresSchedulerClaim(
|
|
|
13694
13877
|
{ logMetadata: options.completionLogMetadata ?? 'attach' },
|
|
13695
13878
|
)
|
|
13696
13879
|
: output;
|
|
13880
|
+
const terminalLogProjection = terminalLogProjectionForOutbox(
|
|
13881
|
+
output,
|
|
13882
|
+
options.terminalLogOutbox,
|
|
13883
|
+
);
|
|
13697
13884
|
const completed = await completePostgresSchedulerAttempt(
|
|
13698
13885
|
client,
|
|
13699
13886
|
{
|
|
@@ -13703,14 +13890,12 @@ export async function executePostgresSchedulerClaim(
|
|
|
13703
13890
|
output: terminalOutput,
|
|
13704
13891
|
rowOutcomes: (output as { rowOutcomes?: unknown } | null)?.rowOutcomes,
|
|
13705
13892
|
runtimeTiming: extractRunnerRuntimeTiming(output),
|
|
13893
|
+
terminalLogLines: terminalLogProjection.lines,
|
|
13894
|
+
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
13706
13895
|
terminalWorkCounterSegment: terminalWorkCounterSegmentFromRunnerResult(
|
|
13707
13896
|
output,
|
|
13708
13897
|
claim.attempt,
|
|
13709
13898
|
),
|
|
13710
|
-
terminalLogLines: terminalLogLinesForOutbox(
|
|
13711
|
-
output,
|
|
13712
|
-
options.terminalLogOutbox,
|
|
13713
|
-
),
|
|
13714
13899
|
},
|
|
13715
13900
|
options,
|
|
13716
13901
|
);
|
|
@@ -13754,6 +13939,10 @@ export async function executePostgresSchedulerClaim(
|
|
|
13754
13939
|
checkpoint: null,
|
|
13755
13940
|
tableNamespace: null,
|
|
13756
13941
|
};
|
|
13942
|
+
const terminalLogProjection = terminalLogProjectionForOutbox(
|
|
13943
|
+
terminalFailureResult,
|
|
13944
|
+
options.terminalLogOutbox,
|
|
13945
|
+
);
|
|
13757
13946
|
const failed = await failPostgresSchedulerAttempt(
|
|
13758
13947
|
client,
|
|
13759
13948
|
{
|
|
@@ -13762,10 +13951,8 @@ export async function executePostgresSchedulerClaim(
|
|
|
13762
13951
|
leaseToken: claim.leaseToken,
|
|
13763
13952
|
error: message,
|
|
13764
13953
|
result: terminalFailureResult,
|
|
13765
|
-
terminalLogLines:
|
|
13766
|
-
|
|
13767
|
-
options.terminalLogOutbox,
|
|
13768
|
-
),
|
|
13954
|
+
terminalLogLines: terminalLogProjection.lines,
|
|
13955
|
+
terminalLogContexts: terminalLogProjection.lineContexts,
|
|
13769
13956
|
sandboxCrashDiagnostic: options.sandboxCrashDiagnostic,
|
|
13770
13957
|
},
|
|
13771
13958
|
options,
|
|
@@ -17032,9 +17219,101 @@ export async function listPostgresSchedulerDeliveredOutboxRetentionCandidates(
|
|
|
17032
17219
|
*/
|
|
17033
17220
|
export async function purgePostgresSchedulerDeliveredOutboxRetentionBatch(
|
|
17034
17221
|
client: PostgresSchedulerQueryClient,
|
|
17035
|
-
input: {
|
|
17222
|
+
input: {
|
|
17223
|
+
olderThan: Date;
|
|
17224
|
+
logBodiesOlderThan?: Date;
|
|
17225
|
+
limit?: number;
|
|
17226
|
+
},
|
|
17036
17227
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
17037
17228
|
): Promise<PostgresSchedulerDeliveredOutboxRow[]> {
|
|
17229
|
+
if (input.logBodiesOlderThan) {
|
|
17230
|
+
const logCutoff = input.logBodiesOlderThan.getTime();
|
|
17231
|
+
if (
|
|
17232
|
+
!Number.isFinite(logCutoff) ||
|
|
17233
|
+
logCutoff > Date.now() - 60 * 60 * 1_000
|
|
17234
|
+
) {
|
|
17235
|
+
throw new TypeError(
|
|
17236
|
+
'Delivered outbox log-body cleanup cutoff must be at least one hour old.',
|
|
17237
|
+
);
|
|
17238
|
+
}
|
|
17239
|
+
const limit = normalizeOutboxRetentionLimit(input.limit);
|
|
17240
|
+
await client.query(
|
|
17241
|
+
`WITH candidates AS (
|
|
17242
|
+
SELECT event_id
|
|
17243
|
+
FROM ${tableName('outbox', options)} AS outbox
|
|
17244
|
+
WHERE delivered_at IS NOT NULL
|
|
17245
|
+
AND failed_at IS NULL
|
|
17246
|
+
AND delivered_at < $1::timestamptz
|
|
17247
|
+
AND (
|
|
17248
|
+
(event_type = 'run.events'
|
|
17249
|
+
AND jsonb_typeof(payload_json->'events') = 'array'
|
|
17250
|
+
AND EXISTS (
|
|
17251
|
+
SELECT 1
|
|
17252
|
+
FROM jsonb_array_elements(
|
|
17253
|
+
CASE WHEN jsonb_typeof(payload_json->'events') = 'array'
|
|
17254
|
+
THEN payload_json->'events' ELSE '[]'::jsonb END
|
|
17255
|
+
) AS item
|
|
17256
|
+
WHERE item->>'type' = 'log.appended'
|
|
17257
|
+
AND (item ? 'lines' OR item ? 'lineContexts')
|
|
17258
|
+
))
|
|
17259
|
+
OR (event_type IN ('run.completed', 'run.failed')
|
|
17260
|
+
AND (
|
|
17261
|
+
payload_json ? 'terminalLogLines'
|
|
17262
|
+
OR (jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
|
|
17263
|
+
AND EXISTS (
|
|
17264
|
+
SELECT 1
|
|
17265
|
+
FROM jsonb_array_elements(
|
|
17266
|
+
CASE WHEN jsonb_typeof(payload_json->'terminalProgressEvents') = 'array'
|
|
17267
|
+
THEN payload_json->'terminalProgressEvents' ELSE '[]'::jsonb END
|
|
17268
|
+
) AS item
|
|
17269
|
+
WHERE item->>'type' = 'log.appended'
|
|
17270
|
+
AND (item ? 'lines' OR item ? 'lineContexts')
|
|
17271
|
+
))
|
|
17272
|
+
))
|
|
17273
|
+
)
|
|
17274
|
+
ORDER BY delivered_at ASC, event_id ASC
|
|
17275
|
+
LIMIT $2::integer
|
|
17276
|
+
FOR UPDATE SKIP LOCKED
|
|
17277
|
+
)
|
|
17278
|
+
UPDATE ${tableName('outbox', options)} AS outbox
|
|
17279
|
+
SET payload_json = CASE
|
|
17280
|
+
WHEN outbox.event_type = 'run.events' THEN jsonb_set(
|
|
17281
|
+
outbox.payload_json,
|
|
17282
|
+
'{events}',
|
|
17283
|
+
(SELECT jsonb_agg(
|
|
17284
|
+
CASE WHEN item.value->>'type' = 'log.appended'
|
|
17285
|
+
THEN item.value - 'lines' - 'lineContexts'
|
|
17286
|
+
ELSE item.value
|
|
17287
|
+
END ORDER BY item.ordinality
|
|
17288
|
+
)
|
|
17289
|
+
FROM jsonb_array_elements(
|
|
17290
|
+
CASE WHEN jsonb_typeof(outbox.payload_json->'events') = 'array'
|
|
17291
|
+
THEN outbox.payload_json->'events' ELSE '[]'::jsonb END
|
|
17292
|
+
)
|
|
17293
|
+
WITH ORDINALITY AS item(value, ordinality)),
|
|
17294
|
+
false
|
|
17295
|
+
)
|
|
17296
|
+
WHEN jsonb_typeof(outbox.payload_json->'terminalProgressEvents') = 'array'
|
|
17297
|
+
THEN jsonb_set(
|
|
17298
|
+
outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts',
|
|
17299
|
+
'{terminalProgressEvents}',
|
|
17300
|
+
coalesce((SELECT jsonb_agg(
|
|
17301
|
+
CASE WHEN item.value->>'type' = 'log.appended'
|
|
17302
|
+
THEN item.value - 'lines' - 'lineContexts'
|
|
17303
|
+
ELSE item.value
|
|
17304
|
+
END ORDER BY item.ordinality
|
|
17305
|
+
)
|
|
17306
|
+
FROM jsonb_array_elements(outbox.payload_json->'terminalProgressEvents')
|
|
17307
|
+
WITH ORDINALITY AS item(value, ordinality)), '[]'::jsonb),
|
|
17308
|
+
false
|
|
17309
|
+
)
|
|
17310
|
+
ELSE outbox.payload_json - 'terminalLogLines' - 'terminalLogContexts'
|
|
17311
|
+
END
|
|
17312
|
+
FROM candidates
|
|
17313
|
+
WHERE outbox.event_id = candidates.event_id`,
|
|
17314
|
+
[input.logBodiesOlderThan.toISOString(), limit],
|
|
17315
|
+
);
|
|
17316
|
+
}
|
|
17038
17317
|
const result = await client.query<{
|
|
17039
17318
|
event_id: string;
|
|
17040
17319
|
run_id: string;
|
|
@@ -17317,9 +17596,11 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
17317
17596
|
const occurredAt = event.createdAt.getTime();
|
|
17318
17597
|
if (event.eventType === 'run.events') {
|
|
17319
17598
|
const events = payloadField(event.payload, 'events');
|
|
17599
|
+
const logsStored =
|
|
17600
|
+
payloadField(event.payload, 'logBodyStore') === 'planet_scale';
|
|
17320
17601
|
if (
|
|
17321
17602
|
!Array.isArray(events) ||
|
|
17322
|
-
events.length === 0 ||
|
|
17603
|
+
(events.length === 0 && !logsStored) ||
|
|
17323
17604
|
events.some(
|
|
17324
17605
|
(value) =>
|
|
17325
17606
|
!recordPayload(value) ||
|
|
@@ -17364,6 +17645,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
17364
17645
|
event.runId,
|
|
17365
17646
|
);
|
|
17366
17647
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
17648
|
+
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
17367
17649
|
const resultSummary = payloadField(event.payload, 'resultSummary');
|
|
17368
17650
|
return [
|
|
17369
17651
|
...progressEvents.map(normalizeOutboxIngressPreview),
|
|
@@ -17376,6 +17658,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
17376
17658
|
occurredAt,
|
|
17377
17659
|
source: 'worker',
|
|
17378
17660
|
lines,
|
|
17661
|
+
...(lineContexts ? { lineContexts } : {}),
|
|
17379
17662
|
} as PlayRunLedgerEvent,
|
|
17380
17663
|
]),
|
|
17381
17664
|
{
|
|
@@ -17404,6 +17687,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
17404
17687
|
event.runId,
|
|
17405
17688
|
);
|
|
17406
17689
|
const lines = stringArrayPayloadField(event.payload, 'terminalLogLines');
|
|
17690
|
+
const lineContexts = terminalLogContextsPayloadField(event.payload, lines);
|
|
17407
17691
|
const result = terminalRunResultForLedger(
|
|
17408
17692
|
payloadField(event.payload, 'result'),
|
|
17409
17693
|
);
|
|
@@ -17419,6 +17703,7 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
|
|
|
17419
17703
|
occurredAt,
|
|
17420
17704
|
source: 'worker',
|
|
17421
17705
|
lines,
|
|
17706
|
+
...(lineContexts ? { lineContexts } : {}),
|
|
17422
17707
|
} as PlayRunLedgerEvent,
|
|
17423
17708
|
]),
|
|
17424
17709
|
{
|
|
@@ -18236,6 +18521,9 @@ async function projectPostgresSchedulerCreatedEvent(
|
|
|
18236
18521
|
runtimeBackend: value('runtimeBackend'),
|
|
18237
18522
|
schedulerBackend: value('schedulerBackend'),
|
|
18238
18523
|
schedulerSchema: value('schedulerSchema'),
|
|
18524
|
+
...(value('runLogStore') === 'planet_scale'
|
|
18525
|
+
? { runLogStore: 'planet_scale' as const }
|
|
18526
|
+
: {}),
|
|
18239
18527
|
executionProfile: value('executionProfile'),
|
|
18240
18528
|
runtimeReleaseId: value('runtimeReleaseId'),
|
|
18241
18529
|
runtimeDeployVersion: value('runtimeDeployVersion'),
|
|
@@ -18666,6 +18954,20 @@ function stringArrayPayloadField(payload: unknown, key: string): string[] {
|
|
|
18666
18954
|
: [];
|
|
18667
18955
|
}
|
|
18668
18956
|
|
|
18957
|
+
function terminalLogContextsPayloadField(
|
|
18958
|
+
payload: unknown,
|
|
18959
|
+
lines: readonly string[],
|
|
18960
|
+
): Array<PlayLogContext | null> | null {
|
|
18961
|
+
const value = payloadField(payload, 'terminalLogContexts');
|
|
18962
|
+
if (value === undefined) return null;
|
|
18963
|
+
if (!Array.isArray(value) || value.length !== lines.length) {
|
|
18964
|
+
throw new TypeError(
|
|
18965
|
+
'Terminal log contexts must align with terminal log lines.',
|
|
18966
|
+
);
|
|
18967
|
+
}
|
|
18968
|
+
return value.map(normalizePlayLogContext);
|
|
18969
|
+
}
|
|
18970
|
+
|
|
18669
18971
|
function ledgerEventsPayloadField(
|
|
18670
18972
|
payload: unknown,
|
|
18671
18973
|
key: string,
|