deepline 0.3.223 → 0.3.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts +24 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-projection-contract.ts +121 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger-projection-diagnostics.ts +151 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +36 -84
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +1 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.225',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -51,9 +51,11 @@ import {
|
|
|
51
51
|
import { PLAY_RUNTIME_API_COMPAT_PATH } from './/runtime-api-paths';
|
|
52
52
|
import {
|
|
53
53
|
buildRunLedgerEventsIdempotencyKey,
|
|
54
|
+
isRunLedgerProjectionMismatchDiagnostic,
|
|
54
55
|
isRunLedgerProjectionLeaseLostReason,
|
|
55
56
|
type RunLedgerProjectionChunk,
|
|
56
57
|
type RunLedgerProjectionLeaseLostReason,
|
|
58
|
+
type RunLedgerProjectionMismatchDiagnostic,
|
|
57
59
|
} from './run-ledger-projection-contract';
|
|
58
60
|
import { partitionRunEventsForAppRuntime } from './run-ledger-event-transport';
|
|
59
61
|
import {
|
|
@@ -851,6 +853,21 @@ function appRuntimeFailureReason(
|
|
|
851
853
|
}
|
|
852
854
|
}
|
|
853
855
|
|
|
856
|
+
function appRuntimeProjectionMismatch(
|
|
857
|
+
responseCode: string | null,
|
|
858
|
+
body: string,
|
|
859
|
+
): RunLedgerProjectionMismatchDiagnostic | null {
|
|
860
|
+
if (responseCode !== 'RUN_LEDGER_DELIVERY_LEASE_LOST') return null;
|
|
861
|
+
try {
|
|
862
|
+
const parsed = JSON.parse(body) as { projectionMismatch?: unknown };
|
|
863
|
+
return isRunLedgerProjectionMismatchDiagnostic(parsed.projectionMismatch)
|
|
864
|
+
? parsed.projectionMismatch
|
|
865
|
+
: null;
|
|
866
|
+
} catch {
|
|
867
|
+
return null;
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
854
871
|
export function isTransientAppRuntimeFailureBody(body: string): boolean {
|
|
855
872
|
return /WorkerOverloaded|"\s*code\s*"\s*:\s*"\s*InternalServerError\s*"|Your request couldn't be completed\. Try again later\.|timeout exceeded when trying to connect|timed out|fetch failed|ECONNRESET|ECONNREFUSED|UND_ERR_CONNECT_TIMEOUT|tuple concurrently updated|RUNTIME_SCHEDULER_(?:SATURATED|UNAVAILABLE|SCHEMA_NOT_READY)|Runtime scheduler DB pool is saturated|Runtime scheduler DB circuit breaker is open|Runtime scheduler schema .* is not ready|OptimisticConcurrencyControlFailure|changed while this mutation was being run|Documents read from or written to the .* table changed while this mutation/i.test(
|
|
856
873
|
body,
|
|
@@ -1264,6 +1281,7 @@ export class AppRuntimeApiResponseError extends Error {
|
|
|
1264
1281
|
readonly status: number;
|
|
1265
1282
|
readonly code: string | null;
|
|
1266
1283
|
readonly failureReason: RunLedgerProjectionLeaseLostReason | null;
|
|
1284
|
+
readonly projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
|
|
1267
1285
|
readonly requestId: string | null;
|
|
1268
1286
|
readonly retryable: boolean;
|
|
1269
1287
|
/** Server-requested delay, retained for durable callers after local retries end. */
|
|
@@ -1275,6 +1293,7 @@ export class AppRuntimeApiResponseError extends Error {
|
|
|
1275
1293
|
status: number;
|
|
1276
1294
|
code?: string | null;
|
|
1277
1295
|
failureReason?: RunLedgerProjectionLeaseLostReason | null;
|
|
1296
|
+
projectionMismatch?: RunLedgerProjectionMismatchDiagnostic | null;
|
|
1278
1297
|
requestId?: string | null;
|
|
1279
1298
|
retryable: boolean;
|
|
1280
1299
|
retryAfterMs?: number | null;
|
|
@@ -1292,6 +1311,7 @@ export class AppRuntimeApiResponseError extends Error {
|
|
|
1292
1311
|
this.status = input.status;
|
|
1293
1312
|
this.code = input.code?.trim() || null;
|
|
1294
1313
|
this.failureReason = input.failureReason ?? null;
|
|
1314
|
+
this.projectionMismatch = input.projectionMismatch ?? null;
|
|
1295
1315
|
this.requestId = input.requestId?.trim() || null;
|
|
1296
1316
|
this.retryable = input.retryable;
|
|
1297
1317
|
this.retryAfterMs =
|
|
@@ -1757,6 +1777,7 @@ async function postAppRuntimeApi<TResponse>(
|
|
|
1757
1777
|
status: response.status,
|
|
1758
1778
|
code,
|
|
1759
1779
|
failureReason: appRuntimeFailureReason(code, responseText),
|
|
1780
|
+
projectionMismatch: appRuntimeProjectionMismatch(code, responseText),
|
|
1760
1781
|
requestId,
|
|
1761
1782
|
// The structured response is authoritative even for a 5xx. Retrying an
|
|
1762
1783
|
// explicitly deterministic database failure here turns one rejected SQL
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { PlayRunLedgerIngressEvent } from './run-ledger';
|
|
2
|
+
import type { RunLedgerProjectionChunk } from './run-ledger-projection-contract';
|
|
2
3
|
|
|
3
4
|
const APP_RUNTIME_LOG_EVENT_MAX_JSON_BYTES = 128 * 1024;
|
|
4
5
|
const APP_RUNTIME_EVENT_BATCH_MAX_JSON_BYTES = 512 * 1024;
|
|
@@ -108,3 +109,26 @@ export function partitionRunEventsForAppRuntime(
|
|
|
108
109
|
if (current.length > 0) batches.push(current);
|
|
109
110
|
return batches;
|
|
110
111
|
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Select a projection chunk from the raw canonical prefix. Callers must apply
|
|
115
|
+
* route sanitization only after this boundary is selected, because sanitizing
|
|
116
|
+
* event payloads can change their encoded sizes and shift transport chunks.
|
|
117
|
+
*/
|
|
118
|
+
export function selectRunEventsForAppRuntimeProjectionChunk(
|
|
119
|
+
events: readonly PlayRunLedgerIngressEvent[],
|
|
120
|
+
chunk: RunLedgerProjectionChunk,
|
|
121
|
+
): PlayRunLedgerIngressEvent[] | undefined {
|
|
122
|
+
if (
|
|
123
|
+
!Number.isSafeInteger(chunk.index) ||
|
|
124
|
+
!Number.isSafeInteger(chunk.count) ||
|
|
125
|
+
chunk.index < 0 ||
|
|
126
|
+
chunk.count < 2 ||
|
|
127
|
+
chunk.index >= chunk.count
|
|
128
|
+
) {
|
|
129
|
+
return undefined;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const batches = partitionRunEventsForAppRuntime(events);
|
|
133
|
+
return batches.length === chunk.count ? batches[chunk.index] : undefined;
|
|
134
|
+
}
|
|
@@ -45,6 +45,124 @@ export const RUN_LEDGER_PROJECTION_LEASE_LOST_REASONS = [
|
|
|
45
45
|
'projected_events_mismatch',
|
|
46
46
|
] as const;
|
|
47
47
|
|
|
48
|
+
export const RUN_LEDGER_PROJECTION_MISMATCH_KINDS = [
|
|
49
|
+
'expected_chunk_unavailable',
|
|
50
|
+
'event_count_mismatch',
|
|
51
|
+
'event_content_mismatch',
|
|
52
|
+
] as const;
|
|
53
|
+
|
|
54
|
+
export const RUN_LEDGER_PROJECTION_MISMATCH_FIELDS = [
|
|
55
|
+
'type',
|
|
56
|
+
'runId',
|
|
57
|
+
'occurredAt',
|
|
58
|
+
'source',
|
|
59
|
+
'opaque',
|
|
60
|
+
'payload',
|
|
61
|
+
'seq',
|
|
62
|
+
'status',
|
|
63
|
+
'playName',
|
|
64
|
+
'runtimeBackend',
|
|
65
|
+
'triggerSource',
|
|
66
|
+
'inputSummary',
|
|
67
|
+
'definitionScope',
|
|
68
|
+
'result',
|
|
69
|
+
'resultSummary',
|
|
70
|
+
'error',
|
|
71
|
+
'errorSummary',
|
|
72
|
+
'retryAfterMs',
|
|
73
|
+
'waitKind',
|
|
74
|
+
'waitUntil',
|
|
75
|
+
'nodeId',
|
|
76
|
+
'progress',
|
|
77
|
+
'stepId',
|
|
78
|
+
'toolId',
|
|
79
|
+
'provider',
|
|
80
|
+
'operation',
|
|
81
|
+
'lines',
|
|
82
|
+
'channelOffset',
|
|
83
|
+
'producerAttempt',
|
|
84
|
+
'other',
|
|
85
|
+
] as const;
|
|
86
|
+
|
|
87
|
+
export type RunLedgerProjectionMismatchKind =
|
|
88
|
+
(typeof RUN_LEDGER_PROJECTION_MISMATCH_KINDS)[number];
|
|
89
|
+
|
|
90
|
+
export type RunLedgerProjectionMismatchField =
|
|
91
|
+
(typeof RUN_LEDGER_PROJECTION_MISMATCH_FIELDS)[number];
|
|
92
|
+
|
|
93
|
+
/** Payload-free comparison facts safe to persist in the operator diagnostics. */
|
|
94
|
+
export type RunLedgerProjectionMismatchDiagnostic = {
|
|
95
|
+
kind: RunLedgerProjectionMismatchKind;
|
|
96
|
+
expectedEventCount: number | null;
|
|
97
|
+
projectedEventCount: number;
|
|
98
|
+
chunkIndex: number | null;
|
|
99
|
+
chunkCount: number | null;
|
|
100
|
+
firstMismatchIndex: number | null;
|
|
101
|
+
eventTypeMatches: boolean | null;
|
|
102
|
+
mismatchedFields: RunLedgerProjectionMismatchField[];
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
export function isRunLedgerProjectionMismatchDiagnostic(
|
|
106
|
+
value: unknown,
|
|
107
|
+
): value is RunLedgerProjectionMismatchDiagnostic {
|
|
108
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
|
109
|
+
const candidate = value as Record<string, unknown>;
|
|
110
|
+
const diagnosticKeys = [
|
|
111
|
+
'kind',
|
|
112
|
+
'expectedEventCount',
|
|
113
|
+
'projectedEventCount',
|
|
114
|
+
'chunkIndex',
|
|
115
|
+
'chunkCount',
|
|
116
|
+
'firstMismatchIndex',
|
|
117
|
+
'eventTypeMatches',
|
|
118
|
+
'mismatchedFields',
|
|
119
|
+
];
|
|
120
|
+
const keys = Object.keys(candidate);
|
|
121
|
+
if (
|
|
122
|
+
keys.length !== diagnosticKeys.length ||
|
|
123
|
+
keys.some((key) => !diagnosticKeys.includes(key))
|
|
124
|
+
) {
|
|
125
|
+
return false;
|
|
126
|
+
}
|
|
127
|
+
return (
|
|
128
|
+
typeof candidate.kind === 'string' &&
|
|
129
|
+
RUN_LEDGER_PROJECTION_MISMATCH_KINDS.includes(
|
|
130
|
+
candidate.kind as RunLedgerProjectionMismatchKind,
|
|
131
|
+
) &&
|
|
132
|
+
(candidate.expectedEventCount === null ||
|
|
133
|
+
(Number.isSafeInteger(candidate.expectedEventCount) &&
|
|
134
|
+
Number(candidate.expectedEventCount) >= 0)) &&
|
|
135
|
+
Number.isSafeInteger(candidate.projectedEventCount) &&
|
|
136
|
+
Number(candidate.projectedEventCount) >= 0 &&
|
|
137
|
+
(candidate.chunkIndex === null ||
|
|
138
|
+
(Number.isSafeInteger(candidate.chunkIndex) &&
|
|
139
|
+
Number(candidate.chunkIndex) >= 0)) &&
|
|
140
|
+
(candidate.chunkCount === null ||
|
|
141
|
+
(Number.isSafeInteger(candidate.chunkCount) &&
|
|
142
|
+
Number(candidate.chunkCount) > 1)) &&
|
|
143
|
+
((candidate.chunkIndex === null && candidate.chunkCount === null) ||
|
|
144
|
+
(Number.isSafeInteger(candidate.chunkIndex) &&
|
|
145
|
+
Number.isSafeInteger(candidate.chunkCount) &&
|
|
146
|
+
Number(candidate.chunkIndex) < Number(candidate.chunkCount))) &&
|
|
147
|
+
(candidate.firstMismatchIndex === null ||
|
|
148
|
+
(Number.isSafeInteger(candidate.firstMismatchIndex) &&
|
|
149
|
+
Number(candidate.firstMismatchIndex) >= 0)) &&
|
|
150
|
+
(candidate.eventTypeMatches === null ||
|
|
151
|
+
typeof candidate.eventTypeMatches === 'boolean') &&
|
|
152
|
+
Array.isArray(candidate.mismatchedFields) &&
|
|
153
|
+
candidate.mismatchedFields.length <= 8 &&
|
|
154
|
+
candidate.mismatchedFields.every(
|
|
155
|
+
(field) =>
|
|
156
|
+
typeof field === 'string' &&
|
|
157
|
+
RUN_LEDGER_PROJECTION_MISMATCH_FIELDS.includes(
|
|
158
|
+
field as RunLedgerProjectionMismatchField,
|
|
159
|
+
),
|
|
160
|
+
) &&
|
|
161
|
+
new Set(candidate.mismatchedFields).size ===
|
|
162
|
+
candidate.mismatchedFields.length
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
48
166
|
export type RunLedgerProjectionLeaseLostReason =
|
|
49
167
|
(typeof RUN_LEDGER_PROJECTION_LEASE_LOST_REASONS)[number];
|
|
50
168
|
|
|
@@ -61,13 +179,16 @@ export function isRunLedgerProjectionLeaseLostReason(
|
|
|
61
179
|
|
|
62
180
|
export class RunLedgerProjectionLeaseLostError extends Error {
|
|
63
181
|
readonly reason: RunLedgerProjectionLeaseLostReason;
|
|
182
|
+
readonly projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
|
|
64
183
|
|
|
65
184
|
constructor(
|
|
66
185
|
reason: RunLedgerProjectionLeaseLostReason = 'delivery_lease_not_current',
|
|
186
|
+
projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null = null,
|
|
67
187
|
) {
|
|
68
188
|
super('Run Ledger projection requires the current outbox delivery lease.');
|
|
69
189
|
this.name = 'RunLedgerProjectionLeaseLostError';
|
|
70
190
|
this.reason = reason;
|
|
191
|
+
this.projectionMismatch = projectionMismatch;
|
|
71
192
|
}
|
|
72
193
|
}
|
|
73
194
|
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { stableStringify } from '../plays/row-identity';
|
|
2
|
+
import type {
|
|
3
|
+
AcceptedPlayRunLedgerEvent,
|
|
4
|
+
PlayRunLedgerIngressEvent,
|
|
5
|
+
} from './run-ledger';
|
|
6
|
+
import { isPlayRunLedgerOpaqueEvent } from './run-ledger';
|
|
7
|
+
import {
|
|
8
|
+
RUN_LEDGER_PROJECTION_MISMATCH_FIELDS,
|
|
9
|
+
type RunLedgerProjectionChunk,
|
|
10
|
+
type RunLedgerProjectionMismatchDiagnostic,
|
|
11
|
+
type RunLedgerProjectionMismatchField,
|
|
12
|
+
} from './run-ledger-projection-contract';
|
|
13
|
+
|
|
14
|
+
type ComparableRunLedgerEvent =
|
|
15
|
+
| AcceptedPlayRunLedgerEvent
|
|
16
|
+
| PlayRunLedgerIngressEvent;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A newer app may receive a projection from a worker that predates optional
|
|
20
|
+
* wait metadata. Normalize only those absent fields before the exact compare.
|
|
21
|
+
*/
|
|
22
|
+
function normalizeLegacyRunWaitingProjectionFields(
|
|
23
|
+
projectedEvents: readonly AcceptedPlayRunLedgerEvent[],
|
|
24
|
+
expectedEvents: readonly ComparableRunLedgerEvent[],
|
|
25
|
+
): AcceptedPlayRunLedgerEvent[] {
|
|
26
|
+
return projectedEvents.map((event, index) => {
|
|
27
|
+
const expected = expectedEvents[index];
|
|
28
|
+
if (
|
|
29
|
+
isPlayRunLedgerOpaqueEvent(event) ||
|
|
30
|
+
event.type !== 'run.waiting' ||
|
|
31
|
+
expected?.type !== 'run.waiting' ||
|
|
32
|
+
('opaque' in expected && expected.opaque === true)
|
|
33
|
+
) {
|
|
34
|
+
return event;
|
|
35
|
+
}
|
|
36
|
+
const expectedEvent = expected as Record<string, unknown>;
|
|
37
|
+
const expectedWaitKind = expectedEvent.waitKind;
|
|
38
|
+
const expectedWaitUntil = expectedEvent.waitUntil;
|
|
39
|
+
return {
|
|
40
|
+
...event,
|
|
41
|
+
...(event.waitKind === undefined &&
|
|
42
|
+
(typeof expectedWaitKind === 'string' || expectedWaitKind === null)
|
|
43
|
+
? { waitKind: expectedWaitKind }
|
|
44
|
+
: {}),
|
|
45
|
+
...(event.waitUntil === undefined &&
|
|
46
|
+
(typeof expectedWaitUntil === 'number' || expectedWaitUntil === null)
|
|
47
|
+
? { waitUntil: expectedWaitUntil }
|
|
48
|
+
: {}),
|
|
49
|
+
};
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function eventType(event: ComparableRunLedgerEvent | undefined): string | null {
|
|
54
|
+
if (!event) return null;
|
|
55
|
+
return typeof event.type === 'string'
|
|
56
|
+
? event.type
|
|
57
|
+
: typeof event.key === 'string'
|
|
58
|
+
? event.key
|
|
59
|
+
: null;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function safeMismatchedFields(
|
|
63
|
+
projected: ComparableRunLedgerEvent,
|
|
64
|
+
expected: ComparableRunLedgerEvent,
|
|
65
|
+
): RunLedgerProjectionMismatchField[] {
|
|
66
|
+
const projectedRecord = projected as Record<string, unknown>;
|
|
67
|
+
const expectedRecord = expected as Record<string, unknown>;
|
|
68
|
+
const allowedFields = new Set<string>(RUN_LEDGER_PROJECTION_MISMATCH_FIELDS);
|
|
69
|
+
const fields = new Set<RunLedgerProjectionMismatchField>();
|
|
70
|
+
const keys = new Set([
|
|
71
|
+
...Object.keys(projectedRecord),
|
|
72
|
+
...Object.keys(expectedRecord),
|
|
73
|
+
]);
|
|
74
|
+
for (const key of [...keys].sort()) {
|
|
75
|
+
if (
|
|
76
|
+
stableStringify(projectedRecord[key]) ===
|
|
77
|
+
stableStringify(expectedRecord[key])
|
|
78
|
+
) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
fields.add(
|
|
82
|
+
allowedFields.has(key)
|
|
83
|
+
? (key as RunLedgerProjectionMismatchField)
|
|
84
|
+
: 'other',
|
|
85
|
+
);
|
|
86
|
+
if (fields.size === 8) break;
|
|
87
|
+
}
|
|
88
|
+
return [...fields];
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Compare one received chunk with its canonical outbox slice and return only
|
|
93
|
+
* bounded structural facts. Event values, IDs, and opaque payload keys never
|
|
94
|
+
* leave this module.
|
|
95
|
+
*/
|
|
96
|
+
export function runLedgerProjectionMismatch(input: {
|
|
97
|
+
projectedEvents: readonly AcceptedPlayRunLedgerEvent[];
|
|
98
|
+
expectedEvents: readonly ComparableRunLedgerEvent[] | null;
|
|
99
|
+
chunk?: RunLedgerProjectionChunk;
|
|
100
|
+
}): RunLedgerProjectionMismatchDiagnostic | null {
|
|
101
|
+
const chunkIndex = input.chunk?.index ?? null;
|
|
102
|
+
const chunkCount = input.chunk?.count ?? null;
|
|
103
|
+
const projectedEventCount = input.projectedEvents.length;
|
|
104
|
+
if (input.expectedEvents === null) {
|
|
105
|
+
return {
|
|
106
|
+
kind: 'expected_chunk_unavailable',
|
|
107
|
+
expectedEventCount: null,
|
|
108
|
+
projectedEventCount,
|
|
109
|
+
chunkIndex,
|
|
110
|
+
chunkCount,
|
|
111
|
+
firstMismatchIndex: null,
|
|
112
|
+
eventTypeMatches: null,
|
|
113
|
+
mismatchedFields: [],
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const normalizedProjectedEvents = normalizeLegacyRunWaitingProjectionFields(
|
|
118
|
+
input.projectedEvents,
|
|
119
|
+
input.expectedEvents,
|
|
120
|
+
);
|
|
121
|
+
const expectedEventCount = input.expectedEvents.length;
|
|
122
|
+
const sharedEventCount = Math.min(projectedEventCount, expectedEventCount);
|
|
123
|
+
for (let index = 0; index < sharedEventCount; index += 1) {
|
|
124
|
+
const projected = normalizedProjectedEvents[index]!;
|
|
125
|
+
const expected = input.expectedEvents[index]!;
|
|
126
|
+
if (stableStringify(projected) === stableStringify(expected)) continue;
|
|
127
|
+
return {
|
|
128
|
+
kind: 'event_content_mismatch',
|
|
129
|
+
expectedEventCount,
|
|
130
|
+
projectedEventCount,
|
|
131
|
+
chunkIndex,
|
|
132
|
+
chunkCount,
|
|
133
|
+
firstMismatchIndex: index,
|
|
134
|
+
eventTypeMatches: eventType(projected) === eventType(expected),
|
|
135
|
+
mismatchedFields: safeMismatchedFields(projected, expected),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
if (projectedEventCount !== expectedEventCount) {
|
|
139
|
+
return {
|
|
140
|
+
kind: 'event_count_mismatch',
|
|
141
|
+
expectedEventCount,
|
|
142
|
+
projectedEventCount,
|
|
143
|
+
chunkIndex,
|
|
144
|
+
chunkCount,
|
|
145
|
+
firstMismatchIndex: sharedEventCount,
|
|
146
|
+
eventTypeMatches: null,
|
|
147
|
+
mismatchedFields: [],
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
@@ -78,7 +78,6 @@ import {
|
|
|
78
78
|
} from '../work-receipts';
|
|
79
79
|
import type { PlayStaticPipeline } from '../../plays/static-pipeline';
|
|
80
80
|
import {
|
|
81
|
-
isPlayRunLedgerOpaqueEvent,
|
|
82
81
|
LOG_TAIL_LIMIT,
|
|
83
82
|
type AcceptedPlayRunLedgerEvent,
|
|
84
83
|
type PlayRunLedgerEvent,
|
|
@@ -98,10 +97,12 @@ import {
|
|
|
98
97
|
RunLedgerProjectionLeaseLostError,
|
|
99
98
|
type RunLedgerProjectionChunk,
|
|
100
99
|
type RunLedgerProjectionLeaseLostReason,
|
|
100
|
+
type RunLedgerProjectionMismatchDiagnostic,
|
|
101
101
|
buildRunLedgerStartIdempotencyKey,
|
|
102
102
|
buildRunLedgerTerminalIdempotencyKey,
|
|
103
103
|
} from '../run-ledger-projection-contract';
|
|
104
|
-
import {
|
|
104
|
+
import { runLedgerProjectionMismatch } from '../run-ledger-projection-diagnostics';
|
|
105
|
+
import { selectRunEventsForAppRuntimeProjectionChunk } from '../run-ledger-event-transport';
|
|
105
106
|
import {
|
|
106
107
|
assertTerminalRunResultWithinLimit,
|
|
107
108
|
isLedgerTerminalResultRef,
|
|
@@ -15938,64 +15939,6 @@ async function appendRunLedgerOutboxFact(
|
|
|
15938
15939
|
|
|
15939
15940
|
export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-contract';
|
|
15940
15941
|
|
|
15941
|
-
/**
|
|
15942
|
-
* A newer app can receive a projection from a worker that predates optional
|
|
15943
|
-
* wait metadata. Treat only those absent fields as their canonical values;
|
|
15944
|
-
* lease identity, event ordering, and every field the worker did send remain
|
|
15945
|
-
* exact.
|
|
15946
|
-
*/
|
|
15947
|
-
function normalizeLegacyRunWaitingProjectionFields(
|
|
15948
|
-
projectedEvents: readonly AcceptedPlayRunLedgerEvent[],
|
|
15949
|
-
expectedEvents: readonly (
|
|
15950
|
-
| AcceptedPlayRunLedgerEvent
|
|
15951
|
-
| PlayRunLedgerIngressEvent
|
|
15952
|
-
)[],
|
|
15953
|
-
): AcceptedPlayRunLedgerEvent[] {
|
|
15954
|
-
return projectedEvents.map((event, index) => {
|
|
15955
|
-
const expected = expectedEvents[index];
|
|
15956
|
-
if (
|
|
15957
|
-
isPlayRunLedgerOpaqueEvent(event) ||
|
|
15958
|
-
event.type !== 'run.waiting' ||
|
|
15959
|
-
expected?.type !== 'run.waiting' ||
|
|
15960
|
-
('opaque' in expected && expected.opaque === true)
|
|
15961
|
-
) {
|
|
15962
|
-
return event;
|
|
15963
|
-
}
|
|
15964
|
-
const expectedEvent = expected as Record<string, unknown>;
|
|
15965
|
-
const expectedWaitKind = expectedEvent.waitKind;
|
|
15966
|
-
const expectedWaitUntil = expectedEvent.waitUntil;
|
|
15967
|
-
return {
|
|
15968
|
-
...event,
|
|
15969
|
-
...(event.waitKind === undefined &&
|
|
15970
|
-
(typeof expectedWaitKind === 'string' || expectedWaitKind === null)
|
|
15971
|
-
? { waitKind: expectedWaitKind }
|
|
15972
|
-
: {}),
|
|
15973
|
-
...(event.waitUntil === undefined &&
|
|
15974
|
-
(typeof expectedWaitUntil === 'number' || expectedWaitUntil === null)
|
|
15975
|
-
? { waitUntil: expectedWaitUntil }
|
|
15976
|
-
: {}),
|
|
15977
|
-
};
|
|
15978
|
-
});
|
|
15979
|
-
}
|
|
15980
|
-
|
|
15981
|
-
function runLedgerProjectionMatchesCanonicalPrefix(
|
|
15982
|
-
projectedEvents: readonly AcceptedPlayRunLedgerEvent[],
|
|
15983
|
-
expectedEvents: readonly (
|
|
15984
|
-
| AcceptedPlayRunLedgerEvent
|
|
15985
|
-
| PlayRunLedgerIngressEvent
|
|
15986
|
-
)[],
|
|
15987
|
-
): boolean {
|
|
15988
|
-
if (projectedEvents.length !== expectedEvents.length) return false;
|
|
15989
|
-
return (
|
|
15990
|
-
stableStringify(
|
|
15991
|
-
normalizeLegacyRunWaitingProjectionFields(
|
|
15992
|
-
projectedEvents,
|
|
15993
|
-
expectedEvents,
|
|
15994
|
-
),
|
|
15995
|
-
) === stableStringify(expectedEvents)
|
|
15996
|
-
);
|
|
15997
|
-
}
|
|
15998
|
-
|
|
15999
15942
|
/** Extend only our still-live lease before another bounded transport piece. */
|
|
16000
15943
|
export async function renewPostgresRunLedgerDelivery(
|
|
16001
15944
|
client: PostgresSchedulerQueryClient,
|
|
@@ -16030,7 +15973,7 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
16030
15973
|
leaseOwner?: string;
|
|
16031
15974
|
projectedEvents?: readonly AcceptedPlayRunLedgerEvent[];
|
|
16032
15975
|
projectionChunk?: RunLedgerProjectionChunk;
|
|
16033
|
-
|
|
15976
|
+
sanitizeExpectedProjectionEvents?: (
|
|
16034
15977
|
events: readonly PlayRunLedgerIngressEvent[],
|
|
16035
15978
|
) => readonly AcceptedPlayRunLedgerEvent[];
|
|
16036
15979
|
},
|
|
@@ -16163,35 +16106,37 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
16163
16106
|
}),
|
|
16164
16107
|
);
|
|
16165
16108
|
}
|
|
16166
|
-
const comparablePrefixEvents: readonly (
|
|
16167
|
-
| AcceptedPlayRunLedgerEvent
|
|
16168
|
-
| PlayRunLedgerIngressEvent
|
|
16169
|
-
)[] = input.normalizeProjectedEvents
|
|
16170
|
-
? input.normalizeProjectedEvents(prefixEvents)
|
|
16171
|
-
: prefixEvents;
|
|
16172
16109
|
let expectedProjectionEvents:
|
|
16173
16110
|
| readonly (AcceptedPlayRunLedgerEvent | PlayRunLedgerIngressEvent)[]
|
|
16174
|
-
| undefined
|
|
16111
|
+
| undefined;
|
|
16175
16112
|
if (input.projectionChunk) {
|
|
16176
16113
|
const chunk = input.projectionChunk;
|
|
16177
|
-
|
|
16178
|
-
|
|
16179
|
-
|
|
16180
|
-
|
|
16181
|
-
: [];
|
|
16114
|
+
// The sender partitions ingress events before the route sanitizes each chunk.
|
|
16115
|
+
const expectedTransportChunk = isRunLedgerProjectionChunk(chunk)
|
|
16116
|
+
? selectRunEventsForAppRuntimeProjectionChunk(prefixEvents, chunk)
|
|
16117
|
+
: undefined;
|
|
16182
16118
|
expectedProjectionEvents =
|
|
16183
|
-
|
|
16184
|
-
|
|
16185
|
-
|
|
16186
|
-
|
|
16187
|
-
|
|
16188
|
-
|
|
16189
|
-
|
|
16190
|
-
|
|
16191
|
-
|
|
16192
|
-
return;
|
|
16119
|
+
expectedTransportChunk === undefined
|
|
16120
|
+
? undefined
|
|
16121
|
+
: input.sanitizeExpectedProjectionEvents
|
|
16122
|
+
? input.sanitizeExpectedProjectionEvents(expectedTransportChunk)
|
|
16123
|
+
: expectedTransportChunk;
|
|
16124
|
+
} else {
|
|
16125
|
+
expectedProjectionEvents = input.sanitizeExpectedProjectionEvents
|
|
16126
|
+
? input.sanitizeExpectedProjectionEvents(prefixEvents)
|
|
16127
|
+
: prefixEvents;
|
|
16193
16128
|
}
|
|
16194
|
-
|
|
16129
|
+
const projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null =
|
|
16130
|
+
runLedgerProjectionMismatch({
|
|
16131
|
+
projectedEvents: input.projectedEvents,
|
|
16132
|
+
expectedEvents: expectedProjectionEvents ?? null,
|
|
16133
|
+
...(input.projectionChunk ? { chunk: input.projectionChunk } : {}),
|
|
16134
|
+
});
|
|
16135
|
+
if (projectionMismatch === null) return;
|
|
16136
|
+
throw new RunLedgerProjectionLeaseLostError(
|
|
16137
|
+
'projected_events_mismatch',
|
|
16138
|
+
projectionMismatch,
|
|
16139
|
+
);
|
|
16195
16140
|
}
|
|
16196
16141
|
|
|
16197
16142
|
if (
|
|
@@ -16727,6 +16672,7 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
16727
16672
|
status: number | null;
|
|
16728
16673
|
retryable: boolean;
|
|
16729
16674
|
failureReason: RunLedgerProjectionLeaseLostReason | null;
|
|
16675
|
+
projectionMismatch: RunLedgerProjectionMismatchDiagnostic | null;
|
|
16730
16676
|
retryAfterMs: number | null;
|
|
16731
16677
|
} {
|
|
16732
16678
|
if (input.error instanceof AppRuntimeApiResponseError) {
|
|
@@ -16747,6 +16693,7 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
16747
16693
|
status: input.error.status,
|
|
16748
16694
|
retryable,
|
|
16749
16695
|
failureReason: input.error.failureReason,
|
|
16696
|
+
projectionMismatch: input.error.projectionMismatch,
|
|
16750
16697
|
retryAfterMs:
|
|
16751
16698
|
input.error.retryAfterMs === null
|
|
16752
16699
|
? null
|
|
@@ -16763,6 +16710,7 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
16763
16710
|
status: null,
|
|
16764
16711
|
retryable: true,
|
|
16765
16712
|
failureReason: null,
|
|
16713
|
+
projectionMismatch: null,
|
|
16766
16714
|
retryAfterMs: null,
|
|
16767
16715
|
};
|
|
16768
16716
|
}
|
|
@@ -16774,6 +16722,7 @@ function runLedgerDeliveryFailure(input: { error: unknown }): {
|
|
|
16774
16722
|
status: null,
|
|
16775
16723
|
retryable: true,
|
|
16776
16724
|
failureReason: null,
|
|
16725
|
+
projectionMismatch: null,
|
|
16777
16726
|
retryAfterMs: null,
|
|
16778
16727
|
};
|
|
16779
16728
|
}
|
|
@@ -16873,6 +16822,9 @@ export async function settlePostgresSchedulerOutboxFailure(
|
|
|
16873
16822
|
...(failure.failureReason
|
|
16874
16823
|
? { failureReason: failure.failureReason }
|
|
16875
16824
|
: {}),
|
|
16825
|
+
...(failure.projectionMismatch
|
|
16826
|
+
? { projectionMismatch: failure.projectionMismatch }
|
|
16827
|
+
: {}),
|
|
16876
16828
|
retryAfterMs: failure.retryAfterMs,
|
|
16877
16829
|
recordedAt: new Date().toISOString(),
|
|
16878
16830
|
}),
|
package/dist/cli/index.js
CHANGED
|
@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
|
|
|
3051
3051
|
// getters keep their established compatibility behavior.
|
|
3052
3052
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3053
3053
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3054
|
-
version: "0.3.
|
|
3054
|
+
version: "0.3.225",
|
|
3055
3055
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3056
3056
|
packageCapabilities: {
|
|
3057
3057
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
|
|
|
3046
3046
|
// getters keep their established compatibility behavior.
|
|
3047
3047
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3048
3048
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3049
|
-
version: "0.3.
|
|
3049
|
+
version: "0.3.225",
|
|
3050
3050
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3051
3051
|
packageCapabilities: {
|
|
3052
3052
|
updatePreferences: 1
|
package/dist/index.js
CHANGED
|
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
|
|
|
864
864
|
// getters keep their established compatibility behavior.
|
|
865
865
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
866
866
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
867
|
-
version: "0.3.
|
|
867
|
+
version: "0.3.225",
|
|
868
868
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
869
869
|
packageCapabilities: {
|
|
870
870
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
|
|
|
768
768
|
// getters keep their established compatibility behavior.
|
|
769
769
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
770
770
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
771
|
-
version: "0.3.
|
|
771
|
+
version: "0.3.225",
|
|
772
772
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
773
773
|
packageCapabilities: {
|
|
774
774
|
updatePreferences: 1
|
|
@@ -295,6 +295,7 @@
|
|
|
295
295
|
"dist/bundling-sources/shared_libs/play-runtime/run-identity.ts",
|
|
296
296
|
"dist/bundling-sources/shared_libs/play-runtime/run-ledger-event-transport.ts",
|
|
297
297
|
"dist/bundling-sources/shared_libs/play-runtime/run-ledger-projection-contract.ts",
|
|
298
|
+
"dist/bundling-sources/shared_libs/play-runtime/run-ledger-projection-diagnostics.ts",
|
|
298
299
|
"dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts",
|
|
299
300
|
"dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts",
|
|
300
301
|
"dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts",
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.225";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.225";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.225",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.225",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|