deepline 0.1.225 → 0.1.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +33 -17
- package/dist/bundling-sources/sdk/src/play.ts +5 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/observability/tracing.ts +147 -22
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +80 -6
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -1
- package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/postgres-json.ts +66 -4
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +103 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +186 -45
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-transition.ts +126 -8
- package/dist/bundling-sources/shared_libs/play-runtime/step-lifecycle-tracker.ts +3 -1
- package/dist/cli/index.js +19 -2
- package/dist/cli/index.mjs +19 -2
- package/dist/index.d.mts +5 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +7 -2
- package/dist/index.mjs +7 -2
- package/package.json +1 -1
|
@@ -2539,6 +2539,7 @@ type WorkerStepProgram = {
|
|
|
2539
2539
|
|
|
2540
2540
|
type WorkerMapOptions = {
|
|
2541
2541
|
description?: string;
|
|
2542
|
+
mode?: 'upsert' | 'net_new';
|
|
2542
2543
|
concurrency?: number;
|
|
2543
2544
|
key?:
|
|
2544
2545
|
| string
|
|
@@ -4870,9 +4871,18 @@ function createMinimalWorkerCtx(
|
|
|
4870
4871
|
runId: req.runId,
|
|
4871
4872
|
inputOffset: 0,
|
|
4872
4873
|
attemptLeaseTtlMs: runtimeSheetAttemptLeaseTtlMs(req),
|
|
4874
|
+
...(opts?.mode ? { mode: opts.mode } : {}),
|
|
4873
4875
|
...mapWriteAttempt,
|
|
4874
4876
|
});
|
|
4875
4877
|
const mapWriteVersion = mapReservation?.writeVersion ?? null;
|
|
4878
|
+
const netNewRowKeys =
|
|
4879
|
+
opts?.mode === 'net_new'
|
|
4880
|
+
? new Set(
|
|
4881
|
+
(mapReservation?.pendingRows ?? [])
|
|
4882
|
+
.map((row) => resolveMapRowOutcomeKey(row))
|
|
4883
|
+
.filter((key): key is string => Boolean(key)),
|
|
4884
|
+
)
|
|
4885
|
+
: null;
|
|
4876
4886
|
|
|
4877
4887
|
let totalRowsWritten = 0;
|
|
4878
4888
|
let datasetLifecycleRegistered = false;
|
|
@@ -4896,7 +4906,7 @@ function createMinimalWorkerCtx(
|
|
|
4896
4906
|
chunkIndex,
|
|
4897
4907
|
});
|
|
4898
4908
|
const keyStartedAt = nowMs();
|
|
4899
|
-
|
|
4909
|
+
let chunkEntries = chunkRows.map((row, localIndex) => {
|
|
4900
4910
|
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
4901
4911
|
const rowKey = resolveRowKey(
|
|
4902
4912
|
row,
|
|
@@ -4906,6 +4916,11 @@ function createMinimalWorkerCtx(
|
|
|
4906
4916
|
);
|
|
4907
4917
|
return { row, absoluteIndex, rowKey };
|
|
4908
4918
|
});
|
|
4919
|
+
if (netNewRowKeys) {
|
|
4920
|
+
chunkEntries = chunkEntries.filter(({ rowKey }) =>
|
|
4921
|
+
netNewRowKeys.has(rowKey),
|
|
4922
|
+
);
|
|
4923
|
+
}
|
|
4909
4924
|
recordRunnerPerfTrace({
|
|
4910
4925
|
req,
|
|
4911
4926
|
phase: 'runner.map_chunk.keys',
|
|
@@ -4968,7 +4983,10 @@ function createMinimalWorkerCtx(
|
|
|
4968
4983
|
},
|
|
4969
4984
|
});
|
|
4970
4985
|
mapLatencyProfile.recordPhase('prepare_rows', nowMs() - prepareStartedAt);
|
|
4971
|
-
|
|
4986
|
+
// net_new reserves the entire source before chunk execution, so its
|
|
4987
|
+
// progress describes the admitted subset rather than the source CSV.
|
|
4988
|
+
const progressTotalRows =
|
|
4989
|
+
netNewRowKeys?.size ?? rowCountHint ?? chunkRows.length;
|
|
4972
4990
|
const preparedCompletedRows = Math.min(
|
|
4973
4991
|
progressTotalRows,
|
|
4974
4992
|
totalRowsWritten,
|
|
@@ -5023,15 +5041,8 @@ function createMinimalWorkerCtx(
|
|
|
5023
5041
|
({ rowKey }) => !preparedKeys.has(rowKey),
|
|
5024
5042
|
);
|
|
5025
5043
|
if (missingPreparedRows.length > 0) {
|
|
5026
|
-
const rowKeySamples = missingPreparedRows
|
|
5027
|
-
.map(({ rowKey }) => rowKey)
|
|
5028
|
-
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
5029
5044
|
throw new Error(
|
|
5030
|
-
`ctx.dataset("${name}")
|
|
5031
|
-
`Every input row must be claimed, completed, or blocked before execution starts.` +
|
|
5032
|
-
(rowKeySamples.length > 0
|
|
5033
|
-
? ` Missing row keys: ${rowKeySamples.join(', ')}`
|
|
5034
|
-
: ''),
|
|
5045
|
+
`ctx.dataset("${name}") preparation omitted ${missingPreparedRows.length} row disposition(s).`,
|
|
5035
5046
|
);
|
|
5036
5047
|
}
|
|
5037
5048
|
const rowsToExecuteEntries = chunkEntries.filter(({ rowKey }) =>
|
|
@@ -5986,9 +5997,8 @@ function createMinimalWorkerCtx(
|
|
|
5986
5997
|
// dataset (their recoverable state lives in the runtime sheet).
|
|
5987
5998
|
if (key && executedRow) resultByKey.set(key, executedRow);
|
|
5988
5999
|
}
|
|
5989
|
-
const outEntries =
|
|
5990
|
-
.map((
|
|
5991
|
-
const entry = chunkEntries[index]!;
|
|
6000
|
+
const outEntries = chunkEntries
|
|
6001
|
+
.map((entry) => {
|
|
5992
6002
|
const row = resultByKey.get(entry.rowKey);
|
|
5993
6003
|
return row
|
|
5994
6004
|
? {
|
|
@@ -6531,11 +6541,11 @@ function createMinimalWorkerCtx(
|
|
|
6531
6541
|
}
|
|
6532
6542
|
enqueueMapProgressUpdate({
|
|
6533
6543
|
completed: totalRowsWritten,
|
|
6534
|
-
total: rowCountHint ?? undefined,
|
|
6544
|
+
total: netNewRowKeys?.size ?? rowCountHint ?? undefined,
|
|
6535
6545
|
...(totalRowsFailed > 0 ? { failed: totalRowsFailed } : {}),
|
|
6536
6546
|
message: formatMapProgressMessage(
|
|
6537
6547
|
totalRowsWritten,
|
|
6538
|
-
rowCountHint ?? undefined,
|
|
6548
|
+
netNewRowKeys?.size ?? rowCountHint ?? undefined,
|
|
6539
6549
|
),
|
|
6540
6550
|
});
|
|
6541
6551
|
if (previewRows.length < WORKER_DATASET_PREVIEW_ROWS) {
|
|
@@ -7836,6 +7846,9 @@ async function executeRunRequest(
|
|
|
7836
7846
|
occurredAt: event.at,
|
|
7837
7847
|
stepId: event.nodeId,
|
|
7838
7848
|
kind: event.type,
|
|
7849
|
+
...(event.transition === 'failed' && event.error
|
|
7850
|
+
? { error: event.error }
|
|
7851
|
+
: {}),
|
|
7839
7852
|
},
|
|
7840
7853
|
];
|
|
7841
7854
|
flushLedgerEvents(false);
|
|
@@ -8317,7 +8330,8 @@ async function executeRunRequest(
|
|
|
8317
8330
|
durationMs: nowMs() - startedAt,
|
|
8318
8331
|
};
|
|
8319
8332
|
} catch (error) {
|
|
8320
|
-
|
|
8333
|
+
const failure = normalizePlayRunFailure(error);
|
|
8334
|
+
stepLifecycle?.markStartedFailed(nowMs(), failure.message);
|
|
8321
8335
|
// A runtime-limit abort is a timeout failure, not a user cancellation, so
|
|
8322
8336
|
// it should be reported as run.failed with the limit message rather than
|
|
8323
8337
|
// run.cancelled.
|
|
@@ -8329,7 +8343,6 @@ async function executeRunRequest(
|
|
|
8329
8343
|
error instanceof Error ? error.message : 'Play run cancelled.',
|
|
8330
8344
|
);
|
|
8331
8345
|
}
|
|
8332
|
-
const failure = normalizePlayRunFailure(error);
|
|
8333
8346
|
const message = failure.message;
|
|
8334
8347
|
// Controlled run-fatal teardown: release the runtime-sheet row leases and
|
|
8335
8348
|
// work-receipt leases this attempt still holds so an immediate rerun is not
|
|
@@ -8369,6 +8382,9 @@ async function executeRunRequest(
|
|
|
8369
8382
|
retryable: failure.retryable,
|
|
8370
8383
|
...(errorBilling ? { billing: errorBilling } : {}),
|
|
8371
8384
|
...(failure.cause ? { cause: failure.cause } : {}),
|
|
8385
|
+
...(failure.name ? { name: failure.name } : {}),
|
|
8386
|
+
...(failure.stack ? { stack: failure.stack } : {}),
|
|
8387
|
+
...(failure.causes ? { causes: failure.causes } : {}),
|
|
8372
8388
|
},
|
|
8373
8389
|
],
|
|
8374
8390
|
});
|
|
@@ -560,6 +560,11 @@ export type DatasetBuilder<
|
|
|
560
560
|
*/
|
|
561
561
|
run(options?: {
|
|
562
562
|
description?: string;
|
|
563
|
+
/**
|
|
564
|
+
* `upsert` (default) returns every input row, reusing persisted work for
|
|
565
|
+
* existing keys. `net_new` atomically inserts and returns only unseen keys.
|
|
566
|
+
*/
|
|
567
|
+
mode?: 'upsert' | 'net_new';
|
|
563
568
|
key?:
|
|
564
569
|
| (keyof InputRow & string)
|
|
565
570
|
| readonly (keyof InputRow & string)[]
|
|
@@ -108,10 +108,10 @@ export const SDK_RELEASE = {
|
|
|
108
108
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
109
109
|
// 0.1.220 deprecates SDK CLI versions below 0.1.219 and updates them
|
|
110
110
|
// automatically without blocking their current command.
|
|
111
|
-
version: '0.1.
|
|
111
|
+
version: '0.1.227',
|
|
112
112
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
113
113
|
supportPolicy: {
|
|
114
|
-
latest: '0.1.
|
|
114
|
+
latest: '0.1.227',
|
|
115
115
|
minimumSupported: '0.1.53',
|
|
116
116
|
deprecatedBelow: '0.1.219',
|
|
117
117
|
commandMinimumSupported: [
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
INVALID_SPAN_CONTEXT,
|
|
2
3
|
SpanKind,
|
|
3
4
|
SpanStatusCode,
|
|
4
5
|
trace,
|
|
@@ -10,6 +11,97 @@ type PrimitiveAttribute = string | number | boolean | null | undefined;
|
|
|
10
11
|
|
|
11
12
|
type TraceAttributes = Record<string, PrimitiveAttribute>;
|
|
12
13
|
|
|
14
|
+
const NOOP_SPAN: Span = {
|
|
15
|
+
spanContext: () => INVALID_SPAN_CONTEXT,
|
|
16
|
+
setAttribute: () => NOOP_SPAN,
|
|
17
|
+
setAttributes: () => NOOP_SPAN,
|
|
18
|
+
addEvent: () => NOOP_SPAN,
|
|
19
|
+
addLink: () => NOOP_SPAN,
|
|
20
|
+
addLinks: () => NOOP_SPAN,
|
|
21
|
+
setStatus: () => NOOP_SPAN,
|
|
22
|
+
updateName: () => NOOP_SPAN,
|
|
23
|
+
end: () => undefined,
|
|
24
|
+
isRecording: () => false,
|
|
25
|
+
recordException: () => undefined,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
function discardTelemetryResult(invoke: () => unknown): void {
|
|
29
|
+
try {
|
|
30
|
+
const result = invoke();
|
|
31
|
+
if (
|
|
32
|
+
result === null ||
|
|
33
|
+
(typeof result !== 'object' && typeof result !== 'function')
|
|
34
|
+
) {
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
let then: unknown;
|
|
38
|
+
try {
|
|
39
|
+
then = Reflect.get(result, 'then');
|
|
40
|
+
} catch {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
if (typeof then === 'function') {
|
|
44
|
+
void Promise.resolve(result).catch(() => undefined);
|
|
45
|
+
}
|
|
46
|
+
} catch {}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function nonBlockingSpan(span: Span): Span {
|
|
50
|
+
const safeSpan: Span = {
|
|
51
|
+
spanContext() {
|
|
52
|
+
try {
|
|
53
|
+
return span.spanContext();
|
|
54
|
+
} catch {
|
|
55
|
+
return INVALID_SPAN_CONTEXT;
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
setAttribute(key, value) {
|
|
59
|
+
discardTelemetryResult(() => span.setAttribute(key, value));
|
|
60
|
+
return safeSpan;
|
|
61
|
+
},
|
|
62
|
+
setAttributes(attributes) {
|
|
63
|
+
discardTelemetryResult(() => span.setAttributes(attributes));
|
|
64
|
+
return safeSpan;
|
|
65
|
+
},
|
|
66
|
+
addEvent(name, attributesOrStartTime, startTime) {
|
|
67
|
+
discardTelemetryResult(() =>
|
|
68
|
+
span.addEvent(name, attributesOrStartTime, startTime),
|
|
69
|
+
);
|
|
70
|
+
return safeSpan;
|
|
71
|
+
},
|
|
72
|
+
addLink(link) {
|
|
73
|
+
discardTelemetryResult(() => span.addLink(link));
|
|
74
|
+
return safeSpan;
|
|
75
|
+
},
|
|
76
|
+
addLinks(links) {
|
|
77
|
+
discardTelemetryResult(() => span.addLinks(links));
|
|
78
|
+
return safeSpan;
|
|
79
|
+
},
|
|
80
|
+
setStatus(status) {
|
|
81
|
+
discardTelemetryResult(() => span.setStatus(status));
|
|
82
|
+
return safeSpan;
|
|
83
|
+
},
|
|
84
|
+
updateName(name) {
|
|
85
|
+
discardTelemetryResult(() => span.updateName(name));
|
|
86
|
+
return safeSpan;
|
|
87
|
+
},
|
|
88
|
+
end(endTime) {
|
|
89
|
+
discardTelemetryResult(() => span.end(endTime));
|
|
90
|
+
},
|
|
91
|
+
isRecording() {
|
|
92
|
+
try {
|
|
93
|
+
return span.isRecording();
|
|
94
|
+
} catch {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
recordException(exception, time) {
|
|
99
|
+
discardTelemetryResult(() => span.recordException(exception, time));
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
return safeSpan;
|
|
103
|
+
}
|
|
104
|
+
|
|
13
105
|
function normalizeAttributes(
|
|
14
106
|
attributes: TraceAttributes | undefined,
|
|
15
107
|
): Attributes | undefined {
|
|
@@ -60,34 +152,67 @@ export async function withActiveSpan<T>(
|
|
|
60
152
|
},
|
|
61
153
|
fn: (span: Span) => Promise<T> | T,
|
|
62
154
|
): Promise<T> {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
{
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
155
|
+
let operationPromise: Promise<T> | null = null;
|
|
156
|
+
const runOperation = async (span: Span): Promise<T> => {
|
|
157
|
+
const safeSpan = nonBlockingSpan(span);
|
|
158
|
+
try {
|
|
159
|
+
const result = await fn(safeSpan);
|
|
160
|
+
safeSpan.setStatus({ code: SpanStatusCode.OK });
|
|
161
|
+
return result;
|
|
162
|
+
} catch (error) {
|
|
71
163
|
try {
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
} catch (error) {
|
|
76
|
-
recordError(span, error);
|
|
77
|
-
throw error;
|
|
78
|
-
} finally {
|
|
79
|
-
span.end();
|
|
164
|
+
recordError(safeSpan, error);
|
|
165
|
+
} catch {
|
|
166
|
+
// Error telemetry must not replace the operation's real error.
|
|
80
167
|
}
|
|
81
|
-
|
|
82
|
-
|
|
168
|
+
throw error;
|
|
169
|
+
} finally {
|
|
170
|
+
safeSpan.end();
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
const startOperation = (span: Span): Promise<T> => {
|
|
174
|
+
if (operationPromise) return operationPromise;
|
|
175
|
+
let resolveOperation!: (value: T | PromiseLike<T>) => void;
|
|
176
|
+
let rejectOperation!: (reason?: unknown) => void;
|
|
177
|
+
operationPromise = new Promise<T>((resolve, reject) => {
|
|
178
|
+
resolveOperation = resolve;
|
|
179
|
+
rejectOperation = reject;
|
|
180
|
+
});
|
|
181
|
+
void runOperation(span).then(resolveOperation, rejectOperation);
|
|
182
|
+
return operationPromise;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
let tracerResult: unknown;
|
|
186
|
+
try {
|
|
187
|
+
const tracer = trace.getTracer(options.tracer ?? 'deepline');
|
|
188
|
+
tracerResult = tracer.startActiveSpan(
|
|
189
|
+
name,
|
|
190
|
+
{
|
|
191
|
+
kind: options.kind,
|
|
192
|
+
attributes: normalizeAttributes(options.attributes),
|
|
193
|
+
},
|
|
194
|
+
(span) => startOperation(span),
|
|
195
|
+
);
|
|
196
|
+
} catch {}
|
|
197
|
+
|
|
198
|
+
// A non-conforming exporter may return work unrelated to the callback.
|
|
199
|
+
// Drain its rejection without awaiting it; only execution owns latency and
|
|
200
|
+
// the returned success/failure contract.
|
|
201
|
+
discardTelemetryResult(() => tracerResult);
|
|
202
|
+
|
|
203
|
+
// A broken tracer may return without invoking the callback. Execution is
|
|
204
|
+
// authoritative, so run it once without telemetry in that case.
|
|
205
|
+
return await startOperation(NOOP_SPAN);
|
|
83
206
|
}
|
|
84
207
|
|
|
85
208
|
export function setSpanAttributes(
|
|
86
209
|
span: Span,
|
|
87
210
|
attributes: TraceAttributes,
|
|
88
211
|
): void {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
212
|
+
try {
|
|
213
|
+
const normalized = normalizeAttributes(attributes);
|
|
214
|
+
if (normalized) {
|
|
215
|
+
discardTelemetryResult(() => span.setAttributes(normalized));
|
|
216
|
+
}
|
|
217
|
+
} catch {}
|
|
93
218
|
}
|
|
@@ -33,6 +33,8 @@ import { createRuntimeReceiptHeartbeatSupervisor } from './receipt-heartbeat-sup
|
|
|
33
33
|
import { dispatchBoundedSettled } from './bounded-dispatch';
|
|
34
34
|
import type { PlayQueueHint } from './governor/rate-state-backend';
|
|
35
35
|
import type { MapRowOutcome } from './durability-store';
|
|
36
|
+
import { stringifyPostgresJson } from './postgres-json';
|
|
37
|
+
import { RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE } from './runtime-sheet-row-transition';
|
|
36
38
|
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
37
39
|
import {
|
|
38
40
|
completedMapRowOutcome,
|
|
@@ -3090,9 +3092,19 @@ export class PlayContextImpl {
|
|
|
3090
3092
|
// are no-ops for any value that is not a tool result.
|
|
3091
3093
|
|
|
3092
3094
|
private serializeCellValue(value: unknown): unknown {
|
|
3093
|
-
|
|
3095
|
+
const serialized = isToolExecuteResult(value)
|
|
3094
3096
|
? serializeToolExecuteResult(value)
|
|
3095
3097
|
: value;
|
|
3098
|
+
try {
|
|
3099
|
+
stringifyPostgresJson(serialized);
|
|
3100
|
+
} catch (error) {
|
|
3101
|
+
throw new Error(
|
|
3102
|
+
`${RUNTIME_ROW_OUTPUT_NOT_JSON_SERIALIZABLE}: Dataset cell output is not JSON-serializable: ${
|
|
3103
|
+
error instanceof Error ? error.message : String(error)
|
|
3104
|
+
}`,
|
|
3105
|
+
);
|
|
3106
|
+
}
|
|
3107
|
+
return serialized;
|
|
3096
3108
|
}
|
|
3097
3109
|
|
|
3098
3110
|
private rehydrateCellValue(value: unknown): unknown {
|
|
@@ -3931,7 +3943,10 @@ export class PlayContextImpl {
|
|
|
3931
3943
|
this.#options.runId,
|
|
3932
3944
|
);
|
|
3933
3945
|
if (shouldStreamRuntimeBackedDataset) {
|
|
3934
|
-
|
|
3946
|
+
// Net-new admission happens one source page at a time. Its progress and
|
|
3947
|
+
// returned dataset describe admitted rows, not every provider/CSV row
|
|
3948
|
+
// supplied to this run, so accumulate the admitted count below.
|
|
3949
|
+
totalInputCount = options?.mode === 'net_new' ? 0 : await items.count();
|
|
3935
3950
|
const mapScope = this.createMapExecutionScope({
|
|
3936
3951
|
logicalNamespace: normalizedMapNamespace,
|
|
3937
3952
|
artifactTableNamespace: resolvedTableNamespace,
|
|
@@ -4085,6 +4100,7 @@ export class PlayContextImpl {
|
|
|
4085
4100
|
staticPipeline: this.currentStaticPipeline,
|
|
4086
4101
|
forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
|
|
4087
4102
|
inputOffset: page.offset,
|
|
4103
|
+
mode: options?.mode,
|
|
4088
4104
|
},
|
|
4089
4105
|
);
|
|
4090
4106
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
@@ -4128,12 +4144,21 @@ export class PlayContextImpl {
|
|
|
4128
4144
|
index: originalIndex,
|
|
4129
4145
|
};
|
|
4130
4146
|
});
|
|
4147
|
+
const admittedSeededItems =
|
|
4148
|
+
options?.mode === 'net_new'
|
|
4149
|
+
? seededItems.filter(({ row, index }) =>
|
|
4150
|
+
pendingRowsByKey.has(seedRowIdentity(row, index)),
|
|
4151
|
+
)
|
|
4152
|
+
: seededItems;
|
|
4153
|
+
if (options?.mode === 'net_new') {
|
|
4154
|
+
totalInputCount += admittedSeededItems.length;
|
|
4155
|
+
}
|
|
4131
4156
|
|
|
4132
4157
|
const rowsToExecuteByKey = new Map<
|
|
4133
4158
|
string,
|
|
4134
4159
|
{ row: Record<string, unknown>; originalIndex: number }
|
|
4135
4160
|
>();
|
|
4136
|
-
for (const { row, index } of
|
|
4161
|
+
for (const { row, index } of admittedSeededItems) {
|
|
4137
4162
|
if (explicitKeyResolver) {
|
|
4138
4163
|
const explicitKey = explicitKeyResolver(row, index);
|
|
4139
4164
|
if (seenExplicitKeys.has(explicitKey)) {
|
|
@@ -4268,6 +4293,45 @@ export class PlayContextImpl {
|
|
|
4268
4293
|
page.rows.length = 0;
|
|
4269
4294
|
}
|
|
4270
4295
|
|
|
4296
|
+
// Isolated failures are a partial result only when at least one admitted
|
|
4297
|
+
// row completed. A fully failed admission run is systemic: preserve the
|
|
4298
|
+
// failed sheet rows for recovery, but fail the enclosing play loudly.
|
|
4299
|
+
if (totalInputCount > 0 && failedCount === totalInputCount) {
|
|
4300
|
+
const completedRows = 0;
|
|
4301
|
+
this.setMapFrame({
|
|
4302
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
4303
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
4304
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
4305
|
+
artifactTableNamespace: resolvedTableNamespace,
|
|
4306
|
+
status: 'failed',
|
|
4307
|
+
totalRows: totalInputCount,
|
|
4308
|
+
completedRowKeys: [],
|
|
4309
|
+
pendingRowKeys: [],
|
|
4310
|
+
completedRowsCount: completedRows,
|
|
4311
|
+
pendingRowsCount: 0,
|
|
4312
|
+
failedRowsCount: failedCount,
|
|
4313
|
+
startedAt:
|
|
4314
|
+
this.checkpoint.mapFrames?.[mapScope.mapInvocationId]?.startedAt ??
|
|
4315
|
+
Date.now(),
|
|
4316
|
+
updatedAt: Date.now(),
|
|
4317
|
+
});
|
|
4318
|
+
this.emitExecutionEvent({
|
|
4319
|
+
type: 'map.failed',
|
|
4320
|
+
mapInvocationId: mapScope.mapInvocationId,
|
|
4321
|
+
mapNodeId: mapScope.mapNodeId ?? null,
|
|
4322
|
+
logicalNamespace: mapScope.logicalNamespace,
|
|
4323
|
+
artifactTableNamespace: resolvedTableNamespace,
|
|
4324
|
+
completedRows,
|
|
4325
|
+
failedRows: failedCount,
|
|
4326
|
+
totalRows: totalInputCount,
|
|
4327
|
+
at: Date.now(),
|
|
4328
|
+
});
|
|
4329
|
+
throw new Error(
|
|
4330
|
+
`ctx.dataset("${normalizedMapNamespace}") failed for every admitted row (${failedCount}/${totalInputCount}). ` +
|
|
4331
|
+
'Failed rows were persisted and will be retried on the next run.',
|
|
4332
|
+
);
|
|
4333
|
+
}
|
|
4334
|
+
|
|
4271
4335
|
this.setMapFrame({
|
|
4272
4336
|
mapInvocationId: mapScope.mapInvocationId,
|
|
4273
4337
|
mapNodeId: mapScope.mapNodeId ?? null,
|
|
@@ -4519,6 +4583,7 @@ export class PlayContextImpl {
|
|
|
4519
4583
|
executorToken: this.#options.executorToken,
|
|
4520
4584
|
staticPipeline: this.currentStaticPipeline,
|
|
4521
4585
|
forceRefresh: this.#options.cachePolicy?.forceToolRefresh === true,
|
|
4586
|
+
mode: options?.mode,
|
|
4522
4587
|
},
|
|
4523
4588
|
);
|
|
4524
4589
|
resolvedTableNamespace = normalizeTableNamespace(
|
|
@@ -4575,8 +4640,16 @@ export class PlayContextImpl {
|
|
|
4575
4640
|
index,
|
|
4576
4641
|
};
|
|
4577
4642
|
});
|
|
4578
|
-
|
|
4579
|
-
|
|
4643
|
+
const admittedItems =
|
|
4644
|
+
options?.mode === 'net_new'
|
|
4645
|
+
? seededItems.filter(({ row, index }) =>
|
|
4646
|
+
pendingRowsByKey.has(seedRowIdentity(row, index)),
|
|
4647
|
+
)
|
|
4648
|
+
: seededItems;
|
|
4649
|
+
rawItems = admittedItems.map((item) => item.row);
|
|
4650
|
+
itemsToProcess = admittedItems.map((item) => item.row);
|
|
4651
|
+
itemOriginalIndexes = admittedItems.map((item) => item.index);
|
|
4652
|
+
totalInputCount = admittedItems.length;
|
|
4580
4653
|
}
|
|
4581
4654
|
|
|
4582
4655
|
const mapScope = this.createMapExecutionScope({
|
|
@@ -5391,6 +5464,7 @@ export class PlayContextImpl {
|
|
|
5391
5464
|
});
|
|
5392
5465
|
|
|
5393
5466
|
let value: unknown;
|
|
5467
|
+
let cellValue: unknown;
|
|
5394
5468
|
try {
|
|
5395
5469
|
value = await rowContext.run(
|
|
5396
5470
|
{
|
|
@@ -5411,6 +5485,7 @@ export class PlayContextImpl {
|
|
|
5411
5485
|
this.previousCellForField(baseRow, fieldName),
|
|
5412
5486
|
),
|
|
5413
5487
|
);
|
|
5488
|
+
cellValue = this.serializeCellValue(value);
|
|
5414
5489
|
} catch (error) {
|
|
5415
5490
|
if (
|
|
5416
5491
|
isPlayRowExecutionSuspendedError(error) ||
|
|
@@ -5482,7 +5557,6 @@ export class PlayContextImpl {
|
|
|
5482
5557
|
});
|
|
5483
5558
|
return FAILED_ROW;
|
|
5484
5559
|
}
|
|
5485
|
-
const cellValue = this.serializeCellValue(value);
|
|
5486
5560
|
computedFields[fieldName] = cellValue;
|
|
5487
5561
|
this.rowStates.get(idx)?.results.set(fieldName, cellValue);
|
|
5488
5562
|
const currentCellMeta =
|
|
@@ -248,6 +248,16 @@ export interface DatasetOptions<TItem = Record<string, unknown>> {
|
|
|
248
248
|
* and are reused on re-run.
|
|
249
249
|
*/
|
|
250
250
|
onRowError?: 'isolate' | 'fail';
|
|
251
|
+
/**
|
|
252
|
+
* Controls how a persisted dataset admits rows whose key already exists.
|
|
253
|
+
*
|
|
254
|
+
* `upsert` (the default) preserves the row-preserving enrichment contract:
|
|
255
|
+
* existing rows are reused and returned with the current input. `net_new`
|
|
256
|
+
* atomically inserts only previously unseen keys and returns/processes only
|
|
257
|
+
* those inserted rows. Use `net_new` for a durable sourcing table, not for
|
|
258
|
+
* ordinary CSV enrichment reruns.
|
|
259
|
+
*/
|
|
260
|
+
mode?: 'upsert' | 'net_new';
|
|
251
261
|
}
|
|
252
262
|
|
|
253
263
|
export type DatasetDefinitionOptions<TItem = Record<string, unknown>> = Omit<
|
|
@@ -257,7 +267,7 @@ export type DatasetDefinitionOptions<TItem = Record<string, unknown>> = Omit<
|
|
|
257
267
|
|
|
258
268
|
export type DatasetRunOptions<TItem = Record<string, unknown>> = Pick<
|
|
259
269
|
DatasetOptions<TItem>,
|
|
260
|
-
'description' | 'key' | 'onRowError'
|
|
270
|
+
'description' | 'key' | 'onRowError' | 'mode'
|
|
261
271
|
>;
|
|
262
272
|
|
|
263
273
|
export interface ToolCallOptions {
|
|
@@ -597,6 +607,7 @@ export interface ContextOptions {
|
|
|
597
607
|
staticPipeline?: PlayStaticPipeline | null;
|
|
598
608
|
forceRefresh?: boolean;
|
|
599
609
|
inputOffset?: number;
|
|
610
|
+
mode?: DatasetRunOptions['mode'];
|
|
600
611
|
},
|
|
601
612
|
) => Promise<MapStartResult>;
|
|
602
613
|
/**
|
|
@@ -35,6 +35,7 @@ export type PlayVisualNodeProgressSnapshot = {
|
|
|
35
35
|
export type PlayVisualNodeStateSnapshot = {
|
|
36
36
|
nodeId: string;
|
|
37
37
|
status: PlayVisualNodeStatus;
|
|
38
|
+
error?: string | null;
|
|
38
39
|
artifactTableNamespace?: string | null;
|
|
39
40
|
progress?: PlayVisualNodeProgressSnapshot | null;
|
|
40
41
|
startedAt?: number | null;
|
|
@@ -1,9 +1,71 @@
|
|
|
1
|
-
const
|
|
1
|
+
const POSTGRES_JSON_REPLACEMENT_CHARACTER = '\uFFFD';
|
|
2
|
+
const POSTGRES_JSON_SUSPECT_CODE_UNIT_RE = /[\u0000\uD800-\uDFFF]/;
|
|
3
|
+
|
|
4
|
+
function sanitizePostgresJsonString(value: string): string {
|
|
5
|
+
if (!POSTGRES_JSON_SUSPECT_CODE_UNIT_RE.test(value)) return value;
|
|
6
|
+
|
|
7
|
+
let sanitized = '';
|
|
8
|
+
let changed = false;
|
|
9
|
+
|
|
10
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
11
|
+
const codeUnit = value.charCodeAt(index);
|
|
12
|
+
if (codeUnit === 0) {
|
|
13
|
+
changed = true;
|
|
14
|
+
continue;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) {
|
|
18
|
+
const nextCodeUnit = value.charCodeAt(index + 1);
|
|
19
|
+
if (nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff) {
|
|
20
|
+
sanitized += value[index] + value[index + 1];
|
|
21
|
+
index += 1;
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
24
|
+
sanitized += POSTGRES_JSON_REPLACEMENT_CHARACTER;
|
|
25
|
+
changed = true;
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) {
|
|
30
|
+
sanitized += POSTGRES_JSON_REPLACEMENT_CHARACTER;
|
|
31
|
+
changed = true;
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
sanitized += value[index];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return changed ? sanitized : value;
|
|
39
|
+
}
|
|
2
40
|
|
|
3
41
|
function postgresJsonReplacer(_key: string, value: unknown): unknown {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
42
|
+
if (typeof value === 'string') {
|
|
43
|
+
return sanitizePostgresJsonString(value);
|
|
44
|
+
}
|
|
45
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const keys = Object.keys(value);
|
|
50
|
+
if (!keys.some((key) => sanitizePostgresJsonString(key) !== key)) {
|
|
51
|
+
return value;
|
|
52
|
+
}
|
|
53
|
+
const sanitizedEntries: Array<[string, unknown]> = [];
|
|
54
|
+
const sanitizedKeys = new Set<string>();
|
|
55
|
+
for (const key of keys) {
|
|
56
|
+
const sanitizedKey = sanitizePostgresJsonString(key);
|
|
57
|
+
if (sanitizedKeys.has(sanitizedKey)) {
|
|
58
|
+
throw new TypeError(
|
|
59
|
+
'Postgres JSON key normalization produced a duplicate object key.',
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
sanitizedKeys.add(sanitizedKey);
|
|
63
|
+
sanitizedEntries.push([
|
|
64
|
+
sanitizedKey,
|
|
65
|
+
(value as Record<string, unknown>)[key],
|
|
66
|
+
]);
|
|
67
|
+
}
|
|
68
|
+
return Object.fromEntries(sanitizedEntries);
|
|
7
69
|
}
|
|
8
70
|
|
|
9
71
|
export function sanitizePostgresJsonValue<T>(value: T): T {
|
|
@@ -12,6 +12,7 @@ import type { PreloadedRuntimeDbSession } from './db-session';
|
|
|
12
12
|
import type { PacingRule } from './governor/rate-state-backend';
|
|
13
13
|
import type { GovernanceSnapshot } from './governor/governor';
|
|
14
14
|
import type { PlayRunInputPayload } from './play-input';
|
|
15
|
+
import type { PlayRunFailureDetails } from './run-failure';
|
|
15
16
|
|
|
16
17
|
export type PlayRunnerRateStateBackendConfig =
|
|
17
18
|
| {
|
|
@@ -165,6 +166,7 @@ export interface PlayRunnerExecutionConfig {
|
|
|
165
166
|
artifactTransport?: {
|
|
166
167
|
bundledCodePath?: string | null;
|
|
167
168
|
bundledCodeEncoding?: 'utf8' | 'gzip';
|
|
169
|
+
sourceMapPath?: string | null;
|
|
168
170
|
} | null;
|
|
169
171
|
input: PlayRunInputPayload;
|
|
170
172
|
checkpoint?: PlayCheckpoint | null;
|
|
@@ -241,6 +243,7 @@ export type PlayRunnerResult =
|
|
|
241
243
|
| {
|
|
242
244
|
status: 'failed';
|
|
243
245
|
error: string;
|
|
246
|
+
errors?: PlayRunFailureDetails[];
|
|
244
247
|
logs: string[];
|
|
245
248
|
stats: Record<string, unknown>;
|
|
246
249
|
steps: PlayStep[];
|