deepline 0.1.210 → 0.1.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +53 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +15 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +90 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +161 -99
- package/dist/bundling-sources/sdk/src/play.ts +2 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +343 -83
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +42 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/single-flight.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +12 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +41 -5
- package/dist/cli/index.js +93 -12
- package/dist/cli/index.mjs +93 -12
- package/dist/index.d.mts +17 -1
- package/dist/index.d.ts +17 -1
- package/dist/index.js +22 -5
- package/dist/index.mjs +22 -5
- package/package.json +1 -1
|
@@ -243,9 +243,11 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
|
|
|
243
243
|
rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
|
|
244
244
|
});
|
|
245
245
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
246
|
+
// markRunning is the authoritative pre-dispatch ownership fence. Renewal is
|
|
247
|
+
// only needed when the lease approaches expiry; an immediate read/renewal
|
|
248
|
+
// adds a round trip to every short tool call without extending ownership.
|
|
249
|
+
// Completion remains conditional on this lease, and long work still renews.
|
|
250
|
+
supervisor.start();
|
|
249
251
|
try {
|
|
250
252
|
return await Promise.race([input.execute(), leaseLost]);
|
|
251
253
|
} finally {
|
|
@@ -294,6 +296,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
294
296
|
maxAttempts?: number;
|
|
295
297
|
delayMs?: number;
|
|
296
298
|
abortSignal?: AbortSignal;
|
|
299
|
+
log?: (message: string) => void;
|
|
297
300
|
}): Promise<{
|
|
298
301
|
completed: Map<string, RuntimeStepReceipt>;
|
|
299
302
|
failed: Map<string, Error>;
|
|
@@ -326,6 +329,20 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
326
329
|
}
|
|
327
330
|
throw error;
|
|
328
331
|
}
|
|
332
|
+
const statuses = [...receipts.values()].reduce<Record<string, number>>(
|
|
333
|
+
(counts, receipt) => {
|
|
334
|
+
counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
|
|
335
|
+
return counts;
|
|
336
|
+
},
|
|
337
|
+
{},
|
|
338
|
+
);
|
|
339
|
+
input.log?.(
|
|
340
|
+
`[perf] runtime-receipt-wait attempt=${attempt} requested=${pending.size} indexed=${receipts.size} statuses=${
|
|
341
|
+
Object.entries(statuses)
|
|
342
|
+
.map(([status, count]) => `${status}:${count}`)
|
|
343
|
+
.join(',') || 'none'
|
|
344
|
+
}`,
|
|
345
|
+
);
|
|
329
346
|
for (const key of [...pending]) {
|
|
330
347
|
const receipt = receipts.get(key);
|
|
331
348
|
if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
|
|
@@ -343,6 +360,9 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
343
360
|
pending.delete(key);
|
|
344
361
|
}
|
|
345
362
|
}
|
|
363
|
+
input.log?.(
|
|
364
|
+
`[perf] runtime-receipt-wait attempt=${attempt} remaining=${pending.size}`,
|
|
365
|
+
);
|
|
346
366
|
}
|
|
347
367
|
|
|
348
368
|
return { completed, failed, timedOut: pending };
|
|
@@ -374,6 +394,16 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
374
394
|
log: (message: string) => void;
|
|
375
395
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
376
396
|
}): Promise<T> {
|
|
397
|
+
const lifecycleStartedAt = Date.now();
|
|
398
|
+
const logPhase = (phase: string, startedAt: number): void => {
|
|
399
|
+
// Keep this at the semantic receipt layer. Transport timing below this
|
|
400
|
+
// function explains HTTP cost; these markers expose gaps before/after the
|
|
401
|
+
// store calls and prove whether user code or receipt orchestration owns the
|
|
402
|
+
// elapsed time. Do not log receipt keys because they encode tool inputs.
|
|
403
|
+
input.log(
|
|
404
|
+
`[perf] durable receipt operation=${input.operation} id=${input.id} phase=${phase} elapsed_ms=${Date.now() - startedAt} lifecycle_ms=${Date.now() - lifecycleStartedAt}`,
|
|
405
|
+
);
|
|
406
|
+
};
|
|
377
407
|
if (!input.store.enabled) {
|
|
378
408
|
return await input.execute({ leaseId: null });
|
|
379
409
|
}
|
|
@@ -483,12 +513,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
483
513
|
);
|
|
484
514
|
};
|
|
485
515
|
|
|
516
|
+
const claimStartedAt = Date.now();
|
|
486
517
|
const claimed = await input.store.claim(
|
|
487
518
|
input.receiptKey,
|
|
488
519
|
input.runId,
|
|
489
520
|
input.reclaimRunning === true || input.force === true,
|
|
490
521
|
input.force === true,
|
|
491
522
|
);
|
|
523
|
+
logPhase('claim', claimStartedAt);
|
|
492
524
|
if (
|
|
493
525
|
input.force !== true &&
|
|
494
526
|
(claimed?.status === 'completed' || claimed?.status === 'skipped')
|
|
@@ -525,11 +557,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
525
557
|
let result: T;
|
|
526
558
|
try {
|
|
527
559
|
if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
|
|
560
|
+
const markRunningStartedAt = Date.now();
|
|
528
561
|
const running = await input.store.markRunning(
|
|
529
562
|
input.receiptKey,
|
|
530
563
|
input.runId,
|
|
531
564
|
ownedLeaseId,
|
|
532
565
|
);
|
|
566
|
+
logPhase('mark_running', markRunningStartedAt);
|
|
533
567
|
if (!running || running.status !== 'running') {
|
|
534
568
|
throw new RuntimeReceiptLeaseLostError({
|
|
535
569
|
receiptKey: input.receiptKey,
|
|
@@ -540,6 +574,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
540
574
|
ownedLeaseId = running.leaseId ?? ownedLeaseId;
|
|
541
575
|
ownedLeaseExpiresAt = running.leaseExpiresAt ?? ownedLeaseExpiresAt;
|
|
542
576
|
}
|
|
577
|
+
const executeStartedAt = Date.now();
|
|
543
578
|
const executed =
|
|
544
579
|
ownedLeaseId && input.store.heartbeat
|
|
545
580
|
? await executeWithDurableRuntimeReceiptHeartbeat({
|
|
@@ -561,6 +596,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
561
596
|
execute: () => input.execute({ leaseId: ownedLeaseId }),
|
|
562
597
|
})
|
|
563
598
|
: await input.execute({ leaseId: ownedLeaseId });
|
|
599
|
+
logPhase('execute', executeStartedAt);
|
|
564
600
|
result = input.onClaimedResult
|
|
565
601
|
? input.onClaimedResult(executed, input.receiptKey)
|
|
566
602
|
: executed;
|
|
@@ -603,12 +639,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
603
639
|
throw error;
|
|
604
640
|
}
|
|
605
641
|
|
|
642
|
+
const completeStartedAt = Date.now();
|
|
606
643
|
const completed = await input.store.complete(
|
|
607
644
|
input.receiptKey,
|
|
608
645
|
input.runId,
|
|
609
646
|
isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
|
|
610
647
|
ownedLeaseId,
|
|
611
648
|
);
|
|
649
|
+
logPhase('complete', completeStartedAt);
|
|
612
650
|
if (
|
|
613
651
|
!completed ||
|
|
614
652
|
(completed.status !== 'completed' && completed.status !== 'skipped')
|
|
@@ -633,6 +671,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
633
671
|
completed &&
|
|
634
672
|
(completed.status === 'completed' || completed.status === 'skipped')
|
|
635
673
|
) {
|
|
674
|
+
logPhase('total', lifecycleStartedAt);
|
|
636
675
|
return await recoverCompletedReceipt(completed, 'owner');
|
|
637
676
|
}
|
|
638
677
|
if (input.store.canPersistCompletion) {
|
|
@@ -344,13 +344,17 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
344
344
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
345
345
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
346
346
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
347
|
+
const runnerTraceEnv =
|
|
348
|
+
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
349
|
+
? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
|
|
350
|
+
: '';
|
|
347
351
|
const runnerCommand = `${nodeMaterializePayloadCommand({
|
|
348
352
|
envelopePath,
|
|
349
353
|
runnerPath,
|
|
350
354
|
configPath,
|
|
351
355
|
artifactCodePath,
|
|
352
356
|
crashPusherPath,
|
|
353
|
-
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
357
|
+
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
354
358
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
355
359
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
356
360
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type SingleFlight<K, V> = {
|
|
2
|
+
run(key: K, create: () => Promise<V>): Promise<V>;
|
|
3
|
+
has(key: K): boolean;
|
|
4
|
+
activeCount(): number;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
/** Coalesce concurrent work for one key without caching its settled result. */
|
|
8
|
+
export function createSingleFlight<K, V>(): SingleFlight<K, V> {
|
|
9
|
+
const active = new Map<K, Promise<V>>();
|
|
10
|
+
|
|
11
|
+
return {
|
|
12
|
+
run(key, create) {
|
|
13
|
+
const existing = active.get(key);
|
|
14
|
+
if (existing) return existing;
|
|
15
|
+
|
|
16
|
+
const pending = Promise.resolve().then(create);
|
|
17
|
+
active.set(key, pending);
|
|
18
|
+
const clear = () => {
|
|
19
|
+
if (active.get(key) === pending) active.delete(key);
|
|
20
|
+
};
|
|
21
|
+
void pending.then(clear, clear);
|
|
22
|
+
return pending;
|
|
23
|
+
},
|
|
24
|
+
has(key) {
|
|
25
|
+
return active.has(key);
|
|
26
|
+
},
|
|
27
|
+
activeCount() {
|
|
28
|
+
return active.size;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -34,6 +34,8 @@ type ValidatedRuntimeTestFaultHeader =
|
|
|
34
34
|
| { ok: false; status: 400 | 403; error: string };
|
|
35
35
|
|
|
36
36
|
export type RuntimeTestPolicyOverrides = {
|
|
37
|
+
/** Opt-in bounded runner map latency profile for local/preview diagnosis. */
|
|
38
|
+
mapLatencyProfile?: boolean;
|
|
37
39
|
receiptLeaseTtlMs?: number;
|
|
38
40
|
sheetAttemptLeaseMs?: number;
|
|
39
41
|
heartbeatIntervalMs?: number;
|
|
@@ -170,7 +172,8 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
170
172
|
const unknownKeys = Object.keys(record).filter(
|
|
171
173
|
(key) =>
|
|
172
174
|
!RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
|
|
173
|
-
key !== 'workBudgetYieldLimits'
|
|
175
|
+
key !== 'workBudgetYieldLimits' &&
|
|
176
|
+
key !== 'mapLatencyProfile',
|
|
174
177
|
);
|
|
175
178
|
if (unknownKeys.length > 0) {
|
|
176
179
|
return {
|
|
@@ -179,6 +182,14 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
179
182
|
}
|
|
180
183
|
|
|
181
184
|
const overrides: RuntimeTestPolicyOverrides = {};
|
|
185
|
+
if ('mapLatencyProfile' in record) {
|
|
186
|
+
if (typeof record.mapLatencyProfile !== 'boolean') {
|
|
187
|
+
return {
|
|
188
|
+
error: 'testPolicyOverrides.mapLatencyProfile must be a boolean.',
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
overrides.mapLatencyProfile = record.mapLatencyProfile;
|
|
192
|
+
}
|
|
182
193
|
for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
|
|
183
194
|
if (!(field in record)) continue;
|
|
184
195
|
const parsed = readPositiveIntegerField({
|
|
@@ -125,7 +125,7 @@ export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
125
125
|
* Large datasets should flow by handle through Neon-backed storage, not
|
|
126
126
|
* through worker memory as giant arrays.
|
|
127
127
|
*/
|
|
128
|
-
materialize(
|
|
128
|
+
materialize(options?: number | PlayDatasetMaterializeOptions): Promise<T[]>;
|
|
129
129
|
toJSON(): {
|
|
130
130
|
kind: 'dataset';
|
|
131
131
|
datasetKind: PlayDatasetKind;
|
|
@@ -146,9 +146,19 @@ type PlayDatasetResolvers<T> = {
|
|
|
146
146
|
count: () => Promise<number>;
|
|
147
147
|
peek: (limit: number) => Promise<T[]>;
|
|
148
148
|
materialize: (limit?: number) => Promise<T[]>;
|
|
149
|
+
materializeFullPersistedDataset?: (limit?: number) => Promise<T[]>;
|
|
149
150
|
iterate: () => AsyncIterable<T>;
|
|
150
151
|
};
|
|
151
152
|
|
|
153
|
+
export type PlayDatasetMaterializeScope = 'result' | 'full_persisted_dataset';
|
|
154
|
+
|
|
155
|
+
export type PlayDatasetMaterializeOptions = {
|
|
156
|
+
/** Rows returned by this operation, or every current row in its persisted dataset. */
|
|
157
|
+
scope?: PlayDatasetMaterializeScope;
|
|
158
|
+
/** Maximum number of rows to load into memory. */
|
|
159
|
+
limit?: number;
|
|
160
|
+
};
|
|
161
|
+
|
|
152
162
|
type PlayDatasetTransform<T, U = T> =
|
|
153
163
|
| {
|
|
154
164
|
kind: 'map';
|
|
@@ -164,7 +174,7 @@ type PlayDatasetTransform<T, U = T> =
|
|
|
164
174
|
end?: number;
|
|
165
175
|
};
|
|
166
176
|
|
|
167
|
-
function resolveMaterializeLimitCap(): number {
|
|
177
|
+
export function resolveMaterializeLimitCap(): number {
|
|
168
178
|
const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
|
|
169
179
|
const parsed = raw ? Number(raw) : NaN;
|
|
170
180
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
@@ -301,10 +311,25 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
301
311
|
return this.slice(0, limit, options);
|
|
302
312
|
}
|
|
303
313
|
|
|
304
|
-
async materialize(
|
|
314
|
+
async materialize(
|
|
315
|
+
options?: number | PlayDatasetMaterializeOptions,
|
|
316
|
+
): Promise<T[]> {
|
|
317
|
+
const scope =
|
|
318
|
+
typeof options === 'object' ? (options.scope ?? 'result') : 'result';
|
|
319
|
+
const limit = typeof options === 'number' ? options : options?.limit;
|
|
305
320
|
const requestedLimit =
|
|
306
321
|
limit !== undefined ? Math.max(0, Math.floor(limit)) : undefined;
|
|
307
322
|
const cap = resolveMaterializeLimitCap();
|
|
323
|
+
const materialize =
|
|
324
|
+
scope === 'full_persisted_dataset'
|
|
325
|
+
? this.resolvers.materializeFullPersistedDataset
|
|
326
|
+
: this.resolvers.materialize;
|
|
327
|
+
if (!materialize) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) is only available ' +
|
|
330
|
+
'for a persisted dataset returned by ctx.dataset(...).run().',
|
|
331
|
+
);
|
|
332
|
+
}
|
|
308
333
|
if (requestedLimit !== undefined) {
|
|
309
334
|
if (requestedLimit > cap) {
|
|
310
335
|
throw new Error(
|
|
@@ -312,7 +337,18 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
312
337
|
'Return the dataset handle instead, or request a smaller bounded slice.',
|
|
313
338
|
);
|
|
314
339
|
}
|
|
315
|
-
return await
|
|
340
|
+
return await materialize(requestedLimit);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (scope === 'full_persisted_dataset') {
|
|
344
|
+
const rows = await materialize(cap + 1);
|
|
345
|
+
if (rows.length > cap) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) refuses to load ' +
|
|
348
|
+
`more than ${cap} rows into memory. Pass an explicit bounded limit.`,
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
return rows;
|
|
316
352
|
}
|
|
317
353
|
|
|
318
354
|
const count = await this.count();
|
|
@@ -322,7 +358,7 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
322
358
|
`The hard limit is ${cap}. Return the dataset handle instead or call materialize(limit).`,
|
|
323
359
|
);
|
|
324
360
|
}
|
|
325
|
-
return await
|
|
361
|
+
return await materialize();
|
|
326
362
|
}
|
|
327
363
|
|
|
328
364
|
async *[Symbol.asyncIterator](): AsyncIterator<T> {
|
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.211",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.211",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -10826,7 +10826,8 @@ var PLAY_RUN_RESERVED_BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
|
|
|
10826
10826
|
"--logs",
|
|
10827
10827
|
"--full",
|
|
10828
10828
|
"--force",
|
|
10829
|
-
"--no-open"
|
|
10829
|
+
"--no-open",
|
|
10830
|
+
"--debug-map-latency"
|
|
10830
10831
|
]);
|
|
10831
10832
|
function traceCliSync(phase, fields, run) {
|
|
10832
10833
|
const startedAt = Date.now();
|
|
@@ -14173,6 +14174,7 @@ function parsePlayRunOptions(args) {
|
|
|
14173
14174
|
const emitLogs = !jsonOutput || args.includes("--logs");
|
|
14174
14175
|
const force = args.includes("--force");
|
|
14175
14176
|
const noOpen = args.includes("--no-open");
|
|
14177
|
+
const debugMapLatency = args.includes("--debug-map-latency");
|
|
14176
14178
|
let waitTimeoutMs = null;
|
|
14177
14179
|
let profile = null;
|
|
14178
14180
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -14291,7 +14293,8 @@ function parsePlayRunOptions(args) {
|
|
|
14291
14293
|
waitTimeoutMs,
|
|
14292
14294
|
force,
|
|
14293
14295
|
noOpen,
|
|
14294
|
-
profile
|
|
14296
|
+
profile,
|
|
14297
|
+
debugMapLatency
|
|
14295
14298
|
};
|
|
14296
14299
|
}
|
|
14297
14300
|
function parsePlayCheckOptions(args) {
|
|
@@ -14776,6 +14779,7 @@ async function handleFileBackedRun(options) {
|
|
|
14776
14779
|
...stagedFileInputs.packagedFiles.length ? { packagedFiles: stagedFileInputs.packagedFiles } : {},
|
|
14777
14780
|
...options.force ? { force: true } : {},
|
|
14778
14781
|
...options.profile ? { profile: options.profile } : {},
|
|
14782
|
+
...options.debugMapLatency ? { testPolicyOverrides: { mapLatencyProfile: true } } : {},
|
|
14779
14783
|
...integrationMode ? { integrationMode } : {}
|
|
14780
14784
|
};
|
|
14781
14785
|
if (options.watch) {
|
|
@@ -14937,6 +14941,7 @@ async function handleNamedRun(options) {
|
|
|
14937
14941
|
...stagedFileInputs.packagedFiles.length ? { packagedFiles: stagedFileInputs.packagedFiles } : {},
|
|
14938
14942
|
...options.force ? { force: true } : {},
|
|
14939
14943
|
...options.profile ? { profile: options.profile } : {},
|
|
14944
|
+
...options.debugMapLatency ? { testPolicyOverrides: { mapLatencyProfile: true } } : {},
|
|
14940
14945
|
...integrationMode ? { integrationMode } : {}
|
|
14941
14946
|
};
|
|
14942
14947
|
if (options.watch) {
|
|
@@ -16185,7 +16190,10 @@ Examples:
|
|
|
16185
16190
|
).option("--watch", "Compatibility alias; run waits by default").option("--wait", "Compatibility alias; run waits by default").option("--no-wait", "Start the run and return immediately").option(
|
|
16186
16191
|
"--logs",
|
|
16187
16192
|
"When output is non-interactive, stream play logs to stderr while waiting"
|
|
16188
|
-
).option("--tail-timeout-ms <ms>", "Timeout while watching the run stream").option("--force", "Start a fresh run graph").option(
|
|
16193
|
+
).option("--tail-timeout-ms <ms>", "Timeout while watching the run stream").option("--force", "Start a fresh run graph").option(
|
|
16194
|
+
"--debug-map-latency",
|
|
16195
|
+
"Internal diagnostics: emit one aggregate latency profile per dataset map"
|
|
16196
|
+
).option("--no-open", "Print the play page URL without opening a browser").option("--json", "Emit JSON output").option("--full", "Debug only: with --json, emit the raw status payload").addHelpText(
|
|
16189
16197
|
"afterAll",
|
|
16190
16198
|
`
|
|
16191
16199
|
Pass-through input flags:
|
|
@@ -16221,6 +16229,7 @@ Pass-through input flags:
|
|
|
16221
16229
|
...options.logs ? ["--logs"] : [],
|
|
16222
16230
|
...options.tailTimeoutMs ? ["--tail-timeout-ms", options.tailTimeoutMs] : [],
|
|
16223
16231
|
...options.force ? ["--force"] : [],
|
|
16232
|
+
...options.debugMapLatency ? ["--debug-map-latency"] : [],
|
|
16224
16233
|
...options.noOpen || options.open === false ? ["--no-open"] : [],
|
|
16225
16234
|
...options.json ? ["--json"] : [],
|
|
16226
16235
|
...options.full ? ["--full"] : [],
|
|
@@ -17270,6 +17279,8 @@ function renderPlayStep(command, options) {
|
|
|
17270
17279
|
` const __dlRunIf = ${runIfJs};`,
|
|
17271
17280
|
` if (!__dlRunIf(templateRow)) return null;`
|
|
17272
17281
|
] : [];
|
|
17282
|
+
const inline = command.play?.inline;
|
|
17283
|
+
const inlineHandler = inline ? `__dlInlinePlay_${inline.sourceHash.slice(0, 16)}` : null;
|
|
17273
17284
|
return [
|
|
17274
17285
|
`async (row, stepCtx) => {`,
|
|
17275
17286
|
...options.precheck ? [` if (${options.precheck}) return null;`] : [],
|
|
@@ -17279,9 +17290,16 @@ function renderPlayStep(command, options) {
|
|
|
17279
17290
|
` if (__dlShouldSkipBlankPlayPayload(payload)) return null;`,
|
|
17280
17291
|
` let result: unknown;`,
|
|
17281
17292
|
` try {`,
|
|
17282
|
-
|
|
17283
|
-
|
|
17284
|
-
|
|
17293
|
+
...inlineHandler ? [
|
|
17294
|
+
// Enrich validates and normalizes this payload against the certified
|
|
17295
|
+
// prebuilt contract before code generation. The attachment retains
|
|
17296
|
+
// its concrete input type, while generated payloads are records.
|
|
17297
|
+
` result = await __dlRunInlinePlay(${inlineHandler}, stepCtx, payload as never, ${options.force});`
|
|
17298
|
+
] : [
|
|
17299
|
+
` result = await stepCtx.runPlay(${callId}, ${playRef}, payload, {`,
|
|
17300
|
+
` description: ${stringLiteral(command.description ?? command.alias)}`,
|
|
17301
|
+
` });`
|
|
17302
|
+
],
|
|
17285
17303
|
` } catch (error) {`,
|
|
17286
17304
|
...options.waterfallSoftFail ? [` return __dlWaterfallToolFailure(error);`] : [` throw error;`],
|
|
17287
17305
|
` }`,
|
|
@@ -17289,6 +17307,19 @@ function renderPlayStep(command, options) {
|
|
|
17289
17307
|
`}`
|
|
17290
17308
|
].join("\n");
|
|
17291
17309
|
}
|
|
17310
|
+
function collectInlinePlayHandlers(commands) {
|
|
17311
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
17312
|
+
const visit = (command) => {
|
|
17313
|
+
if (isWaterfall(command)) {
|
|
17314
|
+
command.commands.forEach(visit);
|
|
17315
|
+
return;
|
|
17316
|
+
}
|
|
17317
|
+
const inline = command.play?.inline;
|
|
17318
|
+
if (inline) handlers.set(inline.sourceHash, inline);
|
|
17319
|
+
};
|
|
17320
|
+
commands.forEach(visit);
|
|
17321
|
+
return [...handlers.values()];
|
|
17322
|
+
}
|
|
17292
17323
|
function renderInlineJavascriptStep(command, options) {
|
|
17293
17324
|
const alias = stringLiteral(command.alias);
|
|
17294
17325
|
const extractJs = renderExtractFunction(command, 4);
|
|
@@ -17489,6 +17520,10 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17489
17520
|
[...options.forceAliases ?? []].map((alias) => normalizeAlias(alias))
|
|
17490
17521
|
);
|
|
17491
17522
|
const columnSteps = [];
|
|
17523
|
+
const inlineHandlers = collectInlinePlayHandlers(config.commands);
|
|
17524
|
+
const inlineTypeImports = [
|
|
17525
|
+
...new Set(inlineHandlers.flatMap((inline) => inline.typeImports ?? []))
|
|
17526
|
+
].sort();
|
|
17492
17527
|
config.commands.forEach((command) => {
|
|
17493
17528
|
if (isWaterfall(command)) {
|
|
17494
17529
|
columnSteps.push(
|
|
@@ -17526,6 +17561,19 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17526
17561
|
const generatedAliases = collectGeneratedAliases(config.commands);
|
|
17527
17562
|
const runOptionsSource = options.failFast ? `{ key: (row, index) => __dlEnrichRowKey(row, index + rowStart), onRowError: 'fail' as const }` : `{ key: (row, index) => __dlEnrichRowKey(row, index + rowStart) }`;
|
|
17528
17563
|
const body = [
|
|
17564
|
+
`function __dlRunInlinePlay<TContext, TInput, TOutput>(handler: (ctx: TContext, input: TInput) => Promise<TOutput>, scope: TContext, payload: TInput, force: boolean): Promise<TOutput> {`,
|
|
17565
|
+
` if (!force) return handler(scope, payload);`,
|
|
17566
|
+
` const runtimeScope = scope as TContext & { __deeplineRunWithForcedTools?: <T>(run: () => Promise<T>) => Promise<T> };`,
|
|
17567
|
+
` if (typeof runtimeScope.__deeplineRunWithForcedTools !== 'function') {`,
|
|
17568
|
+
` throw new Error('This runtime does not support forced certified inline play execution.');`,
|
|
17569
|
+
` }`,
|
|
17570
|
+
` return runtimeScope.__deeplineRunWithForcedTools(() => handler(scope, payload));`,
|
|
17571
|
+
`}`,
|
|
17572
|
+
``,
|
|
17573
|
+
...inlineHandlers.flatMap((inline) => [
|
|
17574
|
+
`const __dlInlinePlay_${inline.sourceHash.slice(0, 16)} = ${inline.functionExpressionSource};`,
|
|
17575
|
+
``
|
|
17576
|
+
]),
|
|
17529
17577
|
`export default definePlay(${stringLiteral(playName)}, async (ctx, input: EnrichInput) => {`,
|
|
17530
17578
|
` const sourceRows = await ctx.csv<Record<string, unknown>>(input.file);`,
|
|
17531
17579
|
` const rowStart = __dlNonNegativeInteger(input.rowStart, 0);`,
|
|
@@ -17550,7 +17598,8 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17550
17598
|
const helpers = idiomaticGetters ? selectUsedHelpers(helperSource(), body.join("\n")) : helperSource();
|
|
17551
17599
|
return [
|
|
17552
17600
|
...inlineRunJavascript && configHasRunJavascript(config) ? ["// @ts-nocheck", "/* eslint-disable */", ""] : [],
|
|
17553
|
-
`import { definePlay } from 'deepline';`,
|
|
17601
|
+
`import { definePlay, steps } from 'deepline';`,
|
|
17602
|
+
...inlineTypeImports.length > 0 ? [`import type { ${inlineTypeImports.join(", ")} } from 'deepline';`] : [],
|
|
17554
17603
|
``,
|
|
17555
17604
|
`type EnrichInput = { file: string; rowStart?: number | null; rowEnd?: number | null };`,
|
|
17556
17605
|
``,
|
|
@@ -18765,7 +18814,8 @@ async function buildPlanArgs(args) {
|
|
|
18765
18814
|
"--fail-fast",
|
|
18766
18815
|
"--all",
|
|
18767
18816
|
"--in-place",
|
|
18768
|
-
"--no-open"
|
|
18817
|
+
"--no-open",
|
|
18818
|
+
"--debug-map-latency"
|
|
18769
18819
|
]);
|
|
18770
18820
|
const planArgs = [];
|
|
18771
18821
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -21318,11 +21368,15 @@ async function fetchBackingRowsForCsvExport(input2) {
|
|
|
21318
21368
|
}
|
|
21319
21369
|
function mergeRowsForCsvExport(enrichedRows, options) {
|
|
21320
21370
|
const compactAliases = options?.config ? enrichAiRuntimeCompactAliases(options.config) : /* @__PURE__ */ new Set();
|
|
21371
|
+
const preservedAliases = /* @__PURE__ */ new Set([
|
|
21372
|
+
...compactAliases,
|
|
21373
|
+
...options?.config ? collectConfigPlayAliases(options.config) : []
|
|
21374
|
+
]);
|
|
21321
21375
|
const rows = dataExportRows(
|
|
21322
21376
|
normalizeEnrichRowsForCsvExport(enrichedRows, options?.config, {
|
|
21323
21377
|
statusFailureMessages: options?.statusFailureMessages
|
|
21324
21378
|
}),
|
|
21325
|
-
{ preserveJsonStringColumns:
|
|
21379
|
+
{ preserveJsonStringColumns: preservedAliases }
|
|
21326
21380
|
);
|
|
21327
21381
|
const range = options?.rows;
|
|
21328
21382
|
if (!options?.sourceCsvPath) {
|
|
@@ -21399,6 +21453,22 @@ function mergeRowsForCsvExport(enrichedRows, options) {
|
|
|
21399
21453
|
}
|
|
21400
21454
|
return { rows: merged, preferredColumns };
|
|
21401
21455
|
}
|
|
21456
|
+
function collectConfigPlayAliases(config) {
|
|
21457
|
+
const aliases = /* @__PURE__ */ new Set();
|
|
21458
|
+
const visit = (commands) => {
|
|
21459
|
+
for (const command of commands) {
|
|
21460
|
+
if ("with_waterfall" in command) {
|
|
21461
|
+
visit(command.commands);
|
|
21462
|
+
continue;
|
|
21463
|
+
}
|
|
21464
|
+
if (!command.disabled && command.play) {
|
|
21465
|
+
aliases.add(normalizeAlias2(command.alias));
|
|
21466
|
+
}
|
|
21467
|
+
}
|
|
21468
|
+
};
|
|
21469
|
+
visit(config.commands);
|
|
21470
|
+
return aliases;
|
|
21471
|
+
}
|
|
21402
21472
|
function collectConfigScalarAliasOrder(config) {
|
|
21403
21473
|
const aliases = [];
|
|
21404
21474
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -21554,6 +21624,9 @@ function materializeEnrichAliasCellForCsv(row, alias, options) {
|
|
|
21554
21624
|
if (Object.prototype.hasOwnProperty.call(direct, "matched_result") || Object.prototype.hasOwnProperty.call(direct, "matchedResult") || Object.prototype.hasOwnProperty.call(direct, "result")) {
|
|
21555
21625
|
return materializeCsvCellValue(direct);
|
|
21556
21626
|
}
|
|
21627
|
+
if (options?.preservePlainObject && !Object.prototype.hasOwnProperty.call(direct, "_metadata")) {
|
|
21628
|
+
return materializeCsvCellValue(direct);
|
|
21629
|
+
}
|
|
21557
21630
|
const directCells = [];
|
|
21558
21631
|
for (const [field, value] of Object.entries(direct)) {
|
|
21559
21632
|
if (!isEnrichAliasPayloadField(field)) {
|
|
@@ -21610,6 +21683,7 @@ function materializeEnrichAliasCellForCsv(row, alias, options) {
|
|
|
21610
21683
|
function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
21611
21684
|
const aliases = config ? collectConfigScalarAliasOrder(config) : [];
|
|
21612
21685
|
const aiCompactAliases = config ? enrichAiRuntimeCompactAliases(config) : null;
|
|
21686
|
+
const playAliases = config ? collectConfigPlayAliases(config) : /* @__PURE__ */ new Set();
|
|
21613
21687
|
const failureOperationByAlias = hardFailureOperationByAlias(config);
|
|
21614
21688
|
return rows.map((row) => {
|
|
21615
21689
|
const metadata = legacyMetadataFromRow(row);
|
|
@@ -21653,7 +21727,8 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
21653
21727
|
}
|
|
21654
21728
|
for (const alias of aliases) {
|
|
21655
21729
|
const value = materializeEnrichAliasCellForCsv(normalized, alias, {
|
|
21656
|
-
compactAiCell: aiCompactAliases?.has(normalizeAlias2(alias)) ?? false
|
|
21730
|
+
compactAiCell: aiCompactAliases?.has(normalizeAlias2(alias)) ?? false,
|
|
21731
|
+
preservePlainObject: playAliases.has(normalizeAlias2(alias))
|
|
21657
21732
|
});
|
|
21658
21733
|
if (value !== void 0) {
|
|
21659
21734
|
normalized[alias] = value;
|
|
@@ -21841,6 +21916,9 @@ function registerEnrichCommand(program) {
|
|
|
21841
21916
|
).option(
|
|
21842
21917
|
"--profile <id>",
|
|
21843
21918
|
"Internal/testing: override the execution profile for the generated play run."
|
|
21919
|
+
).option(
|
|
21920
|
+
"--debug-map-latency",
|
|
21921
|
+
"Internal diagnostics: emit one aggregate latency profile per dataset map."
|
|
21844
21922
|
).option(
|
|
21845
21923
|
"--fail-fast",
|
|
21846
21924
|
"Fail the generated play when any selected row errors, while preserving recovered runtime-sheet rows for export."
|
|
@@ -21986,6 +22064,9 @@ function registerEnrichCommand(program) {
|
|
|
21986
22064
|
if (options.profile) {
|
|
21987
22065
|
runArgs.push("--profile", options.profile);
|
|
21988
22066
|
}
|
|
22067
|
+
if (options.debugMapLatency) {
|
|
22068
|
+
runArgs.push("--debug-map-latency");
|
|
22069
|
+
}
|
|
21989
22070
|
if (options.noOpen || input2.suppressOpen) {
|
|
21990
22071
|
runArgs.push("--no-open");
|
|
21991
22072
|
}
|