deepline 0.1.209 → 0.1.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +155 -139
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +15 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-latency-profile.ts +90 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +173 -103
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +25 -92
- package/dist/bundling-sources/sdk/src/play.ts +4 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +4 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +348 -109
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +1 -2
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +2 -4
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +42 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/single-flight.ts +31 -0
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +12 -1
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +41 -5
- package/dist/cli/index.js +94 -16
- package/dist/cli/index.mjs +94 -16
- package/dist/index.d.mts +18 -3
- package/dist/index.d.ts +18 -3
- package/dist/index.js +22 -5
- package/dist/index.mjs +22 -5
- package/package.json +1 -1
|
@@ -281,7 +281,6 @@ export type CustomerDbQueryHandler = (
|
|
|
281
281
|
|
|
282
282
|
export interface PlayCallOptions {
|
|
283
283
|
description?: string;
|
|
284
|
-
staleAfterSeconds?: number;
|
|
285
284
|
}
|
|
286
285
|
|
|
287
286
|
export interface StepOptions {
|
|
@@ -591,7 +590,7 @@ export interface ContextOptions {
|
|
|
591
590
|
runId?: string;
|
|
592
591
|
/** Physical executor run that owns leases for an inline child context. */
|
|
593
592
|
runtimeReceiptOwnerRunId?: string;
|
|
594
|
-
/** Logical invocation scope for
|
|
593
|
+
/** Logical invocation scope for receipts created inside an inline child. */
|
|
595
594
|
runtimeReceiptScope?: string;
|
|
596
595
|
runAttempt?: number | null;
|
|
597
596
|
/**
|
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
|
|
10
10
|
export const DURABLE_CALL_CACHE_POLICY_VERSION = 'call-cache-v1';
|
|
11
11
|
|
|
12
|
-
export type DurableCallKind = 'tool' | 'step' | 'fetch'
|
|
12
|
+
export type DurableCallKind = 'tool' | 'step' | 'fetch';
|
|
13
13
|
|
|
14
14
|
function validateStaleAfterSeconds(staleAfterSeconds?: number | null): void {
|
|
15
15
|
if (staleAfterSeconds === undefined || staleAfterSeconds === null) {
|
|
@@ -121,16 +121,14 @@ export function buildDurableCtxCallCacheKey(input: {
|
|
|
121
121
|
return `ctx:${orgId}:call:${input.kind}:${normalizedId}:${digest}`;
|
|
122
122
|
}
|
|
123
123
|
|
|
124
|
-
export function
|
|
124
|
+
export function buildDurableRunPlayInvocationScope(input: {
|
|
125
125
|
childPlayName: string;
|
|
126
126
|
input: Record<string, unknown>;
|
|
127
|
-
childRevisionFingerprint?: string | null;
|
|
128
127
|
rowScope?: Record<string, unknown> | null;
|
|
129
128
|
}): string {
|
|
130
129
|
return sha256Hex(
|
|
131
130
|
stableStringify({
|
|
132
131
|
childPlayName: input.childPlayName.trim(),
|
|
133
|
-
childRevisionFingerprint: input.childRevisionFingerprint ?? null,
|
|
134
132
|
input: input.input,
|
|
135
133
|
rowScope: input.rowScope ?? null,
|
|
136
134
|
}),
|
|
@@ -243,9 +243,11 @@ async function executeWithDurableRuntimeReceiptHeartbeat<T>(input: {
|
|
|
243
243
|
rejectLeaseLost(error as RuntimeReceiptLeaseLostError),
|
|
244
244
|
});
|
|
245
245
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
246
|
+
// markRunning is the authoritative pre-dispatch ownership fence. Renewal is
|
|
247
|
+
// only needed when the lease approaches expiry; an immediate read/renewal
|
|
248
|
+
// adds a round trip to every short tool call without extending ownership.
|
|
249
|
+
// Completion remains conditional on this lease, and long work still renews.
|
|
250
|
+
supervisor.start();
|
|
249
251
|
try {
|
|
250
252
|
return await Promise.race([input.execute(), leaseLost]);
|
|
251
253
|
} finally {
|
|
@@ -294,6 +296,7 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
294
296
|
maxAttempts?: number;
|
|
295
297
|
delayMs?: number;
|
|
296
298
|
abortSignal?: AbortSignal;
|
|
299
|
+
log?: (message: string) => void;
|
|
297
300
|
}): Promise<{
|
|
298
301
|
completed: Map<string, RuntimeStepReceipt>;
|
|
299
302
|
failed: Map<string, Error>;
|
|
@@ -326,6 +329,20 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
326
329
|
}
|
|
327
330
|
throw error;
|
|
328
331
|
}
|
|
332
|
+
const statuses = [...receipts.values()].reduce<Record<string, number>>(
|
|
333
|
+
(counts, receipt) => {
|
|
334
|
+
counts[receipt.status] = (counts[receipt.status] ?? 0) + 1;
|
|
335
|
+
return counts;
|
|
336
|
+
},
|
|
337
|
+
{},
|
|
338
|
+
);
|
|
339
|
+
input.log?.(
|
|
340
|
+
`[perf] runtime-receipt-wait attempt=${attempt} requested=${pending.size} indexed=${receipts.size} statuses=${
|
|
341
|
+
Object.entries(statuses)
|
|
342
|
+
.map(([status, count]) => `${status}:${count}`)
|
|
343
|
+
.join(',') || 'none'
|
|
344
|
+
}`,
|
|
345
|
+
);
|
|
329
346
|
for (const key of [...pending]) {
|
|
330
347
|
const receipt = receipts.get(key);
|
|
331
348
|
if (receipt?.status === 'completed' || receipt?.status === 'skipped') {
|
|
@@ -343,6 +360,9 @@ export async function waitForCompletedRuntimeReceipts(input: {
|
|
|
343
360
|
pending.delete(key);
|
|
344
361
|
}
|
|
345
362
|
}
|
|
363
|
+
input.log?.(
|
|
364
|
+
`[perf] runtime-receipt-wait attempt=${attempt} remaining=${pending.size}`,
|
|
365
|
+
);
|
|
346
366
|
}
|
|
347
367
|
|
|
348
368
|
return { completed, failed, timedOut: pending };
|
|
@@ -374,6 +394,16 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
374
394
|
log: (message: string) => void;
|
|
375
395
|
execute: (context: { leaseId: string | null }) => Promise<T>;
|
|
376
396
|
}): Promise<T> {
|
|
397
|
+
const lifecycleStartedAt = Date.now();
|
|
398
|
+
const logPhase = (phase: string, startedAt: number): void => {
|
|
399
|
+
// Keep this at the semantic receipt layer. Transport timing below this
|
|
400
|
+
// function explains HTTP cost; these markers expose gaps before/after the
|
|
401
|
+
// store calls and prove whether user code or receipt orchestration owns the
|
|
402
|
+
// elapsed time. Do not log receipt keys because they encode tool inputs.
|
|
403
|
+
input.log(
|
|
404
|
+
`[perf] durable receipt operation=${input.operation} id=${input.id} phase=${phase} elapsed_ms=${Date.now() - startedAt} lifecycle_ms=${Date.now() - lifecycleStartedAt}`,
|
|
405
|
+
);
|
|
406
|
+
};
|
|
377
407
|
if (!input.store.enabled) {
|
|
378
408
|
return await input.execute({ leaseId: null });
|
|
379
409
|
}
|
|
@@ -483,12 +513,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
483
513
|
);
|
|
484
514
|
};
|
|
485
515
|
|
|
516
|
+
const claimStartedAt = Date.now();
|
|
486
517
|
const claimed = await input.store.claim(
|
|
487
518
|
input.receiptKey,
|
|
488
519
|
input.runId,
|
|
489
520
|
input.reclaimRunning === true || input.force === true,
|
|
490
521
|
input.force === true,
|
|
491
522
|
);
|
|
523
|
+
logPhase('claim', claimStartedAt);
|
|
492
524
|
if (
|
|
493
525
|
input.force !== true &&
|
|
494
526
|
(claimed?.status === 'completed' || claimed?.status === 'skipped')
|
|
@@ -525,11 +557,13 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
525
557
|
let result: T;
|
|
526
558
|
try {
|
|
527
559
|
if (input.markRunningBeforeExecute !== false && input.store.markRunning) {
|
|
560
|
+
const markRunningStartedAt = Date.now();
|
|
528
561
|
const running = await input.store.markRunning(
|
|
529
562
|
input.receiptKey,
|
|
530
563
|
input.runId,
|
|
531
564
|
ownedLeaseId,
|
|
532
565
|
);
|
|
566
|
+
logPhase('mark_running', markRunningStartedAt);
|
|
533
567
|
if (!running || running.status !== 'running') {
|
|
534
568
|
throw new RuntimeReceiptLeaseLostError({
|
|
535
569
|
receiptKey: input.receiptKey,
|
|
@@ -540,6 +574,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
540
574
|
ownedLeaseId = running.leaseId ?? ownedLeaseId;
|
|
541
575
|
ownedLeaseExpiresAt = running.leaseExpiresAt ?? ownedLeaseExpiresAt;
|
|
542
576
|
}
|
|
577
|
+
const executeStartedAt = Date.now();
|
|
543
578
|
const executed =
|
|
544
579
|
ownedLeaseId && input.store.heartbeat
|
|
545
580
|
? await executeWithDurableRuntimeReceiptHeartbeat({
|
|
@@ -561,6 +596,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
561
596
|
execute: () => input.execute({ leaseId: ownedLeaseId }),
|
|
562
597
|
})
|
|
563
598
|
: await input.execute({ leaseId: ownedLeaseId });
|
|
599
|
+
logPhase('execute', executeStartedAt);
|
|
564
600
|
result = input.onClaimedResult
|
|
565
601
|
? input.onClaimedResult(executed, input.receiptKey)
|
|
566
602
|
: executed;
|
|
@@ -603,12 +639,14 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
603
639
|
throw error;
|
|
604
640
|
}
|
|
605
641
|
|
|
642
|
+
const completeStartedAt = Date.now();
|
|
606
643
|
const completed = await input.store.complete(
|
|
607
644
|
input.receiptKey,
|
|
608
645
|
input.runId,
|
|
609
646
|
isToolExecuteResult(result) ? serializeToolExecuteResult(result) : result,
|
|
610
647
|
ownedLeaseId,
|
|
611
648
|
);
|
|
649
|
+
logPhase('complete', completeStartedAt);
|
|
612
650
|
if (
|
|
613
651
|
!completed ||
|
|
614
652
|
(completed.status !== 'completed' && completed.status !== 'skipped')
|
|
@@ -633,6 +671,7 @@ export async function executeWithDurableRuntimeReceipt<T>(input: {
|
|
|
633
671
|
completed &&
|
|
634
672
|
(completed.status === 'completed' || completed.status === 'skipped')
|
|
635
673
|
) {
|
|
674
|
+
logPhase('total', lifecycleStartedAt);
|
|
636
675
|
return await recoverCompletedReceipt(completed, 'owner');
|
|
637
676
|
}
|
|
638
677
|
if (input.store.canPersistCompletion) {
|
|
@@ -344,13 +344,17 @@ export async function stageDaytonaRunnerPayload(input: {
|
|
|
344
344
|
const outputPath = `${workDir}/deepline-play-output-${randomUUID()}.log`;
|
|
345
345
|
const exitCodePath = `${workDir}/deepline-play-exit-${randomUUID()}.txt`;
|
|
346
346
|
const progressEventPath = `${workDir}/deepline-play-progress-${randomUUID()}.jsonl`;
|
|
347
|
+
const runnerTraceEnv =
|
|
348
|
+
process.env.DEEPLINE_RUNTIME_RECEIPT_TRACE === '1'
|
|
349
|
+
? 'DEEPLINE_RUNTIME_RECEIPT_TRACE=1 '
|
|
350
|
+
: '';
|
|
347
351
|
const runnerCommand = `${nodeMaterializePayloadCommand({
|
|
348
352
|
envelopePath,
|
|
349
353
|
runnerPath,
|
|
350
354
|
configPath,
|
|
351
355
|
artifactCodePath,
|
|
352
356
|
crashPusherPath,
|
|
353
|
-
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
357
|
+
})} && DEEPLINE_PLAY_RUNNER_PROGRESS_EVENT_PATH=${shellQuote(progressEventPath)} DEEPLINE_PLAY_RUNNER_STDOUT_EVENT_MODE=compact ${runnerTraceEnv}node ${shellQuote(runnerPath)} ${shellQuote(configPath)}`;
|
|
354
358
|
// Crash-containment epilogue: runs UNCONDITIONALLY after the runner exits and
|
|
355
359
|
// pushes the parsed (or synthesized) terminal to the gateway so the parked
|
|
356
360
|
// worker wakes within seconds of ANY runner death — process.exit abuse, OOM
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export type SingleFlight<K, V> = {
|
|
2
|
+
run(key: K, create: () => Promise<V>): Promise<V>;
|
|
3
|
+
has(key: K): boolean;
|
|
4
|
+
activeCount(): number;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
/** Coalesce concurrent work for one key without caching its settled result. */
|
|
8
|
+
export function createSingleFlight<K, V>(): SingleFlight<K, V> {
|
|
9
|
+
const active = new Map<K, Promise<V>>();
|
|
10
|
+
|
|
11
|
+
return {
|
|
12
|
+
run(key, create) {
|
|
13
|
+
const existing = active.get(key);
|
|
14
|
+
if (existing) return existing;
|
|
15
|
+
|
|
16
|
+
const pending = Promise.resolve().then(create);
|
|
17
|
+
active.set(key, pending);
|
|
18
|
+
const clear = () => {
|
|
19
|
+
if (active.get(key) === pending) active.delete(key);
|
|
20
|
+
};
|
|
21
|
+
void pending.then(clear, clear);
|
|
22
|
+
return pending;
|
|
23
|
+
},
|
|
24
|
+
has(key) {
|
|
25
|
+
return active.has(key);
|
|
26
|
+
},
|
|
27
|
+
activeCount() {
|
|
28
|
+
return active.size;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -34,6 +34,8 @@ type ValidatedRuntimeTestFaultHeader =
|
|
|
34
34
|
| { ok: false; status: 400 | 403; error: string };
|
|
35
35
|
|
|
36
36
|
export type RuntimeTestPolicyOverrides = {
|
|
37
|
+
/** Opt-in bounded runner map latency profile for local/preview diagnosis. */
|
|
38
|
+
mapLatencyProfile?: boolean;
|
|
37
39
|
receiptLeaseTtlMs?: number;
|
|
38
40
|
sheetAttemptLeaseMs?: number;
|
|
39
41
|
heartbeatIntervalMs?: number;
|
|
@@ -170,7 +172,8 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
170
172
|
const unknownKeys = Object.keys(record).filter(
|
|
171
173
|
(key) =>
|
|
172
174
|
!RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
|
|
173
|
-
key !== 'workBudgetYieldLimits'
|
|
175
|
+
key !== 'workBudgetYieldLimits' &&
|
|
176
|
+
key !== 'mapLatencyProfile',
|
|
174
177
|
);
|
|
175
178
|
if (unknownKeys.length > 0) {
|
|
176
179
|
return {
|
|
@@ -179,6 +182,14 @@ function parseRuntimeTestPolicyOverrides(
|
|
|
179
182
|
}
|
|
180
183
|
|
|
181
184
|
const overrides: RuntimeTestPolicyOverrides = {};
|
|
185
|
+
if ('mapLatencyProfile' in record) {
|
|
186
|
+
if (typeof record.mapLatencyProfile !== 'boolean') {
|
|
187
|
+
return {
|
|
188
|
+
error: 'testPolicyOverrides.mapLatencyProfile must be a boolean.',
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
overrides.mapLatencyProfile = record.mapLatencyProfile;
|
|
192
|
+
}
|
|
182
193
|
for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
|
|
183
194
|
if (!(field in record)) continue;
|
|
184
195
|
const parsed = readPositiveIntegerField({
|
|
@@ -125,7 +125,7 @@ export interface PlayDataset<T> extends AsyncIterable<T> {
|
|
|
125
125
|
* Large datasets should flow by handle through Neon-backed storage, not
|
|
126
126
|
* through worker memory as giant arrays.
|
|
127
127
|
*/
|
|
128
|
-
materialize(
|
|
128
|
+
materialize(options?: number | PlayDatasetMaterializeOptions): Promise<T[]>;
|
|
129
129
|
toJSON(): {
|
|
130
130
|
kind: 'dataset';
|
|
131
131
|
datasetKind: PlayDatasetKind;
|
|
@@ -146,9 +146,19 @@ type PlayDatasetResolvers<T> = {
|
|
|
146
146
|
count: () => Promise<number>;
|
|
147
147
|
peek: (limit: number) => Promise<T[]>;
|
|
148
148
|
materialize: (limit?: number) => Promise<T[]>;
|
|
149
|
+
materializeFullPersistedDataset?: (limit?: number) => Promise<T[]>;
|
|
149
150
|
iterate: () => AsyncIterable<T>;
|
|
150
151
|
};
|
|
151
152
|
|
|
153
|
+
export type PlayDatasetMaterializeScope = 'result' | 'full_persisted_dataset';
|
|
154
|
+
|
|
155
|
+
export type PlayDatasetMaterializeOptions = {
|
|
156
|
+
/** Rows returned by this operation, or every current row in its persisted dataset. */
|
|
157
|
+
scope?: PlayDatasetMaterializeScope;
|
|
158
|
+
/** Maximum number of rows to load into memory. */
|
|
159
|
+
limit?: number;
|
|
160
|
+
};
|
|
161
|
+
|
|
152
162
|
type PlayDatasetTransform<T, U = T> =
|
|
153
163
|
| {
|
|
154
164
|
kind: 'map';
|
|
@@ -164,7 +174,7 @@ type PlayDatasetTransform<T, U = T> =
|
|
|
164
174
|
end?: number;
|
|
165
175
|
};
|
|
166
176
|
|
|
167
|
-
function resolveMaterializeLimitCap(): number {
|
|
177
|
+
export function resolveMaterializeLimitCap(): number {
|
|
168
178
|
const raw = process.env.DEEPLINE_PLAY_DATASET_MATERIALIZE_LIMIT;
|
|
169
179
|
const parsed = raw ? Number(raw) : NaN;
|
|
170
180
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
@@ -301,10 +311,25 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
301
311
|
return this.slice(0, limit, options);
|
|
302
312
|
}
|
|
303
313
|
|
|
304
|
-
async materialize(
|
|
314
|
+
async materialize(
|
|
315
|
+
options?: number | PlayDatasetMaterializeOptions,
|
|
316
|
+
): Promise<T[]> {
|
|
317
|
+
const scope =
|
|
318
|
+
typeof options === 'object' ? (options.scope ?? 'result') : 'result';
|
|
319
|
+
const limit = typeof options === 'number' ? options : options?.limit;
|
|
305
320
|
const requestedLimit =
|
|
306
321
|
limit !== undefined ? Math.max(0, Math.floor(limit)) : undefined;
|
|
307
322
|
const cap = resolveMaterializeLimitCap();
|
|
323
|
+
const materialize =
|
|
324
|
+
scope === 'full_persisted_dataset'
|
|
325
|
+
? this.resolvers.materializeFullPersistedDataset
|
|
326
|
+
: this.resolvers.materialize;
|
|
327
|
+
if (!materialize) {
|
|
328
|
+
throw new Error(
|
|
329
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) is only available ' +
|
|
330
|
+
'for a persisted dataset returned by ctx.dataset(...).run().',
|
|
331
|
+
);
|
|
332
|
+
}
|
|
308
333
|
if (requestedLimit !== undefined) {
|
|
309
334
|
if (requestedLimit > cap) {
|
|
310
335
|
throw new Error(
|
|
@@ -312,7 +337,18 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
312
337
|
'Return the dataset handle instead, or request a smaller bounded slice.',
|
|
313
338
|
);
|
|
314
339
|
}
|
|
315
|
-
return await
|
|
340
|
+
return await materialize(requestedLimit);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (scope === 'full_persisted_dataset') {
|
|
344
|
+
const rows = await materialize(cap + 1);
|
|
345
|
+
if (rows.length > cap) {
|
|
346
|
+
throw new Error(
|
|
347
|
+
'PlayDataset.materialize({ scope: "full_persisted_dataset" }) refuses to load ' +
|
|
348
|
+
`more than ${cap} rows into memory. Pass an explicit bounded limit.`,
|
|
349
|
+
);
|
|
350
|
+
}
|
|
351
|
+
return rows;
|
|
316
352
|
}
|
|
317
353
|
|
|
318
354
|
const count = await this.count();
|
|
@@ -322,7 +358,7 @@ class DeferredPlayDataset<T> implements PlayDataset<T> {
|
|
|
322
358
|
`The hard limit is ${cap}. Return the dataset handle instead or call materialize(limit).`,
|
|
323
359
|
);
|
|
324
360
|
}
|
|
325
|
-
return await
|
|
361
|
+
return await materialize();
|
|
326
362
|
}
|
|
327
363
|
|
|
328
364
|
async *[Symbol.asyncIterator](): AsyncIterator<T> {
|
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.211",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.211",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -10826,7 +10826,8 @@ var PLAY_RUN_RESERVED_BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
|
|
|
10826
10826
|
"--logs",
|
|
10827
10827
|
"--full",
|
|
10828
10828
|
"--force",
|
|
10829
|
-
"--no-open"
|
|
10829
|
+
"--no-open",
|
|
10830
|
+
"--debug-map-latency"
|
|
10830
10831
|
]);
|
|
10831
10832
|
function traceCliSync(phase, fields, run) {
|
|
10832
10833
|
const startedAt = Date.now();
|
|
@@ -14173,6 +14174,7 @@ function parsePlayRunOptions(args) {
|
|
|
14173
14174
|
const emitLogs = !jsonOutput || args.includes("--logs");
|
|
14174
14175
|
const force = args.includes("--force");
|
|
14175
14176
|
const noOpen = args.includes("--no-open");
|
|
14177
|
+
const debugMapLatency = args.includes("--debug-map-latency");
|
|
14176
14178
|
let waitTimeoutMs = null;
|
|
14177
14179
|
let profile = null;
|
|
14178
14180
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -14291,7 +14293,8 @@ function parsePlayRunOptions(args) {
|
|
|
14291
14293
|
waitTimeoutMs,
|
|
14292
14294
|
force,
|
|
14293
14295
|
noOpen,
|
|
14294
|
-
profile
|
|
14296
|
+
profile,
|
|
14297
|
+
debugMapLatency
|
|
14295
14298
|
};
|
|
14296
14299
|
}
|
|
14297
14300
|
function parsePlayCheckOptions(args) {
|
|
@@ -14776,6 +14779,7 @@ async function handleFileBackedRun(options) {
|
|
|
14776
14779
|
...stagedFileInputs.packagedFiles.length ? { packagedFiles: stagedFileInputs.packagedFiles } : {},
|
|
14777
14780
|
...options.force ? { force: true } : {},
|
|
14778
14781
|
...options.profile ? { profile: options.profile } : {},
|
|
14782
|
+
...options.debugMapLatency ? { testPolicyOverrides: { mapLatencyProfile: true } } : {},
|
|
14779
14783
|
...integrationMode ? { integrationMode } : {}
|
|
14780
14784
|
};
|
|
14781
14785
|
if (options.watch) {
|
|
@@ -14937,6 +14941,7 @@ async function handleNamedRun(options) {
|
|
|
14937
14941
|
...stagedFileInputs.packagedFiles.length ? { packagedFiles: stagedFileInputs.packagedFiles } : {},
|
|
14938
14942
|
...options.force ? { force: true } : {},
|
|
14939
14943
|
...options.profile ? { profile: options.profile } : {},
|
|
14944
|
+
...options.debugMapLatency ? { testPolicyOverrides: { mapLatencyProfile: true } } : {},
|
|
14940
14945
|
...integrationMode ? { integrationMode } : {}
|
|
14941
14946
|
};
|
|
14942
14947
|
if (options.watch) {
|
|
@@ -16185,7 +16190,10 @@ Examples:
|
|
|
16185
16190
|
).option("--watch", "Compatibility alias; run waits by default").option("--wait", "Compatibility alias; run waits by default").option("--no-wait", "Start the run and return immediately").option(
|
|
16186
16191
|
"--logs",
|
|
16187
16192
|
"When output is non-interactive, stream play logs to stderr while waiting"
|
|
16188
|
-
).option("--tail-timeout-ms <ms>", "Timeout while watching the run stream").option("--force", "Start a fresh run graph").option(
|
|
16193
|
+
).option("--tail-timeout-ms <ms>", "Timeout while watching the run stream").option("--force", "Start a fresh run graph").option(
|
|
16194
|
+
"--debug-map-latency",
|
|
16195
|
+
"Internal diagnostics: emit one aggregate latency profile per dataset map"
|
|
16196
|
+
).option("--no-open", "Print the play page URL without opening a browser").option("--json", "Emit JSON output").option("--full", "Debug only: with --json, emit the raw status payload").addHelpText(
|
|
16189
16197
|
"afterAll",
|
|
16190
16198
|
`
|
|
16191
16199
|
Pass-through input flags:
|
|
@@ -16221,6 +16229,7 @@ Pass-through input flags:
|
|
|
16221
16229
|
...options.logs ? ["--logs"] : [],
|
|
16222
16230
|
...options.tailTimeoutMs ? ["--tail-timeout-ms", options.tailTimeoutMs] : [],
|
|
16223
16231
|
...options.force ? ["--force"] : [],
|
|
16232
|
+
...options.debugMapLatency ? ["--debug-map-latency"] : [],
|
|
16224
16233
|
...options.noOpen || options.open === false ? ["--no-open"] : [],
|
|
16225
16234
|
...options.json ? ["--json"] : [],
|
|
16226
16235
|
...options.full ? ["--full"] : [],
|
|
@@ -17215,12 +17224,9 @@ function renderColumnRunIfFunction(command) {
|
|
|
17215
17224
|
if (!command.run_if_js) {
|
|
17216
17225
|
return null;
|
|
17217
17226
|
}
|
|
17218
|
-
|
|
17219
|
-
return `(__dlRawRow) => { const row = __dlPrepareEnrichRow(__dlRawRow, [${stringLiteral(command.alias)}]); const input = row; const context = row;
|
|
17227
|
+
return `(__dlRawRow) => { const row = __dlPrepareEnrichRow(__dlRawRow, [${stringLiteral(command.alias)}]); const input = row; const context = row;
|
|
17220
17228
|
${indent(renderJavascriptBody(command.run_if_js), 6)}
|
|
17221
17229
|
}`;
|
|
17222
|
-
}
|
|
17223
|
-
return renderRunIfFunction(command);
|
|
17224
17230
|
}
|
|
17225
17231
|
function renderCombinedRunIfFunction(precheck, runIfSource) {
|
|
17226
17232
|
if (!runIfSource) {
|
|
@@ -17273,6 +17279,8 @@ function renderPlayStep(command, options) {
|
|
|
17273
17279
|
` const __dlRunIf = ${runIfJs};`,
|
|
17274
17280
|
` if (!__dlRunIf(templateRow)) return null;`
|
|
17275
17281
|
] : [];
|
|
17282
|
+
const inline = command.play?.inline;
|
|
17283
|
+
const inlineHandler = inline ? `__dlInlinePlay_${inline.sourceHash.slice(0, 16)}` : null;
|
|
17276
17284
|
return [
|
|
17277
17285
|
`async (row, stepCtx) => {`,
|
|
17278
17286
|
...options.precheck ? [` if (${options.precheck}) return null;`] : [],
|
|
@@ -17282,9 +17290,16 @@ function renderPlayStep(command, options) {
|
|
|
17282
17290
|
` if (__dlShouldSkipBlankPlayPayload(payload)) return null;`,
|
|
17283
17291
|
` let result: unknown;`,
|
|
17284
17292
|
` try {`,
|
|
17285
|
-
|
|
17286
|
-
|
|
17287
|
-
|
|
17293
|
+
...inlineHandler ? [
|
|
17294
|
+
// Enrich validates and normalizes this payload against the certified
|
|
17295
|
+
// prebuilt contract before code generation. The attachment retains
|
|
17296
|
+
// its concrete input type, while generated payloads are records.
|
|
17297
|
+
` result = await __dlRunInlinePlay(${inlineHandler}, stepCtx, payload as never, ${options.force});`
|
|
17298
|
+
] : [
|
|
17299
|
+
` result = await stepCtx.runPlay(${callId}, ${playRef}, payload, {`,
|
|
17300
|
+
` description: ${stringLiteral(command.description ?? command.alias)}`,
|
|
17301
|
+
` });`
|
|
17302
|
+
],
|
|
17288
17303
|
` } catch (error) {`,
|
|
17289
17304
|
...options.waterfallSoftFail ? [` return __dlWaterfallToolFailure(error);`] : [` throw error;`],
|
|
17290
17305
|
` }`,
|
|
@@ -17292,6 +17307,19 @@ function renderPlayStep(command, options) {
|
|
|
17292
17307
|
`}`
|
|
17293
17308
|
].join("\n");
|
|
17294
17309
|
}
|
|
17310
|
+
function collectInlinePlayHandlers(commands) {
|
|
17311
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
17312
|
+
const visit = (command) => {
|
|
17313
|
+
if (isWaterfall(command)) {
|
|
17314
|
+
command.commands.forEach(visit);
|
|
17315
|
+
return;
|
|
17316
|
+
}
|
|
17317
|
+
const inline = command.play?.inline;
|
|
17318
|
+
if (inline) handlers.set(inline.sourceHash, inline);
|
|
17319
|
+
};
|
|
17320
|
+
commands.forEach(visit);
|
|
17321
|
+
return [...handlers.values()];
|
|
17322
|
+
}
|
|
17295
17323
|
function renderInlineJavascriptStep(command, options) {
|
|
17296
17324
|
const alias = stringLiteral(command.alias);
|
|
17297
17325
|
const extractJs = renderExtractFunction(command, 4);
|
|
@@ -17492,6 +17520,10 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17492
17520
|
[...options.forceAliases ?? []].map((alias) => normalizeAlias(alias))
|
|
17493
17521
|
);
|
|
17494
17522
|
const columnSteps = [];
|
|
17523
|
+
const inlineHandlers = collectInlinePlayHandlers(config.commands);
|
|
17524
|
+
const inlineTypeImports = [
|
|
17525
|
+
...new Set(inlineHandlers.flatMap((inline) => inline.typeImports ?? []))
|
|
17526
|
+
].sort();
|
|
17495
17527
|
config.commands.forEach((command) => {
|
|
17496
17528
|
if (isWaterfall(command)) {
|
|
17497
17529
|
columnSteps.push(
|
|
@@ -17529,6 +17561,19 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17529
17561
|
const generatedAliases = collectGeneratedAliases(config.commands);
|
|
17530
17562
|
const runOptionsSource = options.failFast ? `{ key: (row, index) => __dlEnrichRowKey(row, index + rowStart), onRowError: 'fail' as const }` : `{ key: (row, index) => __dlEnrichRowKey(row, index + rowStart) }`;
|
|
17531
17563
|
const body = [
|
|
17564
|
+
`function __dlRunInlinePlay<TContext, TInput, TOutput>(handler: (ctx: TContext, input: TInput) => Promise<TOutput>, scope: TContext, payload: TInput, force: boolean): Promise<TOutput> {`,
|
|
17565
|
+
` if (!force) return handler(scope, payload);`,
|
|
17566
|
+
` const runtimeScope = scope as TContext & { __deeplineRunWithForcedTools?: <T>(run: () => Promise<T>) => Promise<T> };`,
|
|
17567
|
+
` if (typeof runtimeScope.__deeplineRunWithForcedTools !== 'function') {`,
|
|
17568
|
+
` throw new Error('This runtime does not support forced certified inline play execution.');`,
|
|
17569
|
+
` }`,
|
|
17570
|
+
` return runtimeScope.__deeplineRunWithForcedTools(() => handler(scope, payload));`,
|
|
17571
|
+
`}`,
|
|
17572
|
+
``,
|
|
17573
|
+
...inlineHandlers.flatMap((inline) => [
|
|
17574
|
+
`const __dlInlinePlay_${inline.sourceHash.slice(0, 16)} = ${inline.functionExpressionSource};`,
|
|
17575
|
+
``
|
|
17576
|
+
]),
|
|
17532
17577
|
`export default definePlay(${stringLiteral(playName)}, async (ctx, input: EnrichInput) => {`,
|
|
17533
17578
|
` const sourceRows = await ctx.csv<Record<string, unknown>>(input.file);`,
|
|
17534
17579
|
` const rowStart = __dlNonNegativeInteger(input.rowStart, 0);`,
|
|
@@ -17553,7 +17598,8 @@ function compileEnrichConfigToPlaySource(config, options = {}) {
|
|
|
17553
17598
|
const helpers = idiomaticGetters ? selectUsedHelpers(helperSource(), body.join("\n")) : helperSource();
|
|
17554
17599
|
return [
|
|
17555
17600
|
...inlineRunJavascript && configHasRunJavascript(config) ? ["// @ts-nocheck", "/* eslint-disable */", ""] : [],
|
|
17556
|
-
`import { definePlay } from 'deepline';`,
|
|
17601
|
+
`import { definePlay, steps } from 'deepline';`,
|
|
17602
|
+
...inlineTypeImports.length > 0 ? [`import type { ${inlineTypeImports.join(", ")} } from 'deepline';`] : [],
|
|
17557
17603
|
``,
|
|
17558
17604
|
`type EnrichInput = { file: string; rowStart?: number | null; rowEnd?: number | null };`,
|
|
17559
17605
|
``,
|
|
@@ -18768,7 +18814,8 @@ async function buildPlanArgs(args) {
|
|
|
18768
18814
|
"--fail-fast",
|
|
18769
18815
|
"--all",
|
|
18770
18816
|
"--in-place",
|
|
18771
|
-
"--no-open"
|
|
18817
|
+
"--no-open",
|
|
18818
|
+
"--debug-map-latency"
|
|
18772
18819
|
]);
|
|
18773
18820
|
const planArgs = [];
|
|
18774
18821
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -21321,11 +21368,15 @@ async function fetchBackingRowsForCsvExport(input2) {
|
|
|
21321
21368
|
}
|
|
21322
21369
|
function mergeRowsForCsvExport(enrichedRows, options) {
|
|
21323
21370
|
const compactAliases = options?.config ? enrichAiRuntimeCompactAliases(options.config) : /* @__PURE__ */ new Set();
|
|
21371
|
+
const preservedAliases = /* @__PURE__ */ new Set([
|
|
21372
|
+
...compactAliases,
|
|
21373
|
+
...options?.config ? collectConfigPlayAliases(options.config) : []
|
|
21374
|
+
]);
|
|
21324
21375
|
const rows = dataExportRows(
|
|
21325
21376
|
normalizeEnrichRowsForCsvExport(enrichedRows, options?.config, {
|
|
21326
21377
|
statusFailureMessages: options?.statusFailureMessages
|
|
21327
21378
|
}),
|
|
21328
|
-
{ preserveJsonStringColumns:
|
|
21379
|
+
{ preserveJsonStringColumns: preservedAliases }
|
|
21329
21380
|
);
|
|
21330
21381
|
const range = options?.rows;
|
|
21331
21382
|
if (!options?.sourceCsvPath) {
|
|
@@ -21402,6 +21453,22 @@ function mergeRowsForCsvExport(enrichedRows, options) {
|
|
|
21402
21453
|
}
|
|
21403
21454
|
return { rows: merged, preferredColumns };
|
|
21404
21455
|
}
|
|
21456
|
+
function collectConfigPlayAliases(config) {
|
|
21457
|
+
const aliases = /* @__PURE__ */ new Set();
|
|
21458
|
+
const visit = (commands) => {
|
|
21459
|
+
for (const command of commands) {
|
|
21460
|
+
if ("with_waterfall" in command) {
|
|
21461
|
+
visit(command.commands);
|
|
21462
|
+
continue;
|
|
21463
|
+
}
|
|
21464
|
+
if (!command.disabled && command.play) {
|
|
21465
|
+
aliases.add(normalizeAlias2(command.alias));
|
|
21466
|
+
}
|
|
21467
|
+
}
|
|
21468
|
+
};
|
|
21469
|
+
visit(config.commands);
|
|
21470
|
+
return aliases;
|
|
21471
|
+
}
|
|
21405
21472
|
function collectConfigScalarAliasOrder(config) {
|
|
21406
21473
|
const aliases = [];
|
|
21407
21474
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -21557,6 +21624,9 @@ function materializeEnrichAliasCellForCsv(row, alias, options) {
|
|
|
21557
21624
|
if (Object.prototype.hasOwnProperty.call(direct, "matched_result") || Object.prototype.hasOwnProperty.call(direct, "matchedResult") || Object.prototype.hasOwnProperty.call(direct, "result")) {
|
|
21558
21625
|
return materializeCsvCellValue(direct);
|
|
21559
21626
|
}
|
|
21627
|
+
if (options?.preservePlainObject && !Object.prototype.hasOwnProperty.call(direct, "_metadata")) {
|
|
21628
|
+
return materializeCsvCellValue(direct);
|
|
21629
|
+
}
|
|
21560
21630
|
const directCells = [];
|
|
21561
21631
|
for (const [field, value] of Object.entries(direct)) {
|
|
21562
21632
|
if (!isEnrichAliasPayloadField(field)) {
|
|
@@ -21613,6 +21683,7 @@ function materializeEnrichAliasCellForCsv(row, alias, options) {
|
|
|
21613
21683
|
function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
21614
21684
|
const aliases = config ? collectConfigScalarAliasOrder(config) : [];
|
|
21615
21685
|
const aiCompactAliases = config ? enrichAiRuntimeCompactAliases(config) : null;
|
|
21686
|
+
const playAliases = config ? collectConfigPlayAliases(config) : /* @__PURE__ */ new Set();
|
|
21616
21687
|
const failureOperationByAlias = hardFailureOperationByAlias(config);
|
|
21617
21688
|
return rows.map((row) => {
|
|
21618
21689
|
const metadata = legacyMetadataFromRow(row);
|
|
@@ -21656,7 +21727,8 @@ function normalizeEnrichRowsForCsvExport(rows, config, options) {
|
|
|
21656
21727
|
}
|
|
21657
21728
|
for (const alias of aliases) {
|
|
21658
21729
|
const value = materializeEnrichAliasCellForCsv(normalized, alias, {
|
|
21659
|
-
compactAiCell: aiCompactAliases?.has(normalizeAlias2(alias)) ?? false
|
|
21730
|
+
compactAiCell: aiCompactAliases?.has(normalizeAlias2(alias)) ?? false,
|
|
21731
|
+
preservePlainObject: playAliases.has(normalizeAlias2(alias))
|
|
21660
21732
|
});
|
|
21661
21733
|
if (value !== void 0) {
|
|
21662
21734
|
normalized[alias] = value;
|
|
@@ -21844,6 +21916,9 @@ function registerEnrichCommand(program) {
|
|
|
21844
21916
|
).option(
|
|
21845
21917
|
"--profile <id>",
|
|
21846
21918
|
"Internal/testing: override the execution profile for the generated play run."
|
|
21919
|
+
).option(
|
|
21920
|
+
"--debug-map-latency",
|
|
21921
|
+
"Internal diagnostics: emit one aggregate latency profile per dataset map."
|
|
21847
21922
|
).option(
|
|
21848
21923
|
"--fail-fast",
|
|
21849
21924
|
"Fail the generated play when any selected row errors, while preserving recovered runtime-sheet rows for export."
|
|
@@ -21989,6 +22064,9 @@ function registerEnrichCommand(program) {
|
|
|
21989
22064
|
if (options.profile) {
|
|
21990
22065
|
runArgs.push("--profile", options.profile);
|
|
21991
22066
|
}
|
|
22067
|
+
if (options.debugMapLatency) {
|
|
22068
|
+
runArgs.push("--debug-map-latency");
|
|
22069
|
+
}
|
|
21992
22070
|
if (options.noOpen || input2.suppressOpen) {
|
|
21993
22071
|
runArgs.push("--no-open");
|
|
21994
22072
|
}
|