deepline 0.1.160 → 0.1.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +263 -29
- package/dist/bundling-sources/sdk/src/config.ts +0 -49
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +8 -0
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/map-row-outcome.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +260 -16
- package/dist/cli/index.js +113 -62
- package/dist/cli/index.mjs +113 -62
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/package.json +1 -1
|
@@ -169,6 +169,7 @@ import {
|
|
|
169
169
|
// re-bundle harness internals into per-play. Keep that in mind.
|
|
170
170
|
import {
|
|
171
171
|
harnessPersistCompletedSheetRows,
|
|
172
|
+
harnessReadSheetDatasetRowKeys,
|
|
172
173
|
harnessReadSheetDatasetRows,
|
|
173
174
|
harnessReadStagedFileChunk,
|
|
174
175
|
harnessStartSheetDataset,
|
|
@@ -183,7 +184,6 @@ import {
|
|
|
183
184
|
import { collectStableResultDatasetPersistenceContract } from './runtime/result-dataset-persistence';
|
|
184
185
|
import {
|
|
185
186
|
runtimeCsvExecutionRow,
|
|
186
|
-
publicCsvInputRow,
|
|
187
187
|
publicCsvOutputRow,
|
|
188
188
|
publicCsvStorageRow,
|
|
189
189
|
runtimeCsvStorageRow,
|
|
@@ -191,6 +191,7 @@ import {
|
|
|
191
191
|
import {
|
|
192
192
|
completedMapRowOutcome,
|
|
193
193
|
failedMapRowOutcome,
|
|
194
|
+
MAP_ROW_OUTCOME_RUNTIME_FIELDS,
|
|
194
195
|
mapRowOutcomeRuntimeRow,
|
|
195
196
|
mapRowOutcomeRuntimeFields,
|
|
196
197
|
resolveMapRowOutcomeKey,
|
|
@@ -201,6 +202,7 @@ import { chooseWorkerMapRowsPerChunk } from './runtime/map-chunk-plan';
|
|
|
201
202
|
import {
|
|
202
203
|
applyCsvRenameProjection,
|
|
203
204
|
cloneCsvAliasedRow,
|
|
205
|
+
toSerializableCsvAliasedRow,
|
|
204
206
|
type CsvRenameOptions,
|
|
205
207
|
} from '../../../shared_libs/play-runtime/csv-rename';
|
|
206
208
|
import { coordinatorRequestHeaders } from '../../../shared_libs/play-runtime/coordinator-headers';
|
|
@@ -1515,6 +1517,7 @@ type RecordedStepProgramOutput = {
|
|
|
1515
1517
|
columnName: string;
|
|
1516
1518
|
stepId: string;
|
|
1517
1519
|
value: unknown;
|
|
1520
|
+
durationMs: number;
|
|
1518
1521
|
status?: 'skipped';
|
|
1519
1522
|
};
|
|
1520
1523
|
|
|
@@ -2536,6 +2539,8 @@ type WorkerMapChunkSummary<T extends Record<string, unknown>> = {
|
|
|
2536
2539
|
rowsFailed: number;
|
|
2537
2540
|
/** Bounded sample of row failures for the partial-failure summary. */
|
|
2538
2541
|
rowFailureSamples: Array<{ rowKey: string; error: string }>;
|
|
2542
|
+
stepCellsCompleted: number;
|
|
2543
|
+
stepCellsSkipped: number;
|
|
2539
2544
|
outputDatasetId: string;
|
|
2540
2545
|
hash: string;
|
|
2541
2546
|
preview: T[];
|
|
@@ -2763,7 +2768,9 @@ async function executeWorkerStepProgram(
|
|
|
2763
2768
|
) => Promise<WorkerStepResolution>
|
|
2764
2769
|
)(stepId, runStep)
|
|
2765
2770
|
: await runStep();
|
|
2771
|
+
const stepStartedAt = nowMs();
|
|
2766
2772
|
const resolution = await executeStep();
|
|
2773
|
+
const durationMs = nowMs() - stepStartedAt;
|
|
2767
2774
|
const value = deserializeDurableStepValue(resolution.value);
|
|
2768
2775
|
currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
|
|
2769
2776
|
if (recorder) {
|
|
@@ -2772,6 +2779,7 @@ async function executeWorkerStepProgram(
|
|
|
2772
2779
|
columnName: stepProgramColumnName(recorder.parentField, stepId),
|
|
2773
2780
|
stepId,
|
|
2774
2781
|
value,
|
|
2782
|
+
durationMs,
|
|
2775
2783
|
...(resolution.status ? { status: resolution.status } : {}),
|
|
2776
2784
|
};
|
|
2777
2785
|
recorder.outputs.push(output);
|
|
@@ -3581,32 +3589,115 @@ async function persistCompletedMapRows(input: {
|
|
|
3581
3589
|
rows: Record<string, unknown>[];
|
|
3582
3590
|
outputFields: string[];
|
|
3583
3591
|
extraOutputFields?: string[];
|
|
3584
|
-
}): Promise<
|
|
3585
|
-
|
|
3592
|
+
}): Promise<{
|
|
3593
|
+
rows: number;
|
|
3594
|
+
written: number;
|
|
3595
|
+
visible: number;
|
|
3596
|
+
retryWritten: number | null;
|
|
3597
|
+
retryVisible: number | null;
|
|
3598
|
+
}> {
|
|
3599
|
+
if (input.rows.length === 0) {
|
|
3600
|
+
return {
|
|
3601
|
+
rows: 0,
|
|
3602
|
+
written: 0,
|
|
3603
|
+
visible: 0,
|
|
3604
|
+
retryWritten: null,
|
|
3605
|
+
retryVisible: null,
|
|
3606
|
+
};
|
|
3607
|
+
}
|
|
3608
|
+
const { req, tableNamespace } = input;
|
|
3586
3609
|
const outputFields = [
|
|
3587
3610
|
...input.outputFields,
|
|
3588
3611
|
...(input.extraOutputFields ?? []).filter(
|
|
3589
3612
|
(field) => !input.outputFields.includes(field),
|
|
3590
3613
|
),
|
|
3591
3614
|
];
|
|
3592
|
-
const sessionScope = runtimeSheetSessionScope(
|
|
3615
|
+
const sessionScope = runtimeSheetSessionScope(req);
|
|
3593
3616
|
const rows = input.rows.map((row) => publicCsvStorageRow(row));
|
|
3594
|
-
const
|
|
3595
|
-
|
|
3596
|
-
tableNamespace: input.tableNamespace,
|
|
3597
|
-
sheetContract: augmentSheetContractWithDatasetFields({
|
|
3598
|
-
contract: requireSheetContract(input.req, input.tableNamespace),
|
|
3599
|
-
rows,
|
|
3600
|
-
outputFields,
|
|
3601
|
-
}),
|
|
3617
|
+
const sheetContract = augmentSheetContractWithDatasetFields({
|
|
3618
|
+
contract: requireSheetContract(req, tableNamespace),
|
|
3602
3619
|
rows,
|
|
3603
3620
|
outputFields,
|
|
3604
3621
|
});
|
|
3622
|
+
const persistRequest = {
|
|
3623
|
+
...sessionScope,
|
|
3624
|
+
tableNamespace,
|
|
3625
|
+
sheetContract,
|
|
3626
|
+
rows,
|
|
3627
|
+
outputFields,
|
|
3628
|
+
};
|
|
3629
|
+
const expectedKeys = [
|
|
3630
|
+
...new Set(
|
|
3631
|
+
input.rows
|
|
3632
|
+
.map((row) => resolveMapRowOutcomeKey(row))
|
|
3633
|
+
.filter((key): key is string => Boolean(key)),
|
|
3634
|
+
),
|
|
3635
|
+
];
|
|
3636
|
+
const readVisibleRowCount = async () => {
|
|
3637
|
+
if (expectedKeys.length > 0) {
|
|
3638
|
+
const result = await harnessReadSheetDatasetRowKeys({
|
|
3639
|
+
...sessionScope,
|
|
3640
|
+
tableNamespace,
|
|
3641
|
+
runId: req.runId,
|
|
3642
|
+
rowMode: 'all',
|
|
3643
|
+
keys: expectedKeys,
|
|
3644
|
+
});
|
|
3645
|
+
return result.keys.length;
|
|
3646
|
+
}
|
|
3647
|
+
const result = await harnessReadSheetDatasetRows({
|
|
3648
|
+
...sessionScope,
|
|
3649
|
+
tableNamespace,
|
|
3650
|
+
runId: req.runId,
|
|
3651
|
+
rowMode: 'all',
|
|
3652
|
+
limit: rows.length,
|
|
3653
|
+
offset: 0,
|
|
3654
|
+
});
|
|
3655
|
+
return result.rows.length;
|
|
3656
|
+
};
|
|
3657
|
+
const result = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3658
|
+
let visibleRows = -1;
|
|
3659
|
+
let retryWritten: number | null = null;
|
|
3660
|
+
let retryVisible: number | null = null;
|
|
3661
|
+
if (rows.length <= MAP_INCREMENTAL_PERSIST_CHUNK_ROWS) {
|
|
3662
|
+
visibleRows = await readVisibleRowCount();
|
|
3663
|
+
if (visibleRows < rows.length) {
|
|
3664
|
+
await harnessStartSheetDataset({
|
|
3665
|
+
...sessionScope,
|
|
3666
|
+
tableNamespace,
|
|
3667
|
+
sheetContract,
|
|
3668
|
+
rows: input.rows.map((row) => runtimeCsvStorageRow(row)),
|
|
3669
|
+
runId: req.runId,
|
|
3670
|
+
inputOffset: 0,
|
|
3671
|
+
});
|
|
3672
|
+
const retry = await harnessPersistCompletedSheetRows(persistRequest);
|
|
3673
|
+
retryWritten = retry.rowsWritten;
|
|
3674
|
+
retryVisible = await readVisibleRowCount();
|
|
3675
|
+
if (retryVisible >= rows.length) {
|
|
3676
|
+
return {
|
|
3677
|
+
rows: rows.length,
|
|
3678
|
+
written: result.rowsWritten,
|
|
3679
|
+
visible: visibleRows,
|
|
3680
|
+
retryWritten,
|
|
3681
|
+
retryVisible,
|
|
3682
|
+
};
|
|
3683
|
+
}
|
|
3684
|
+
throw new Error(
|
|
3685
|
+
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; visible ${visibleRows}; retry wrote ${retryWritten}/${rows.length}; retry visible ${retryVisible}; run ${req.runId}.`,
|
|
3686
|
+
);
|
|
3687
|
+
}
|
|
3688
|
+
}
|
|
3605
3689
|
if (result.rowsWritten !== rows.length) {
|
|
3606
3690
|
throw new Error(
|
|
3607
|
-
`Runtime sheet persistence mismatch for
|
|
3691
|
+
`Runtime sheet persistence mismatch for ${tableNamespace}: wrote ${result.rowsWritten}/${rows.length}; run ${req.runId}.`,
|
|
3608
3692
|
);
|
|
3609
3693
|
}
|
|
3694
|
+
return {
|
|
3695
|
+
rows: rows.length,
|
|
3696
|
+
written: result.rowsWritten,
|
|
3697
|
+
visible: visibleRows,
|
|
3698
|
+
retryWritten,
|
|
3699
|
+
retryVisible,
|
|
3700
|
+
};
|
|
3610
3701
|
}
|
|
3611
3702
|
|
|
3612
3703
|
async function applyLiveMapRowUpdates(input: {
|
|
@@ -4270,13 +4361,50 @@ function createMinimalWorkerCtx(
|
|
|
4270
4361
|
startedAt: mapStartedAt,
|
|
4271
4362
|
message: formatMapPreparingMessage(rowCountHint ?? undefined),
|
|
4272
4363
|
});
|
|
4364
|
+
const stableMapInputRow = (row: Record<string, unknown>) => {
|
|
4365
|
+
const stableRow = toSerializableCsvAliasedRow(row);
|
|
4366
|
+
for (const fieldName of outputFields) {
|
|
4367
|
+
delete stableRow[fieldName];
|
|
4368
|
+
}
|
|
4369
|
+
return stableRow;
|
|
4370
|
+
};
|
|
4371
|
+
const stableDefaultMapIdentityRow = (
|
|
4372
|
+
row: Record<string, unknown>,
|
|
4373
|
+
index: number,
|
|
4374
|
+
chunkIndex?: number,
|
|
4375
|
+
chunkLocalIndex?: number,
|
|
4376
|
+
) => ({
|
|
4377
|
+
__deeplineMapRunId: req.runId,
|
|
4378
|
+
...stableMapInputRow(row),
|
|
4379
|
+
__deeplineMapInputIndex: index,
|
|
4380
|
+
...(chunkIndex === undefined ? {} : { ci: chunkIndex }),
|
|
4381
|
+
...(chunkLocalIndex === undefined ? {} : { li: chunkLocalIndex }),
|
|
4382
|
+
});
|
|
4383
|
+
const resolveRuntimeInputIndex = (
|
|
4384
|
+
row: Record<string, unknown>,
|
|
4385
|
+
): number | null => {
|
|
4386
|
+
const value = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex];
|
|
4387
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
4388
|
+
? value
|
|
4389
|
+
: null;
|
|
4390
|
+
};
|
|
4391
|
+
const deriveDefaultMapRowIdentity = (
|
|
4392
|
+
row: Record<string, unknown>,
|
|
4393
|
+
index: number,
|
|
4394
|
+
chunkIndex?: number,
|
|
4395
|
+
chunkLocalIndex?: number,
|
|
4396
|
+
) =>
|
|
4397
|
+
derivePlayRowIdentity(
|
|
4398
|
+
stableDefaultMapIdentityRow(row, index, chunkIndex, chunkLocalIndex),
|
|
4399
|
+
name,
|
|
4400
|
+
);
|
|
4273
4401
|
const explicitRowKeysSeen =
|
|
4274
4402
|
opts?.key === undefined ? null : new Map<string, number>();
|
|
4275
4403
|
const resolveExplicitKeyValue = (
|
|
4276
4404
|
row: Record<string, unknown>,
|
|
4277
4405
|
index: number,
|
|
4278
4406
|
): string | null => {
|
|
4279
|
-
const inputRow =
|
|
4407
|
+
const inputRow = stableMapInputRow(row);
|
|
4280
4408
|
const keyOption = opts?.key;
|
|
4281
4409
|
if (keyOption === undefined) {
|
|
4282
4410
|
return null;
|
|
@@ -4315,11 +4443,17 @@ function createMinimalWorkerCtx(
|
|
|
4315
4443
|
const resolveRowKey = (
|
|
4316
4444
|
row: Record<string, unknown>,
|
|
4317
4445
|
index: number,
|
|
4446
|
+
chunkIndex?: number,
|
|
4447
|
+
chunkLocalIndex?: number,
|
|
4318
4448
|
): string => {
|
|
4319
|
-
const inputRow = publicCsvInputRow(row);
|
|
4320
4449
|
const explicitKeyValue = resolveExplicitKeyValue(row, index);
|
|
4321
4450
|
return explicitKeyValue == null
|
|
4322
|
-
?
|
|
4451
|
+
? deriveDefaultMapRowIdentity(
|
|
4452
|
+
row,
|
|
4453
|
+
index,
|
|
4454
|
+
chunkIndex,
|
|
4455
|
+
chunkLocalIndex,
|
|
4456
|
+
)
|
|
4323
4457
|
: derivePlayRowIdentityFromKey(explicitKeyValue, name);
|
|
4324
4458
|
};
|
|
4325
4459
|
// Cross-chunk dedupe accumulators for the single end-of-map log line.
|
|
@@ -4387,7 +4521,12 @@ function createMinimalWorkerCtx(
|
|
|
4387
4521
|
const keyStartedAt = nowMs();
|
|
4388
4522
|
const chunkEntries = chunkRows.map((row, localIndex) => {
|
|
4389
4523
|
const absoluteIndex = baseOffset + chunkStart + localIndex;
|
|
4390
|
-
const rowKey = resolveRowKey(
|
|
4524
|
+
const rowKey = resolveRowKey(
|
|
4525
|
+
row,
|
|
4526
|
+
absoluteIndex,
|
|
4527
|
+
chunkIndex,
|
|
4528
|
+
localIndex,
|
|
4529
|
+
);
|
|
4391
4530
|
return { row, absoluteIndex, rowKey };
|
|
4392
4531
|
});
|
|
4393
4532
|
recordRunnerPerfTrace({
|
|
@@ -4443,7 +4582,7 @@ function createMinimalWorkerCtx(
|
|
|
4443
4582
|
for (const row of prepared.pendingRows) {
|
|
4444
4583
|
const key =
|
|
4445
4584
|
resolveMapRowOutcomeKey(row) ??
|
|
4446
|
-
|
|
4585
|
+
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
4447
4586
|
if (key) {
|
|
4448
4587
|
pendingKeys.add(key);
|
|
4449
4588
|
pendingRowsByKey.set(key, row);
|
|
@@ -4453,7 +4592,7 @@ function createMinimalWorkerCtx(
|
|
|
4453
4592
|
for (const row of prepared.completedRows) {
|
|
4454
4593
|
const key =
|
|
4455
4594
|
resolveMapRowOutcomeKey(row) ??
|
|
4456
|
-
|
|
4595
|
+
deriveDefaultMapRowIdentity(row, resolveRuntimeInputIndex(row) ?? 0);
|
|
4457
4596
|
if (key) {
|
|
4458
4597
|
completedKeys.add(key);
|
|
4459
4598
|
preparedKeys.add(key);
|
|
@@ -4592,6 +4731,8 @@ function createMinimalWorkerCtx(
|
|
|
4592
4731
|
receiptStore,
|
|
4593
4732
|
false,
|
|
4594
4733
|
);
|
|
4734
|
+
let stepCellsCompleted = 0;
|
|
4735
|
+
let stepCellsSkipped = 0;
|
|
4595
4736
|
const generatedOutputFields = new Set<string>();
|
|
4596
4737
|
const pendingLiveRowUpdates: Array<
|
|
4597
4738
|
Omit<PlayRowUpdate, 'rowId'> & { runId?: string }
|
|
@@ -4663,7 +4804,7 @@ function createMinimalWorkerCtx(
|
|
|
4663
4804
|
if (rowsToPersist.length === 0 && failedRowsToPersist.length === 0) {
|
|
4664
4805
|
return;
|
|
4665
4806
|
}
|
|
4666
|
-
await persistCompletedMapRows({
|
|
4807
|
+
const persistStats = await persistCompletedMapRows({
|
|
4667
4808
|
req,
|
|
4668
4809
|
tableNamespace: name,
|
|
4669
4810
|
outputFields,
|
|
@@ -4673,6 +4814,8 @@ function createMinimalWorkerCtx(
|
|
|
4673
4814
|
mapRowOutcomeRuntimeRow(
|
|
4674
4815
|
completedMapRowOutcome({
|
|
4675
4816
|
key: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
|
|
4817
|
+
inputIndex:
|
|
4818
|
+
uniqueRowsToExecuteEntries[executedIndex]!.absoluteIndex,
|
|
4676
4819
|
data: row,
|
|
4677
4820
|
cellMetaPatch: executedCellMetaPatches[executedIndex],
|
|
4678
4821
|
}),
|
|
@@ -4684,6 +4827,8 @@ function createMinimalWorkerCtx(
|
|
|
4684
4827
|
mapRowOutcomeRuntimeRow(
|
|
4685
4828
|
failedMapRowOutcome({
|
|
4686
4829
|
key: uniqueRowsToExecuteEntries[executedIndex]!.rowKey,
|
|
4830
|
+
inputIndex:
|
|
4831
|
+
uniqueRowsToExecuteEntries[executedIndex]!.absoluteIndex,
|
|
4687
4832
|
data: failure.row,
|
|
4688
4833
|
cellMetaPatch: executedCellMetaPatches[executedIndex],
|
|
4689
4834
|
error: failure.error,
|
|
@@ -4692,6 +4837,23 @@ function createMinimalWorkerCtx(
|
|
|
4692
4837
|
),
|
|
4693
4838
|
],
|
|
4694
4839
|
});
|
|
4840
|
+
if (persistStats.rows > 0) {
|
|
4841
|
+
emitEvent({
|
|
4842
|
+
type: 'log',
|
|
4843
|
+
level: 'info',
|
|
4844
|
+
message:
|
|
4845
|
+
`Runtime sheet persisted ${persistStats.rows} rows for ctx.dataset("${name}") ` +
|
|
4846
|
+
`(written=${persistStats.written}, visible=${persistStats.visible}` +
|
|
4847
|
+
(persistStats.retryWritten === null
|
|
4848
|
+
? ''
|
|
4849
|
+
: `, retryWritten=${persistStats.retryWritten}`) +
|
|
4850
|
+
(persistStats.retryVisible === null
|
|
4851
|
+
? ''
|
|
4852
|
+
: `, retryVisible=${persistStats.retryVisible}`) +
|
|
4853
|
+
')',
|
|
4854
|
+
ts: nowMs(),
|
|
4855
|
+
});
|
|
4856
|
+
}
|
|
4695
4857
|
for (const { executedIndex } of rowsToPersist) {
|
|
4696
4858
|
persistedExecutedIndexes.add(executedIndex);
|
|
4697
4859
|
}
|
|
@@ -4881,6 +5043,8 @@ function createMinimalWorkerCtx(
|
|
|
4881
5043
|
onOutput: async (stepOutput) => {
|
|
4882
5044
|
generatedOutputFields.add(stepOutput.columnName);
|
|
4883
5045
|
const status = stepOutput.status ?? 'completed';
|
|
5046
|
+
if (status === 'skipped') stepCellsSkipped += 1;
|
|
5047
|
+
else stepCellsCompleted += 1;
|
|
4884
5048
|
await enqueueLiveRowUpdate({
|
|
4885
5049
|
key: entry.rowKey,
|
|
4886
5050
|
tableNamespace: name,
|
|
@@ -5096,7 +5260,10 @@ function createMinimalWorkerCtx(
|
|
|
5096
5260
|
for (const completedRow of prepared.completedRows) {
|
|
5097
5261
|
const key =
|
|
5098
5262
|
resolveMapRowOutcomeKey(completedRow) ??
|
|
5099
|
-
|
|
5263
|
+
deriveDefaultMapRowIdentity(
|
|
5264
|
+
completedRow,
|
|
5265
|
+
resolveRuntimeInputIndex(completedRow) ?? 0,
|
|
5266
|
+
);
|
|
5100
5267
|
if (key) {
|
|
5101
5268
|
const cleanedRow = stripMapRowOutcomeRuntimeFields(
|
|
5102
5269
|
publicCsvOutputRow(completedRow),
|
|
@@ -5115,12 +5282,28 @@ function createMinimalWorkerCtx(
|
|
|
5115
5282
|
// dataset (their recoverable state lives in the runtime sheet).
|
|
5116
5283
|
if (key && executedRow) resultByKey.set(key, executedRow);
|
|
5117
5284
|
}
|
|
5118
|
-
const
|
|
5285
|
+
const outEntries = chunkRows
|
|
5119
5286
|
.map((_row, index) => {
|
|
5120
|
-
const
|
|
5121
|
-
|
|
5287
|
+
const entry = chunkEntries[index]!;
|
|
5288
|
+
const row = resultByKey.get(entry.rowKey);
|
|
5289
|
+
return row
|
|
5290
|
+
? {
|
|
5291
|
+
key: entry.rowKey,
|
|
5292
|
+
inputIndex: entry.absoluteIndex,
|
|
5293
|
+
row,
|
|
5294
|
+
}
|
|
5295
|
+
: null;
|
|
5122
5296
|
})
|
|
5123
|
-
.filter(
|
|
5297
|
+
.filter(
|
|
5298
|
+
(
|
|
5299
|
+
entry,
|
|
5300
|
+
): entry is {
|
|
5301
|
+
key: string;
|
|
5302
|
+
inputIndex: number;
|
|
5303
|
+
row: T & Record<string, unknown>;
|
|
5304
|
+
} => entry !== null,
|
|
5305
|
+
);
|
|
5306
|
+
const out = outEntries.map((entry) => entry.row);
|
|
5124
5307
|
const executedSuccessCount = Math.max(
|
|
5125
5308
|
0,
|
|
5126
5309
|
executedRows.length - failedExecutedRows,
|
|
@@ -5140,6 +5323,10 @@ function createMinimalWorkerCtx(
|
|
|
5140
5323
|
)
|
|
5141
5324
|
.slice(0, MAP_ROW_FAILURE_SAMPLE_LIMIT);
|
|
5142
5325
|
const publicOut = out.map((row) => publicCsvOutputRow(row));
|
|
5326
|
+
const keyedOut = outEntries.map(({ key, inputIndex, row }) => ({
|
|
5327
|
+
...row,
|
|
5328
|
+
...mapRowOutcomeRuntimeFields({ key, inputIndex }),
|
|
5329
|
+
}));
|
|
5143
5330
|
const hashStartedAt = nowMs();
|
|
5144
5331
|
const hash = await hashJson(publicOut);
|
|
5145
5332
|
const includeCachedRowsInChunkResult = !workflowStep;
|
|
@@ -5150,7 +5337,7 @@ function createMinimalWorkerCtx(
|
|
|
5150
5337
|
) {
|
|
5151
5338
|
volatileWorkflowChunkRows.set(
|
|
5152
5339
|
chunkIndex,
|
|
5153
|
-
serializeDurableStepValue(
|
|
5340
|
+
serializeDurableStepValue(keyedOut),
|
|
5154
5341
|
);
|
|
5155
5342
|
}
|
|
5156
5343
|
recordRunnerPerfTrace({
|
|
@@ -5186,6 +5373,8 @@ function createMinimalWorkerCtx(
|
|
|
5186
5373
|
rowsSkipped,
|
|
5187
5374
|
rowsFailed: failedExecutedRows,
|
|
5188
5375
|
rowFailureSamples,
|
|
5376
|
+
stepCellsCompleted,
|
|
5377
|
+
stepCellsSkipped,
|
|
5189
5378
|
outputDatasetId: `map:${name}`,
|
|
5190
5379
|
hash,
|
|
5191
5380
|
// Runtime Sheet owns the full row payloads. Native Workflow step
|
|
@@ -5196,7 +5385,7 @@ function createMinimalWorkerCtx(
|
|
|
5196
5385
|
cachedRows:
|
|
5197
5386
|
includeCachedRowsInChunkResult &&
|
|
5198
5387
|
out.length <= WORKER_DATASET_IN_MEMORY_ROWS
|
|
5199
|
-
? serializeDurableStepValue(
|
|
5388
|
+
? serializeDurableStepValue(keyedOut)
|
|
5200
5389
|
: undefined,
|
|
5201
5390
|
};
|
|
5202
5391
|
};
|
|
@@ -5210,6 +5399,8 @@ function createMinimalWorkerCtx(
|
|
|
5210
5399
|
let totalRowsInserted = 0;
|
|
5211
5400
|
let totalRowsSkipped = 0;
|
|
5212
5401
|
let totalRowsFailed = 0;
|
|
5402
|
+
let totalStepCellsCompleted = 0;
|
|
5403
|
+
let totalStepCellsSkipped = 0;
|
|
5213
5404
|
const totalRowFailureSamples: Array<{ rowKey: string; error: string }> = [];
|
|
5214
5405
|
|
|
5215
5406
|
const runChunkStep = async (
|
|
@@ -5256,7 +5447,7 @@ function createMinimalWorkerCtx(
|
|
|
5256
5447
|
return result.rows as Array<T & Record<string, unknown>>;
|
|
5257
5448
|
};
|
|
5258
5449
|
|
|
5259
|
-
const finalize = (totalRowsWritten: number) => {
|
|
5450
|
+
const finalize = async (totalRowsWritten: number) => {
|
|
5260
5451
|
const failureSampleSummary =
|
|
5261
5452
|
totalRowFailureSamples.length > 0
|
|
5262
5453
|
? ` First error: ${totalRowFailureSamples[0]!.error}`
|
|
@@ -5271,6 +5462,11 @@ function createMinimalWorkerCtx(
|
|
|
5271
5462
|
`(${totalRowsExecuted} executed, ${totalRowsCached} already satisfied) ` +
|
|
5272
5463
|
`inserted=${totalRowsInserted} skipped=${totalRowsSkipped}`;
|
|
5273
5464
|
const completedAt = nowMs();
|
|
5465
|
+
const totalStepCells = totalStepCellsCompleted + totalStepCellsSkipped;
|
|
5466
|
+
const mapDiag =
|
|
5467
|
+
totalStepCells > 0
|
|
5468
|
+
? ` cells=${totalStepCells}/${totalStepCellsCompleted}/${totalStepCellsSkipped}`
|
|
5469
|
+
: '';
|
|
5274
5470
|
callbacks?.onMapCompleted?.(mapNodeId, completedAt);
|
|
5275
5471
|
void updateMapProgress({
|
|
5276
5472
|
completed: totalRowsWritten,
|
|
@@ -5286,9 +5482,45 @@ function createMinimalWorkerCtx(
|
|
|
5286
5482
|
emitEvent({
|
|
5287
5483
|
type: 'log',
|
|
5288
5484
|
level: totalRowsFailed > 0 ? 'warn' : 'info',
|
|
5289
|
-
message: cacheSummary,
|
|
5485
|
+
message: cacheSummary + mapDiag,
|
|
5290
5486
|
ts: nowMs(),
|
|
5291
5487
|
});
|
|
5488
|
+
if (
|
|
5489
|
+
totalRowsWritten > 0 &&
|
|
5490
|
+
totalRowsFailed === 0 &&
|
|
5491
|
+
canCacheRows &&
|
|
5492
|
+
cachedRows.length === totalRowsWritten
|
|
5493
|
+
) {
|
|
5494
|
+
const persistedRows = await readPersistedRows({
|
|
5495
|
+
limit: totalRowsWritten,
|
|
5496
|
+
offset: 0,
|
|
5497
|
+
});
|
|
5498
|
+
if (persistedRows.length < totalRowsWritten) {
|
|
5499
|
+
const repair = await persistCompletedMapRows({
|
|
5500
|
+
req,
|
|
5501
|
+
tableNamespace: name,
|
|
5502
|
+
outputFields,
|
|
5503
|
+
extraOutputFields: [],
|
|
5504
|
+
rows: cachedRows,
|
|
5505
|
+
});
|
|
5506
|
+
emitEvent({
|
|
5507
|
+
type: 'log',
|
|
5508
|
+
level: 'warn',
|
|
5509
|
+
message:
|
|
5510
|
+
`Runtime sheet finalization repaired ctx.dataset("${name}") ` +
|
|
5511
|
+
`from ${persistedRows.length}/${totalRowsWritten} visible row(s) ` +
|
|
5512
|
+
`(written=${repair.written}, visible=${repair.visible}` +
|
|
5513
|
+
(repair.retryWritten === null
|
|
5514
|
+
? ''
|
|
5515
|
+
: `, retryWritten=${repair.retryWritten}`) +
|
|
5516
|
+
(repair.retryVisible === null
|
|
5517
|
+
? ''
|
|
5518
|
+
: `, retryVisible=${repair.retryVisible}`) +
|
|
5519
|
+
')',
|
|
5520
|
+
ts: nowMs(),
|
|
5521
|
+
});
|
|
5522
|
+
}
|
|
5523
|
+
}
|
|
5292
5524
|
return createPersistedDatasetHandle({
|
|
5293
5525
|
playName: req.playName,
|
|
5294
5526
|
name,
|
|
@@ -5345,6 +5577,8 @@ function createMinimalWorkerCtx(
|
|
|
5345
5577
|
totalRowsInserted += chunkResult.rowsInserted;
|
|
5346
5578
|
totalRowsSkipped += chunkResult.rowsSkipped;
|
|
5347
5579
|
totalRowsFailed += chunkResult.rowsFailed ?? 0;
|
|
5580
|
+
totalStepCellsCompleted += chunkResult.stepCellsCompleted ?? 0;
|
|
5581
|
+
totalStepCellsSkipped += chunkResult.stepCellsSkipped ?? 0;
|
|
5348
5582
|
for (const sample of chunkResult.rowFailureSamples ?? []) {
|
|
5349
5583
|
if (totalRowFailureSamples.length >= MAP_ROW_FAILURE_SAMPLE_LIMIT) {
|
|
5350
5584
|
break;
|
|
@@ -58,8 +58,6 @@ const DEFAULT_TIMEOUT = 60_000;
|
|
|
58
58
|
const DEFAULT_MAX_RETRIES = 3;
|
|
59
59
|
|
|
60
60
|
const PROJECT_DEEPLINE_ENV_FILE = '.env.deepline';
|
|
61
|
-
const WORKSPACE_RESTORE_ENV_DIR = '.deepline';
|
|
62
|
-
const WORKSPACE_RESTORE_ENV_FILE = '.env';
|
|
63
61
|
const COWORK_IGNORED_WORKSPACE_DIRS = new Set([
|
|
64
62
|
'.auto-memory',
|
|
65
63
|
'.claude',
|
|
@@ -507,27 +505,6 @@ function ensureProjectEnvIsIgnored(dir: string): void {
|
|
|
507
505
|
writeFileSync(gitignorePath, `${existing}${prefix}${entry}\n`, 'utf-8');
|
|
508
506
|
}
|
|
509
507
|
|
|
510
|
-
function ensureWorkspaceRestoreEnvIsIgnored(workspaceDir: string): void {
|
|
511
|
-
const gitignorePath = join(workspaceDir, '.gitignore');
|
|
512
|
-
const entry = `${WORKSPACE_RESTORE_ENV_DIR}/`;
|
|
513
|
-
const existing = existsSync(gitignorePath)
|
|
514
|
-
? readFileSync(gitignorePath, 'utf-8')
|
|
515
|
-
: '';
|
|
516
|
-
const alreadyIgnored = existing
|
|
517
|
-
.split(/\r?\n/)
|
|
518
|
-
.map((line) => line.trim())
|
|
519
|
-
.some(
|
|
520
|
-
(line) =>
|
|
521
|
-
line === entry ||
|
|
522
|
-
line === `/${entry}` ||
|
|
523
|
-
line === WORKSPACE_RESTORE_ENV_DIR ||
|
|
524
|
-
line === `/${WORKSPACE_RESTORE_ENV_DIR}`,
|
|
525
|
-
);
|
|
526
|
-
if (alreadyIgnored) return;
|
|
527
|
-
const prefix = existing && !existing.endsWith('\n') ? '\n' : '';
|
|
528
|
-
writeFileSync(gitignorePath, `${existing}${prefix}${entry}\n`, 'utf-8');
|
|
529
|
-
}
|
|
530
|
-
|
|
531
508
|
export function saveProjectDeeplineEnvValues(
|
|
532
509
|
values: EnvValues,
|
|
533
510
|
startDir: string = process.cwd(),
|
|
@@ -545,32 +522,6 @@ export function saveProjectDeeplineEnvValues(
|
|
|
545
522
|
return [filePath];
|
|
546
523
|
}
|
|
547
524
|
|
|
548
|
-
export function saveCoworkWorkspaceRestoreEnvValues(
|
|
549
|
-
values: EnvValues,
|
|
550
|
-
startDir: string = process.cwd(),
|
|
551
|
-
): string[] {
|
|
552
|
-
if (!isCoworkLikeSandbox()) return [];
|
|
553
|
-
const target = resolveProjectPinTarget(startDir);
|
|
554
|
-
if (!target.ok || target.source === 'cwd') return [];
|
|
555
|
-
const workspaceDir = target.dir;
|
|
556
|
-
const filePath = join(
|
|
557
|
-
workspaceDir,
|
|
558
|
-
WORKSPACE_RESTORE_ENV_DIR,
|
|
559
|
-
WORKSPACE_RESTORE_ENV_FILE,
|
|
560
|
-
);
|
|
561
|
-
const existing = parseEnvFile(filePath);
|
|
562
|
-
const merged = { ...existing, ...values };
|
|
563
|
-
const dir = dirname(filePath);
|
|
564
|
-
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
565
|
-
ensureWorkspaceRestoreEnvIsIgnored(workspaceDir);
|
|
566
|
-
const allowedKeys = new Set([HOST_URL_ENV, API_KEY_ENV]);
|
|
567
|
-
const lines = Object.entries(merged)
|
|
568
|
-
.filter(([key, value]) => allowedKeys.has(key) && value !== '')
|
|
569
|
-
.map(([key, value]) => `${key}=${value}`);
|
|
570
|
-
writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf-8');
|
|
571
|
-
return [filePath];
|
|
572
|
-
}
|
|
573
|
-
|
|
574
525
|
export function resolveProjectPinDir(startDir: string = process.cwd()): string {
|
|
575
526
|
const target = resolveProjectPinTarget(startDir);
|
|
576
527
|
if (!target.ok) {
|
|
@@ -40,6 +40,8 @@ import type {
|
|
|
40
40
|
RuntimeApiCallResult,
|
|
41
41
|
RuntimeReceiptInput,
|
|
42
42
|
RuntimeReceiptsInput,
|
|
43
|
+
SheetDatasetRowKeysInput,
|
|
44
|
+
SheetDatasetRowKeysResult,
|
|
43
45
|
SheetDatasetRowsInput,
|
|
44
46
|
SheetDatasetRowsResult,
|
|
45
47
|
StagedFileChunkInput,
|
|
@@ -189,6 +191,12 @@ export async function harnessReadSheetDatasetRows(
|
|
|
189
191
|
return requireBinding().readSheetDatasetRows(input);
|
|
190
192
|
}
|
|
191
193
|
|
|
194
|
+
export async function harnessReadSheetDatasetRowKeys(
|
|
195
|
+
input: SheetDatasetRowKeysInput,
|
|
196
|
+
): Promise<SheetDatasetRowKeysResult> {
|
|
197
|
+
return requireBinding().readSheetDatasetRowKeys(input);
|
|
198
|
+
}
|
|
199
|
+
|
|
192
200
|
/**
|
|
193
201
|
* Warm Postgres sessions in the long-lived harness Worker. This preserves the
|
|
194
202
|
* map fast path without bundling the Neon client into every dynamic play.
|
|
@@ -104,10 +104,10 @@ export const SDK_RELEASE = {
|
|
|
104
104
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
|
-
version: '0.1.
|
|
107
|
+
version: '0.1.162',
|
|
108
108
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
109
109
|
supportPolicy: {
|
|
110
|
-
latest: '0.1.
|
|
110
|
+
latest: '0.1.162',
|
|
111
111
|
minimumSupported: '0.1.53',
|
|
112
112
|
deprecatedBelow: '0.1.53',
|
|
113
113
|
commandMinimumSupported: [
|
|
@@ -2,6 +2,7 @@ import type { MapRowOutcome, MapRowOutcomeStatus } from './durability-store';
|
|
|
2
2
|
|
|
3
3
|
export const MAP_ROW_OUTCOME_RUNTIME_FIELDS = {
|
|
4
4
|
rowKey: '__deeplineRowKey',
|
|
5
|
+
inputIndex: '__deeplineInputIndex',
|
|
5
6
|
cellMetaPatch: '__deeplineCellMetaPatch',
|
|
6
7
|
rowStatus: '__deeplineRowStatus',
|
|
7
8
|
rowError: '__deeplineRowError',
|
|
@@ -26,12 +27,16 @@ export function resolveMapRowOutcomeKey(
|
|
|
26
27
|
|
|
27
28
|
export function mapRowOutcomeRuntimeFields(input: {
|
|
28
29
|
key: string;
|
|
30
|
+
inputIndex?: number | null;
|
|
29
31
|
cellMetaPatch?: Record<string, unknown> | null;
|
|
30
32
|
status?: MapRowOutcomeStatus | null;
|
|
31
33
|
error?: string | null;
|
|
32
34
|
}): Record<string, unknown> {
|
|
33
35
|
return {
|
|
34
36
|
[MAP_ROW_OUTCOME_RUNTIME_FIELDS.rowKey]: input.key,
|
|
37
|
+
...(typeof input.inputIndex === 'number' && Number.isFinite(input.inputIndex)
|
|
38
|
+
? { [MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex]: input.inputIndex }
|
|
39
|
+
: {}),
|
|
35
40
|
...(input.cellMetaPatch
|
|
36
41
|
? { [MAP_ROW_OUTCOME_RUNTIME_FIELDS.cellMetaPatch]: input.cellMetaPatch }
|
|
37
42
|
: {}),
|
|
@@ -79,6 +84,7 @@ export function mapRowOutcomeRuntimeRow(
|
|
|
79
84
|
...outcome.data,
|
|
80
85
|
...mapRowOutcomeRuntimeFields({
|
|
81
86
|
key: outcome.key,
|
|
87
|
+
inputIndex: outcome.inputIndex,
|
|
82
88
|
cellMetaPatch: outcome.cellMetaPatch,
|
|
83
89
|
status: outcome.status,
|
|
84
90
|
error: outcome.error,
|
|
@@ -120,9 +126,13 @@ export function mapRowOutcomeFromRuntimeRow(
|
|
|
120
126
|
row: MapRowOutcomeRuntimeRow,
|
|
121
127
|
): MapRowOutcome | null {
|
|
122
128
|
if (typeof row.key === 'string' && isRecord(row.data)) {
|
|
129
|
+
const inputIndex = row.inputIndex;
|
|
123
130
|
return {
|
|
124
131
|
key: row.key,
|
|
125
132
|
data: row.data,
|
|
133
|
+
...(typeof inputIndex === 'number' && Number.isFinite(inputIndex)
|
|
134
|
+
? { inputIndex }
|
|
135
|
+
: {}),
|
|
126
136
|
...(isRecord(row.cellMetaPatch)
|
|
127
137
|
? { cellMetaPatch: row.cellMetaPatch }
|
|
128
138
|
: {}),
|
|
@@ -137,11 +147,15 @@ export function mapRowOutcomeFromRuntimeRow(
|
|
|
137
147
|
|
|
138
148
|
const key = resolveMapRowOutcomeKey(row);
|
|
139
149
|
if (!key) return null;
|
|
150
|
+
const inputIndex = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.inputIndex];
|
|
140
151
|
const cellMetaPatch = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.cellMetaPatch];
|
|
141
152
|
const rowStatus = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.rowStatus];
|
|
142
153
|
const rowError = row[MAP_ROW_OUTCOME_RUNTIME_FIELDS.rowError];
|
|
143
154
|
return {
|
|
144
155
|
key,
|
|
156
|
+
...(typeof inputIndex === 'number' && Number.isFinite(inputIndex)
|
|
157
|
+
? { inputIndex }
|
|
158
|
+
: {}),
|
|
145
159
|
data: stripMapRowOutcomeRuntimeFields(row),
|
|
146
160
|
...(isRecord(cellMetaPatch) ? { cellMetaPatch } : {}),
|
|
147
161
|
...(rowStatus === 'failed'
|