deepline 0.1.189 → 0.1.191
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1393,6 +1393,13 @@ function isRetryableRuntimeApiError(error: unknown): boolean {
|
|
|
1393
1393
|
);
|
|
1394
1394
|
}
|
|
1395
1395
|
|
|
1396
|
+
function isRetryableRuntimePersistenceError(error: unknown): boolean {
|
|
1397
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1398
|
+
return /deadlock detected|Runtime Postgres connection timed out|could not serialize access|tuple concurrently updated/i.test(
|
|
1399
|
+
message,
|
|
1400
|
+
);
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1396
1403
|
function isRetryableRuntimeApiResponse(status: number, body: string): boolean {
|
|
1397
1404
|
if (
|
|
1398
1405
|
status === 408 ||
|
|
@@ -4495,6 +4502,10 @@ function createMinimalWorkerCtx(
|
|
|
4495
4502
|
executionPlan: plan,
|
|
4496
4503
|
staticPipeline: staticPipelineFromReq(req),
|
|
4497
4504
|
});
|
|
4505
|
+
const mapHasExternalSideEffects =
|
|
4506
|
+
mapDispatchPlan.workEstimate.providerToolCallsPerRow > 0 ||
|
|
4507
|
+
mapDispatchPlan.workEstimate.childPlaySubmitsPerRow > 0 ||
|
|
4508
|
+
mapDispatchPlan.workEstimate.eventWaitsPerRow > 0;
|
|
4498
4509
|
const rowsPerChunk = mapDispatchPlan.rowsPerChunk;
|
|
4499
4510
|
recordRunnerPerfTrace({
|
|
4500
4511
|
req,
|
|
@@ -5765,6 +5776,13 @@ function createMinimalWorkerCtx(
|
|
|
5765
5776
|
},
|
|
5766
5777
|
});
|
|
5767
5778
|
} catch (error) {
|
|
5779
|
+
if (
|
|
5780
|
+
workflowStep &&
|
|
5781
|
+
!mapHasExternalSideEffects &&
|
|
5782
|
+
isRetryableRuntimePersistenceError(error)
|
|
5783
|
+
) {
|
|
5784
|
+
throw error;
|
|
5785
|
+
}
|
|
5768
5786
|
tripRuntimePersistenceLatch(persistenceLatch, error);
|
|
5769
5787
|
recordRunnerPerfTrace({
|
|
5770
5788
|
req,
|
|
@@ -105,10 +105,10 @@ export const SDK_RELEASE = {
|
|
|
105
105
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
106
106
|
// fields shipped in 0.1.153.
|
|
107
107
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
108
|
-
version: '0.1.
|
|
108
|
+
version: '0.1.191',
|
|
109
109
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
110
110
|
supportPolicy: {
|
|
111
|
-
latest: '0.1.
|
|
111
|
+
latest: '0.1.191',
|
|
112
112
|
minimumSupported: '0.1.53',
|
|
113
113
|
deprecatedBelow: '0.1.53',
|
|
114
114
|
commandMinimumSupported: [
|
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.191",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.191",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -4829,14 +4829,64 @@ function readCsvRows(csvPath) {
|
|
|
4829
4829
|
});
|
|
4830
4830
|
}
|
|
4831
4831
|
function csvStringFromRows(rows, columns) {
|
|
4832
|
-
return (0, import_sync2.stringify)(rows, {
|
|
4832
|
+
return (0, import_sync2.stringify)(rows.map(csvSafeRow), {
|
|
4833
4833
|
header: true,
|
|
4834
4834
|
cast: {
|
|
4835
|
-
boolean: (value) => value ? "true" : "false"
|
|
4835
|
+
boolean: (value) => value ? "true" : "false",
|
|
4836
|
+
object: (value) => {
|
|
4837
|
+
const cell = csvSafeCell(value);
|
|
4838
|
+
return typeof cell === "string" ? cell : null;
|
|
4839
|
+
}
|
|
4836
4840
|
},
|
|
4837
4841
|
...columns?.length ? { columns } : {}
|
|
4838
4842
|
});
|
|
4839
4843
|
}
|
|
4844
|
+
function csvSafeRow(row) {
|
|
4845
|
+
return Object.fromEntries(
|
|
4846
|
+
Object.entries(row).map(([key, value]) => [key, csvSafeCell(value)])
|
|
4847
|
+
);
|
|
4848
|
+
}
|
|
4849
|
+
function csvSafeCell(value) {
|
|
4850
|
+
if (value === void 0) {
|
|
4851
|
+
return null;
|
|
4852
|
+
}
|
|
4853
|
+
if (value === null) {
|
|
4854
|
+
return null;
|
|
4855
|
+
}
|
|
4856
|
+
if (typeof value === "bigint") {
|
|
4857
|
+
return value.toString();
|
|
4858
|
+
}
|
|
4859
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
4860
|
+
return value;
|
|
4861
|
+
}
|
|
4862
|
+
if (value instanceof Date) {
|
|
4863
|
+
return value.toISOString();
|
|
4864
|
+
}
|
|
4865
|
+
if (typeof value === "symbol" || typeof value === "function") {
|
|
4866
|
+
return null;
|
|
4867
|
+
}
|
|
4868
|
+
if (typeof value !== "object") {
|
|
4869
|
+
return String(value);
|
|
4870
|
+
}
|
|
4871
|
+
return csvSafeJsonString(value);
|
|
4872
|
+
}
|
|
4873
|
+
function csvSafeJsonReplacer(_key, nested) {
|
|
4874
|
+
if (typeof nested === "bigint") {
|
|
4875
|
+
return nested.toString();
|
|
4876
|
+
}
|
|
4877
|
+
if (nested === void 0 || typeof nested === "symbol" || typeof nested === "function") {
|
|
4878
|
+
return null;
|
|
4879
|
+
}
|
|
4880
|
+
return nested;
|
|
4881
|
+
}
|
|
4882
|
+
function csvSafeJsonString(value) {
|
|
4883
|
+
try {
|
|
4884
|
+
const serialized = JSON.stringify(value, csvSafeJsonReplacer);
|
|
4885
|
+
return serialized ?? null;
|
|
4886
|
+
} catch {
|
|
4887
|
+
return String(value);
|
|
4888
|
+
}
|
|
4889
|
+
}
|
|
4840
4890
|
function parseMaybeJsonObject(value) {
|
|
4841
4891
|
if (typeof value !== "string") {
|
|
4842
4892
|
return value;
|
|
@@ -4877,14 +4927,14 @@ function flattenObjectColumns(row, options = {}) {
|
|
|
4877
4927
|
const value = parseMaybeJsonObject(rawValue);
|
|
4878
4928
|
const parsedFromString = typeof rawValue === "string" && value !== rawValue;
|
|
4879
4929
|
if (key === "_metadata") {
|
|
4880
|
-
flattened[key] = value && typeof value === "object" ?
|
|
4930
|
+
flattened[key] = value && typeof value === "object" ? csvSafeJsonString(value) : value;
|
|
4881
4931
|
continue;
|
|
4882
4932
|
}
|
|
4883
4933
|
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
4884
4934
|
const record = value;
|
|
4885
4935
|
const hasMatchedEnvelope = Object.prototype.hasOwnProperty.call(record, "matched_result") || Object.prototype.hasOwnProperty.call(record, "matchedResult");
|
|
4886
4936
|
if (hasMatchedEnvelope) {
|
|
4887
|
-
flattened[key] =
|
|
4937
|
+
flattened[key] = csvSafeJsonString(record);
|
|
4888
4938
|
continue;
|
|
4889
4939
|
}
|
|
4890
4940
|
if (parsedFromString && shouldPreserveJsonStringColumn(options.preserveJsonStringColumns, key)) {
|
|
@@ -4896,16 +4946,16 @@ function flattenObjectColumns(row, options = {}) {
|
|
|
4896
4946
|
flattened[key] = failureMessage;
|
|
4897
4947
|
continue;
|
|
4898
4948
|
} else if (Object.prototype.hasOwnProperty.call(record, "result")) {
|
|
4899
|
-
flattened[key] =
|
|
4949
|
+
flattened[key] = csvSafeJsonString(record);
|
|
4900
4950
|
continue;
|
|
4901
4951
|
}
|
|
4902
4952
|
}
|
|
4903
4953
|
for (const [nestedKey, nestedValue] of Object.entries(record)) {
|
|
4904
|
-
flattened[`${key}.${nestedKey}`] = nestedValue && typeof nestedValue === "object" ?
|
|
4954
|
+
flattened[`${key}.${nestedKey}`] = nestedValue && typeof nestedValue === "object" ? csvSafeJsonString(nestedValue) : nestedValue;
|
|
4905
4955
|
}
|
|
4906
4956
|
continue;
|
|
4907
4957
|
}
|
|
4908
|
-
flattened[key] = Array.isArray(value) ?
|
|
4958
|
+
flattened[key] = Array.isArray(value) ? csvSafeJsonString(value) : value;
|
|
4909
4959
|
}
|
|
4910
4960
|
return flattened;
|
|
4911
4961
|
}
|
|
@@ -7393,6 +7443,9 @@ async function waitForRenderHealth(url, state) {
|
|
|
7393
7443
|
if (isOwnedCsvRenderHealth(await fetchCsvRenderHealth(url), state)) {
|
|
7394
7444
|
return true;
|
|
7395
7445
|
}
|
|
7446
|
+
if (state.pid && !processAlive(state.pid)) {
|
|
7447
|
+
return false;
|
|
7448
|
+
}
|
|
7396
7449
|
await new Promise((resolveDelay) => setTimeout(resolveDelay, 100));
|
|
7397
7450
|
}
|
|
7398
7451
|
return false;
|
|
@@ -7455,6 +7508,10 @@ const server = http.createServer((req, res) => {
|
|
|
7455
7508
|
});
|
|
7456
7509
|
|
|
7457
7510
|
server.listen(port, '127.0.0.1');
|
|
7511
|
+
server.on('error', (error) => {
|
|
7512
|
+
console.error(error && error.stack ? error.stack : error);
|
|
7513
|
+
process.exit(1);
|
|
7514
|
+
});
|
|
7458
7515
|
process.on('SIGTERM', () => server.close(() => process.exit(0)));
|
|
7459
7516
|
process.on('SIGINT', () => server.close(() => process.exit(0)));
|
|
7460
7517
|
`;
|
|
@@ -7510,17 +7567,23 @@ async function handleCsvRenderStart(options) {
|
|
|
7510
7567
|
}
|
|
7511
7568
|
ensureCsvRenderStateDir();
|
|
7512
7569
|
const logPath = csvRenderLogPath();
|
|
7570
|
+
const logFd = (0, import_node_fs7.openSync)(logPath, "w");
|
|
7513
7571
|
const token = (0, import_node_crypto2.randomUUID)();
|
|
7514
|
-
const child = (0, import_node_child_process.spawn)(
|
|
7515
|
-
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
|
|
7521
|
-
|
|
7572
|
+
const child = (0, import_node_child_process.spawn)(
|
|
7573
|
+
process.execPath,
|
|
7574
|
+
["-e", CSV_RENDER_SERVER_SOURCE],
|
|
7575
|
+
{
|
|
7576
|
+
detached: true,
|
|
7577
|
+
stdio: ["ignore", logFd, logFd],
|
|
7578
|
+
env: {
|
|
7579
|
+
...process.env,
|
|
7580
|
+
DEEPLINE_CSV_RENDER_PORT: String(port),
|
|
7581
|
+
DEEPLINE_CSV_RENDER_CSV: csvPath,
|
|
7582
|
+
DEEPLINE_CSV_RENDER_TOKEN: token
|
|
7583
|
+
}
|
|
7522
7584
|
}
|
|
7523
|
-
|
|
7585
|
+
);
|
|
7586
|
+
(0, import_node_fs7.closeSync)(logFd);
|
|
7524
7587
|
child.unref();
|
|
7525
7588
|
const state = {
|
|
7526
7589
|
pid: child.pid ?? null,
|
|
@@ -7536,7 +7599,19 @@ async function handleCsvRenderStart(options) {
|
|
|
7536
7599
|
if (processAlive(child.pid)) {
|
|
7537
7600
|
process.kill(child.pid, "SIGTERM");
|
|
7538
7601
|
}
|
|
7539
|
-
|
|
7602
|
+
let detail = "";
|
|
7603
|
+
try {
|
|
7604
|
+
const log = (0, import_node_fs7.readFileSync)(logPath, "utf8").trim();
|
|
7605
|
+
if (log) {
|
|
7606
|
+
detail = `
|
|
7607
|
+
CSV render log:
|
|
7608
|
+
${clip(log, 2e3)}`;
|
|
7609
|
+
}
|
|
7610
|
+
} catch {
|
|
7611
|
+
}
|
|
7612
|
+
throw new Error(
|
|
7613
|
+
`Timed out waiting for CSV render to start at ${url}.${detail}`
|
|
7614
|
+
);
|
|
7540
7615
|
}
|
|
7541
7616
|
writeCsvRenderState(state);
|
|
7542
7617
|
(0, import_node_fs7.writeFileSync)(
|
|
@@ -16968,16 +17043,39 @@ function helperSource() {
|
|
|
16968
17043
|
``,
|
|
16969
17044
|
`function __dlGetByPath(root: unknown, path: string): unknown {`,
|
|
16970
17045
|
` let cursor = root;`,
|
|
16971
|
-
`
|
|
17046
|
+
` const parts = __dlPathParts(path);`,
|
|
17047
|
+
` for (let index = 0; index < parts.length; index += 1) {`,
|
|
17048
|
+
` const part = parts[index] || '';`,
|
|
17049
|
+
` cursor = __dlParseJsonContainer(cursor);`,
|
|
16972
17050
|
` if (!cursor || typeof cursor !== 'object') return undefined;`,
|
|
16973
17051
|
` const record = cursor as Record<string, unknown>;`,
|
|
16974
|
-
`
|
|
17052
|
+
` let field = { found: false, value: undefined as unknown };`,
|
|
17053
|
+
` for (let end = parts.length; end > index + 1; end -= 1) {`,
|
|
17054
|
+
` const dottedPart = parts.slice(index, end).join('.');`,
|
|
17055
|
+
` field = __dlGetRecordField(record, dottedPart);`,
|
|
17056
|
+
` if (field.found) {`,
|
|
17057
|
+
` index = end - 1;`,
|
|
17058
|
+
` break;`,
|
|
17059
|
+
` }`,
|
|
17060
|
+
` }`,
|
|
17061
|
+
` if (!field.found) field = __dlGetRecordField(record, part);`,
|
|
16975
17062
|
` if (!field.found) return undefined;`,
|
|
16976
17063
|
` cursor = field.value;`,
|
|
16977
17064
|
` }`,
|
|
16978
17065
|
` return cursor;`,
|
|
16979
17066
|
`}`,
|
|
16980
17067
|
``,
|
|
17068
|
+
`function __dlParseJsonContainer(value: unknown): unknown {`,
|
|
17069
|
+
` if (typeof value !== 'string') return value;`,
|
|
17070
|
+
` const trimmed = value.trim();`,
|
|
17071
|
+
` if ((!trimmed.startsWith('{') || !trimmed.endsWith('}')) && (!trimmed.startsWith('[') || !trimmed.endsWith(']'))) return value;`,
|
|
17072
|
+
` try {`,
|
|
17073
|
+
` return JSON.parse(trimmed);`,
|
|
17074
|
+
` } catch {`,
|
|
17075
|
+
` return value;`,
|
|
17076
|
+
` }`,
|
|
17077
|
+
`}`,
|
|
17078
|
+
``,
|
|
16981
17079
|
`function __dlMeaningful(value: unknown): boolean {`,
|
|
16982
17080
|
` if (value && typeof value === 'object' && !Array.isArray(value)) {`,
|
|
16983
17081
|
` const record = value as Record<string, unknown>;`,
|
|
@@ -19998,6 +20096,12 @@ function sidecarEnrichRowsExportPath(outputPath) {
|
|
|
19998
20096
|
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
19999
20097
|
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-rows${ext}`);
|
|
20000
20098
|
}
|
|
20099
|
+
function sidecarEnrichBatchManifestPath(outputPath) {
|
|
20100
|
+
const resolved = (0, import_node_path12.resolve)(outputPath);
|
|
20101
|
+
const ext = (0, import_node_path12.extname)(resolved) || ".csv";
|
|
20102
|
+
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
20103
|
+
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-batches.json`);
|
|
20104
|
+
}
|
|
20001
20105
|
function collectDatasetFollowUpCommands(value, state) {
|
|
20002
20106
|
if (state.depth > 12 || !value || typeof value !== "object" || state.commands.length >= 8) {
|
|
20003
20107
|
return;
|
|
@@ -20503,7 +20607,7 @@ var ENRICH_FLATTENED_CONTROL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
20503
20607
|
]);
|
|
20504
20608
|
function materializeCsvCellValue(value) {
|
|
20505
20609
|
if (value && typeof value === "object") {
|
|
20506
|
-
return
|
|
20610
|
+
return csvSafeJsonString(value);
|
|
20507
20611
|
}
|
|
20508
20612
|
return value;
|
|
20509
20613
|
}
|
|
@@ -20905,13 +21009,35 @@ function registerEnrichCommand(program) {
|
|
|
20905
21009
|
const inPlaceCommitOutputPath = options.inPlace ? (await (0, import_promises3.lstat)(inputCsv)).isSymbolicLink() ? await (0, import_promises3.realpath)(inputCsv) : inPlaceFinalOutputPath : null;
|
|
20906
21010
|
const failureReportOutputPath = options.inPlace ? inPlaceFinalOutputPath : null;
|
|
20907
21011
|
const enrichIssueFollowUpOutputPath = options.inPlace && inPlaceFinalOutputPath ? sidecarEnrichRowsExportPath(inPlaceFinalOutputPath) : null;
|
|
20908
|
-
|
|
21012
|
+
const prepareInPlaceOutput = async () => {
|
|
21013
|
+
if (!options.inPlace) {
|
|
21014
|
+
return;
|
|
21015
|
+
}
|
|
21016
|
+
if (inPlaceTempDir) {
|
|
21017
|
+
await (0, import_promises3.rm)(inPlaceTempDir, { recursive: true, force: true });
|
|
21018
|
+
}
|
|
21019
|
+
inPlaceTempDir = await (0, import_promises3.mkdtemp)(
|
|
21020
|
+
(0, import_node_path12.join)(
|
|
21021
|
+
(0, import_node_path12.dirname)(inPlaceCommitOutputPath ?? (0, import_node_path12.resolve)(inputCsv)),
|
|
21022
|
+
".deepline-enrich-in-place-"
|
|
21023
|
+
)
|
|
21024
|
+
);
|
|
21025
|
+
inPlaceTempOutputPath = (0, import_node_path12.join)(inPlaceTempDir, "output.csv");
|
|
21026
|
+
await (0, import_promises3.copyFile)((0, import_node_path12.resolve)(inputCsv), inPlaceTempOutputPath);
|
|
21027
|
+
outputPath = inPlaceTempOutputPath;
|
|
21028
|
+
};
|
|
20909
21029
|
const commitInPlaceOutput = async (exportResult) => {
|
|
20910
21030
|
if (!inPlaceTempOutputPath || !inPlaceCommitOutputPath || !inPlaceFinalOutputPath) {
|
|
20911
21031
|
return exportResult;
|
|
20912
21032
|
}
|
|
21033
|
+
const committedTempDir = inPlaceTempDir;
|
|
20913
21034
|
await (0, import_promises3.rename)(inPlaceTempOutputPath, inPlaceCommitOutputPath);
|
|
20914
|
-
|
|
21035
|
+
inPlaceTempDir = null;
|
|
21036
|
+
inPlaceTempOutputPath = null;
|
|
21037
|
+
outputPath = inPlaceFinalOutputPath;
|
|
21038
|
+
if (committedTempDir) {
|
|
21039
|
+
await (0, import_promises3.rm)(committedTempDir, { recursive: true, force: true });
|
|
21040
|
+
}
|
|
20915
21041
|
if (!exportResult) {
|
|
20916
21042
|
return null;
|
|
20917
21043
|
}
|
|
@@ -20923,15 +21049,7 @@ function registerEnrichCommand(program) {
|
|
|
20923
21049
|
try {
|
|
20924
21050
|
await (0, import_promises3.writeFile)(tempPlay, playSource, "utf8");
|
|
20925
21051
|
if (options.inPlace) {
|
|
20926
|
-
|
|
20927
|
-
(0, import_node_path12.join)(
|
|
20928
|
-
(0, import_node_path12.dirname)(inPlaceCommitOutputPath ?? (0, import_node_path12.resolve)(inputCsv)),
|
|
20929
|
-
".deepline-enrich-in-place-"
|
|
20930
|
-
)
|
|
20931
|
-
);
|
|
20932
|
-
inPlaceTempOutputPath = (0, import_node_path12.join)(inPlaceTempDir, "output.csv");
|
|
20933
|
-
await (0, import_promises3.copyFile)((0, import_node_path12.resolve)(inputCsv), inPlaceTempOutputPath);
|
|
20934
|
-
outputPath = inPlaceTempOutputPath;
|
|
21052
|
+
await prepareInPlaceOutput();
|
|
20935
21053
|
}
|
|
20936
21054
|
const runOne = async (input2) => {
|
|
20937
21055
|
const runtimeInput = {
|
|
@@ -20986,12 +21104,62 @@ function registerEnrichCommand(program) {
|
|
|
20986
21104
|
const autoBatchRows = enrichAutoBatchRowsForConfig(config);
|
|
20987
21105
|
if (outputPath && selectedRange.count > autoBatchRows) {
|
|
20988
21106
|
const chunkCount = Math.ceil(selectedRange.count / autoBatchRows);
|
|
21107
|
+
const batchManifestPath = sidecarEnrichBatchManifestPath(
|
|
21108
|
+
inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath)
|
|
21109
|
+
);
|
|
21110
|
+
const batchManifest = {
|
|
21111
|
+
version: 1,
|
|
21112
|
+
kind: "deepline_enrich_batch_manifest",
|
|
21113
|
+
input: (0, import_node_path12.resolve)(inputCsv),
|
|
21114
|
+
output: inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath),
|
|
21115
|
+
selectedRows: selectedRange.count,
|
|
21116
|
+
sourceCsvRows: selectedRange.sourceRows,
|
|
21117
|
+
chunkRows: autoBatchRows,
|
|
21118
|
+
chunks: chunkCount,
|
|
21119
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21120
|
+
batches: []
|
|
21121
|
+
};
|
|
21122
|
+
const writeBatchManifest = async () => {
|
|
21123
|
+
batchManifest.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
21124
|
+
await (0, import_promises3.writeFile)(
|
|
21125
|
+
batchManifestPath,
|
|
21126
|
+
`${JSON.stringify(batchManifest, null, 2)}
|
|
21127
|
+
`,
|
|
21128
|
+
"utf8"
|
|
21129
|
+
);
|
|
21130
|
+
};
|
|
21131
|
+
const appendBatchManifestEntry = async (input2) => {
|
|
21132
|
+
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
21133
|
+
batchManifest.batches.push({
|
|
21134
|
+
chunk: input2.chunkIndex + 1,
|
|
21135
|
+
chunks: chunkCount,
|
|
21136
|
+
rows: {
|
|
21137
|
+
rowStart: input2.rows.rowStart,
|
|
21138
|
+
rowEnd: input2.rows.rowEnd
|
|
21139
|
+
},
|
|
21140
|
+
status: readEnrichRunStatus(input2.status),
|
|
21141
|
+
runId,
|
|
21142
|
+
customer_db: enrichCustomerDbJson({
|
|
21143
|
+
status: input2.status,
|
|
21144
|
+
client: client2,
|
|
21145
|
+
outputPath: enrichIssueFollowUpOutputPath ?? input2.exportResult?.path ?? outputPath
|
|
21146
|
+
}),
|
|
21147
|
+
output: enrichOutputJson(input2.exportResult),
|
|
21148
|
+
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21149
|
+
});
|
|
21150
|
+
await writeBatchManifest();
|
|
21151
|
+
};
|
|
21152
|
+
await writeBatchManifest();
|
|
20989
21153
|
if (!options.json) {
|
|
20990
21154
|
process.stderr.write(
|
|
20991
21155
|
`Large enrich input selected ${selectedRange.count.toLocaleString()} rows. Running ${chunkCount.toLocaleString()} sequential batch runs of up to ${autoBatchRows.toLocaleString()} rows and merging ${(0, import_node_path12.resolve)(outputPath)}.
|
|
20992
21156
|
`
|
|
20993
21157
|
);
|
|
20994
21158
|
}
|
|
21159
|
+
process.stderr.write(
|
|
21160
|
+
`Batch recovery manifest: ${batchManifestPath}
|
|
21161
|
+
`
|
|
21162
|
+
);
|
|
20995
21163
|
let workingMergeSourceCsvPath = sourceCsvPath;
|
|
20996
21164
|
let lastStatus = null;
|
|
20997
21165
|
const batchStatuses = [];
|
|
@@ -21034,6 +21202,7 @@ function registerEnrichCommand(program) {
|
|
|
21034
21202
|
lastStatus = chunk.status;
|
|
21035
21203
|
batchStatuses.push(chunk.status);
|
|
21036
21204
|
if (chunk.captured.result !== 0) {
|
|
21205
|
+
let currentChunkCommittedExportResult = null;
|
|
21037
21206
|
if (chunk.exportResult) {
|
|
21038
21207
|
finalExportResult = chunk.exportResult;
|
|
21039
21208
|
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
@@ -21057,37 +21226,48 @@ function registerEnrichCommand(program) {
|
|
|
21057
21226
|
waterfallStatus: batchStatuses,
|
|
21058
21227
|
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
21059
21228
|
});
|
|
21060
|
-
|
|
21229
|
+
if (chunk.exportResult) {
|
|
21230
|
+
currentChunkCommittedExportResult = await commitInPlaceOutput(chunk.exportResult);
|
|
21231
|
+
finalExportResult = currentChunkCommittedExportResult;
|
|
21232
|
+
}
|
|
21233
|
+
const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;
|
|
21234
|
+
await appendBatchManifestEntry({
|
|
21235
|
+
chunkIndex,
|
|
21236
|
+
rows: chunkRows,
|
|
21237
|
+
status: chunk.status,
|
|
21238
|
+
exportResult: currentChunkCommittedExportResult
|
|
21239
|
+
});
|
|
21061
21240
|
if (options.json) {
|
|
21062
21241
|
printJson({
|
|
21063
21242
|
ok: false,
|
|
21064
21243
|
batch: {
|
|
21065
21244
|
chunk: chunkIndex + 1,
|
|
21066
21245
|
chunks: chunkCount,
|
|
21067
|
-
rows: chunkRows
|
|
21246
|
+
rows: chunkRows,
|
|
21247
|
+
manifest: batchManifestPath
|
|
21068
21248
|
},
|
|
21069
21249
|
result: chunk.status,
|
|
21070
|
-
output:
|
|
21250
|
+
output: reportedExportResult ? {
|
|
21071
21251
|
sourceCsvRows: selectedRange.sourceRows,
|
|
21072
21252
|
selectedRows: selectedRange.count,
|
|
21073
21253
|
enrichedRows: totalEnrichedRows,
|
|
21074
|
-
path:
|
|
21075
|
-
...
|
|
21076
|
-
rows:
|
|
21254
|
+
path: reportedExportResult.path,
|
|
21255
|
+
...reportedExportResult.partial ? {
|
|
21256
|
+
rows: reportedExportResult.rows,
|
|
21077
21257
|
partial: true
|
|
21078
21258
|
} : {}
|
|
21079
21259
|
} : null,
|
|
21080
21260
|
customer_db: enrichCustomerDbJson({
|
|
21081
21261
|
status: chunk.status,
|
|
21082
21262
|
client: client2,
|
|
21083
|
-
outputPath: enrichIssueFollowUpOutputPath ??
|
|
21263
|
+
outputPath: enrichIssueFollowUpOutputPath ?? reportedExportResult?.path ?? outputPath
|
|
21084
21264
|
}),
|
|
21085
21265
|
...enrichReportJson(failureReport3)
|
|
21086
21266
|
});
|
|
21087
21267
|
} else {
|
|
21088
|
-
if (
|
|
21268
|
+
if (reportedExportResult) {
|
|
21089
21269
|
process.stderr.write(
|
|
21090
|
-
`Wrote ${
|
|
21270
|
+
`Wrote ${reportedExportResult.rows} row(s) to ${reportedExportResult.path}${reportedExportResult.partial ? " (partial run output)" : ""}
|
|
21091
21271
|
`
|
|
21092
21272
|
);
|
|
21093
21273
|
}
|
|
@@ -21102,13 +21282,22 @@ function registerEnrichCommand(program) {
|
|
|
21102
21282
|
return;
|
|
21103
21283
|
}
|
|
21104
21284
|
if (chunk.exportResult) {
|
|
21105
|
-
finalExportResult = chunk.exportResult;
|
|
21106
21285
|
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
21107
21286
|
chunk.exportResult,
|
|
21108
21287
|
chunkRows
|
|
21109
21288
|
);
|
|
21110
21289
|
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
21290
|
+
finalExportResult = options.inPlace ? await commitInPlaceOutput(chunk.exportResult) : chunk.exportResult;
|
|
21291
|
+
await appendBatchManifestEntry({
|
|
21292
|
+
chunkIndex,
|
|
21293
|
+
rows: chunkRows,
|
|
21294
|
+
status: chunk.status,
|
|
21295
|
+
exportResult: finalExportResult
|
|
21296
|
+
});
|
|
21111
21297
|
workingMergeSourceCsvPath = outputPath;
|
|
21298
|
+
if (options.inPlace && (chunkRows.rowEnd ?? selectedRange.end) < selectedRange.end) {
|
|
21299
|
+
await prepareInPlaceOutput();
|
|
21300
|
+
}
|
|
21112
21301
|
}
|
|
21113
21302
|
}
|
|
21114
21303
|
const outputRows = readCsvRows(outputPath);
|
|
@@ -21148,7 +21337,8 @@ function registerEnrichCommand(program) {
|
|
|
21148
21337
|
batch: {
|
|
21149
21338
|
chunks: chunkCount,
|
|
21150
21339
|
chunkRows: autoBatchRows,
|
|
21151
|
-
selectedRows: selectedRange.count
|
|
21340
|
+
selectedRows: selectedRange.count,
|
|
21341
|
+
manifest: batchManifestPath
|
|
21152
21342
|
},
|
|
21153
21343
|
output: finalExportResult ? {
|
|
21154
21344
|
sourceCsvRows: selectedRange.sourceRows,
|
|
@@ -21278,7 +21468,7 @@ function registerEnrichCommand(program) {
|
|
|
21278
21468
|
} finally {
|
|
21279
21469
|
if (inPlaceTempDir) {
|
|
21280
21470
|
await (0, import_promises3.rm)(inPlaceTempDir, { recursive: true, force: true });
|
|
21281
|
-
} else if (inPlaceTempOutputPath
|
|
21471
|
+
} else if (inPlaceTempOutputPath) {
|
|
21282
21472
|
await (0, import_promises3.rm)(inPlaceTempOutputPath, { force: true });
|
|
21283
21473
|
}
|
|
21284
21474
|
await (0, import_promises3.rm)(tempDir, { recursive: true, force: true });
|
package/dist/cli/index.mjs
CHANGED
|
@@ -608,10 +608,10 @@ var SDK_RELEASE = {
|
|
|
608
608
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
609
609
|
// fields shipped in 0.1.153.
|
|
610
610
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
611
|
-
version: "0.1.
|
|
611
|
+
version: "0.1.191",
|
|
612
612
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
613
613
|
supportPolicy: {
|
|
614
|
-
latest: "0.1.
|
|
614
|
+
latest: "0.1.191",
|
|
615
615
|
minimumSupported: "0.1.53",
|
|
616
616
|
deprecatedBelow: "0.1.53",
|
|
617
617
|
commandMinimumSupported: [
|
|
@@ -4826,14 +4826,64 @@ function readCsvRows(csvPath) {
|
|
|
4826
4826
|
});
|
|
4827
4827
|
}
|
|
4828
4828
|
function csvStringFromRows(rows, columns) {
|
|
4829
|
-
return stringify(rows, {
|
|
4829
|
+
return stringify(rows.map(csvSafeRow), {
|
|
4830
4830
|
header: true,
|
|
4831
4831
|
cast: {
|
|
4832
|
-
boolean: (value) => value ? "true" : "false"
|
|
4832
|
+
boolean: (value) => value ? "true" : "false",
|
|
4833
|
+
object: (value) => {
|
|
4834
|
+
const cell = csvSafeCell(value);
|
|
4835
|
+
return typeof cell === "string" ? cell : null;
|
|
4836
|
+
}
|
|
4833
4837
|
},
|
|
4834
4838
|
...columns?.length ? { columns } : {}
|
|
4835
4839
|
});
|
|
4836
4840
|
}
|
|
4841
|
+
function csvSafeRow(row) {
|
|
4842
|
+
return Object.fromEntries(
|
|
4843
|
+
Object.entries(row).map(([key, value]) => [key, csvSafeCell(value)])
|
|
4844
|
+
);
|
|
4845
|
+
}
|
|
4846
|
+
function csvSafeCell(value) {
|
|
4847
|
+
if (value === void 0) {
|
|
4848
|
+
return null;
|
|
4849
|
+
}
|
|
4850
|
+
if (value === null) {
|
|
4851
|
+
return null;
|
|
4852
|
+
}
|
|
4853
|
+
if (typeof value === "bigint") {
|
|
4854
|
+
return value.toString();
|
|
4855
|
+
}
|
|
4856
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
4857
|
+
return value;
|
|
4858
|
+
}
|
|
4859
|
+
if (value instanceof Date) {
|
|
4860
|
+
return value.toISOString();
|
|
4861
|
+
}
|
|
4862
|
+
if (typeof value === "symbol" || typeof value === "function") {
|
|
4863
|
+
return null;
|
|
4864
|
+
}
|
|
4865
|
+
if (typeof value !== "object") {
|
|
4866
|
+
return String(value);
|
|
4867
|
+
}
|
|
4868
|
+
return csvSafeJsonString(value);
|
|
4869
|
+
}
|
|
4870
|
+
function csvSafeJsonReplacer(_key, nested) {
|
|
4871
|
+
if (typeof nested === "bigint") {
|
|
4872
|
+
return nested.toString();
|
|
4873
|
+
}
|
|
4874
|
+
if (nested === void 0 || typeof nested === "symbol" || typeof nested === "function") {
|
|
4875
|
+
return null;
|
|
4876
|
+
}
|
|
4877
|
+
return nested;
|
|
4878
|
+
}
|
|
4879
|
+
function csvSafeJsonString(value) {
|
|
4880
|
+
try {
|
|
4881
|
+
const serialized = JSON.stringify(value, csvSafeJsonReplacer);
|
|
4882
|
+
return serialized ?? null;
|
|
4883
|
+
} catch {
|
|
4884
|
+
return String(value);
|
|
4885
|
+
}
|
|
4886
|
+
}
|
|
4837
4887
|
function parseMaybeJsonObject(value) {
|
|
4838
4888
|
if (typeof value !== "string") {
|
|
4839
4889
|
return value;
|
|
@@ -4874,14 +4924,14 @@ function flattenObjectColumns(row, options = {}) {
|
|
|
4874
4924
|
const value = parseMaybeJsonObject(rawValue);
|
|
4875
4925
|
const parsedFromString = typeof rawValue === "string" && value !== rawValue;
|
|
4876
4926
|
if (key === "_metadata") {
|
|
4877
|
-
flattened[key] = value && typeof value === "object" ?
|
|
4927
|
+
flattened[key] = value && typeof value === "object" ? csvSafeJsonString(value) : value;
|
|
4878
4928
|
continue;
|
|
4879
4929
|
}
|
|
4880
4930
|
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
4881
4931
|
const record = value;
|
|
4882
4932
|
const hasMatchedEnvelope = Object.prototype.hasOwnProperty.call(record, "matched_result") || Object.prototype.hasOwnProperty.call(record, "matchedResult");
|
|
4883
4933
|
if (hasMatchedEnvelope) {
|
|
4884
|
-
flattened[key] =
|
|
4934
|
+
flattened[key] = csvSafeJsonString(record);
|
|
4885
4935
|
continue;
|
|
4886
4936
|
}
|
|
4887
4937
|
if (parsedFromString && shouldPreserveJsonStringColumn(options.preserveJsonStringColumns, key)) {
|
|
@@ -4893,16 +4943,16 @@ function flattenObjectColumns(row, options = {}) {
|
|
|
4893
4943
|
flattened[key] = failureMessage;
|
|
4894
4944
|
continue;
|
|
4895
4945
|
} else if (Object.prototype.hasOwnProperty.call(record, "result")) {
|
|
4896
|
-
flattened[key] =
|
|
4946
|
+
flattened[key] = csvSafeJsonString(record);
|
|
4897
4947
|
continue;
|
|
4898
4948
|
}
|
|
4899
4949
|
}
|
|
4900
4950
|
for (const [nestedKey, nestedValue] of Object.entries(record)) {
|
|
4901
|
-
flattened[`${key}.${nestedKey}`] = nestedValue && typeof nestedValue === "object" ?
|
|
4951
|
+
flattened[`${key}.${nestedKey}`] = nestedValue && typeof nestedValue === "object" ? csvSafeJsonString(nestedValue) : nestedValue;
|
|
4902
4952
|
}
|
|
4903
4953
|
continue;
|
|
4904
4954
|
}
|
|
4905
|
-
flattened[key] = Array.isArray(value) ?
|
|
4955
|
+
flattened[key] = Array.isArray(value) ? csvSafeJsonString(value) : value;
|
|
4906
4956
|
}
|
|
4907
4957
|
return flattened;
|
|
4908
4958
|
}
|
|
@@ -6620,8 +6670,10 @@ Examples:
|
|
|
6620
6670
|
import { spawn } from "child_process";
|
|
6621
6671
|
import { randomUUID } from "crypto";
|
|
6622
6672
|
import {
|
|
6673
|
+
closeSync,
|
|
6623
6674
|
existsSync as existsSync6,
|
|
6624
6675
|
mkdirSync as mkdirSync5,
|
|
6676
|
+
openSync,
|
|
6625
6677
|
readFileSync as readFileSync6,
|
|
6626
6678
|
rmSync as rmSync3,
|
|
6627
6679
|
writeFileSync as writeFileSync6
|
|
@@ -7396,6 +7448,9 @@ async function waitForRenderHealth(url, state) {
|
|
|
7396
7448
|
if (isOwnedCsvRenderHealth(await fetchCsvRenderHealth(url), state)) {
|
|
7397
7449
|
return true;
|
|
7398
7450
|
}
|
|
7451
|
+
if (state.pid && !processAlive(state.pid)) {
|
|
7452
|
+
return false;
|
|
7453
|
+
}
|
|
7399
7454
|
await new Promise((resolveDelay) => setTimeout(resolveDelay, 100));
|
|
7400
7455
|
}
|
|
7401
7456
|
return false;
|
|
@@ -7458,6 +7513,10 @@ const server = http.createServer((req, res) => {
|
|
|
7458
7513
|
});
|
|
7459
7514
|
|
|
7460
7515
|
server.listen(port, '127.0.0.1');
|
|
7516
|
+
server.on('error', (error) => {
|
|
7517
|
+
console.error(error && error.stack ? error.stack : error);
|
|
7518
|
+
process.exit(1);
|
|
7519
|
+
});
|
|
7461
7520
|
process.on('SIGTERM', () => server.close(() => process.exit(0)));
|
|
7462
7521
|
process.on('SIGINT', () => server.close(() => process.exit(0)));
|
|
7463
7522
|
`;
|
|
@@ -7513,17 +7572,23 @@ async function handleCsvRenderStart(options) {
|
|
|
7513
7572
|
}
|
|
7514
7573
|
ensureCsvRenderStateDir();
|
|
7515
7574
|
const logPath = csvRenderLogPath();
|
|
7575
|
+
const logFd = openSync(logPath, "w");
|
|
7516
7576
|
const token = randomUUID();
|
|
7517
|
-
const child = spawn(
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
|
|
7521
|
-
|
|
7522
|
-
|
|
7523
|
-
|
|
7524
|
-
|
|
7577
|
+
const child = spawn(
|
|
7578
|
+
process.execPath,
|
|
7579
|
+
["-e", CSV_RENDER_SERVER_SOURCE],
|
|
7580
|
+
{
|
|
7581
|
+
detached: true,
|
|
7582
|
+
stdio: ["ignore", logFd, logFd],
|
|
7583
|
+
env: {
|
|
7584
|
+
...process.env,
|
|
7585
|
+
DEEPLINE_CSV_RENDER_PORT: String(port),
|
|
7586
|
+
DEEPLINE_CSV_RENDER_CSV: csvPath,
|
|
7587
|
+
DEEPLINE_CSV_RENDER_TOKEN: token
|
|
7588
|
+
}
|
|
7525
7589
|
}
|
|
7526
|
-
|
|
7590
|
+
);
|
|
7591
|
+
closeSync(logFd);
|
|
7527
7592
|
child.unref();
|
|
7528
7593
|
const state = {
|
|
7529
7594
|
pid: child.pid ?? null,
|
|
@@ -7539,7 +7604,19 @@ async function handleCsvRenderStart(options) {
|
|
|
7539
7604
|
if (processAlive(child.pid)) {
|
|
7540
7605
|
process.kill(child.pid, "SIGTERM");
|
|
7541
7606
|
}
|
|
7542
|
-
|
|
7607
|
+
let detail = "";
|
|
7608
|
+
try {
|
|
7609
|
+
const log = readFileSync6(logPath, "utf8").trim();
|
|
7610
|
+
if (log) {
|
|
7611
|
+
detail = `
|
|
7612
|
+
CSV render log:
|
|
7613
|
+
${clip(log, 2e3)}`;
|
|
7614
|
+
}
|
|
7615
|
+
} catch {
|
|
7616
|
+
}
|
|
7617
|
+
throw new Error(
|
|
7618
|
+
`Timed out waiting for CSV render to start at ${url}.${detail}`
|
|
7619
|
+
);
|
|
7543
7620
|
}
|
|
7544
7621
|
writeCsvRenderState(state);
|
|
7545
7622
|
writeFileSync6(
|
|
@@ -8105,8 +8182,8 @@ import { parse as parseCsvSync2 } from "csv-parse/sync";
|
|
|
8105
8182
|
|
|
8106
8183
|
// src/cli/commands/plays/bootstrap.ts
|
|
8107
8184
|
import {
|
|
8108
|
-
closeSync,
|
|
8109
|
-
openSync,
|
|
8185
|
+
closeSync as closeSync2,
|
|
8186
|
+
openSync as openSync2,
|
|
8110
8187
|
readSync,
|
|
8111
8188
|
statSync as statSync2,
|
|
8112
8189
|
writeFileSync as writeFileSync8
|
|
@@ -8684,11 +8761,11 @@ function inferCsvColumnSpecs(headers, rows) {
|
|
|
8684
8761
|
function readCsvSample(csvPath) {
|
|
8685
8762
|
const resolvedPath = resolve7(csvPath);
|
|
8686
8763
|
const size = statSync2(resolvedPath).size;
|
|
8687
|
-
const fd =
|
|
8764
|
+
const fd = openSync2(resolvedPath, "r");
|
|
8688
8765
|
const byteLength = Math.min(size, CSV_HEADER_SAMPLE_BYTES);
|
|
8689
8766
|
const buffer = Buffer.alloc(byteLength);
|
|
8690
8767
|
const bytesRead = readSync(fd, buffer, 0, byteLength, 0);
|
|
8691
|
-
|
|
8768
|
+
closeSync2(fd);
|
|
8692
8769
|
if (bytesRead === 0) {
|
|
8693
8770
|
throw new PlayBootstrapUsageError(`--from csv:${csvPath} is empty.`);
|
|
8694
8771
|
}
|
|
@@ -16995,16 +17072,39 @@ function helperSource() {
|
|
|
16995
17072
|
``,
|
|
16996
17073
|
`function __dlGetByPath(root: unknown, path: string): unknown {`,
|
|
16997
17074
|
` let cursor = root;`,
|
|
16998
|
-
`
|
|
17075
|
+
` const parts = __dlPathParts(path);`,
|
|
17076
|
+
` for (let index = 0; index < parts.length; index += 1) {`,
|
|
17077
|
+
` const part = parts[index] || '';`,
|
|
17078
|
+
` cursor = __dlParseJsonContainer(cursor);`,
|
|
16999
17079
|
` if (!cursor || typeof cursor !== 'object') return undefined;`,
|
|
17000
17080
|
` const record = cursor as Record<string, unknown>;`,
|
|
17001
|
-
`
|
|
17081
|
+
` let field = { found: false, value: undefined as unknown };`,
|
|
17082
|
+
` for (let end = parts.length; end > index + 1; end -= 1) {`,
|
|
17083
|
+
` const dottedPart = parts.slice(index, end).join('.');`,
|
|
17084
|
+
` field = __dlGetRecordField(record, dottedPart);`,
|
|
17085
|
+
` if (field.found) {`,
|
|
17086
|
+
` index = end - 1;`,
|
|
17087
|
+
` break;`,
|
|
17088
|
+
` }`,
|
|
17089
|
+
` }`,
|
|
17090
|
+
` if (!field.found) field = __dlGetRecordField(record, part);`,
|
|
17002
17091
|
` if (!field.found) return undefined;`,
|
|
17003
17092
|
` cursor = field.value;`,
|
|
17004
17093
|
` }`,
|
|
17005
17094
|
` return cursor;`,
|
|
17006
17095
|
`}`,
|
|
17007
17096
|
``,
|
|
17097
|
+
`function __dlParseJsonContainer(value: unknown): unknown {`,
|
|
17098
|
+
` if (typeof value !== 'string') return value;`,
|
|
17099
|
+
` const trimmed = value.trim();`,
|
|
17100
|
+
` if ((!trimmed.startsWith('{') || !trimmed.endsWith('}')) && (!trimmed.startsWith('[') || !trimmed.endsWith(']'))) return value;`,
|
|
17101
|
+
` try {`,
|
|
17102
|
+
` return JSON.parse(trimmed);`,
|
|
17103
|
+
` } catch {`,
|
|
17104
|
+
` return value;`,
|
|
17105
|
+
` }`,
|
|
17106
|
+
`}`,
|
|
17107
|
+
``,
|
|
17008
17108
|
`function __dlMeaningful(value: unknown): boolean {`,
|
|
17009
17109
|
` if (value && typeof value === 'object' && !Array.isArray(value)) {`,
|
|
17010
17110
|
` const record = value as Record<string, unknown>;`,
|
|
@@ -20025,6 +20125,12 @@ function sidecarEnrichRowsExportPath(outputPath) {
|
|
|
20025
20125
|
const stem = basename2(resolved, ext);
|
|
20026
20126
|
return join7(dirname7(resolved), `${stem}.deepline-enrich-rows${ext}`);
|
|
20027
20127
|
}
|
|
20128
|
+
function sidecarEnrichBatchManifestPath(outputPath) {
|
|
20129
|
+
const resolved = resolve9(outputPath);
|
|
20130
|
+
const ext = extname(resolved) || ".csv";
|
|
20131
|
+
const stem = basename2(resolved, ext);
|
|
20132
|
+
return join7(dirname7(resolved), `${stem}.deepline-enrich-batches.json`);
|
|
20133
|
+
}
|
|
20028
20134
|
function collectDatasetFollowUpCommands(value, state) {
|
|
20029
20135
|
if (state.depth > 12 || !value || typeof value !== "object" || state.commands.length >= 8) {
|
|
20030
20136
|
return;
|
|
@@ -20530,7 +20636,7 @@ var ENRICH_FLATTENED_CONTROL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
20530
20636
|
]);
|
|
20531
20637
|
function materializeCsvCellValue(value) {
|
|
20532
20638
|
if (value && typeof value === "object") {
|
|
20533
|
-
return
|
|
20639
|
+
return csvSafeJsonString(value);
|
|
20534
20640
|
}
|
|
20535
20641
|
return value;
|
|
20536
20642
|
}
|
|
@@ -20932,13 +21038,35 @@ function registerEnrichCommand(program) {
|
|
|
20932
21038
|
const inPlaceCommitOutputPath = options.inPlace ? (await lstat(inputCsv)).isSymbolicLink() ? await realpath(inputCsv) : inPlaceFinalOutputPath : null;
|
|
20933
21039
|
const failureReportOutputPath = options.inPlace ? inPlaceFinalOutputPath : null;
|
|
20934
21040
|
const enrichIssueFollowUpOutputPath = options.inPlace && inPlaceFinalOutputPath ? sidecarEnrichRowsExportPath(inPlaceFinalOutputPath) : null;
|
|
20935
|
-
|
|
21041
|
+
const prepareInPlaceOutput = async () => {
|
|
21042
|
+
if (!options.inPlace) {
|
|
21043
|
+
return;
|
|
21044
|
+
}
|
|
21045
|
+
if (inPlaceTempDir) {
|
|
21046
|
+
await rm(inPlaceTempDir, { recursive: true, force: true });
|
|
21047
|
+
}
|
|
21048
|
+
inPlaceTempDir = await mkdtemp(
|
|
21049
|
+
join7(
|
|
21050
|
+
dirname7(inPlaceCommitOutputPath ?? resolve9(inputCsv)),
|
|
21051
|
+
".deepline-enrich-in-place-"
|
|
21052
|
+
)
|
|
21053
|
+
);
|
|
21054
|
+
inPlaceTempOutputPath = join7(inPlaceTempDir, "output.csv");
|
|
21055
|
+
await copyFile(resolve9(inputCsv), inPlaceTempOutputPath);
|
|
21056
|
+
outputPath = inPlaceTempOutputPath;
|
|
21057
|
+
};
|
|
20936
21058
|
const commitInPlaceOutput = async (exportResult) => {
|
|
20937
21059
|
if (!inPlaceTempOutputPath || !inPlaceCommitOutputPath || !inPlaceFinalOutputPath) {
|
|
20938
21060
|
return exportResult;
|
|
20939
21061
|
}
|
|
21062
|
+
const committedTempDir = inPlaceTempDir;
|
|
20940
21063
|
await rename(inPlaceTempOutputPath, inPlaceCommitOutputPath);
|
|
20941
|
-
|
|
21064
|
+
inPlaceTempDir = null;
|
|
21065
|
+
inPlaceTempOutputPath = null;
|
|
21066
|
+
outputPath = inPlaceFinalOutputPath;
|
|
21067
|
+
if (committedTempDir) {
|
|
21068
|
+
await rm(committedTempDir, { recursive: true, force: true });
|
|
21069
|
+
}
|
|
20942
21070
|
if (!exportResult) {
|
|
20943
21071
|
return null;
|
|
20944
21072
|
}
|
|
@@ -20950,15 +21078,7 @@ function registerEnrichCommand(program) {
|
|
|
20950
21078
|
try {
|
|
20951
21079
|
await writeFile3(tempPlay, playSource, "utf8");
|
|
20952
21080
|
if (options.inPlace) {
|
|
20953
|
-
|
|
20954
|
-
join7(
|
|
20955
|
-
dirname7(inPlaceCommitOutputPath ?? resolve9(inputCsv)),
|
|
20956
|
-
".deepline-enrich-in-place-"
|
|
20957
|
-
)
|
|
20958
|
-
);
|
|
20959
|
-
inPlaceTempOutputPath = join7(inPlaceTempDir, "output.csv");
|
|
20960
|
-
await copyFile(resolve9(inputCsv), inPlaceTempOutputPath);
|
|
20961
|
-
outputPath = inPlaceTempOutputPath;
|
|
21081
|
+
await prepareInPlaceOutput();
|
|
20962
21082
|
}
|
|
20963
21083
|
const runOne = async (input2) => {
|
|
20964
21084
|
const runtimeInput = {
|
|
@@ -21013,12 +21133,62 @@ function registerEnrichCommand(program) {
|
|
|
21013
21133
|
const autoBatchRows = enrichAutoBatchRowsForConfig(config);
|
|
21014
21134
|
if (outputPath && selectedRange.count > autoBatchRows) {
|
|
21015
21135
|
const chunkCount = Math.ceil(selectedRange.count / autoBatchRows);
|
|
21136
|
+
const batchManifestPath = sidecarEnrichBatchManifestPath(
|
|
21137
|
+
inPlaceFinalOutputPath ?? resolve9(outputPath)
|
|
21138
|
+
);
|
|
21139
|
+
const batchManifest = {
|
|
21140
|
+
version: 1,
|
|
21141
|
+
kind: "deepline_enrich_batch_manifest",
|
|
21142
|
+
input: resolve9(inputCsv),
|
|
21143
|
+
output: inPlaceFinalOutputPath ?? resolve9(outputPath),
|
|
21144
|
+
selectedRows: selectedRange.count,
|
|
21145
|
+
sourceCsvRows: selectedRange.sourceRows,
|
|
21146
|
+
chunkRows: autoBatchRows,
|
|
21147
|
+
chunks: chunkCount,
|
|
21148
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21149
|
+
batches: []
|
|
21150
|
+
};
|
|
21151
|
+
const writeBatchManifest = async () => {
|
|
21152
|
+
batchManifest.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
21153
|
+
await writeFile3(
|
|
21154
|
+
batchManifestPath,
|
|
21155
|
+
`${JSON.stringify(batchManifest, null, 2)}
|
|
21156
|
+
`,
|
|
21157
|
+
"utf8"
|
|
21158
|
+
);
|
|
21159
|
+
};
|
|
21160
|
+
const appendBatchManifestEntry = async (input2) => {
|
|
21161
|
+
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
21162
|
+
batchManifest.batches.push({
|
|
21163
|
+
chunk: input2.chunkIndex + 1,
|
|
21164
|
+
chunks: chunkCount,
|
|
21165
|
+
rows: {
|
|
21166
|
+
rowStart: input2.rows.rowStart,
|
|
21167
|
+
rowEnd: input2.rows.rowEnd
|
|
21168
|
+
},
|
|
21169
|
+
status: readEnrichRunStatus(input2.status),
|
|
21170
|
+
runId,
|
|
21171
|
+
customer_db: enrichCustomerDbJson({
|
|
21172
|
+
status: input2.status,
|
|
21173
|
+
client: client2,
|
|
21174
|
+
outputPath: enrichIssueFollowUpOutputPath ?? input2.exportResult?.path ?? outputPath
|
|
21175
|
+
}),
|
|
21176
|
+
output: enrichOutputJson(input2.exportResult),
|
|
21177
|
+
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
21178
|
+
});
|
|
21179
|
+
await writeBatchManifest();
|
|
21180
|
+
};
|
|
21181
|
+
await writeBatchManifest();
|
|
21016
21182
|
if (!options.json) {
|
|
21017
21183
|
process.stderr.write(
|
|
21018
21184
|
`Large enrich input selected ${selectedRange.count.toLocaleString()} rows. Running ${chunkCount.toLocaleString()} sequential batch runs of up to ${autoBatchRows.toLocaleString()} rows and merging ${resolve9(outputPath)}.
|
|
21019
21185
|
`
|
|
21020
21186
|
);
|
|
21021
21187
|
}
|
|
21188
|
+
process.stderr.write(
|
|
21189
|
+
`Batch recovery manifest: ${batchManifestPath}
|
|
21190
|
+
`
|
|
21191
|
+
);
|
|
21022
21192
|
let workingMergeSourceCsvPath = sourceCsvPath;
|
|
21023
21193
|
let lastStatus = null;
|
|
21024
21194
|
const batchStatuses = [];
|
|
@@ -21061,6 +21231,7 @@ function registerEnrichCommand(program) {
|
|
|
21061
21231
|
lastStatus = chunk.status;
|
|
21062
21232
|
batchStatuses.push(chunk.status);
|
|
21063
21233
|
if (chunk.captured.result !== 0) {
|
|
21234
|
+
let currentChunkCommittedExportResult = null;
|
|
21064
21235
|
if (chunk.exportResult) {
|
|
21065
21236
|
finalExportResult = chunk.exportResult;
|
|
21066
21237
|
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
@@ -21084,37 +21255,48 @@ function registerEnrichCommand(program) {
|
|
|
21084
21255
|
waterfallStatus: batchStatuses,
|
|
21085
21256
|
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
21086
21257
|
});
|
|
21087
|
-
|
|
21258
|
+
if (chunk.exportResult) {
|
|
21259
|
+
currentChunkCommittedExportResult = await commitInPlaceOutput(chunk.exportResult);
|
|
21260
|
+
finalExportResult = currentChunkCommittedExportResult;
|
|
21261
|
+
}
|
|
21262
|
+
const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;
|
|
21263
|
+
await appendBatchManifestEntry({
|
|
21264
|
+
chunkIndex,
|
|
21265
|
+
rows: chunkRows,
|
|
21266
|
+
status: chunk.status,
|
|
21267
|
+
exportResult: currentChunkCommittedExportResult
|
|
21268
|
+
});
|
|
21088
21269
|
if (options.json) {
|
|
21089
21270
|
printJson({
|
|
21090
21271
|
ok: false,
|
|
21091
21272
|
batch: {
|
|
21092
21273
|
chunk: chunkIndex + 1,
|
|
21093
21274
|
chunks: chunkCount,
|
|
21094
|
-
rows: chunkRows
|
|
21275
|
+
rows: chunkRows,
|
|
21276
|
+
manifest: batchManifestPath
|
|
21095
21277
|
},
|
|
21096
21278
|
result: chunk.status,
|
|
21097
|
-
output:
|
|
21279
|
+
output: reportedExportResult ? {
|
|
21098
21280
|
sourceCsvRows: selectedRange.sourceRows,
|
|
21099
21281
|
selectedRows: selectedRange.count,
|
|
21100
21282
|
enrichedRows: totalEnrichedRows,
|
|
21101
|
-
path:
|
|
21102
|
-
...
|
|
21103
|
-
rows:
|
|
21283
|
+
path: reportedExportResult.path,
|
|
21284
|
+
...reportedExportResult.partial ? {
|
|
21285
|
+
rows: reportedExportResult.rows,
|
|
21104
21286
|
partial: true
|
|
21105
21287
|
} : {}
|
|
21106
21288
|
} : null,
|
|
21107
21289
|
customer_db: enrichCustomerDbJson({
|
|
21108
21290
|
status: chunk.status,
|
|
21109
21291
|
client: client2,
|
|
21110
|
-
outputPath: enrichIssueFollowUpOutputPath ??
|
|
21292
|
+
outputPath: enrichIssueFollowUpOutputPath ?? reportedExportResult?.path ?? outputPath
|
|
21111
21293
|
}),
|
|
21112
21294
|
...enrichReportJson(failureReport3)
|
|
21113
21295
|
});
|
|
21114
21296
|
} else {
|
|
21115
|
-
if (
|
|
21297
|
+
if (reportedExportResult) {
|
|
21116
21298
|
process.stderr.write(
|
|
21117
|
-
`Wrote ${
|
|
21299
|
+
`Wrote ${reportedExportResult.rows} row(s) to ${reportedExportResult.path}${reportedExportResult.partial ? " (partial run output)" : ""}
|
|
21118
21300
|
`
|
|
21119
21301
|
);
|
|
21120
21302
|
}
|
|
@@ -21129,13 +21311,22 @@ function registerEnrichCommand(program) {
|
|
|
21129
21311
|
return;
|
|
21130
21312
|
}
|
|
21131
21313
|
if (chunk.exportResult) {
|
|
21132
|
-
finalExportResult = chunk.exportResult;
|
|
21133
21314
|
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
21134
21315
|
chunk.exportResult,
|
|
21135
21316
|
chunkRows
|
|
21136
21317
|
);
|
|
21137
21318
|
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
21319
|
+
finalExportResult = options.inPlace ? await commitInPlaceOutput(chunk.exportResult) : chunk.exportResult;
|
|
21320
|
+
await appendBatchManifestEntry({
|
|
21321
|
+
chunkIndex,
|
|
21322
|
+
rows: chunkRows,
|
|
21323
|
+
status: chunk.status,
|
|
21324
|
+
exportResult: finalExportResult
|
|
21325
|
+
});
|
|
21138
21326
|
workingMergeSourceCsvPath = outputPath;
|
|
21327
|
+
if (options.inPlace && (chunkRows.rowEnd ?? selectedRange.end) < selectedRange.end) {
|
|
21328
|
+
await prepareInPlaceOutput();
|
|
21329
|
+
}
|
|
21139
21330
|
}
|
|
21140
21331
|
}
|
|
21141
21332
|
const outputRows = readCsvRows(outputPath);
|
|
@@ -21175,7 +21366,8 @@ function registerEnrichCommand(program) {
|
|
|
21175
21366
|
batch: {
|
|
21176
21367
|
chunks: chunkCount,
|
|
21177
21368
|
chunkRows: autoBatchRows,
|
|
21178
|
-
selectedRows: selectedRange.count
|
|
21369
|
+
selectedRows: selectedRange.count,
|
|
21370
|
+
manifest: batchManifestPath
|
|
21179
21371
|
},
|
|
21180
21372
|
output: finalExportResult ? {
|
|
21181
21373
|
sourceCsvRows: selectedRange.sourceRows,
|
|
@@ -21305,7 +21497,7 @@ function registerEnrichCommand(program) {
|
|
|
21305
21497
|
} finally {
|
|
21306
21498
|
if (inPlaceTempDir) {
|
|
21307
21499
|
await rm(inPlaceTempDir, { recursive: true, force: true });
|
|
21308
|
-
} else if (inPlaceTempOutputPath
|
|
21500
|
+
} else if (inPlaceTempOutputPath) {
|
|
21309
21501
|
await rm(inPlaceTempOutputPath, { force: true });
|
|
21310
21502
|
}
|
|
21311
21503
|
await rm(tempDir, { recursive: true, force: true });
|
|
@@ -24189,9 +24381,9 @@ import { join as join11, resolve as resolve11 } from "path";
|
|
|
24189
24381
|
|
|
24190
24382
|
// src/tool-output.ts
|
|
24191
24383
|
import {
|
|
24192
|
-
closeSync as
|
|
24384
|
+
closeSync as closeSync3,
|
|
24193
24385
|
mkdirSync as mkdirSync8,
|
|
24194
|
-
openSync as
|
|
24386
|
+
openSync as openSync3,
|
|
24195
24387
|
writeFileSync as writeFileSync12,
|
|
24196
24388
|
writeSync
|
|
24197
24389
|
} from "fs";
|
|
@@ -24344,7 +24536,7 @@ function writeCsvOutputFile(rows, stem, options) {
|
|
|
24344
24536
|
}
|
|
24345
24537
|
return normalized;
|
|
24346
24538
|
};
|
|
24347
|
-
const fd =
|
|
24539
|
+
const fd = openSync3(outputPath, "w");
|
|
24348
24540
|
try {
|
|
24349
24541
|
writeSync(fd, `${columns.map(escapeCell).join(",")}
|
|
24350
24542
|
`);
|
|
@@ -24356,7 +24548,7 @@ function writeCsvOutputFile(rows, stem, options) {
|
|
|
24356
24548
|
);
|
|
24357
24549
|
}
|
|
24358
24550
|
} finally {
|
|
24359
|
-
|
|
24551
|
+
closeSync3(fd);
|
|
24360
24552
|
}
|
|
24361
24553
|
const previewRows = rows.slice(0, 5);
|
|
24362
24554
|
const previewColumns = columns.slice(0, 5);
|
package/dist/index.js
CHANGED
|
@@ -422,10 +422,10 @@ var SDK_RELEASE = {
|
|
|
422
422
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
423
423
|
// fields shipped in 0.1.153.
|
|
424
424
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
425
|
-
version: "0.1.
|
|
425
|
+
version: "0.1.191",
|
|
426
426
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
427
427
|
supportPolicy: {
|
|
428
|
-
latest: "0.1.
|
|
428
|
+
latest: "0.1.191",
|
|
429
429
|
minimumSupported: "0.1.53",
|
|
430
430
|
deprecatedBelow: "0.1.53",
|
|
431
431
|
commandMinimumSupported: [
|
package/dist/index.mjs
CHANGED
|
@@ -352,10 +352,10 @@ var SDK_RELEASE = {
|
|
|
352
352
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
353
353
|
// fields shipped in 0.1.153.
|
|
354
354
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
355
|
-
version: "0.1.
|
|
355
|
+
version: "0.1.191",
|
|
356
356
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
357
357
|
supportPolicy: {
|
|
358
|
-
latest: "0.1.
|
|
358
|
+
latest: "0.1.191",
|
|
359
359
|
minimumSupported: "0.1.53",
|
|
360
360
|
deprecatedBelow: "0.1.53",
|
|
361
361
|
commandMinimumSupported: [
|