deepline 0.1.176 → 0.1.178
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +5 -4
- package/dist/cli/index.js +456 -56
- package/dist/cli/index.mjs +464 -64
- package/dist/index.js +5 -4
- package/dist/index.mjs +5 -4
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -622,17 +622,18 @@ var SDK_RELEASE = {
|
|
|
622
622
|
// 0.1.111 ships dataset-native tool list getters and result row datasets.
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
|
-
|
|
625
|
+
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
+
version: "0.1.178",
|
|
626
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
627
628
|
supportPolicy: {
|
|
628
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.178",
|
|
629
630
|
minimumSupported: "0.1.53",
|
|
630
631
|
deprecatedBelow: "0.1.53",
|
|
631
632
|
commandMinimumSupported: [
|
|
632
633
|
{
|
|
633
634
|
command: "enrich",
|
|
634
|
-
minimumSupported: "0.1.
|
|
635
|
-
reason: "Older SDK CLI enrich generated stale play source; SDK CLI enrich 0.1.153 generated removed StepOptions fields in play source."
|
|
635
|
+
minimumSupported: "0.1.175",
|
|
636
|
+
reason: "Older SDK CLI enrich generated stale play source; SDK CLI enrich 0.1.153 generated removed StepOptions fields in play source; SDK CLI enrich before 0.1.175 could silently succeed on the empty-waterfall bug when a requested waterfall returned no values for selected rows."
|
|
636
637
|
},
|
|
637
638
|
{
|
|
638
639
|
command: "plays",
|
|
@@ -16928,7 +16929,8 @@ var HEAVY_ENRICH_AUTO_BATCH_PLAYS = /* @__PURE__ */ new Set([
|
|
|
16928
16929
|
var HEAVY_ENRICH_AUTO_BATCH_TOOLS = /* @__PURE__ */ new Set([
|
|
16929
16930
|
"ai_inference",
|
|
16930
16931
|
"aiinference",
|
|
16931
|
-
"deeplineagent"
|
|
16932
|
+
"deeplineagent",
|
|
16933
|
+
"exa_answer"
|
|
16932
16934
|
]);
|
|
16933
16935
|
var ENRICH_TEST_AI_INFERENCE_PAYLOAD_MARKER = "__deeplineTestAiInference";
|
|
16934
16936
|
var PLAN_SHAPING_OPTION_NAMES = [
|
|
@@ -17839,14 +17841,39 @@ function enrichOutputJson(exportResult) {
|
|
|
17839
17841
|
...exportResult.partial ? { rows: exportResult.rows, partial: true } : {}
|
|
17840
17842
|
};
|
|
17841
17843
|
}
|
|
17844
|
+
function enrichReportJson(report) {
|
|
17845
|
+
if (!report) {
|
|
17846
|
+
return {};
|
|
17847
|
+
}
|
|
17848
|
+
return {
|
|
17849
|
+
...report.jobs.length > 0 ? {
|
|
17850
|
+
failure_report: {
|
|
17851
|
+
path: report.path,
|
|
17852
|
+
jobs: report.jobs.length
|
|
17853
|
+
}
|
|
17854
|
+
} : {},
|
|
17855
|
+
...report.issues.length > 0 ? {
|
|
17856
|
+
enrich_issues: {
|
|
17857
|
+
path: report.path,
|
|
17858
|
+
issues: report.issues.length,
|
|
17859
|
+
items: report.issues
|
|
17860
|
+
}
|
|
17861
|
+
} : {}
|
|
17862
|
+
};
|
|
17863
|
+
}
|
|
17842
17864
|
async function buildEnrichFailureReport(input2) {
|
|
17865
|
+
const fallbackRowsInfo = input2.exportResult ? null : extractCanonicalRowsInfo(input2.status);
|
|
17843
17866
|
return maybeEmitEnrichFailureReport({
|
|
17844
17867
|
config: input2.config,
|
|
17845
17868
|
rows: input2.rows,
|
|
17846
17869
|
rowRange: input2.rowRange,
|
|
17847
17870
|
client: input2.client,
|
|
17848
17871
|
outputPath: input2.reportOutputPath ?? input2.exportResult?.path ?? input2.outputPath ?? null,
|
|
17849
|
-
|
|
17872
|
+
followUpOutputPath: input2.followUpOutputPath,
|
|
17873
|
+
status: input2.status,
|
|
17874
|
+
rowSetComplete: input2.exportResult ? !input2.exportResult.partial : Boolean(
|
|
17875
|
+
fallbackRowsInfo?.complete && input2.rows.length >= fallbackRowsInfo.totalRows
|
|
17876
|
+
)
|
|
17850
17877
|
});
|
|
17851
17878
|
}
|
|
17852
17879
|
function recordField(value, key) {
|
|
@@ -18123,7 +18150,7 @@ function emitPlainBatchRunFailure(input2) {
|
|
|
18123
18150
|
`Batch ${input2.chunkIndex + 1}/${input2.chunkCount} failed for rows ${input2.rows.rowStart}:${input2.rows.rowEnd}.
|
|
18124
18151
|
`
|
|
18125
18152
|
);
|
|
18126
|
-
const runId = extractRunId(input2.status);
|
|
18153
|
+
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
18127
18154
|
const error = firstCollectedStringField(input2.status, "error") ?? firstCollectedStringField(input2.status, "message");
|
|
18128
18155
|
if (runId) {
|
|
18129
18156
|
process.stderr.write(`Run id: ${runId}
|
|
@@ -18339,6 +18366,38 @@ function collectHardFailureAliasSpecs(config) {
|
|
|
18339
18366
|
}
|
|
18340
18367
|
return aliases;
|
|
18341
18368
|
}
|
|
18369
|
+
function collectWaterfallAliasSpecs(config) {
|
|
18370
|
+
const aliases = [];
|
|
18371
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18372
|
+
for (const command of config.commands) {
|
|
18373
|
+
if (!("with_waterfall" in command)) {
|
|
18374
|
+
continue;
|
|
18375
|
+
}
|
|
18376
|
+
const alias = command.with_waterfall.trim();
|
|
18377
|
+
if (!alias || seen.has(alias)) {
|
|
18378
|
+
continue;
|
|
18379
|
+
}
|
|
18380
|
+
const activeChildren = command.commands.filter(
|
|
18381
|
+
(child) => !("with_waterfall" in child) && !child.disabled
|
|
18382
|
+
);
|
|
18383
|
+
if (activeChildren.length === 0) {
|
|
18384
|
+
continue;
|
|
18385
|
+
}
|
|
18386
|
+
seen.add(alias);
|
|
18387
|
+
const childOperations = activeChildren.map((child) => child.operation?.trim() || child.tool.trim()).filter(Boolean);
|
|
18388
|
+
aliases.push({
|
|
18389
|
+
alias,
|
|
18390
|
+
childAliases: activeChildren.map((child) => child.alias).filter(Boolean),
|
|
18391
|
+
childToolsByAlias: new Map(
|
|
18392
|
+
activeChildren.map(
|
|
18393
|
+
(child) => [child.alias, child.tool.trim()]
|
|
18394
|
+
)
|
|
18395
|
+
),
|
|
18396
|
+
operation: childOperations.length > 0 ? childOperations.join(", ") : void 0
|
|
18397
|
+
});
|
|
18398
|
+
}
|
|
18399
|
+
return aliases;
|
|
18400
|
+
}
|
|
18342
18401
|
function hardFailureOperationByAlias(config) {
|
|
18343
18402
|
const operations = /* @__PURE__ */ new Map();
|
|
18344
18403
|
if (!config) {
|
|
@@ -18351,6 +18410,60 @@ function hardFailureOperationByAlias(config) {
|
|
|
18351
18410
|
}
|
|
18352
18411
|
return operations;
|
|
18353
18412
|
}
|
|
18413
|
+
function isWaterfallResultMeaningful(value) {
|
|
18414
|
+
if (Array.isArray(value)) {
|
|
18415
|
+
return value.some(isWaterfallResultMeaningful);
|
|
18416
|
+
}
|
|
18417
|
+
if (value && typeof value === "object") {
|
|
18418
|
+
const record = value;
|
|
18419
|
+
const status = typeof record.status === "string" ? record.status.trim().toLowerCase() : "";
|
|
18420
|
+
if (status === "error" || status === "failed") {
|
|
18421
|
+
return false;
|
|
18422
|
+
}
|
|
18423
|
+
if (typeof record.error === "string" && record.error.trim()) {
|
|
18424
|
+
return false;
|
|
18425
|
+
}
|
|
18426
|
+
if (Object.prototype.hasOwnProperty.call(record, "matched_result")) {
|
|
18427
|
+
return isWaterfallResultMeaningful(record.matched_result);
|
|
18428
|
+
}
|
|
18429
|
+
for (const key of [
|
|
18430
|
+
"value",
|
|
18431
|
+
"email",
|
|
18432
|
+
"phone",
|
|
18433
|
+
"phone_number",
|
|
18434
|
+
"mobile_phone",
|
|
18435
|
+
"mobile_phone_number",
|
|
18436
|
+
"url",
|
|
18437
|
+
"domain",
|
|
18438
|
+
"name",
|
|
18439
|
+
"result",
|
|
18440
|
+
"data"
|
|
18441
|
+
]) {
|
|
18442
|
+
if (isWaterfallResultMeaningful(record[key])) {
|
|
18443
|
+
return true;
|
|
18444
|
+
}
|
|
18445
|
+
}
|
|
18446
|
+
return Object.entries(record).some(
|
|
18447
|
+
([key, entry]) => !ENRICH_FLATTENED_CONTROL_FIELDS.has(key) && isWaterfallResultMeaningful(entry)
|
|
18448
|
+
);
|
|
18449
|
+
}
|
|
18450
|
+
return isNonEmptyCsvCell(value);
|
|
18451
|
+
}
|
|
18452
|
+
function aliasHasMeaningfulResult(row, alias) {
|
|
18453
|
+
return aliasFailureCellCandidates(row, alias).some(
|
|
18454
|
+
isWaterfallResultMeaningful
|
|
18455
|
+
);
|
|
18456
|
+
}
|
|
18457
|
+
function stableRowSnapshot(value) {
|
|
18458
|
+
if (Array.isArray(value)) {
|
|
18459
|
+
return `[${value.map(stableRowSnapshot).join(",")}]`;
|
|
18460
|
+
}
|
|
18461
|
+
if (value && typeof value === "object") {
|
|
18462
|
+
const record = value;
|
|
18463
|
+
return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${stableRowSnapshot(record[key])}`).join(",")}}`;
|
|
18464
|
+
}
|
|
18465
|
+
return JSON.stringify(value);
|
|
18466
|
+
}
|
|
18354
18467
|
function cellFailureError(value) {
|
|
18355
18468
|
const parsed = parseMaybeJsonObject(value);
|
|
18356
18469
|
if (!isRecord7(parsed)) {
|
|
@@ -18496,6 +18609,112 @@ function collectEnrichFailureJobs(input2) {
|
|
|
18496
18609
|
});
|
|
18497
18610
|
return jobs;
|
|
18498
18611
|
}
|
|
18612
|
+
function collectEmptyWaterfallIssues(input2) {
|
|
18613
|
+
const aliases = collectWaterfallAliasSpecs(input2.config);
|
|
18614
|
+
if (aliases.length === 0 || input2.rows.length === 0 || !input2.rowSetComplete) {
|
|
18615
|
+
return [];
|
|
18616
|
+
}
|
|
18617
|
+
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
18618
|
+
const followUpCommands = collectEnrichFollowUpCommands({
|
|
18619
|
+
status: input2.status,
|
|
18620
|
+
runId,
|
|
18621
|
+
outputPath: input2.followUpOutputPath ?? input2.outputPath
|
|
18622
|
+
});
|
|
18623
|
+
const issues = [];
|
|
18624
|
+
aliases.forEach((spec) => {
|
|
18625
|
+
if (waterfallExecutionSignal(input2.status, spec) === "all_condition_skipped") {
|
|
18626
|
+
return;
|
|
18627
|
+
}
|
|
18628
|
+
const logicalRows = /* @__PURE__ */ new Map();
|
|
18629
|
+
input2.rows.forEach((row, rowIndex) => {
|
|
18630
|
+
const sourceRowId = sourceRowIndexFromEnrichRow(
|
|
18631
|
+
row,
|
|
18632
|
+
0,
|
|
18633
|
+
Number.MAX_SAFE_INTEGER
|
|
18634
|
+
);
|
|
18635
|
+
const rowKey = sourceRowId !== null ? `source:${sourceRowId}` : `snapshot:${stableRowSnapshot(row)}`;
|
|
18636
|
+
const existing = logicalRows.get(rowKey);
|
|
18637
|
+
const hasMeaningfulResult = aliasHasMeaningfulResult(row, spec.alias);
|
|
18638
|
+
logicalRows.set(rowKey, {
|
|
18639
|
+
hasMeaningfulResult: (existing?.hasMeaningfulResult ?? false) || hasMeaningfulResult
|
|
18640
|
+
});
|
|
18641
|
+
});
|
|
18642
|
+
const emptyRows = Array.from(logicalRows.values()).filter(
|
|
18643
|
+
(row) => !row.hasMeaningfulResult
|
|
18644
|
+
);
|
|
18645
|
+
if (emptyRows.length !== logicalRows.size) {
|
|
18646
|
+
return;
|
|
18647
|
+
}
|
|
18648
|
+
const selectedRows = logicalRows.size;
|
|
18649
|
+
const message = `Waterfall ${spec.alias} produced 0 values for ${selectedRows} selected row(s).`;
|
|
18650
|
+
issues.push({
|
|
18651
|
+
issue_id: `${spec.alias}-empty-waterfall`,
|
|
18652
|
+
kind: "empty_waterfall",
|
|
18653
|
+
severity: "needs_attention",
|
|
18654
|
+
column: spec.alias,
|
|
18655
|
+
selected_rows: selectedRows,
|
|
18656
|
+
rows_with_value: 0,
|
|
18657
|
+
rows_without_value: selectedRows,
|
|
18658
|
+
message,
|
|
18659
|
+
next_steps: enrichIssueNextSteps({ runId }),
|
|
18660
|
+
follow_up_commands: followUpCommands,
|
|
18661
|
+
...spec.operation ? { operation: spec.operation } : {},
|
|
18662
|
+
...runId ? { run_id: runId } : {}
|
|
18663
|
+
});
|
|
18664
|
+
});
|
|
18665
|
+
return issues;
|
|
18666
|
+
}
|
|
18667
|
+
function waterfallExecutionSignal(status, spec) {
|
|
18668
|
+
const childAliases = spec.childAliases ?? [];
|
|
18669
|
+
if (childAliases.length === 0) {
|
|
18670
|
+
return "unknown";
|
|
18671
|
+
}
|
|
18672
|
+
const summaries = collectColumnStats(status);
|
|
18673
|
+
let sawChildStats = false;
|
|
18674
|
+
let everyChildStatOnlyConditionSkipped = true;
|
|
18675
|
+
const childAliasesWithStats = /* @__PURE__ */ new Set();
|
|
18676
|
+
for (const childAlias of childAliases) {
|
|
18677
|
+
for (const stat2 of summaries.map((summary) => summary[childAlias]).filter(isRecord7)) {
|
|
18678
|
+
sawChildStats = true;
|
|
18679
|
+
childAliasesWithStats.add(childAlias);
|
|
18680
|
+
const execution = isRecord7(stat2.execution) ? stat2.execution : null;
|
|
18681
|
+
const executedSummary = parseExecutionSummary(
|
|
18682
|
+
execution?.["completed:executed"]
|
|
18683
|
+
);
|
|
18684
|
+
const skippedConditionSummary = parseExecutionSummary(
|
|
18685
|
+
execution?.["skipped:condition"]
|
|
18686
|
+
);
|
|
18687
|
+
const executedCount = executedSummary?.count ?? 0;
|
|
18688
|
+
const skippedConditionCount = skippedConditionSummary?.count ?? 0;
|
|
18689
|
+
const totalCount = executedSummary?.total ?? skippedConditionSummary?.total;
|
|
18690
|
+
const onlyConditionSkips = totalCount !== void 0 && skippedConditionCount >= totalCount && executedCount <= skippedConditionCount;
|
|
18691
|
+
if (executedCount > 0 && !onlyConditionSkips || numericFailedRowSignal(execution?.failed) > 0 || numericFailedRowSignal(stat2.failed) > 0 || numericFailedRowSignal(stat2.failedRows) > 0) {
|
|
18692
|
+
return "executed";
|
|
18693
|
+
}
|
|
18694
|
+
if (!onlyConditionSkips) {
|
|
18695
|
+
everyChildStatOnlyConditionSkipped = false;
|
|
18696
|
+
}
|
|
18697
|
+
}
|
|
18698
|
+
}
|
|
18699
|
+
if (sawChildStats && childAliasesWithStats.size === childAliases.length && everyChildStatOnlyConditionSkipped) {
|
|
18700
|
+
return "all_condition_skipped";
|
|
18701
|
+
}
|
|
18702
|
+
return "unknown";
|
|
18703
|
+
}
|
|
18704
|
+
function mergeEnrichFailureJobs(...jobGroups) {
|
|
18705
|
+
const merged = [];
|
|
18706
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18707
|
+
for (const jobs of jobGroups) {
|
|
18708
|
+
for (const job of jobs) {
|
|
18709
|
+
if (seen.has(job.job_id)) {
|
|
18710
|
+
continue;
|
|
18711
|
+
}
|
|
18712
|
+
seen.add(job.job_id);
|
|
18713
|
+
merged.push(job);
|
|
18714
|
+
}
|
|
18715
|
+
}
|
|
18716
|
+
return merged;
|
|
18717
|
+
}
|
|
18499
18718
|
function collectStatusFailureJobs(input2) {
|
|
18500
18719
|
const aliases = collectHardFailureAliasSpecs(input2.config);
|
|
18501
18720
|
if (aliases.length === 0) {
|
|
@@ -18589,14 +18808,23 @@ function collectCompleteStatusFailureMessages(input2) {
|
|
|
18589
18808
|
return complete;
|
|
18590
18809
|
}
|
|
18591
18810
|
function parseExecutionCount(value) {
|
|
18811
|
+
return parseExecutionSummary(value)?.count ?? null;
|
|
18812
|
+
}
|
|
18813
|
+
function parseExecutionSummary(value) {
|
|
18592
18814
|
if (typeof value === "number" && Number.isFinite(value)) {
|
|
18593
|
-
return Math.max(0, Math.trunc(value));
|
|
18815
|
+
return { count: Math.max(0, Math.trunc(value)) };
|
|
18594
18816
|
}
|
|
18595
18817
|
if (typeof value !== "string") {
|
|
18596
18818
|
return null;
|
|
18597
18819
|
}
|
|
18598
|
-
const match = value.trim().match(/^(\d+)\
|
|
18599
|
-
|
|
18820
|
+
const match = value.trim().match(/^(\d+)(?:\s*\/\s*(\d+))?/);
|
|
18821
|
+
if (!match?.[1]) {
|
|
18822
|
+
return null;
|
|
18823
|
+
}
|
|
18824
|
+
return {
|
|
18825
|
+
count: Number.parseInt(match[1], 10),
|
|
18826
|
+
...match[2] ? { total: Number.parseInt(match[2], 10) } : {}
|
|
18827
|
+
};
|
|
18600
18828
|
}
|
|
18601
18829
|
function executionSummaryText(count, total) {
|
|
18602
18830
|
const percent = total > 0 ? Math.round(count / total * 100) : 0;
|
|
@@ -18722,19 +18950,145 @@ function summarizeFailedJobError(value) {
|
|
|
18722
18950
|
const text = String(value ?? "").trim();
|
|
18723
18951
|
return text.length > 500 ? `${text.slice(0, 497)}...` : text;
|
|
18724
18952
|
}
|
|
18953
|
+
function shellSingleQuote2(value) {
|
|
18954
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
18955
|
+
}
|
|
18956
|
+
function addEnrichFollowUpCommand(commands, seen, command) {
|
|
18957
|
+
const normalized = command.command.trim();
|
|
18958
|
+
if (!normalized || seen.has(normalized)) {
|
|
18959
|
+
return;
|
|
18960
|
+
}
|
|
18961
|
+
seen.add(normalized);
|
|
18962
|
+
commands.push({ ...command, command: normalized });
|
|
18963
|
+
}
|
|
18964
|
+
function datasetSelectorForEnrichCommand(record, fallbackPath) {
|
|
18965
|
+
const candidates = [record.path, record.tableNamespace, fallbackPath];
|
|
18966
|
+
for (const candidate of candidates) {
|
|
18967
|
+
if (typeof candidate === "string" && candidate.trim()) {
|
|
18968
|
+
return candidate.trim();
|
|
18969
|
+
}
|
|
18970
|
+
}
|
|
18971
|
+
return null;
|
|
18972
|
+
}
|
|
18973
|
+
function collectEnrichFollowUpCommands(input2) {
|
|
18974
|
+
const commands = [];
|
|
18975
|
+
const seen = /* @__PURE__ */ new Set();
|
|
18976
|
+
if (input2.runId) {
|
|
18977
|
+
addEnrichFollowUpCommand(commands, seen, {
|
|
18978
|
+
label: "inspect run",
|
|
18979
|
+
command: `deepline runs get ${input2.runId} --full --json`
|
|
18980
|
+
});
|
|
18981
|
+
}
|
|
18982
|
+
collectDatasetFollowUpCommands(input2.status, {
|
|
18983
|
+
runId: input2.runId,
|
|
18984
|
+
outputPath: input2.outputPath,
|
|
18985
|
+
commands,
|
|
18986
|
+
seen,
|
|
18987
|
+
path: "result",
|
|
18988
|
+
depth: 0
|
|
18989
|
+
});
|
|
18990
|
+
if (input2.runId && input2.outputPath && !commands.some((command) => command.label === "export enrich rows")) {
|
|
18991
|
+
addEnrichFollowUpCommand(commands, seen, {
|
|
18992
|
+
label: "export enrich rows",
|
|
18993
|
+
path: GENERATED_ENRICH_ROWS_TABLE_NAMESPACE,
|
|
18994
|
+
command: `deepline runs export ${input2.runId} --dataset ${GENERATED_ENRICH_ROWS_TABLE_NAMESPACE} --out ${shellSingleQuote2(
|
|
18995
|
+
(0, import_node_path12.resolve)(input2.outputPath)
|
|
18996
|
+
)}`
|
|
18997
|
+
});
|
|
18998
|
+
}
|
|
18999
|
+
return commands.slice(0, 8);
|
|
19000
|
+
}
|
|
19001
|
+
function sidecarEnrichRowsExportPath(outputPath) {
|
|
19002
|
+
const resolved = (0, import_node_path12.resolve)(outputPath);
|
|
19003
|
+
const ext = (0, import_node_path12.extname)(resolved) || ".csv";
|
|
19004
|
+
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
19005
|
+
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-rows${ext}`);
|
|
19006
|
+
}
|
|
19007
|
+
function collectDatasetFollowUpCommands(value, state) {
|
|
19008
|
+
if (state.depth > 12 || !value || typeof value !== "object" || state.commands.length >= 8) {
|
|
19009
|
+
return;
|
|
19010
|
+
}
|
|
19011
|
+
if (Array.isArray(value)) {
|
|
19012
|
+
value.slice(0, 50).forEach(
|
|
19013
|
+
(entry, index) => collectDatasetFollowUpCommands(entry, {
|
|
19014
|
+
...state,
|
|
19015
|
+
path: `${state.path}.${index}`,
|
|
19016
|
+
depth: state.depth + 1
|
|
19017
|
+
})
|
|
19018
|
+
);
|
|
19019
|
+
return;
|
|
19020
|
+
}
|
|
19021
|
+
const record = value;
|
|
19022
|
+
const isDataset = record.kind === "dataset";
|
|
19023
|
+
if (isDataset) {
|
|
19024
|
+
const selector = datasetSelectorForEnrichCommand(record, state.path);
|
|
19025
|
+
const labelPath = typeof record.path === "string" && record.path.trim() ? record.path.trim() : selector ?? state.path;
|
|
19026
|
+
if (typeof record.queryDatasetCommand === "string") {
|
|
19027
|
+
addEnrichFollowUpCommand(state.commands, state.seen, {
|
|
19028
|
+
label: `query ${labelPath}`,
|
|
19029
|
+
path: labelPath,
|
|
19030
|
+
command: record.queryDatasetCommand
|
|
19031
|
+
});
|
|
19032
|
+
}
|
|
19033
|
+
const exportCommand = typeof record.slowExportAsCsvCommand === "string" ? record.slowExportAsCsvCommand : typeof record.fullExportCommand === "string" ? record.fullExportCommand : null;
|
|
19034
|
+
if (exportCommand) {
|
|
19035
|
+
addEnrichFollowUpCommand(state.commands, state.seen, {
|
|
19036
|
+
label: `export ${labelPath}`,
|
|
19037
|
+
path: labelPath,
|
|
19038
|
+
command: exportCommand
|
|
19039
|
+
});
|
|
19040
|
+
} else if (state.runId && state.outputPath && selector) {
|
|
19041
|
+
addEnrichFollowUpCommand(state.commands, state.seen, {
|
|
19042
|
+
label: selector === GENERATED_ENRICH_ROWS_TABLE_NAMESPACE ? "export enrich rows" : `export ${labelPath}`,
|
|
19043
|
+
path: selector,
|
|
19044
|
+
command: `deepline runs export ${state.runId} --dataset ${shellSingleQuote2(
|
|
19045
|
+
selector
|
|
19046
|
+
)} --out ${shellSingleQuote2((0, import_node_path12.resolve)(state.outputPath))}`
|
|
19047
|
+
});
|
|
19048
|
+
}
|
|
19049
|
+
}
|
|
19050
|
+
for (const [key, child] of Object.entries(record)) {
|
|
19051
|
+
if (key === "preview" || key === "access") {
|
|
19052
|
+
continue;
|
|
19053
|
+
}
|
|
19054
|
+
collectDatasetFollowUpCommands(child, {
|
|
19055
|
+
...state,
|
|
19056
|
+
path: `${state.path}.${key}`,
|
|
19057
|
+
depth: state.depth + 1
|
|
19058
|
+
});
|
|
19059
|
+
if (state.commands.length >= 8) {
|
|
19060
|
+
return;
|
|
19061
|
+
}
|
|
19062
|
+
}
|
|
19063
|
+
}
|
|
19064
|
+
function enrichIssueNextSteps(input2) {
|
|
19065
|
+
const steps = [
|
|
19066
|
+
"Inspect the run output and returned dataset handles.",
|
|
19067
|
+
"If runs get returns queryDatasetCommand or slowExportAsCsvCommand, run that command to fetch durable rows from the customer DB.",
|
|
19068
|
+
"For provider-backed waterfalls, execute the child provider tool on a sample row to confirm the provider returns the expected field."
|
|
19069
|
+
];
|
|
19070
|
+
if (!input2.runId) {
|
|
19071
|
+
return steps;
|
|
19072
|
+
}
|
|
19073
|
+
return [
|
|
19074
|
+
`Inspect run output: deepline runs get ${input2.runId} --full --json`,
|
|
19075
|
+
...steps.slice(1)
|
|
19076
|
+
];
|
|
19077
|
+
}
|
|
18725
19078
|
function formatFailureReportCommand(reportPath) {
|
|
18726
19079
|
const quoted = reportPath.replace(/'/g, `'\\''`);
|
|
18727
19080
|
return `Inspect failed jobs: jq -r '.jobs[] | [.job_id,.row_id,.col_index,.column,.status,.last_error] | @tsv' '${quoted}'`;
|
|
18728
19081
|
}
|
|
18729
19082
|
async function persistEnrichFailureReport(input2) {
|
|
18730
|
-
if (input2.jobs.length === 0) {
|
|
19083
|
+
if (input2.jobs.length === 0 && input2.issues.length === 0) {
|
|
18731
19084
|
return null;
|
|
18732
19085
|
}
|
|
18733
19086
|
const stateDir = (0, import_node_path12.join)((0, import_node_os8.homedir)(), ".local", "deepline", "runtime", "state");
|
|
19087
|
+
const reportPrefix = input2.jobs.length > 0 ? "run-block-failures" : "enrich-issues";
|
|
18734
19088
|
await (0, import_promises3.mkdir)(stateDir, { recursive: true });
|
|
18735
19089
|
const reportPath = (0, import_node_path12.join)(
|
|
18736
19090
|
stateDir,
|
|
18737
|
-
|
|
19091
|
+
`${reportPrefix}-${Math.floor(Date.now() / 1e3)}-${process.pid}.json`
|
|
18738
19092
|
);
|
|
18739
19093
|
const report = {
|
|
18740
19094
|
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -18749,6 +19103,7 @@ async function persistEnrichFailureReport(input2) {
|
|
|
18749
19103
|
},
|
|
18750
19104
|
jobs: input2.jobs,
|
|
18751
19105
|
failed_jobs: input2.jobs,
|
|
19106
|
+
enrich_issues: input2.issues,
|
|
18752
19107
|
canceled_jobs: [],
|
|
18753
19108
|
pending_jobs: [],
|
|
18754
19109
|
missing_job_ids: []
|
|
@@ -18761,48 +19116,100 @@ async function persistEnrichFailureReport(input2) {
|
|
|
18761
19116
|
return reportPath;
|
|
18762
19117
|
}
|
|
18763
19118
|
async function maybeEmitEnrichFailureReport(input2) {
|
|
19119
|
+
const fallbackRowsInfo = extractCanonicalRowsInfo(input2.status);
|
|
19120
|
+
const rowSetComplete = input2.rowSetComplete ?? Boolean(
|
|
19121
|
+
fallbackRowsInfo?.complete && input2.rows.length >= fallbackRowsInfo.totalRows
|
|
19122
|
+
);
|
|
18764
19123
|
const rowJobs = collectEnrichFailureJobs({
|
|
18765
19124
|
config: input2.config,
|
|
18766
19125
|
rows: input2.rows,
|
|
18767
19126
|
rowStart: input2.rowRange.rowStart
|
|
18768
19127
|
});
|
|
18769
|
-
const
|
|
19128
|
+
const enrichIssues = collectEmptyWaterfallIssues({
|
|
19129
|
+
config: input2.config,
|
|
19130
|
+
rows: input2.rows,
|
|
19131
|
+
status: input2.waterfallStatus ?? input2.status,
|
|
19132
|
+
outputPath: input2.outputPath,
|
|
19133
|
+
followUpOutputPath: input2.followUpOutputPath,
|
|
19134
|
+
rowSetComplete
|
|
19135
|
+
});
|
|
19136
|
+
const statusJobs = input2.status === void 0 || rowJobs.length > 0 ? [] : collectStatusFailureJobs({
|
|
18770
19137
|
config: input2.config,
|
|
18771
19138
|
status: input2.status,
|
|
18772
19139
|
rowRange: input2.statusRowRange ?? input2.rowRange
|
|
18773
19140
|
});
|
|
18774
|
-
|
|
19141
|
+
const jobs = rowJobs.length > 0 || statusJobs.length > 0 || input2.status === void 0 ? mergeEnrichFailureJobs(rowJobs, statusJobs) : [];
|
|
19142
|
+
if (jobs.length === 0 && enrichIssues.length === 0) {
|
|
18775
19143
|
return null;
|
|
18776
19144
|
}
|
|
18777
19145
|
const reportPath = await persistEnrichFailureReport({
|
|
18778
19146
|
jobs,
|
|
19147
|
+
issues: enrichIssues,
|
|
18779
19148
|
apiUrl: input2.client.baseUrl,
|
|
18780
19149
|
outputPath: input2.outputPath,
|
|
18781
19150
|
rows: input2.rowRange
|
|
18782
19151
|
});
|
|
18783
|
-
|
|
18784
|
-
|
|
18785
|
-
|
|
18786
|
-
|
|
18787
|
-
|
|
18788
|
-
|
|
19152
|
+
if (jobs.length > 0) {
|
|
19153
|
+
process.stderr.write("Execution failed.\n");
|
|
19154
|
+
process.stderr.write("Failure preview (up to 5 failed jobs):\n");
|
|
19155
|
+
process.stderr.write("job_id row_id col_index column status error\n");
|
|
19156
|
+
for (const job of jobs.slice(0, 5)) {
|
|
19157
|
+
process.stderr.write(
|
|
19158
|
+
`${job.job_id} ${job.row_id} ${job.col_index} ${job.column} ${job.status} ${summarizeFailedJobError(job.last_error)}
|
|
18789
19159
|
`
|
|
18790
|
-
|
|
19160
|
+
);
|
|
19161
|
+
}
|
|
19162
|
+
if (jobs.length > 5) {
|
|
19163
|
+
process.stderr.write(`Showing 5 of ${jobs.length} failed jobs.
|
|
19164
|
+
`);
|
|
19165
|
+
}
|
|
18791
19166
|
}
|
|
18792
|
-
if (
|
|
18793
|
-
process.stderr.write(
|
|
19167
|
+
if (enrichIssues.length > 0) {
|
|
19168
|
+
process.stderr.write("Enrichment needs attention.\n");
|
|
19169
|
+
process.stderr.write("Issue preview (up to 5 enrichment issues):\n");
|
|
19170
|
+
process.stderr.write(
|
|
19171
|
+
"issue_id column severity selected_rows rows_with_value message\n"
|
|
19172
|
+
);
|
|
19173
|
+
for (const issue of enrichIssues.slice(0, 5)) {
|
|
19174
|
+
process.stderr.write(
|
|
19175
|
+
`${issue.issue_id} ${issue.column} ${issue.severity} ${issue.selected_rows} ${issue.rows_with_value} ${issue.message}
|
|
19176
|
+
`
|
|
19177
|
+
);
|
|
19178
|
+
for (const step of issue.next_steps.slice(0, 2)) {
|
|
19179
|
+
process.stderr.write(`Next: ${step}
|
|
19180
|
+
`);
|
|
19181
|
+
}
|
|
19182
|
+
for (const command of issue.follow_up_commands.slice(0, 4)) {
|
|
19183
|
+
process.stderr.write(`Command: ${command.label}: ${command.command}
|
|
18794
19184
|
`);
|
|
19185
|
+
}
|
|
19186
|
+
}
|
|
19187
|
+
if (enrichIssues.length > 5) {
|
|
19188
|
+
process.stderr.write(
|
|
19189
|
+
`Showing 5 of ${enrichIssues.length} enrichment issues.
|
|
19190
|
+
`
|
|
19191
|
+
);
|
|
19192
|
+
}
|
|
18795
19193
|
}
|
|
18796
19194
|
if (reportPath) {
|
|
18797
|
-
|
|
19195
|
+
if (jobs.length > 0) {
|
|
19196
|
+
process.stderr.write(`Full failure report: ${reportPath}
|
|
18798
19197
|
`);
|
|
18799
|
-
|
|
19198
|
+
process.stderr.write(`${formatFailureReportCommand(reportPath)}
|
|
18800
19199
|
`);
|
|
18801
|
-
|
|
18802
|
-
|
|
19200
|
+
} else {
|
|
19201
|
+
process.stderr.write(`Full enrich report: ${reportPath}
|
|
19202
|
+
`);
|
|
19203
|
+
}
|
|
19204
|
+
process.stderr.write(
|
|
19205
|
+
jobs.length > 0 ? "Enrichment failed. See report above.\n" : "Enrichment produced no values for at least one requested waterfall. See issue preview above.\n"
|
|
19206
|
+
);
|
|
19207
|
+
return { path: reportPath, jobs, issues: enrichIssues };
|
|
18803
19208
|
}
|
|
18804
|
-
process.stderr.write(
|
|
18805
|
-
|
|
19209
|
+
process.stderr.write(
|
|
19210
|
+
jobs.length > 0 ? "Enrichment failed.\n" : "Enrichment needs attention.\n"
|
|
19211
|
+
);
|
|
19212
|
+
return { path: "", jobs, issues: enrichIssues };
|
|
18806
19213
|
}
|
|
18807
19214
|
function mergePreferredColumns(preferredColumns, rows) {
|
|
18808
19215
|
const columns = [];
|
|
@@ -19468,6 +19875,7 @@ function registerEnrichCommand(program) {
|
|
|
19468
19875
|
const inPlaceFinalOutputPath = options.inPlace ? (0, import_node_path12.resolve)(inputCsv) : null;
|
|
19469
19876
|
const inPlaceCommitOutputPath = options.inPlace ? (await (0, import_promises3.lstat)(inputCsv)).isSymbolicLink() ? await (0, import_promises3.realpath)(inputCsv) : inPlaceFinalOutputPath : null;
|
|
19470
19877
|
const failureReportOutputPath = options.inPlace ? inPlaceFinalOutputPath : null;
|
|
19878
|
+
const enrichIssueFollowUpOutputPath = options.inPlace && inPlaceFinalOutputPath ? sidecarEnrichRowsExportPath(inPlaceFinalOutputPath) : null;
|
|
19471
19879
|
let inPlaceCommitted = false;
|
|
19472
19880
|
const commitInPlaceOutput = async (exportResult) => {
|
|
19473
19881
|
if (!inPlaceTempOutputPath || !inPlaceCommitOutputPath || !inPlaceFinalOutputPath) {
|
|
@@ -19556,6 +19964,7 @@ function registerEnrichCommand(program) {
|
|
|
19556
19964
|
}
|
|
19557
19965
|
let workingMergeSourceCsvPath = sourceCsvPath;
|
|
19558
19966
|
let lastStatus = null;
|
|
19967
|
+
const batchStatuses = [];
|
|
19559
19968
|
let finalExportResult = null;
|
|
19560
19969
|
let totalEnrichedRows = 0;
|
|
19561
19970
|
const batchFailureRows = [];
|
|
@@ -19593,6 +20002,7 @@ function registerEnrichCommand(program) {
|
|
|
19593
20002
|
suppressOpen: true
|
|
19594
20003
|
});
|
|
19595
20004
|
lastStatus = chunk.status;
|
|
20005
|
+
batchStatuses.push(chunk.status);
|
|
19596
20006
|
if (chunk.captured.result !== 0) {
|
|
19597
20007
|
if (chunk.exportResult) {
|
|
19598
20008
|
finalExportResult = chunk.exportResult;
|
|
@@ -19612,7 +20022,10 @@ function registerEnrichCommand(program) {
|
|
|
19612
20022
|
statusRowRange: chunkRows,
|
|
19613
20023
|
client: client2,
|
|
19614
20024
|
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
19615
|
-
|
|
20025
|
+
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
20026
|
+
status: chunk.status,
|
|
20027
|
+
waterfallStatus: batchStatuses,
|
|
20028
|
+
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
19616
20029
|
});
|
|
19617
20030
|
const committedExportResult2 = finalExportResult ? await commitInPlaceOutput(finalExportResult) : null;
|
|
19618
20031
|
if (options.json) {
|
|
@@ -19634,12 +20047,7 @@ function registerEnrichCommand(program) {
|
|
|
19634
20047
|
partial: true
|
|
19635
20048
|
} : {}
|
|
19636
20049
|
} : null,
|
|
19637
|
-
...failureReport3
|
|
19638
|
-
failure_report: {
|
|
19639
|
-
path: failureReport3.path,
|
|
19640
|
-
jobs: failureReport3.jobs.length
|
|
19641
|
-
}
|
|
19642
|
-
} : {}
|
|
20050
|
+
...enrichReportJson(failureReport3)
|
|
19643
20051
|
});
|
|
19644
20052
|
} else {
|
|
19645
20053
|
if (committedExportResult2) {
|
|
@@ -19678,7 +20086,10 @@ function registerEnrichCommand(program) {
|
|
|
19678
20086
|
},
|
|
19679
20087
|
client: client2,
|
|
19680
20088
|
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
19681
|
-
|
|
20089
|
+
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
20090
|
+
status: lastStatus,
|
|
20091
|
+
waterfallStatus: batchStatuses,
|
|
20092
|
+
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
19682
20093
|
});
|
|
19683
20094
|
finalExportResult = await commitInPlaceOutput(finalExportResult);
|
|
19684
20095
|
if (options.json) {
|
|
@@ -19710,12 +20121,7 @@ function registerEnrichCommand(program) {
|
|
|
19710
20121
|
enrichedRows: totalEnrichedRows,
|
|
19711
20122
|
path: finalExportResult.path
|
|
19712
20123
|
} : null,
|
|
19713
|
-
...failureReport2
|
|
19714
|
-
failure_report: {
|
|
19715
|
-
path: failureReport2.path,
|
|
19716
|
-
jobs: failureReport2.jobs.length
|
|
19717
|
-
}
|
|
19718
|
-
} : {}
|
|
20124
|
+
...enrichReportJson(failureReport2)
|
|
19719
20125
|
});
|
|
19720
20126
|
}
|
|
19721
20127
|
if (failureReport2) {
|
|
@@ -19746,6 +20152,7 @@ function registerEnrichCommand(program) {
|
|
|
19746
20152
|
exportResult: committedExportResult2,
|
|
19747
20153
|
outputPath,
|
|
19748
20154
|
reportOutputPath: failureReportOutputPath,
|
|
20155
|
+
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
19749
20156
|
status
|
|
19750
20157
|
});
|
|
19751
20158
|
if (options.json) {
|
|
@@ -19753,12 +20160,7 @@ function registerEnrichCommand(program) {
|
|
|
19753
20160
|
ok: false,
|
|
19754
20161
|
run: status,
|
|
19755
20162
|
output: enrichOutputJson(committedExportResult2),
|
|
19756
|
-
...failureReport2
|
|
19757
|
-
failure_report: {
|
|
19758
|
-
path: failureReport2.path,
|
|
19759
|
-
jobs: failureReport2.jobs.length
|
|
19760
|
-
}
|
|
19761
|
-
} : {}
|
|
20163
|
+
...enrichReportJson(failureReport2)
|
|
19762
20164
|
});
|
|
19763
20165
|
}
|
|
19764
20166
|
process.exitCode = failureReport2 ? EXIT_SERVER2 : captured.result;
|
|
@@ -19773,6 +20175,7 @@ function registerEnrichCommand(program) {
|
|
|
19773
20175
|
exportResult,
|
|
19774
20176
|
outputPath,
|
|
19775
20177
|
reportOutputPath: failureReportOutputPath,
|
|
20178
|
+
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
19776
20179
|
status
|
|
19777
20180
|
});
|
|
19778
20181
|
const committedExportResult2 = await commitInPlaceOutput(exportResult);
|
|
@@ -19787,12 +20190,7 @@ function registerEnrichCommand(program) {
|
|
|
19787
20190
|
ok: !failureReport2,
|
|
19788
20191
|
run,
|
|
19789
20192
|
output: enrichOutputJson(committedExportResult2),
|
|
19790
|
-
...failureReport2
|
|
19791
|
-
failure_report: {
|
|
19792
|
-
path: failureReport2.path,
|
|
19793
|
-
jobs: failureReport2.jobs.length
|
|
19794
|
-
}
|
|
19795
|
-
} : {}
|
|
20193
|
+
...enrichReportJson(failureReport2)
|
|
19796
20194
|
});
|
|
19797
20195
|
if (failureReport2) {
|
|
19798
20196
|
process.exitCode = EXIT_SERVER2;
|
|
@@ -19805,7 +20203,9 @@ function registerEnrichCommand(program) {
|
|
|
19805
20203
|
rowRange: rows,
|
|
19806
20204
|
client: client2,
|
|
19807
20205
|
outputPath: failureReportOutputPath ?? exportResult?.path ?? null,
|
|
19808
|
-
|
|
20206
|
+
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
20207
|
+
status,
|
|
20208
|
+
...exportResult ? { rowSetComplete: !exportResult.partial } : {}
|
|
19809
20209
|
});
|
|
19810
20210
|
const committedExportResult = await commitInPlaceOutput(exportResult);
|
|
19811
20211
|
if (committedExportResult) {
|