deepline 0.1.212 → 0.1.214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +167 -329
- package/dist/bundling-sources/sdk/src/client.ts +2 -2
- package/dist/bundling-sources/sdk/src/index.ts +2 -0
- package/dist/bundling-sources/sdk/src/play.ts +8 -1
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +11 -1
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +3 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +49 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +57 -23
- package/dist/bundling-sources/shared_libs/play-runtime/completed-receipt-cache.ts +166 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +209 -123
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +30 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +160 -2
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-auth-session.ts +93 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-completion-sink.ts +16 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-state-sink.ts +10 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +80 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +94 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +54 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +477 -85
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-postgres-admission.ts +162 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +16 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +1 -0
- package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -4
- package/dist/cli/index.js +50 -436
- package/dist/cli/index.mjs +50 -436
- package/dist/{compiler-manifest-j6z8qzpd.d.mts → compiler-manifest-BhgZ23A4.d.mts} +1 -0
- package/dist/{compiler-manifest-j6z8qzpd.d.ts → compiler-manifest-BhgZ23A4.d.ts} +1 -0
- package/dist/index.d.mts +11 -8
- package/dist/index.d.ts +11 -8
- package/dist/index.js +4 -4
- package/dist/index.mjs +4 -4
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.214",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.214",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -3047,8 +3047,8 @@ var DeeplineClient = class {
|
|
|
3047
3047
|
...forceToolRefresh ? { forceToolRefresh: true } : {},
|
|
3048
3048
|
...typeof request.waitForCompletionMs === "number" ? { waitForCompletionMs: request.waitForCompletionMs } : {},
|
|
3049
3049
|
// Profile selection is the API's job, not the CLI's. The server
|
|
3050
|
-
// defaults to
|
|
3051
|
-
//
|
|
3050
|
+
// defaults to absurd; callers that need workers_edge pass
|
|
3051
|
+
// `request.profile` explicitly.
|
|
3052
3052
|
...request.profile ? { profile: request.profile } : {},
|
|
3053
3053
|
...integrationMode ? { integrationMode } : {},
|
|
3054
3054
|
...testPolicyOverrides ? { testPolicyOverrides } : {}
|
|
@@ -10785,7 +10785,7 @@ var PLAY_RUNTIME_PROVIDERS = {
|
|
|
10785
10785
|
}
|
|
10786
10786
|
};
|
|
10787
10787
|
function defaultPlayRuntimeProvider() {
|
|
10788
|
-
return PLAY_RUNTIME_PROVIDERS.
|
|
10788
|
+
return PLAY_RUNTIME_PROVIDERS.absurd;
|
|
10789
10789
|
}
|
|
10790
10790
|
function resolvePlayRuntimeProvider(override) {
|
|
10791
10791
|
if (override?.trim()) {
|
|
@@ -14473,7 +14473,7 @@ function writeStartedPlayRun(input2) {
|
|
|
14473
14473
|
);
|
|
14474
14474
|
}
|
|
14475
14475
|
function parsePlayRunOptions(args) {
|
|
14476
|
-
const usage = "Usage: deepline plays run <play-name> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run <play-file.ts> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --file <play-file.ts> [--input '{...}'] [--profile <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --name <name> [--input '{...}'] [--profile <id>] [--live|--latest|--revision-id <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--no-open] [--json] [--full] [--<input> value]\n Production defaults to
|
|
14476
|
+
const usage = "Usage: deepline plays run <play-name> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run <play-file.ts> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --file <play-file.ts> [--input '{...}'] [--profile <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --name <name> [--input '{...}'] [--profile <id>] [--live|--latest|--revision-id <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--no-open] [--json] [--full] [--<input> value]\n Production defaults to absurd. Use --profile workers_edge only for an explicit Cloudflare run.\n Unknown --<input> value flags, such as --limit 5, are passed into play input.\nRun `deepline plays run --help` for idempotent call caching and ctx.dataset guidance.";
|
|
14477
14477
|
let filePath = null;
|
|
14478
14478
|
let playName = null;
|
|
14479
14479
|
let input2 = null;
|
|
@@ -14988,7 +14988,7 @@ async function handlePlayCheck(args) {
|
|
|
14988
14988
|
}
|
|
14989
14989
|
return enrichedResult.valid ? 0 : 1;
|
|
14990
14990
|
}
|
|
14991
|
-
async function handleFileBackedRun(options) {
|
|
14991
|
+
async function handleFileBackedRun(options, hooks) {
|
|
14992
14992
|
if (options.target.kind !== "file") {
|
|
14993
14993
|
throw new Error("Expected a file-backed play run target.");
|
|
14994
14994
|
}
|
|
@@ -15128,6 +15128,7 @@ async function handleFileBackedRun(options) {
|
|
|
15128
15128
|
),
|
|
15129
15129
|
options
|
|
15130
15130
|
);
|
|
15131
|
+
hooks?.onTerminalStatus?.(outputStatus);
|
|
15131
15132
|
traceCliSync(
|
|
15132
15133
|
"cli.play_write_result",
|
|
15133
15134
|
{ targetKind: "file", playName },
|
|
@@ -15177,7 +15178,7 @@ async function resolveNamedRunRevisionId(input2) {
|
|
|
15177
15178
|
}
|
|
15178
15179
|
return null;
|
|
15179
15180
|
}
|
|
15180
|
-
async function handleNamedRun(options) {
|
|
15181
|
+
async function handleNamedRun(options, hooks) {
|
|
15181
15182
|
if (options.target.kind !== "name") {
|
|
15182
15183
|
throw new Error("Expected a named play run target.");
|
|
15183
15184
|
}
|
|
@@ -15296,6 +15297,7 @@ async function handleNamedRun(options) {
|
|
|
15296
15297
|
options,
|
|
15297
15298
|
finalStatus.revisionId ?? selectedRevisionId
|
|
15298
15299
|
);
|
|
15300
|
+
hooks?.onTerminalStatus?.(outputStatus);
|
|
15299
15301
|
traceCliSync(
|
|
15300
15302
|
"cli.play_write_result",
|
|
15301
15303
|
{ targetKind: "name", playName },
|
|
@@ -15333,11 +15335,11 @@ async function handleNamedRun(options) {
|
|
|
15333
15335
|
});
|
|
15334
15336
|
return 0;
|
|
15335
15337
|
}
|
|
15336
|
-
async function handlePlayRun(args) {
|
|
15338
|
+
async function handlePlayRun(args, hooks) {
|
|
15337
15339
|
const options = parsePlayRunOptions(args);
|
|
15338
15340
|
if (options.target.kind === "file") {
|
|
15339
15341
|
if (isFileTarget(options.target.path)) {
|
|
15340
|
-
return handleFileBackedRun(options);
|
|
15342
|
+
return handleFileBackedRun(options, hooks);
|
|
15341
15343
|
}
|
|
15342
15344
|
const resolved = (0, import_node_path11.resolve)(options.target.path);
|
|
15343
15345
|
console.error(`File not found: ${resolved}`);
|
|
@@ -15359,7 +15361,7 @@ async function handlePlayRun(args) {
|
|
|
15359
15361
|
}
|
|
15360
15362
|
return 1;
|
|
15361
15363
|
}
|
|
15362
|
-
return handleNamedRun(options);
|
|
15364
|
+
return handleNamedRun(options, hooks);
|
|
15363
15365
|
}
|
|
15364
15366
|
function parseRunIdPositional(args, usage) {
|
|
15365
15367
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -18786,9 +18788,6 @@ function helperSource() {
|
|
|
18786
18788
|
|
|
18787
18789
|
// src/cli/commands/enrich.ts
|
|
18788
18790
|
var ENRICH_EXPORT_PAGE_SIZE = 5e3;
|
|
18789
|
-
var ENRICH_AUTO_BATCH_ROWS = 200;
|
|
18790
|
-
var ENRICH_HEAVY_PLAY_AUTO_BATCH_ROWS = 25;
|
|
18791
|
-
var ENRICH_NESTED_PROVIDER_AUTO_BATCH_ROWS = 50;
|
|
18792
18791
|
var ENRICH_EXPORT_BACKING_ROWS_WAIT_MS = 6e4;
|
|
18793
18792
|
var ENRICH_EXPORT_BACKING_ROWS_POLL_MS = 1e3;
|
|
18794
18793
|
var ENRICH_CUSTOMER_DB_FAILURE_LOOKUP_RETRY_DELAYS_MS = [1e3, 3e3, 7e3];
|
|
@@ -18801,12 +18800,7 @@ var ENRICH_DEBUG_T0 = Date.now();
|
|
|
18801
18800
|
var GENERATED_ENRICH_ROWS_TABLE_NAMESPACE = "deepline_enrich_rows";
|
|
18802
18801
|
var SDK_ENRICH_TELEMETRY_TIMEOUT_MS = 1e4;
|
|
18803
18802
|
var SDK_ENRICH_TELEMETRY_COMMAND_MAX_LENGTH = 2e4;
|
|
18804
|
-
var
|
|
18805
|
-
"company-to-contact",
|
|
18806
|
-
"company-to-contact-by-role-waterfall",
|
|
18807
|
-
"company_to_contact_by_role_waterfall"
|
|
18808
|
-
]);
|
|
18809
|
-
var HEAVY_ENRICH_AUTO_BATCH_TOOLS = /* @__PURE__ */ new Set([
|
|
18803
|
+
var ENRICH_AI_RUNTIME_COMPACT_TOOLS = /* @__PURE__ */ new Set([
|
|
18810
18804
|
"ai_inference",
|
|
18811
18805
|
"aiinference",
|
|
18812
18806
|
"deeplineagent",
|
|
@@ -19633,9 +19627,18 @@ async function stopGeneratedEnrichRunAfterWatchError(client2, error) {
|
|
|
19633
19627
|
async function runGeneratedEnrichPlay(client2, runArgs, options = {}) {
|
|
19634
19628
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
19635
19629
|
try {
|
|
19636
|
-
|
|
19637
|
-
|
|
19638
|
-
|
|
19630
|
+
let terminalStatus2 = null;
|
|
19631
|
+
const captured2 = await captureStdout(
|
|
19632
|
+
() => handlePlayRun(runArgs, {
|
|
19633
|
+
onTerminalStatus: (status) => {
|
|
19634
|
+
terminalStatus2 = status;
|
|
19635
|
+
}
|
|
19636
|
+
}),
|
|
19637
|
+
{
|
|
19638
|
+
passthrough: options.passthroughStdout
|
|
19639
|
+
}
|
|
19640
|
+
);
|
|
19641
|
+
return { ...captured2, terminalStatus: terminalStatus2 };
|
|
19639
19642
|
} catch (error) {
|
|
19640
19643
|
await stopGeneratedEnrichRunAfterWatchError(client2, error);
|
|
19641
19644
|
if (attempt === 0 && isPlayStartStreamEndedError(error)) {
|
|
@@ -19645,9 +19648,16 @@ async function runGeneratedEnrichPlay(client2, runArgs, options = {}) {
|
|
|
19645
19648
|
throw error;
|
|
19646
19649
|
}
|
|
19647
19650
|
}
|
|
19648
|
-
|
|
19649
|
-
|
|
19650
|
-
|
|
19651
|
+
let terminalStatus = null;
|
|
19652
|
+
const captured = await captureStdout(
|
|
19653
|
+
() => handlePlayRun(runArgs, {
|
|
19654
|
+
onTerminalStatus: (status) => {
|
|
19655
|
+
terminalStatus = status;
|
|
19656
|
+
}
|
|
19657
|
+
}),
|
|
19658
|
+
{ passthrough: options.passthroughStdout }
|
|
19659
|
+
);
|
|
19660
|
+
return { ...captured, terminalStatus };
|
|
19651
19661
|
}
|
|
19652
19662
|
async function writeOutputCsv(outputPath, status, options) {
|
|
19653
19663
|
let rowsInfo = extractCanonicalRowsInfo(status);
|
|
@@ -19730,71 +19740,6 @@ function normalizeEnrichPlayRef(value) {
|
|
|
19730
19740
|
const normalized = value.trim().toLowerCase().replace(/^prebuilt\//, "");
|
|
19731
19741
|
return normalized || null;
|
|
19732
19742
|
}
|
|
19733
|
-
function configContainsHeavyEnrichPlay(config) {
|
|
19734
|
-
const visit = (commands) => {
|
|
19735
|
-
for (const command of commands) {
|
|
19736
|
-
if ("with_waterfall" in command) {
|
|
19737
|
-
if (visit(command.commands)) {
|
|
19738
|
-
return true;
|
|
19739
|
-
}
|
|
19740
|
-
continue;
|
|
19741
|
-
}
|
|
19742
|
-
if (command.disabled) {
|
|
19743
|
-
continue;
|
|
19744
|
-
}
|
|
19745
|
-
const candidates = [
|
|
19746
|
-
command.tool,
|
|
19747
|
-
command.operation,
|
|
19748
|
-
command.play?.ref,
|
|
19749
|
-
command.play?.ref?.replace(/^prebuilt\//, "")
|
|
19750
|
-
];
|
|
19751
|
-
if (candidates.map(normalizeEnrichPlayRef).some(
|
|
19752
|
-
(candidate) => candidate !== null && HEAVY_ENRICH_AUTO_BATCH_PLAYS.has(candidate)
|
|
19753
|
-
)) {
|
|
19754
|
-
return true;
|
|
19755
|
-
}
|
|
19756
|
-
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19757
|
-
const normalizedOperation = normalizeEnrichPlayRef(command.operation);
|
|
19758
|
-
if (isMarkedTestAiInferenceCommand(command) || normalizedTool && HEAVY_ENRICH_AUTO_BATCH_TOOLS.has(normalizedTool) || normalizedOperation && HEAVY_ENRICH_AUTO_BATCH_TOOLS.has(normalizedOperation)) {
|
|
19759
|
-
return true;
|
|
19760
|
-
}
|
|
19761
|
-
}
|
|
19762
|
-
return false;
|
|
19763
|
-
};
|
|
19764
|
-
return visit(config.commands);
|
|
19765
|
-
}
|
|
19766
|
-
function configContainsNestedProviderEnrichWorkload(config) {
|
|
19767
|
-
let hasNestedPlayStep = false;
|
|
19768
|
-
let hasSubrequestProviderStep = false;
|
|
19769
|
-
const visit = (commands) => {
|
|
19770
|
-
for (const command of commands) {
|
|
19771
|
-
if ("with_waterfall" in command) {
|
|
19772
|
-
visit(command.commands);
|
|
19773
|
-
continue;
|
|
19774
|
-
}
|
|
19775
|
-
if (command.disabled) {
|
|
19776
|
-
continue;
|
|
19777
|
-
}
|
|
19778
|
-
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19779
|
-
if (command.play || normalizedTool === "test-play") {
|
|
19780
|
-
hasNestedPlayStep = true;
|
|
19781
|
-
} else if (normalizedTool !== null && normalizedTool !== "run_javascript") {
|
|
19782
|
-
hasSubrequestProviderStep = true;
|
|
19783
|
-
}
|
|
19784
|
-
}
|
|
19785
|
-
};
|
|
19786
|
-
visit(config.commands);
|
|
19787
|
-
return hasNestedPlayStep && hasSubrequestProviderStep;
|
|
19788
|
-
}
|
|
19789
|
-
function enrichAutoBatchRowsForConfig(config) {
|
|
19790
|
-
if (configContainsHeavyEnrichPlay(config)) {
|
|
19791
|
-
return ENRICH_HEAVY_PLAY_AUTO_BATCH_ROWS;
|
|
19792
|
-
}
|
|
19793
|
-
if (configContainsNestedProviderEnrichWorkload(config)) {
|
|
19794
|
-
return ENRICH_NESTED_PROVIDER_AUTO_BATCH_ROWS;
|
|
19795
|
-
}
|
|
19796
|
-
return ENRICH_AUTO_BATCH_ROWS;
|
|
19797
|
-
}
|
|
19798
19743
|
function enrichAiRuntimeCompactAliases(config) {
|
|
19799
19744
|
const aliases = /* @__PURE__ */ new Set();
|
|
19800
19745
|
const visit = (commands) => {
|
|
@@ -19808,7 +19753,7 @@ function enrichAiRuntimeCompactAliases(config) {
|
|
|
19808
19753
|
}
|
|
19809
19754
|
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19810
19755
|
const normalizedOperation = normalizeEnrichPlayRef(command.operation);
|
|
19811
|
-
if (isMarkedTestAiInferenceCommand(command) || normalizedTool &&
|
|
19756
|
+
if (isMarkedTestAiInferenceCommand(command) || normalizedTool && ENRICH_AI_RUNTIME_COMPACT_TOOLS.has(normalizedTool) || normalizedOperation && ENRICH_AI_RUNTIME_COMPACT_TOOLS.has(normalizedOperation)) {
|
|
19812
19757
|
aliases.add(normalizeAlias2(command.alias));
|
|
19813
19758
|
}
|
|
19814
19759
|
}
|
|
@@ -19819,13 +19764,14 @@ function enrichAiRuntimeCompactAliases(config) {
|
|
|
19819
19764
|
async function maybeWriteOutputCsv(input2) {
|
|
19820
19765
|
if (!input2.outputPath) return null;
|
|
19821
19766
|
try {
|
|
19767
|
+
const terminalStatus = readEnrichRunStatus(input2.status);
|
|
19822
19768
|
return await writeOutputCsv(input2.outputPath, input2.status, {
|
|
19823
19769
|
client: input2.client,
|
|
19824
19770
|
config: input2.config,
|
|
19825
19771
|
sourceCsvPath: input2.sourceCsvPath,
|
|
19826
19772
|
rows: input2.rows,
|
|
19827
19773
|
inPlace: input2.inPlace,
|
|
19828
|
-
allowPartial: input2.capturedResult !== 0
|
|
19774
|
+
allowPartial: input2.capturedResult !== 0 || terminalStatus === "failed" || terminalStatus === "cancelled" || hasFailedRowSignal(input2.status)
|
|
19829
19775
|
});
|
|
19830
19776
|
} catch (error) {
|
|
19831
19777
|
if (input2.capturedResult !== 0 && isMissingRowsForCsvExportError(error)) {
|
|
@@ -20218,55 +20164,14 @@ function selectedSourceCsvRange(sourceCsvPath, rows) {
|
|
|
20218
20164
|
sourceRows
|
|
20219
20165
|
};
|
|
20220
20166
|
}
|
|
20221
|
-
function compactRuntimeCsvCell(value) {
|
|
20222
|
-
const parsed = parseMaybeJsonObject(value);
|
|
20223
|
-
const compacted = compactAiInferenceCellForCsv(parsed);
|
|
20224
|
-
return materializeCsvCellValue(compacted);
|
|
20225
|
-
}
|
|
20226
|
-
async function writeSlimBatchedRuntimeCsv(input2) {
|
|
20227
|
-
const cleanRows = readCsvRows(input2.cleanSourceCsvPath).map(
|
|
20228
|
-
(row) => ({ ...row })
|
|
20229
|
-
);
|
|
20230
|
-
const mergeRows = readCsvRows(input2.mergeSourceCsvPath);
|
|
20231
|
-
const preserveJsonStringColumns = /* @__PURE__ */ new Set([
|
|
20232
|
-
...cleanRows.flatMap((row) => Object.keys(row)),
|
|
20233
|
-
...mergeRows.flatMap((row) => Object.keys(row))
|
|
20234
|
-
]);
|
|
20235
|
-
const selectedRange = selectedSourceCsvRange(
|
|
20236
|
-
input2.mergeSourceCsvPath,
|
|
20237
|
-
input2.rows
|
|
20238
|
-
);
|
|
20239
|
-
for (let rowIndex = selectedRange.start; rowIndex <= selectedRange.end; rowIndex += 1) {
|
|
20240
|
-
const mergeRow = mergeRows[rowIndex];
|
|
20241
|
-
if (!mergeRow) {
|
|
20242
|
-
continue;
|
|
20243
|
-
}
|
|
20244
|
-
const baseRow = cleanRows[rowIndex] ?? {};
|
|
20245
|
-
const compactOverlayCell = ([key, value]) => {
|
|
20246
|
-
return [
|
|
20247
|
-
key,
|
|
20248
|
-
input2.compactAliases.has(normalizeAlias2(key)) ? compactRuntimeCsvCell(value) : value
|
|
20249
|
-
];
|
|
20250
|
-
};
|
|
20251
|
-
cleanRows[rowIndex] = {
|
|
20252
|
-
...baseRow,
|
|
20253
|
-
...Object.fromEntries(Object.entries(mergeRow).map(compactOverlayCell))
|
|
20254
|
-
};
|
|
20255
|
-
}
|
|
20256
|
-
const columns = dataExportColumns(
|
|
20257
|
-
dataExportRows(cleanRows, { preserveJsonStringColumns })
|
|
20258
|
-
);
|
|
20259
|
-
await (0, import_promises3.writeFile)(
|
|
20260
|
-
input2.outputPath,
|
|
20261
|
-
csvStringFromRows(cleanRows, columns),
|
|
20262
|
-
"utf8"
|
|
20263
|
-
);
|
|
20264
|
-
}
|
|
20265
20167
|
function selectedSourceCsvRowCount(options) {
|
|
20266
20168
|
if (!options?.sourceCsvPath) {
|
|
20267
20169
|
return null;
|
|
20268
20170
|
}
|
|
20269
|
-
|
|
20171
|
+
const totalRows = readCsvRows(options.sourceCsvPath).length;
|
|
20172
|
+
const start = Math.max(0, options.rows?.rowStart ?? 0);
|
|
20173
|
+
const end = options.rows?.rowEnd === null || options.rows?.rowEnd === void 0 ? totalRows - 1 : Math.min(totalRows - 1, options.rows.rowEnd);
|
|
20174
|
+
return Math.max(0, end - start + 1);
|
|
20270
20175
|
}
|
|
20271
20176
|
function collectStringFields(value, key, output2, depth = 0) {
|
|
20272
20177
|
if (depth > 12 || !value || typeof value !== "object") {
|
|
@@ -20447,13 +20352,9 @@ function firstCollectedStringField(value, key) {
|
|
|
20447
20352
|
collectStringFields(value, key, fields);
|
|
20448
20353
|
return fields.values().next().value ?? null;
|
|
20449
20354
|
}
|
|
20450
|
-
function
|
|
20451
|
-
|
|
20452
|
-
|
|
20453
|
-
`
|
|
20454
|
-
);
|
|
20455
|
-
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
20456
|
-
const error = firstCollectedStringField(input2.status, "error") ?? firstCollectedStringField(input2.status, "message");
|
|
20355
|
+
function emitPlainRunFailure(status) {
|
|
20356
|
+
const runId = extractRunId(status) ?? firstCollectedStringField(status, "runId");
|
|
20357
|
+
const error = firstCollectedStringField(status, "error") ?? firstCollectedStringField(status, "message");
|
|
20457
20358
|
if (runId) {
|
|
20458
20359
|
process.stderr.write(`Run id: ${runId}
|
|
20459
20360
|
`);
|
|
@@ -20584,22 +20485,6 @@ function mergeExportedSheetRow(target, next) {
|
|
|
20584
20485
|
target._metadata = metadata;
|
|
20585
20486
|
}
|
|
20586
20487
|
}
|
|
20587
|
-
function countEnrichedRowsInSourceRange(exportResult, rows) {
|
|
20588
|
-
const start = Math.max(0, rows.rowStart ?? 0);
|
|
20589
|
-
const end = rows.rowEnd === null || rows.rowEnd === void 0 ? Number.MAX_SAFE_INTEGER : Math.max(start, rows.rowEnd);
|
|
20590
|
-
const rowIndexes = /* @__PURE__ */ new Set();
|
|
20591
|
-
for (const row of exportResult.enrichedDataRows) {
|
|
20592
|
-
const rowIndex = sourceRowIndexFromEnrichRow(row, start, end);
|
|
20593
|
-
if (rowIndex !== null) {
|
|
20594
|
-
rowIndexes.add(rowIndex);
|
|
20595
|
-
}
|
|
20596
|
-
}
|
|
20597
|
-
if (rowIndexes.size > 0) {
|
|
20598
|
-
return rowIndexes.size;
|
|
20599
|
-
}
|
|
20600
|
-
const selectedRows = end === Number.MAX_SAFE_INTEGER ? exportResult.enrichedRows : end - start + 1;
|
|
20601
|
-
return Math.min(exportResult.enrichedRows, Math.max(0, selectedRows));
|
|
20602
|
-
}
|
|
20603
20488
|
function coalesceExportableSheetRows(rows, sourceRowStart = 0) {
|
|
20604
20489
|
const mergedRows = [];
|
|
20605
20490
|
const bySourceRowIndex = /* @__PURE__ */ new Map();
|
|
@@ -21425,12 +21310,6 @@ function sidecarEnrichRowsExportPath(outputPath) {
|
|
|
21425
21310
|
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
21426
21311
|
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-rows${ext}`);
|
|
21427
21312
|
}
|
|
21428
|
-
function sidecarEnrichBatchManifestPath(outputPath) {
|
|
21429
|
-
const resolved = (0, import_node_path12.resolve)(outputPath);
|
|
21430
|
-
const ext = (0, import_node_path12.extname)(resolved) || ".csv";
|
|
21431
|
-
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
21432
|
-
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-batches.json`);
|
|
21433
|
-
}
|
|
21434
21313
|
function collectDatasetFollowUpCommands(value, state) {
|
|
21435
21314
|
if (state.depth > 12 || !value || typeof value !== "object" || state.commands.length >= 8) {
|
|
21436
21315
|
return;
|
|
@@ -22487,7 +22366,7 @@ function registerEnrichCommand(program) {
|
|
|
22487
22366
|
const captured2 = await runGeneratedEnrichPlay(client2, runArgs, {
|
|
22488
22367
|
passthroughStdout: input2.passthroughStdout
|
|
22489
22368
|
});
|
|
22490
|
-
const generatedPlayStatus = input2.json ? parseJsonOutput(captured2.stdout) : await resolveWatchedGeneratedPlayStatus({
|
|
22369
|
+
const generatedPlayStatus = input2.json ? parseJsonOutput(captured2.stdout) : captured2.terminalStatus ?? await resolveWatchedGeneratedPlayStatus({
|
|
22491
22370
|
client: client2,
|
|
22492
22371
|
stdout: captured2.stdout,
|
|
22493
22372
|
exitCode: captured2.result
|
|
@@ -22508,274 +22387,6 @@ function registerEnrichCommand(program) {
|
|
|
22508
22387
|
});
|
|
22509
22388
|
return { captured: captured2, status: status2, exportResult: exportResult2 };
|
|
22510
22389
|
};
|
|
22511
|
-
const autoBatchRows = enrichAutoBatchRowsForConfig(config);
|
|
22512
|
-
if (outputPath && selectedRange.count > autoBatchRows) {
|
|
22513
|
-
const chunkCount = Math.ceil(selectedRange.count / autoBatchRows);
|
|
22514
|
-
const batchManifestPath = sidecarEnrichBatchManifestPath(
|
|
22515
|
-
inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath)
|
|
22516
|
-
);
|
|
22517
|
-
const batchManifest = {
|
|
22518
|
-
version: 1,
|
|
22519
|
-
kind: "deepline_enrich_batch_manifest",
|
|
22520
|
-
input: (0, import_node_path12.resolve)(inputCsv),
|
|
22521
|
-
output: inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath),
|
|
22522
|
-
selectedRows: selectedRange.count,
|
|
22523
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22524
|
-
chunkRows: autoBatchRows,
|
|
22525
|
-
chunks: chunkCount,
|
|
22526
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22527
|
-
batches: []
|
|
22528
|
-
};
|
|
22529
|
-
const writeBatchManifest = async () => {
|
|
22530
|
-
batchManifest.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
22531
|
-
await (0, import_promises3.writeFile)(
|
|
22532
|
-
batchManifestPath,
|
|
22533
|
-
`${JSON.stringify(batchManifest, null, 2)}
|
|
22534
|
-
`,
|
|
22535
|
-
"utf8"
|
|
22536
|
-
);
|
|
22537
|
-
};
|
|
22538
|
-
const appendBatchManifestEntry = async (input2) => {
|
|
22539
|
-
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
22540
|
-
batchManifest.batches.push({
|
|
22541
|
-
chunk: input2.chunkIndex + 1,
|
|
22542
|
-
chunks: chunkCount,
|
|
22543
|
-
rows: {
|
|
22544
|
-
rowStart: input2.rows.rowStart,
|
|
22545
|
-
rowEnd: input2.rows.rowEnd
|
|
22546
|
-
},
|
|
22547
|
-
status: readEnrichRunStatus(input2.status),
|
|
22548
|
-
runId,
|
|
22549
|
-
customer_db: enrichCustomerDbJson({
|
|
22550
|
-
status: input2.status,
|
|
22551
|
-
client: client2,
|
|
22552
|
-
outputPath: enrichIssueFollowUpOutputPath ?? input2.exportResult?.path ?? outputPath
|
|
22553
|
-
}),
|
|
22554
|
-
output: enrichOutputJson(input2.exportResult),
|
|
22555
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22556
|
-
});
|
|
22557
|
-
await writeBatchManifest();
|
|
22558
|
-
};
|
|
22559
|
-
await writeBatchManifest();
|
|
22560
|
-
if (!options.json) {
|
|
22561
|
-
process.stderr.write(
|
|
22562
|
-
`Large enrich input selected ${selectedRange.count.toLocaleString()} rows. Running ${chunkCount.toLocaleString()} sequential batch runs of up to ${autoBatchRows.toLocaleString()} rows and merging ${(0, import_node_path12.resolve)(outputPath)}.
|
|
22563
|
-
`
|
|
22564
|
-
);
|
|
22565
|
-
}
|
|
22566
|
-
process.stderr.write(
|
|
22567
|
-
`Batch recovery manifest: ${batchManifestPath}
|
|
22568
|
-
`
|
|
22569
|
-
);
|
|
22570
|
-
let workingMergeSourceCsvPath = sourceCsvPath;
|
|
22571
|
-
let lastStatus = null;
|
|
22572
|
-
const batchStatuses = [];
|
|
22573
|
-
let finalExportResult = null;
|
|
22574
|
-
let totalEnrichedRows = 0;
|
|
22575
|
-
const batchFailureRows = [];
|
|
22576
|
-
const runtimeCompactAliases = enrichAiRuntimeCompactAliases(config);
|
|
22577
|
-
for (let chunkStart = selectedRange.start, chunkIndex = 0; chunkStart <= selectedRange.end; chunkStart += autoBatchRows, chunkIndex += 1) {
|
|
22578
|
-
const chunkRows = {
|
|
22579
|
-
rowStart: chunkStart,
|
|
22580
|
-
rowEnd: Math.min(
|
|
22581
|
-
selectedRange.end,
|
|
22582
|
-
chunkStart + autoBatchRows - 1
|
|
22583
|
-
)
|
|
22584
|
-
};
|
|
22585
|
-
if (!options.json) {
|
|
22586
|
-
process.stderr.write(
|
|
22587
|
-
`Batch ${chunkIndex + 1}/${chunkCount}: rows ${chunkRows.rowStart}:${chunkRows.rowEnd}
|
|
22588
|
-
`
|
|
22589
|
-
);
|
|
22590
|
-
}
|
|
22591
|
-
const runtimeSourceCsvPath = options.inPlace ? sourceCsvPath : workingMergeSourceCsvPath === inputCsv ? inputCsv : (0, import_node_path12.join)(tempDir, `runtime-batch-${chunkIndex}.csv`);
|
|
22592
|
-
if (!options.inPlace && runtimeSourceCsvPath !== inputCsv) {
|
|
22593
|
-
await writeSlimBatchedRuntimeCsv({
|
|
22594
|
-
cleanSourceCsvPath: inputCsv,
|
|
22595
|
-
mergeSourceCsvPath: workingMergeSourceCsvPath,
|
|
22596
|
-
rows: chunkRows,
|
|
22597
|
-
compactAliases: runtimeCompactAliases,
|
|
22598
|
-
outputPath: runtimeSourceCsvPath
|
|
22599
|
-
});
|
|
22600
|
-
}
|
|
22601
|
-
const chunk = await runOne({
|
|
22602
|
-
sourceCsvPath: runtimeSourceCsvPath,
|
|
22603
|
-
mergeSourceCsvPath: workingMergeSourceCsvPath,
|
|
22604
|
-
rows: chunkRows,
|
|
22605
|
-
json: true,
|
|
22606
|
-
passthroughStdout: false,
|
|
22607
|
-
suppressOpen: true
|
|
22608
|
-
});
|
|
22609
|
-
lastStatus = chunk.status;
|
|
22610
|
-
batchStatuses.push(chunk.status);
|
|
22611
|
-
if (chunk.captured.result !== 0) {
|
|
22612
|
-
let currentChunkCommittedExportResult = null;
|
|
22613
|
-
if (chunk.exportResult) {
|
|
22614
|
-
finalExportResult = chunk.exportResult;
|
|
22615
|
-
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
22616
|
-
chunk.exportResult,
|
|
22617
|
-
chunkRows
|
|
22618
|
-
);
|
|
22619
|
-
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
22620
|
-
}
|
|
22621
|
-
const failureReport3 = await maybeEmitEnrichFailureReport({
|
|
22622
|
-
config,
|
|
22623
|
-
rows: batchFailureRows,
|
|
22624
|
-
rowRange: {
|
|
22625
|
-
rowStart: selectedRange.start,
|
|
22626
|
-
rowEnd: selectedRange.end
|
|
22627
|
-
},
|
|
22628
|
-
statusRowRange: chunkRows,
|
|
22629
|
-
client: client2,
|
|
22630
|
-
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
22631
|
-
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
22632
|
-
status: chunk.status,
|
|
22633
|
-
waterfallStatus: batchStatuses,
|
|
22634
|
-
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
22635
|
-
});
|
|
22636
|
-
if (chunk.exportResult) {
|
|
22637
|
-
currentChunkCommittedExportResult = await commitInPlaceOutput(
|
|
22638
|
-
chunk.exportResult
|
|
22639
|
-
);
|
|
22640
|
-
finalExportResult = currentChunkCommittedExportResult;
|
|
22641
|
-
}
|
|
22642
|
-
const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;
|
|
22643
|
-
await appendBatchManifestEntry({
|
|
22644
|
-
chunkIndex,
|
|
22645
|
-
rows: chunkRows,
|
|
22646
|
-
status: chunk.status,
|
|
22647
|
-
exportResult: currentChunkCommittedExportResult
|
|
22648
|
-
});
|
|
22649
|
-
if (options.json) {
|
|
22650
|
-
printJson({
|
|
22651
|
-
ok: false,
|
|
22652
|
-
batch: {
|
|
22653
|
-
chunk: chunkIndex + 1,
|
|
22654
|
-
chunks: chunkCount,
|
|
22655
|
-
rows: chunkRows,
|
|
22656
|
-
manifest: batchManifestPath
|
|
22657
|
-
},
|
|
22658
|
-
result: chunk.status,
|
|
22659
|
-
output: reportedExportResult ? {
|
|
22660
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22661
|
-
selectedRows: selectedRange.count,
|
|
22662
|
-
enrichedRows: totalEnrichedRows,
|
|
22663
|
-
path: reportedExportResult.path,
|
|
22664
|
-
...reportedExportResult.partial ? {
|
|
22665
|
-
rows: reportedExportResult.rows,
|
|
22666
|
-
partial: true
|
|
22667
|
-
} : {}
|
|
22668
|
-
} : null,
|
|
22669
|
-
customer_db: enrichCustomerDbJson({
|
|
22670
|
-
status: chunk.status,
|
|
22671
|
-
client: client2,
|
|
22672
|
-
outputPath: enrichIssueFollowUpOutputPath ?? reportedExportResult?.path ?? outputPath
|
|
22673
|
-
}),
|
|
22674
|
-
...enrichReportJson(failureReport3)
|
|
22675
|
-
});
|
|
22676
|
-
} else {
|
|
22677
|
-
if (reportedExportResult) {
|
|
22678
|
-
process.stderr.write(
|
|
22679
|
-
`Wrote ${reportedExportResult.rows} row(s) to ${reportedExportResult.path}${reportedExportResult.partial ? " (partial run output)" : ""}
|
|
22680
|
-
`
|
|
22681
|
-
);
|
|
22682
|
-
}
|
|
22683
|
-
emitPlainBatchRunFailure({
|
|
22684
|
-
chunkIndex,
|
|
22685
|
-
chunkCount,
|
|
22686
|
-
rows: chunkRows,
|
|
22687
|
-
status: chunk.status
|
|
22688
|
-
});
|
|
22689
|
-
}
|
|
22690
|
-
process.exitCode = failureReport3 ? EXIT_SERVER2 : chunk.captured.result;
|
|
22691
|
-
await completeTelemetry({
|
|
22692
|
-
status: "failed",
|
|
22693
|
-
failedJobs: failureReport3?.jobs.length ?? 1
|
|
22694
|
-
});
|
|
22695
|
-
return;
|
|
22696
|
-
}
|
|
22697
|
-
if (chunk.exportResult) {
|
|
22698
|
-
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
22699
|
-
chunk.exportResult,
|
|
22700
|
-
chunkRows
|
|
22701
|
-
);
|
|
22702
|
-
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
22703
|
-
finalExportResult = options.inPlace ? await commitInPlaceOutput(chunk.exportResult) : chunk.exportResult;
|
|
22704
|
-
await appendBatchManifestEntry({
|
|
22705
|
-
chunkIndex,
|
|
22706
|
-
rows: chunkRows,
|
|
22707
|
-
status: chunk.status,
|
|
22708
|
-
exportResult: finalExportResult
|
|
22709
|
-
});
|
|
22710
|
-
workingMergeSourceCsvPath = outputPath;
|
|
22711
|
-
if (options.inPlace && (chunkRows.rowEnd ?? selectedRange.end) < selectedRange.end) {
|
|
22712
|
-
await prepareInPlaceOutput();
|
|
22713
|
-
}
|
|
22714
|
-
}
|
|
22715
|
-
}
|
|
22716
|
-
const outputRows = readCsvRows(outputPath);
|
|
22717
|
-
const failureReport2 = await maybeEmitEnrichFailureReport({
|
|
22718
|
-
config,
|
|
22719
|
-
rows: batchFailureRows,
|
|
22720
|
-
rowRange: {
|
|
22721
|
-
rowStart: selectedRange.start,
|
|
22722
|
-
rowEnd: selectedRange.end
|
|
22723
|
-
},
|
|
22724
|
-
client: client2,
|
|
22725
|
-
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
22726
|
-
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
22727
|
-
status: lastStatus,
|
|
22728
|
-
waterfallStatus: batchStatuses,
|
|
22729
|
-
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
22730
|
-
});
|
|
22731
|
-
finalExportResult = await commitInPlaceOutput(finalExportResult);
|
|
22732
|
-
if (options.json) {
|
|
22733
|
-
const run = rewriteEnrichJsonStatus({
|
|
22734
|
-
status: lastStatus,
|
|
22735
|
-
config,
|
|
22736
|
-
forceAliases,
|
|
22737
|
-
output: finalExportResult ? {
|
|
22738
|
-
...finalExportResult,
|
|
22739
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22740
|
-
selectedRows: selectedRange.count,
|
|
22741
|
-
enrichedRows: totalEnrichedRows,
|
|
22742
|
-
rows: outputRows.length,
|
|
22743
|
-
enrichedDataRows: outputRows
|
|
22744
|
-
} : null,
|
|
22745
|
-
failureReport: failureReport2
|
|
22746
|
-
});
|
|
22747
|
-
printJson({
|
|
22748
|
-
ok: !failureReport2,
|
|
22749
|
-
run,
|
|
22750
|
-
batch: {
|
|
22751
|
-
chunks: chunkCount,
|
|
22752
|
-
chunkRows: autoBatchRows,
|
|
22753
|
-
selectedRows: selectedRange.count,
|
|
22754
|
-
manifest: batchManifestPath
|
|
22755
|
-
},
|
|
22756
|
-
output: finalExportResult ? {
|
|
22757
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22758
|
-
selectedRows: selectedRange.count,
|
|
22759
|
-
enrichedRows: totalEnrichedRows,
|
|
22760
|
-
path: finalExportResult.path
|
|
22761
|
-
} : null,
|
|
22762
|
-
customer_db: enrichCustomerDbJson({
|
|
22763
|
-
status: lastStatus,
|
|
22764
|
-
client: client2,
|
|
22765
|
-
outputPath: enrichIssueFollowUpOutputPath ?? finalExportResult?.path ?? outputPath
|
|
22766
|
-
}),
|
|
22767
|
-
...enrichReportJson(failureReport2)
|
|
22768
|
-
});
|
|
22769
|
-
}
|
|
22770
|
-
if (failureReport2) {
|
|
22771
|
-
process.exitCode = EXIT_SERVER2;
|
|
22772
|
-
}
|
|
22773
|
-
await completeTelemetry({
|
|
22774
|
-
status: failureReport2 ? "completed_with_failures" : "completed",
|
|
22775
|
-
failedJobs: failureReport2?.jobs.length ?? 0
|
|
22776
|
-
});
|
|
22777
|
-
return;
|
|
22778
|
-
}
|
|
22779
22390
|
const { captured, status, exportResult } = await runOne({
|
|
22780
22391
|
sourceCsvPath,
|
|
22781
22392
|
rows,
|
|
@@ -22791,6 +22402,9 @@ function registerEnrichCommand(program) {
|
|
|
22791
22402
|
`
|
|
22792
22403
|
);
|
|
22793
22404
|
}
|
|
22405
|
+
if (!options.json) {
|
|
22406
|
+
emitPlainRunFailure(status);
|
|
22407
|
+
}
|
|
22794
22408
|
const failureReport2 = await buildEnrichFailureReport({
|
|
22795
22409
|
config,
|
|
22796
22410
|
rows: rowsForFailureReport,
|