deepline 0.1.213 → 0.1.215
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +107 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +85 -10
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-runtime-proxy.ts +34 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +50 -36
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +12 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry.ts +4 -0
- package/dist/bundling-sources/sdk/src/client.ts +1 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -3
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +21 -3
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +73 -28
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +14 -3
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/profiles.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +5 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +66 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +18 -55
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +57 -25
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +17 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/transient-service-error.ts +10 -0
- package/dist/cli/index.js +80 -443
- package/dist/cli/index.mjs +80 -443
- package/dist/index.d.mts +2 -3
- package/dist/index.d.ts +2 -3
- package/dist/index.js +7 -6
- package/dist/index.mjs +7 -6
- package/dist/plays/bundle-play-file.mjs +1 -1
- package/package.json +1 -1
package/dist/cli/index.js
CHANGED
|
@@ -623,10 +623,10 @@ var SDK_RELEASE = {
|
|
|
623
623
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
624
624
|
// fields shipped in 0.1.153.
|
|
625
625
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
626
|
-
version: "0.1.
|
|
626
|
+
version: "0.1.215",
|
|
627
627
|
apiContract: "2026-06-dataset-handle-results-hard-cutover",
|
|
628
628
|
supportPolicy: {
|
|
629
|
-
latest: "0.1.
|
|
629
|
+
latest: "0.1.215",
|
|
630
630
|
minimumSupported: "0.1.53",
|
|
631
631
|
deprecatedBelow: "0.1.53",
|
|
632
632
|
commandMinimumSupported: [
|
|
@@ -1364,8 +1364,10 @@ var SECRET_REDACTION_PLACEHOLDER = "[REDACTED_SECRET]";
|
|
|
1364
1364
|
var BEARER_TOKEN_RE = /\bBearer\s+[A-Za-z0-9._~+/=-]{12,}\b/g;
|
|
1365
1365
|
var JWT_RE = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
|
|
1366
1366
|
var PRIVATE_KEY_RE = /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g;
|
|
1367
|
-
var COMMON_SECRET_RE = /\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs]
|
|
1367
|
+
var COMMON_SECRET_RE = /\b(?:sk|pk|rk|pat|ghp|github_pat|xox[baprs])[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
1368
|
+
var GENERIC_PREFIXED_SECRET_RE = /\b(?:key|token|secret|api[_-]?key)_[A-Za-z0-9_./+=:-]{12,}\b/gi;
|
|
1368
1369
|
var URL_SECRET_VALUE_RE = /\b(?:sk|pk|rk|pat|ghp|github_pat|xox)[A-Za-z0-9_./+=:-]{12,}\b/i;
|
|
1370
|
+
var GENERIC_SECRET_ASSIGNMENT_RE = /\b(?:api[_-]?key|token|secret|password)\b\s*[:=]\s*["']?[^"',\s]{12,}["']?/gi;
|
|
1369
1371
|
var HIGH_ENTROPY_ASSIGNMENT_RE = /\b(?:api[_-]?key|token|secret|password|authorization|access[_-]?token|refresh[_-]?token)\b\s*[:=]\s*["']?[^"',\s]{16,}["']?/gi;
|
|
1370
1372
|
var SENSITIVE_URL_PARAM_RE = /[?#&](?:\w*(?:key|token)|secret|password|authorization|credential|signature|sig)(?:=|$)/i;
|
|
1371
1373
|
function escapeRegExp(value) {
|
|
@@ -1382,7 +1384,7 @@ function redactSecretLikeString(value) {
|
|
|
1382
1384
|
}
|
|
1383
1385
|
if (!URL_SECRET_VALUE_RE.test(value)) return value;
|
|
1384
1386
|
}
|
|
1385
|
-
return value.replace(PRIVATE_KEY_RE, SECRET_REDACTION_PLACEHOLDER).replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`).replace(JWT_RE, SECRET_REDACTION_PLACEHOLDER).replace(COMMON_SECRET_RE, SECRET_REDACTION_PLACEHOLDER).replace(HIGH_ENTROPY_ASSIGNMENT_RE, (match) => {
|
|
1387
|
+
return value.replace(PRIVATE_KEY_RE, SECRET_REDACTION_PLACEHOLDER).replace(BEARER_TOKEN_RE, `Bearer ${SECRET_REDACTION_PLACEHOLDER}`).replace(JWT_RE, SECRET_REDACTION_PLACEHOLDER).replace(COMMON_SECRET_RE, SECRET_REDACTION_PLACEHOLDER).replace(GENERIC_PREFIXED_SECRET_RE, SECRET_REDACTION_PLACEHOLDER).replace(GENERIC_SECRET_ASSIGNMENT_RE, SECRET_REDACTION_PLACEHOLDER).replace(HIGH_ENTROPY_ASSIGNMENT_RE, (match) => {
|
|
1386
1388
|
const separator = match.includes("=") ? "=" : ":";
|
|
1387
1389
|
const [key] = match.split(separator, 1);
|
|
1388
1390
|
return `${key.trim()}${separator}${SECRET_REDACTION_PLACEHOLDER}`;
|
|
@@ -3047,8 +3049,7 @@ var DeeplineClient = class {
|
|
|
3047
3049
|
...forceToolRefresh ? { forceToolRefresh: true } : {},
|
|
3048
3050
|
...typeof request.waitForCompletionMs === "number" ? { waitForCompletionMs: request.waitForCompletionMs } : {},
|
|
3049
3051
|
// Profile selection is the API's job, not the CLI's. The server
|
|
3050
|
-
// defaults to
|
|
3051
|
-
// different profile pass `request.profile` explicitly.
|
|
3052
|
+
// defaults to absurd; callers normally omit this field.
|
|
3052
3053
|
...request.profile ? { profile: request.profile } : {},
|
|
3053
3054
|
...integrationMode ? { integrationMode } : {},
|
|
3054
3055
|
...testPolicyOverrides ? { testPolicyOverrides } : {}
|
|
@@ -10763,6 +10764,18 @@ var PLAY_RUNTIME_PROVIDER_IDS = {
|
|
|
10763
10764
|
workersEdge: "workers_edge",
|
|
10764
10765
|
absurd: "absurd"
|
|
10765
10766
|
};
|
|
10767
|
+
var PLAY_RUNTIME_PROVIDER_DISABLED_CODE = "PLAY_RUNTIME_PROVIDER_DISABLED";
|
|
10768
|
+
var PlayRuntimeProviderDisabledError = class extends Error {
|
|
10769
|
+
code = PLAY_RUNTIME_PROVIDER_DISABLED_CODE;
|
|
10770
|
+
profile;
|
|
10771
|
+
constructor(profile) {
|
|
10772
|
+
super(
|
|
10773
|
+
`Play runtime profile "${profile}" is disabled. Deepline runs now execute on the "absurd" profile.`
|
|
10774
|
+
);
|
|
10775
|
+
this.name = "PlayRuntimeProviderDisabledError";
|
|
10776
|
+
this.profile = profile;
|
|
10777
|
+
}
|
|
10778
|
+
};
|
|
10766
10779
|
var PLAY_RUNTIME_PROVIDERS = {
|
|
10767
10780
|
workers_edge: {
|
|
10768
10781
|
id: PLAY_RUNTIME_PROVIDER_IDS.workersEdge,
|
|
@@ -10785,7 +10798,12 @@ var PLAY_RUNTIME_PROVIDERS = {
|
|
|
10785
10798
|
}
|
|
10786
10799
|
};
|
|
10787
10800
|
function defaultPlayRuntimeProvider() {
|
|
10788
|
-
return PLAY_RUNTIME_PROVIDERS.
|
|
10801
|
+
return PLAY_RUNTIME_PROVIDERS.absurd;
|
|
10802
|
+
}
|
|
10803
|
+
function assertPlayRuntimeProviderEnabled(provider) {
|
|
10804
|
+
if (provider.id === PLAY_RUNTIME_PROVIDER_IDS.workersEdge) {
|
|
10805
|
+
throw new PlayRuntimeProviderDisabledError(provider.id);
|
|
10806
|
+
}
|
|
10789
10807
|
}
|
|
10790
10808
|
function resolvePlayRuntimeProvider(override) {
|
|
10791
10809
|
if (override?.trim()) {
|
|
@@ -10801,17 +10819,22 @@ function resolvePlayRuntimeProvider(override) {
|
|
|
10801
10819
|
}
|
|
10802
10820
|
return defaultPlayRuntimeProvider();
|
|
10803
10821
|
}
|
|
10822
|
+
function resolveEnabledPlayRuntimeProvider(override) {
|
|
10823
|
+
const provider = resolvePlayRuntimeProvider(override);
|
|
10824
|
+
assertPlayRuntimeProviderEnabled(provider);
|
|
10825
|
+
return provider;
|
|
10826
|
+
}
|
|
10804
10827
|
|
|
10805
10828
|
// ../shared_libs/play-runtime/profiles.ts
|
|
10806
10829
|
var PLAY_EXECUTION_PROFILE_IDS = {
|
|
10807
10830
|
...PLAY_RUNTIME_PROVIDER_IDS
|
|
10808
10831
|
};
|
|
10809
10832
|
var PLAY_EXECUTION_PROFILES = PLAY_RUNTIME_PROVIDERS;
|
|
10810
|
-
function
|
|
10833
|
+
function resolveEnabledExecutionProfile(override) {
|
|
10811
10834
|
try {
|
|
10812
|
-
return
|
|
10835
|
+
return resolveEnabledPlayRuntimeProvider(override);
|
|
10813
10836
|
} catch (error) {
|
|
10814
|
-
if (override?.trim()) {
|
|
10837
|
+
if (override?.trim() && error instanceof Error && /Unsupported play runtime provider/.test(error.message)) {
|
|
10815
10838
|
throw new Error(
|
|
10816
10839
|
`Unknown execution profile "${override.trim()}". Expected one of: ${Object.keys(
|
|
10817
10840
|
PLAY_EXECUTION_PROFILES
|
|
@@ -11759,7 +11782,7 @@ async function collectBundledPlayGraph(entryFile, profile = null) {
|
|
|
11759
11782
|
const playBundler = await loadPlayBundler();
|
|
11760
11783
|
const nodes = /* @__PURE__ */ new Map();
|
|
11761
11784
|
const visiting = /* @__PURE__ */ new Set();
|
|
11762
|
-
const artifactKind =
|
|
11785
|
+
const artifactKind = resolveEnabledExecutionProfile(profile).artifactKind;
|
|
11763
11786
|
const visit = async (filePath) => {
|
|
11764
11787
|
const absolutePath = normalizePlayPath(filePath);
|
|
11765
11788
|
const cached = nodes.get(absolutePath);
|
|
@@ -14473,7 +14496,7 @@ function writeStartedPlayRun(input2) {
|
|
|
14473
14496
|
);
|
|
14474
14497
|
}
|
|
14475
14498
|
function parsePlayRunOptions(args) {
|
|
14476
|
-
const usage = "Usage: deepline plays run <play-name> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run <play-file.ts> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --file <play-file.ts> [--input '{...}'] [--profile <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --name <name> [--input '{...}'] [--profile <id>] [--live|--latest|--revision-id <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--no-open] [--json] [--full] [--<input> value]\n
|
|
14499
|
+
const usage = "Usage: deepline plays run <play-name> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run <play-file.ts> [--input '{...}'] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --file <play-file.ts> [--input '{...}'] [--profile <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--full] [--<input> value]\n deepline plays run --name <name> [--input '{...}'] [--profile <id>] [--live|--latest|--revision-id <id>] [--no-wait] [--tail-timeout-ms 30000] [--force] [--no-open] [--json] [--full] [--<input> value]\n Runs default to absurd. The workers_edge profile is disabled.\n Unknown --<input> value flags, such as --limit 5, are passed into play input.\nRun `deepline plays run --help` for idempotent call caching and ctx.dataset guidance.";
|
|
14477
14500
|
let filePath = null;
|
|
14478
14501
|
let playName = null;
|
|
14479
14502
|
let input2 = null;
|
|
@@ -14512,7 +14535,7 @@ function parsePlayRunOptions(args) {
|
|
|
14512
14535
|
throw new Error("--profile requires an execution profile id.");
|
|
14513
14536
|
}
|
|
14514
14537
|
profile = value.trim();
|
|
14515
|
-
|
|
14538
|
+
resolveEnabledExecutionProfile(profile);
|
|
14516
14539
|
index += 1;
|
|
14517
14540
|
continue;
|
|
14518
14541
|
}
|
|
@@ -14988,7 +15011,7 @@ async function handlePlayCheck(args) {
|
|
|
14988
15011
|
}
|
|
14989
15012
|
return enrichedResult.valid ? 0 : 1;
|
|
14990
15013
|
}
|
|
14991
|
-
async function handleFileBackedRun(options) {
|
|
15014
|
+
async function handleFileBackedRun(options, hooks) {
|
|
14992
15015
|
if (options.target.kind !== "file") {
|
|
14993
15016
|
throw new Error("Expected a file-backed play run target.");
|
|
14994
15017
|
}
|
|
@@ -15128,6 +15151,7 @@ async function handleFileBackedRun(options) {
|
|
|
15128
15151
|
),
|
|
15129
15152
|
options
|
|
15130
15153
|
);
|
|
15154
|
+
hooks?.onTerminalStatus?.(outputStatus);
|
|
15131
15155
|
traceCliSync(
|
|
15132
15156
|
"cli.play_write_result",
|
|
15133
15157
|
{ targetKind: "file", playName },
|
|
@@ -15177,7 +15201,7 @@ async function resolveNamedRunRevisionId(input2) {
|
|
|
15177
15201
|
}
|
|
15178
15202
|
return null;
|
|
15179
15203
|
}
|
|
15180
|
-
async function handleNamedRun(options) {
|
|
15204
|
+
async function handleNamedRun(options, hooks) {
|
|
15181
15205
|
if (options.target.kind !== "name") {
|
|
15182
15206
|
throw new Error("Expected a named play run target.");
|
|
15183
15207
|
}
|
|
@@ -15296,6 +15320,7 @@ async function handleNamedRun(options) {
|
|
|
15296
15320
|
options,
|
|
15297
15321
|
finalStatus.revisionId ?? selectedRevisionId
|
|
15298
15322
|
);
|
|
15323
|
+
hooks?.onTerminalStatus?.(outputStatus);
|
|
15299
15324
|
traceCliSync(
|
|
15300
15325
|
"cli.play_write_result",
|
|
15301
15326
|
{ targetKind: "name", playName },
|
|
@@ -15333,11 +15358,11 @@ async function handleNamedRun(options) {
|
|
|
15333
15358
|
});
|
|
15334
15359
|
return 0;
|
|
15335
15360
|
}
|
|
15336
|
-
async function handlePlayRun(args) {
|
|
15361
|
+
async function handlePlayRun(args, hooks) {
|
|
15337
15362
|
const options = parsePlayRunOptions(args);
|
|
15338
15363
|
if (options.target.kind === "file") {
|
|
15339
15364
|
if (isFileTarget(options.target.path)) {
|
|
15340
|
-
return handleFileBackedRun(options);
|
|
15365
|
+
return handleFileBackedRun(options, hooks);
|
|
15341
15366
|
}
|
|
15342
15367
|
const resolved = (0, import_node_path11.resolve)(options.target.path);
|
|
15343
15368
|
console.error(`File not found: ${resolved}`);
|
|
@@ -15359,7 +15384,7 @@ async function handlePlayRun(args) {
|
|
|
15359
15384
|
}
|
|
15360
15385
|
return 1;
|
|
15361
15386
|
}
|
|
15362
|
-
return handleNamedRun(options);
|
|
15387
|
+
return handleNamedRun(options, hooks);
|
|
15363
15388
|
}
|
|
15364
15389
|
function parseRunIdPositional(args, usage) {
|
|
15365
15390
|
for (let index = 0; index < args.length; index += 1) {
|
|
@@ -18786,9 +18811,6 @@ function helperSource() {
|
|
|
18786
18811
|
|
|
18787
18812
|
// src/cli/commands/enrich.ts
|
|
18788
18813
|
var ENRICH_EXPORT_PAGE_SIZE = 5e3;
|
|
18789
|
-
var ENRICH_AUTO_BATCH_ROWS = 200;
|
|
18790
|
-
var ENRICH_HEAVY_PLAY_AUTO_BATCH_ROWS = 25;
|
|
18791
|
-
var ENRICH_NESTED_PROVIDER_AUTO_BATCH_ROWS = 50;
|
|
18792
18814
|
var ENRICH_EXPORT_BACKING_ROWS_WAIT_MS = 6e4;
|
|
18793
18815
|
var ENRICH_EXPORT_BACKING_ROWS_POLL_MS = 1e3;
|
|
18794
18816
|
var ENRICH_CUSTOMER_DB_FAILURE_LOOKUP_RETRY_DELAYS_MS = [1e3, 3e3, 7e3];
|
|
@@ -18801,12 +18823,7 @@ var ENRICH_DEBUG_T0 = Date.now();
|
|
|
18801
18823
|
var GENERATED_ENRICH_ROWS_TABLE_NAMESPACE = "deepline_enrich_rows";
|
|
18802
18824
|
var SDK_ENRICH_TELEMETRY_TIMEOUT_MS = 1e4;
|
|
18803
18825
|
var SDK_ENRICH_TELEMETRY_COMMAND_MAX_LENGTH = 2e4;
|
|
18804
|
-
var
|
|
18805
|
-
"company-to-contact",
|
|
18806
|
-
"company-to-contact-by-role-waterfall",
|
|
18807
|
-
"company_to_contact_by_role_waterfall"
|
|
18808
|
-
]);
|
|
18809
|
-
var HEAVY_ENRICH_AUTO_BATCH_TOOLS = /* @__PURE__ */ new Set([
|
|
18826
|
+
var ENRICH_AI_RUNTIME_COMPACT_TOOLS = /* @__PURE__ */ new Set([
|
|
18810
18827
|
"ai_inference",
|
|
18811
18828
|
"aiinference",
|
|
18812
18829
|
"deeplineagent",
|
|
@@ -19633,9 +19650,18 @@ async function stopGeneratedEnrichRunAfterWatchError(client2, error) {
|
|
|
19633
19650
|
async function runGeneratedEnrichPlay(client2, runArgs, options = {}) {
|
|
19634
19651
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
19635
19652
|
try {
|
|
19636
|
-
|
|
19637
|
-
|
|
19638
|
-
|
|
19653
|
+
let terminalStatus2 = null;
|
|
19654
|
+
const captured2 = await captureStdout(
|
|
19655
|
+
() => handlePlayRun(runArgs, {
|
|
19656
|
+
onTerminalStatus: (status) => {
|
|
19657
|
+
terminalStatus2 = status;
|
|
19658
|
+
}
|
|
19659
|
+
}),
|
|
19660
|
+
{
|
|
19661
|
+
passthrough: options.passthroughStdout
|
|
19662
|
+
}
|
|
19663
|
+
);
|
|
19664
|
+
return { ...captured2, terminalStatus: terminalStatus2 };
|
|
19639
19665
|
} catch (error) {
|
|
19640
19666
|
await stopGeneratedEnrichRunAfterWatchError(client2, error);
|
|
19641
19667
|
if (attempt === 0 && isPlayStartStreamEndedError(error)) {
|
|
@@ -19645,9 +19671,16 @@ async function runGeneratedEnrichPlay(client2, runArgs, options = {}) {
|
|
|
19645
19671
|
throw error;
|
|
19646
19672
|
}
|
|
19647
19673
|
}
|
|
19648
|
-
|
|
19649
|
-
|
|
19650
|
-
|
|
19674
|
+
let terminalStatus = null;
|
|
19675
|
+
const captured = await captureStdout(
|
|
19676
|
+
() => handlePlayRun(runArgs, {
|
|
19677
|
+
onTerminalStatus: (status) => {
|
|
19678
|
+
terminalStatus = status;
|
|
19679
|
+
}
|
|
19680
|
+
}),
|
|
19681
|
+
{ passthrough: options.passthroughStdout }
|
|
19682
|
+
);
|
|
19683
|
+
return { ...captured, terminalStatus };
|
|
19651
19684
|
}
|
|
19652
19685
|
async function writeOutputCsv(outputPath, status, options) {
|
|
19653
19686
|
let rowsInfo = extractCanonicalRowsInfo(status);
|
|
@@ -19730,71 +19763,6 @@ function normalizeEnrichPlayRef(value) {
|
|
|
19730
19763
|
const normalized = value.trim().toLowerCase().replace(/^prebuilt\//, "");
|
|
19731
19764
|
return normalized || null;
|
|
19732
19765
|
}
|
|
19733
|
-
function configContainsHeavyEnrichPlay(config) {
|
|
19734
|
-
const visit = (commands) => {
|
|
19735
|
-
for (const command of commands) {
|
|
19736
|
-
if ("with_waterfall" in command) {
|
|
19737
|
-
if (visit(command.commands)) {
|
|
19738
|
-
return true;
|
|
19739
|
-
}
|
|
19740
|
-
continue;
|
|
19741
|
-
}
|
|
19742
|
-
if (command.disabled) {
|
|
19743
|
-
continue;
|
|
19744
|
-
}
|
|
19745
|
-
const candidates = [
|
|
19746
|
-
command.tool,
|
|
19747
|
-
command.operation,
|
|
19748
|
-
command.play?.ref,
|
|
19749
|
-
command.play?.ref?.replace(/^prebuilt\//, "")
|
|
19750
|
-
];
|
|
19751
|
-
if (candidates.map(normalizeEnrichPlayRef).some(
|
|
19752
|
-
(candidate) => candidate !== null && HEAVY_ENRICH_AUTO_BATCH_PLAYS.has(candidate)
|
|
19753
|
-
)) {
|
|
19754
|
-
return true;
|
|
19755
|
-
}
|
|
19756
|
-
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19757
|
-
const normalizedOperation = normalizeEnrichPlayRef(command.operation);
|
|
19758
|
-
if (isMarkedTestAiInferenceCommand(command) || normalizedTool && HEAVY_ENRICH_AUTO_BATCH_TOOLS.has(normalizedTool) || normalizedOperation && HEAVY_ENRICH_AUTO_BATCH_TOOLS.has(normalizedOperation)) {
|
|
19759
|
-
return true;
|
|
19760
|
-
}
|
|
19761
|
-
}
|
|
19762
|
-
return false;
|
|
19763
|
-
};
|
|
19764
|
-
return visit(config.commands);
|
|
19765
|
-
}
|
|
19766
|
-
function configContainsNestedProviderEnrichWorkload(config) {
|
|
19767
|
-
let hasNestedPlayStep = false;
|
|
19768
|
-
let hasSubrequestProviderStep = false;
|
|
19769
|
-
const visit = (commands) => {
|
|
19770
|
-
for (const command of commands) {
|
|
19771
|
-
if ("with_waterfall" in command) {
|
|
19772
|
-
visit(command.commands);
|
|
19773
|
-
continue;
|
|
19774
|
-
}
|
|
19775
|
-
if (command.disabled) {
|
|
19776
|
-
continue;
|
|
19777
|
-
}
|
|
19778
|
-
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19779
|
-
if (command.play || normalizedTool === "test-play") {
|
|
19780
|
-
hasNestedPlayStep = true;
|
|
19781
|
-
} else if (normalizedTool !== null && normalizedTool !== "run_javascript") {
|
|
19782
|
-
hasSubrequestProviderStep = true;
|
|
19783
|
-
}
|
|
19784
|
-
}
|
|
19785
|
-
};
|
|
19786
|
-
visit(config.commands);
|
|
19787
|
-
return hasNestedPlayStep && hasSubrequestProviderStep;
|
|
19788
|
-
}
|
|
19789
|
-
function enrichAutoBatchRowsForConfig(config) {
|
|
19790
|
-
if (configContainsHeavyEnrichPlay(config)) {
|
|
19791
|
-
return ENRICH_HEAVY_PLAY_AUTO_BATCH_ROWS;
|
|
19792
|
-
}
|
|
19793
|
-
if (configContainsNestedProviderEnrichWorkload(config)) {
|
|
19794
|
-
return ENRICH_NESTED_PROVIDER_AUTO_BATCH_ROWS;
|
|
19795
|
-
}
|
|
19796
|
-
return ENRICH_AUTO_BATCH_ROWS;
|
|
19797
|
-
}
|
|
19798
19766
|
function enrichAiRuntimeCompactAliases(config) {
|
|
19799
19767
|
const aliases = /* @__PURE__ */ new Set();
|
|
19800
19768
|
const visit = (commands) => {
|
|
@@ -19808,7 +19776,7 @@ function enrichAiRuntimeCompactAliases(config) {
|
|
|
19808
19776
|
}
|
|
19809
19777
|
const normalizedTool = normalizeEnrichPlayRef(command.tool);
|
|
19810
19778
|
const normalizedOperation = normalizeEnrichPlayRef(command.operation);
|
|
19811
|
-
if (isMarkedTestAiInferenceCommand(command) || normalizedTool &&
|
|
19779
|
+
if (isMarkedTestAiInferenceCommand(command) || normalizedTool && ENRICH_AI_RUNTIME_COMPACT_TOOLS.has(normalizedTool) || normalizedOperation && ENRICH_AI_RUNTIME_COMPACT_TOOLS.has(normalizedOperation)) {
|
|
19812
19780
|
aliases.add(normalizeAlias2(command.alias));
|
|
19813
19781
|
}
|
|
19814
19782
|
}
|
|
@@ -19819,13 +19787,14 @@ function enrichAiRuntimeCompactAliases(config) {
|
|
|
19819
19787
|
async function maybeWriteOutputCsv(input2) {
|
|
19820
19788
|
if (!input2.outputPath) return null;
|
|
19821
19789
|
try {
|
|
19790
|
+
const terminalStatus = readEnrichRunStatus(input2.status);
|
|
19822
19791
|
return await writeOutputCsv(input2.outputPath, input2.status, {
|
|
19823
19792
|
client: input2.client,
|
|
19824
19793
|
config: input2.config,
|
|
19825
19794
|
sourceCsvPath: input2.sourceCsvPath,
|
|
19826
19795
|
rows: input2.rows,
|
|
19827
19796
|
inPlace: input2.inPlace,
|
|
19828
|
-
allowPartial: input2.capturedResult !== 0
|
|
19797
|
+
allowPartial: input2.capturedResult !== 0 || terminalStatus === "failed" || terminalStatus === "cancelled" || hasFailedRowSignal(input2.status)
|
|
19829
19798
|
});
|
|
19830
19799
|
} catch (error) {
|
|
19831
19800
|
if (input2.capturedResult !== 0 && isMissingRowsForCsvExportError(error)) {
|
|
@@ -20218,55 +20187,14 @@ function selectedSourceCsvRange(sourceCsvPath, rows) {
|
|
|
20218
20187
|
sourceRows
|
|
20219
20188
|
};
|
|
20220
20189
|
}
|
|
20221
|
-
function compactRuntimeCsvCell(value) {
|
|
20222
|
-
const parsed = parseMaybeJsonObject(value);
|
|
20223
|
-
const compacted = compactAiInferenceCellForCsv(parsed);
|
|
20224
|
-
return materializeCsvCellValue(compacted);
|
|
20225
|
-
}
|
|
20226
|
-
async function writeSlimBatchedRuntimeCsv(input2) {
|
|
20227
|
-
const cleanRows = readCsvRows(input2.cleanSourceCsvPath).map(
|
|
20228
|
-
(row) => ({ ...row })
|
|
20229
|
-
);
|
|
20230
|
-
const mergeRows = readCsvRows(input2.mergeSourceCsvPath);
|
|
20231
|
-
const preserveJsonStringColumns = /* @__PURE__ */ new Set([
|
|
20232
|
-
...cleanRows.flatMap((row) => Object.keys(row)),
|
|
20233
|
-
...mergeRows.flatMap((row) => Object.keys(row))
|
|
20234
|
-
]);
|
|
20235
|
-
const selectedRange = selectedSourceCsvRange(
|
|
20236
|
-
input2.mergeSourceCsvPath,
|
|
20237
|
-
input2.rows
|
|
20238
|
-
);
|
|
20239
|
-
for (let rowIndex = selectedRange.start; rowIndex <= selectedRange.end; rowIndex += 1) {
|
|
20240
|
-
const mergeRow = mergeRows[rowIndex];
|
|
20241
|
-
if (!mergeRow) {
|
|
20242
|
-
continue;
|
|
20243
|
-
}
|
|
20244
|
-
const baseRow = cleanRows[rowIndex] ?? {};
|
|
20245
|
-
const compactOverlayCell = ([key, value]) => {
|
|
20246
|
-
return [
|
|
20247
|
-
key,
|
|
20248
|
-
input2.compactAliases.has(normalizeAlias2(key)) ? compactRuntimeCsvCell(value) : value
|
|
20249
|
-
];
|
|
20250
|
-
};
|
|
20251
|
-
cleanRows[rowIndex] = {
|
|
20252
|
-
...baseRow,
|
|
20253
|
-
...Object.fromEntries(Object.entries(mergeRow).map(compactOverlayCell))
|
|
20254
|
-
};
|
|
20255
|
-
}
|
|
20256
|
-
const columns = dataExportColumns(
|
|
20257
|
-
dataExportRows(cleanRows, { preserveJsonStringColumns })
|
|
20258
|
-
);
|
|
20259
|
-
await (0, import_promises3.writeFile)(
|
|
20260
|
-
input2.outputPath,
|
|
20261
|
-
csvStringFromRows(cleanRows, columns),
|
|
20262
|
-
"utf8"
|
|
20263
|
-
);
|
|
20264
|
-
}
|
|
20265
20190
|
function selectedSourceCsvRowCount(options) {
|
|
20266
20191
|
if (!options?.sourceCsvPath) {
|
|
20267
20192
|
return null;
|
|
20268
20193
|
}
|
|
20269
|
-
|
|
20194
|
+
const totalRows = readCsvRows(options.sourceCsvPath).length;
|
|
20195
|
+
const start = Math.max(0, options.rows?.rowStart ?? 0);
|
|
20196
|
+
const end = options.rows?.rowEnd === null || options.rows?.rowEnd === void 0 ? totalRows - 1 : Math.min(totalRows - 1, options.rows.rowEnd);
|
|
20197
|
+
return Math.max(0, end - start + 1);
|
|
20270
20198
|
}
|
|
20271
20199
|
function collectStringFields(value, key, output2, depth = 0) {
|
|
20272
20200
|
if (depth > 12 || !value || typeof value !== "object") {
|
|
@@ -20447,13 +20375,9 @@ function firstCollectedStringField(value, key) {
|
|
|
20447
20375
|
collectStringFields(value, key, fields);
|
|
20448
20376
|
return fields.values().next().value ?? null;
|
|
20449
20377
|
}
|
|
20450
|
-
function
|
|
20451
|
-
|
|
20452
|
-
|
|
20453
|
-
`
|
|
20454
|
-
);
|
|
20455
|
-
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
20456
|
-
const error = firstCollectedStringField(input2.status, "error") ?? firstCollectedStringField(input2.status, "message");
|
|
20378
|
+
function emitPlainRunFailure(status) {
|
|
20379
|
+
const runId = extractRunId(status) ?? firstCollectedStringField(status, "runId");
|
|
20380
|
+
const error = firstCollectedStringField(status, "error") ?? firstCollectedStringField(status, "message");
|
|
20457
20381
|
if (runId) {
|
|
20458
20382
|
process.stderr.write(`Run id: ${runId}
|
|
20459
20383
|
`);
|
|
@@ -20584,22 +20508,6 @@ function mergeExportedSheetRow(target, next) {
|
|
|
20584
20508
|
target._metadata = metadata;
|
|
20585
20509
|
}
|
|
20586
20510
|
}
|
|
20587
|
-
function countEnrichedRowsInSourceRange(exportResult, rows) {
|
|
20588
|
-
const start = Math.max(0, rows.rowStart ?? 0);
|
|
20589
|
-
const end = rows.rowEnd === null || rows.rowEnd === void 0 ? Number.MAX_SAFE_INTEGER : Math.max(start, rows.rowEnd);
|
|
20590
|
-
const rowIndexes = /* @__PURE__ */ new Set();
|
|
20591
|
-
for (const row of exportResult.enrichedDataRows) {
|
|
20592
|
-
const rowIndex = sourceRowIndexFromEnrichRow(row, start, end);
|
|
20593
|
-
if (rowIndex !== null) {
|
|
20594
|
-
rowIndexes.add(rowIndex);
|
|
20595
|
-
}
|
|
20596
|
-
}
|
|
20597
|
-
if (rowIndexes.size > 0) {
|
|
20598
|
-
return rowIndexes.size;
|
|
20599
|
-
}
|
|
20600
|
-
const selectedRows = end === Number.MAX_SAFE_INTEGER ? exportResult.enrichedRows : end - start + 1;
|
|
20601
|
-
return Math.min(exportResult.enrichedRows, Math.max(0, selectedRows));
|
|
20602
|
-
}
|
|
20603
20511
|
function coalesceExportableSheetRows(rows, sourceRowStart = 0) {
|
|
20604
20512
|
const mergedRows = [];
|
|
20605
20513
|
const bySourceRowIndex = /* @__PURE__ */ new Map();
|
|
@@ -21425,12 +21333,6 @@ function sidecarEnrichRowsExportPath(outputPath) {
|
|
|
21425
21333
|
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
21426
21334
|
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-rows${ext}`);
|
|
21427
21335
|
}
|
|
21428
|
-
function sidecarEnrichBatchManifestPath(outputPath) {
|
|
21429
|
-
const resolved = (0, import_node_path12.resolve)(outputPath);
|
|
21430
|
-
const ext = (0, import_node_path12.extname)(resolved) || ".csv";
|
|
21431
|
-
const stem = (0, import_node_path12.basename)(resolved, ext);
|
|
21432
|
-
return (0, import_node_path12.join)((0, import_node_path12.dirname)(resolved), `${stem}.deepline-enrich-batches.json`);
|
|
21433
|
-
}
|
|
21434
21336
|
function collectDatasetFollowUpCommands(value, state) {
|
|
21435
21337
|
if (state.depth > 12 || !value || typeof value !== "object" || state.commands.length >= 8) {
|
|
21436
21338
|
return;
|
|
@@ -22487,7 +22389,7 @@ function registerEnrichCommand(program) {
|
|
|
22487
22389
|
const captured2 = await runGeneratedEnrichPlay(client2, runArgs, {
|
|
22488
22390
|
passthroughStdout: input2.passthroughStdout
|
|
22489
22391
|
});
|
|
22490
|
-
const generatedPlayStatus = input2.json ? parseJsonOutput(captured2.stdout) : await resolveWatchedGeneratedPlayStatus({
|
|
22392
|
+
const generatedPlayStatus = input2.json ? parseJsonOutput(captured2.stdout) : captured2.terminalStatus ?? await resolveWatchedGeneratedPlayStatus({
|
|
22491
22393
|
client: client2,
|
|
22492
22394
|
stdout: captured2.stdout,
|
|
22493
22395
|
exitCode: captured2.result
|
|
@@ -22508,274 +22410,6 @@ function registerEnrichCommand(program) {
|
|
|
22508
22410
|
});
|
|
22509
22411
|
return { captured: captured2, status: status2, exportResult: exportResult2 };
|
|
22510
22412
|
};
|
|
22511
|
-
const autoBatchRows = enrichAutoBatchRowsForConfig(config);
|
|
22512
|
-
if (outputPath && selectedRange.count > autoBatchRows) {
|
|
22513
|
-
const chunkCount = Math.ceil(selectedRange.count / autoBatchRows);
|
|
22514
|
-
const batchManifestPath = sidecarEnrichBatchManifestPath(
|
|
22515
|
-
inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath)
|
|
22516
|
-
);
|
|
22517
|
-
const batchManifest = {
|
|
22518
|
-
version: 1,
|
|
22519
|
-
kind: "deepline_enrich_batch_manifest",
|
|
22520
|
-
input: (0, import_node_path12.resolve)(inputCsv),
|
|
22521
|
-
output: inPlaceFinalOutputPath ?? (0, import_node_path12.resolve)(outputPath),
|
|
22522
|
-
selectedRows: selectedRange.count,
|
|
22523
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22524
|
-
chunkRows: autoBatchRows,
|
|
22525
|
-
chunks: chunkCount,
|
|
22526
|
-
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
22527
|
-
batches: []
|
|
22528
|
-
};
|
|
22529
|
-
const writeBatchManifest = async () => {
|
|
22530
|
-
batchManifest.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
22531
|
-
await (0, import_promises3.writeFile)(
|
|
22532
|
-
batchManifestPath,
|
|
22533
|
-
`${JSON.stringify(batchManifest, null, 2)}
|
|
22534
|
-
`,
|
|
22535
|
-
"utf8"
|
|
22536
|
-
);
|
|
22537
|
-
};
|
|
22538
|
-
const appendBatchManifestEntry = async (input2) => {
|
|
22539
|
-
const runId = extractRunId(input2.status) ?? firstCollectedStringField(input2.status, "runId");
|
|
22540
|
-
batchManifest.batches.push({
|
|
22541
|
-
chunk: input2.chunkIndex + 1,
|
|
22542
|
-
chunks: chunkCount,
|
|
22543
|
-
rows: {
|
|
22544
|
-
rowStart: input2.rows.rowStart,
|
|
22545
|
-
rowEnd: input2.rows.rowEnd
|
|
22546
|
-
},
|
|
22547
|
-
status: readEnrichRunStatus(input2.status),
|
|
22548
|
-
runId,
|
|
22549
|
-
customer_db: enrichCustomerDbJson({
|
|
22550
|
-
status: input2.status,
|
|
22551
|
-
client: client2,
|
|
22552
|
-
outputPath: enrichIssueFollowUpOutputPath ?? input2.exportResult?.path ?? outputPath
|
|
22553
|
-
}),
|
|
22554
|
-
output: enrichOutputJson(input2.exportResult),
|
|
22555
|
-
completedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
22556
|
-
});
|
|
22557
|
-
await writeBatchManifest();
|
|
22558
|
-
};
|
|
22559
|
-
await writeBatchManifest();
|
|
22560
|
-
if (!options.json) {
|
|
22561
|
-
process.stderr.write(
|
|
22562
|
-
`Large enrich input selected ${selectedRange.count.toLocaleString()} rows. Running ${chunkCount.toLocaleString()} sequential batch runs of up to ${autoBatchRows.toLocaleString()} rows and merging ${(0, import_node_path12.resolve)(outputPath)}.
|
|
22563
|
-
`
|
|
22564
|
-
);
|
|
22565
|
-
}
|
|
22566
|
-
process.stderr.write(
|
|
22567
|
-
`Batch recovery manifest: ${batchManifestPath}
|
|
22568
|
-
`
|
|
22569
|
-
);
|
|
22570
|
-
let workingMergeSourceCsvPath = sourceCsvPath;
|
|
22571
|
-
let lastStatus = null;
|
|
22572
|
-
const batchStatuses = [];
|
|
22573
|
-
let finalExportResult = null;
|
|
22574
|
-
let totalEnrichedRows = 0;
|
|
22575
|
-
const batchFailureRows = [];
|
|
22576
|
-
const runtimeCompactAliases = enrichAiRuntimeCompactAliases(config);
|
|
22577
|
-
for (let chunkStart = selectedRange.start, chunkIndex = 0; chunkStart <= selectedRange.end; chunkStart += autoBatchRows, chunkIndex += 1) {
|
|
22578
|
-
const chunkRows = {
|
|
22579
|
-
rowStart: chunkStart,
|
|
22580
|
-
rowEnd: Math.min(
|
|
22581
|
-
selectedRange.end,
|
|
22582
|
-
chunkStart + autoBatchRows - 1
|
|
22583
|
-
)
|
|
22584
|
-
};
|
|
22585
|
-
if (!options.json) {
|
|
22586
|
-
process.stderr.write(
|
|
22587
|
-
`Batch ${chunkIndex + 1}/${chunkCount}: rows ${chunkRows.rowStart}:${chunkRows.rowEnd}
|
|
22588
|
-
`
|
|
22589
|
-
);
|
|
22590
|
-
}
|
|
22591
|
-
const runtimeSourceCsvPath = options.inPlace ? sourceCsvPath : workingMergeSourceCsvPath === inputCsv ? inputCsv : (0, import_node_path12.join)(tempDir, `runtime-batch-${chunkIndex}.csv`);
|
|
22592
|
-
if (!options.inPlace && runtimeSourceCsvPath !== inputCsv) {
|
|
22593
|
-
await writeSlimBatchedRuntimeCsv({
|
|
22594
|
-
cleanSourceCsvPath: inputCsv,
|
|
22595
|
-
mergeSourceCsvPath: workingMergeSourceCsvPath,
|
|
22596
|
-
rows: chunkRows,
|
|
22597
|
-
compactAliases: runtimeCompactAliases,
|
|
22598
|
-
outputPath: runtimeSourceCsvPath
|
|
22599
|
-
});
|
|
22600
|
-
}
|
|
22601
|
-
const chunk = await runOne({
|
|
22602
|
-
sourceCsvPath: runtimeSourceCsvPath,
|
|
22603
|
-
mergeSourceCsvPath: workingMergeSourceCsvPath,
|
|
22604
|
-
rows: chunkRows,
|
|
22605
|
-
json: true,
|
|
22606
|
-
passthroughStdout: false,
|
|
22607
|
-
suppressOpen: true
|
|
22608
|
-
});
|
|
22609
|
-
lastStatus = chunk.status;
|
|
22610
|
-
batchStatuses.push(chunk.status);
|
|
22611
|
-
if (chunk.captured.result !== 0) {
|
|
22612
|
-
let currentChunkCommittedExportResult = null;
|
|
22613
|
-
if (chunk.exportResult) {
|
|
22614
|
-
finalExportResult = chunk.exportResult;
|
|
22615
|
-
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
22616
|
-
chunk.exportResult,
|
|
22617
|
-
chunkRows
|
|
22618
|
-
);
|
|
22619
|
-
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
22620
|
-
}
|
|
22621
|
-
const failureReport3 = await maybeEmitEnrichFailureReport({
|
|
22622
|
-
config,
|
|
22623
|
-
rows: batchFailureRows,
|
|
22624
|
-
rowRange: {
|
|
22625
|
-
rowStart: selectedRange.start,
|
|
22626
|
-
rowEnd: selectedRange.end
|
|
22627
|
-
},
|
|
22628
|
-
statusRowRange: chunkRows,
|
|
22629
|
-
client: client2,
|
|
22630
|
-
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
22631
|
-
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
22632
|
-
status: chunk.status,
|
|
22633
|
-
waterfallStatus: batchStatuses,
|
|
22634
|
-
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
22635
|
-
});
|
|
22636
|
-
if (chunk.exportResult) {
|
|
22637
|
-
currentChunkCommittedExportResult = await commitInPlaceOutput(
|
|
22638
|
-
chunk.exportResult
|
|
22639
|
-
);
|
|
22640
|
-
finalExportResult = currentChunkCommittedExportResult;
|
|
22641
|
-
}
|
|
22642
|
-
const reportedExportResult = currentChunkCommittedExportResult ?? finalExportResult;
|
|
22643
|
-
await appendBatchManifestEntry({
|
|
22644
|
-
chunkIndex,
|
|
22645
|
-
rows: chunkRows,
|
|
22646
|
-
status: chunk.status,
|
|
22647
|
-
exportResult: currentChunkCommittedExportResult
|
|
22648
|
-
});
|
|
22649
|
-
if (options.json) {
|
|
22650
|
-
printJson({
|
|
22651
|
-
ok: false,
|
|
22652
|
-
batch: {
|
|
22653
|
-
chunk: chunkIndex + 1,
|
|
22654
|
-
chunks: chunkCount,
|
|
22655
|
-
rows: chunkRows,
|
|
22656
|
-
manifest: batchManifestPath
|
|
22657
|
-
},
|
|
22658
|
-
result: chunk.status,
|
|
22659
|
-
output: reportedExportResult ? {
|
|
22660
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22661
|
-
selectedRows: selectedRange.count,
|
|
22662
|
-
enrichedRows: totalEnrichedRows,
|
|
22663
|
-
path: reportedExportResult.path,
|
|
22664
|
-
...reportedExportResult.partial ? {
|
|
22665
|
-
rows: reportedExportResult.rows,
|
|
22666
|
-
partial: true
|
|
22667
|
-
} : {}
|
|
22668
|
-
} : null,
|
|
22669
|
-
customer_db: enrichCustomerDbJson({
|
|
22670
|
-
status: chunk.status,
|
|
22671
|
-
client: client2,
|
|
22672
|
-
outputPath: enrichIssueFollowUpOutputPath ?? reportedExportResult?.path ?? outputPath
|
|
22673
|
-
}),
|
|
22674
|
-
...enrichReportJson(failureReport3)
|
|
22675
|
-
});
|
|
22676
|
-
} else {
|
|
22677
|
-
if (reportedExportResult) {
|
|
22678
|
-
process.stderr.write(
|
|
22679
|
-
`Wrote ${reportedExportResult.rows} row(s) to ${reportedExportResult.path}${reportedExportResult.partial ? " (partial run output)" : ""}
|
|
22680
|
-
`
|
|
22681
|
-
);
|
|
22682
|
-
}
|
|
22683
|
-
emitPlainBatchRunFailure({
|
|
22684
|
-
chunkIndex,
|
|
22685
|
-
chunkCount,
|
|
22686
|
-
rows: chunkRows,
|
|
22687
|
-
status: chunk.status
|
|
22688
|
-
});
|
|
22689
|
-
}
|
|
22690
|
-
process.exitCode = failureReport3 ? EXIT_SERVER2 : chunk.captured.result;
|
|
22691
|
-
await completeTelemetry({
|
|
22692
|
-
status: "failed",
|
|
22693
|
-
failedJobs: failureReport3?.jobs.length ?? 1
|
|
22694
|
-
});
|
|
22695
|
-
return;
|
|
22696
|
-
}
|
|
22697
|
-
if (chunk.exportResult) {
|
|
22698
|
-
totalEnrichedRows += countEnrichedRowsInSourceRange(
|
|
22699
|
-
chunk.exportResult,
|
|
22700
|
-
chunkRows
|
|
22701
|
-
);
|
|
22702
|
-
batchFailureRows.push(...chunk.exportResult.enrichedDataRows);
|
|
22703
|
-
finalExportResult = options.inPlace ? await commitInPlaceOutput(chunk.exportResult) : chunk.exportResult;
|
|
22704
|
-
await appendBatchManifestEntry({
|
|
22705
|
-
chunkIndex,
|
|
22706
|
-
rows: chunkRows,
|
|
22707
|
-
status: chunk.status,
|
|
22708
|
-
exportResult: finalExportResult
|
|
22709
|
-
});
|
|
22710
|
-
workingMergeSourceCsvPath = outputPath;
|
|
22711
|
-
if (options.inPlace && (chunkRows.rowEnd ?? selectedRange.end) < selectedRange.end) {
|
|
22712
|
-
await prepareInPlaceOutput();
|
|
22713
|
-
}
|
|
22714
|
-
}
|
|
22715
|
-
}
|
|
22716
|
-
const outputRows = readCsvRows(outputPath);
|
|
22717
|
-
const failureReport2 = await maybeEmitEnrichFailureReport({
|
|
22718
|
-
config,
|
|
22719
|
-
rows: batchFailureRows,
|
|
22720
|
-
rowRange: {
|
|
22721
|
-
rowStart: selectedRange.start,
|
|
22722
|
-
rowEnd: selectedRange.end
|
|
22723
|
-
},
|
|
22724
|
-
client: client2,
|
|
22725
|
-
outputPath: failureReportOutputPath ?? finalExportResult?.path ?? outputPath,
|
|
22726
|
-
followUpOutputPath: enrichIssueFollowUpOutputPath,
|
|
22727
|
-
status: lastStatus,
|
|
22728
|
-
waterfallStatus: batchStatuses,
|
|
22729
|
-
...finalExportResult ? { rowSetComplete: !finalExportResult.partial } : {}
|
|
22730
|
-
});
|
|
22731
|
-
finalExportResult = await commitInPlaceOutput(finalExportResult);
|
|
22732
|
-
if (options.json) {
|
|
22733
|
-
const run = rewriteEnrichJsonStatus({
|
|
22734
|
-
status: lastStatus,
|
|
22735
|
-
config,
|
|
22736
|
-
forceAliases,
|
|
22737
|
-
output: finalExportResult ? {
|
|
22738
|
-
...finalExportResult,
|
|
22739
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22740
|
-
selectedRows: selectedRange.count,
|
|
22741
|
-
enrichedRows: totalEnrichedRows,
|
|
22742
|
-
rows: outputRows.length,
|
|
22743
|
-
enrichedDataRows: outputRows
|
|
22744
|
-
} : null,
|
|
22745
|
-
failureReport: failureReport2
|
|
22746
|
-
});
|
|
22747
|
-
printJson({
|
|
22748
|
-
ok: !failureReport2,
|
|
22749
|
-
run,
|
|
22750
|
-
batch: {
|
|
22751
|
-
chunks: chunkCount,
|
|
22752
|
-
chunkRows: autoBatchRows,
|
|
22753
|
-
selectedRows: selectedRange.count,
|
|
22754
|
-
manifest: batchManifestPath
|
|
22755
|
-
},
|
|
22756
|
-
output: finalExportResult ? {
|
|
22757
|
-
sourceCsvRows: selectedRange.sourceRows,
|
|
22758
|
-
selectedRows: selectedRange.count,
|
|
22759
|
-
enrichedRows: totalEnrichedRows,
|
|
22760
|
-
path: finalExportResult.path
|
|
22761
|
-
} : null,
|
|
22762
|
-
customer_db: enrichCustomerDbJson({
|
|
22763
|
-
status: lastStatus,
|
|
22764
|
-
client: client2,
|
|
22765
|
-
outputPath: enrichIssueFollowUpOutputPath ?? finalExportResult?.path ?? outputPath
|
|
22766
|
-
}),
|
|
22767
|
-
...enrichReportJson(failureReport2)
|
|
22768
|
-
});
|
|
22769
|
-
}
|
|
22770
|
-
if (failureReport2) {
|
|
22771
|
-
process.exitCode = EXIT_SERVER2;
|
|
22772
|
-
}
|
|
22773
|
-
await completeTelemetry({
|
|
22774
|
-
status: failureReport2 ? "completed_with_failures" : "completed",
|
|
22775
|
-
failedJobs: failureReport2?.jobs.length ?? 0
|
|
22776
|
-
});
|
|
22777
|
-
return;
|
|
22778
|
-
}
|
|
22779
22413
|
const { captured, status, exportResult } = await runOne({
|
|
22780
22414
|
sourceCsvPath,
|
|
22781
22415
|
rows,
|
|
@@ -22791,6 +22425,9 @@ function registerEnrichCommand(program) {
|
|
|
22791
22425
|
`
|
|
22792
22426
|
);
|
|
22793
22427
|
}
|
|
22428
|
+
if (!options.json) {
|
|
22429
|
+
emitPlainRunFailure(status);
|
|
22430
|
+
}
|
|
22794
22431
|
const failureReport2 = await buildEnrichFailureReport({
|
|
22795
22432
|
config,
|
|
22796
22433
|
rows: rowsForFailureReport,
|