botmux-workflow-core 3.56.0 → 3.57.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/control.cjs +9 -0
- package/dist/cjs/control.cjs.map +2 -2
- package/dist/cjs/engine.cjs.map +2 -2
- package/dist/cjs/gate-policy.cjs.map +1 -1
- package/dist/cjs/index.cjs +44 -1
- package/dist/cjs/index.cjs.map +2 -2
- package/dist/cjs/runtime.cjs +2028 -356
- package/dist/cjs/runtime.cjs.map +4 -4
- package/dist/cjs/schema.cjs +35 -1
- package/dist/cjs/schema.cjs.map +2 -2
- package/dist/esm/control.js +9 -0
- package/dist/esm/control.js.map +2 -2
- package/dist/esm/engine.js.map +2 -2
- package/dist/esm/gate-policy.js.map +1 -1
- package/dist/esm/index.js +44 -1
- package/dist/esm/index.js.map +2 -2
- package/dist/esm/runtime.js +2033 -352
- package/dist/esm/runtime.js.map +4 -4
- package/dist/esm/schema.js +35 -1
- package/dist/esm/schema.js.map +2 -2
- package/dist/types/src/workflows/v3/dag.d.ts +16 -0
- package/dist/types/src/workflows/v3/event-contract.d.ts +263 -0
- package/package.json +1 -1
package/dist/esm/runtime.js
CHANGED
|
@@ -445,6 +445,12 @@ function validateDag(raw) {
|
|
|
445
445
|
if (n.bot !== void 0) problems.push(`host node "${id}".bot is not supported \u2014 host nodes do not spawn a CLI`);
|
|
446
446
|
if (n.override !== void 0) problems.push(`host node "${id}".override is not supported`);
|
|
447
447
|
if (n.revisitTo !== void 0) problems.push(`host node "${id}".revisitTo is not supported`);
|
|
448
|
+
if (n.checkpoint !== void 0) {
|
|
449
|
+
problems.push(`host node "${id}".checkpoint is not supported \u2014 only goal nodes can be checkpoints`);
|
|
450
|
+
}
|
|
451
|
+
if (n.placement !== void 0) {
|
|
452
|
+
problems.push(`host node "${id}".placement is not supported \u2014 host nodes are executed by the main bot, never handed off`);
|
|
453
|
+
}
|
|
448
454
|
if (n.resultSchema !== void 0) {
|
|
449
455
|
problems.push(`host node "${id}".resultSchema is not supported \u2014 host output uses the trusted executor result contract`);
|
|
450
456
|
}
|
|
@@ -516,6 +522,8 @@ function validateDag(raw) {
|
|
|
516
522
|
const humanGate = normHumanGate(n.humanGate, `node "${id}"`, problems);
|
|
517
523
|
const override = normOverride(n.override, `node "${id}"`, problems);
|
|
518
524
|
const revisitTo = normRevisitTo(n.revisitTo, id, problems);
|
|
525
|
+
const checkpoint = normCheckpoint(n.checkpoint, `goal node "${id}"`, problems);
|
|
526
|
+
const placement = normPlacement(n.placement, `goal node "${id}"`, problems);
|
|
519
527
|
if (schemaVersion !== 2 && n.outputs !== void 0) {
|
|
520
528
|
problems.push(`goal node "${id}".outputs requires schemaVersion 2`);
|
|
521
529
|
}
|
|
@@ -533,7 +541,9 @@ function validateDag(raw) {
|
|
|
533
541
|
timeoutSec,
|
|
534
542
|
humanGate,
|
|
535
543
|
resultSchema,
|
|
536
|
-
...revisitTo ? { revisitTo } : {}
|
|
544
|
+
...revisitTo ? { revisitTo } : {},
|
|
545
|
+
...checkpoint ? { checkpoint } : {},
|
|
546
|
+
...placement ? { placement } : {}
|
|
537
547
|
});
|
|
538
548
|
}
|
|
539
549
|
for (const node of nodes) {
|
|
@@ -903,6 +913,18 @@ function normTimeoutSec(v, where, problems) {
|
|
|
903
913
|
}
|
|
904
914
|
return v;
|
|
905
915
|
}
|
|
916
|
+
function normPlacement(v, where, problems) {
|
|
917
|
+
if (v === void 0 || v === "topic") return void 0;
|
|
918
|
+
if (v === "main") return "main";
|
|
919
|
+
problems.push(`${where}.placement must be "topic" or "main"`);
|
|
920
|
+
return void 0;
|
|
921
|
+
}
|
|
922
|
+
function normCheckpoint(v, where, problems) {
|
|
923
|
+
if (v === void 0 || v === false) return void 0;
|
|
924
|
+
if (v === true) return true;
|
|
925
|
+
problems.push(`${where}.checkpoint must be a boolean`);
|
|
926
|
+
return void 0;
|
|
927
|
+
}
|
|
906
928
|
function normRevisitTo(v, id, problems) {
|
|
907
929
|
if (v === void 0) return void 0;
|
|
908
930
|
if (!Array.isArray(v)) {
|
|
@@ -971,6 +993,12 @@ function normLoopFields(n, id, problems, schemaVersion) {
|
|
|
971
993
|
if (n.override !== void 0) {
|
|
972
994
|
problems.push(`${where}.override is not supported \u2014 a loop composite never spawns a worker; set override on body nodes instead`);
|
|
973
995
|
}
|
|
996
|
+
if (n.checkpoint !== void 0) {
|
|
997
|
+
problems.push(`${where}.checkpoint is not supported \u2014 mark a goal node outside the loop instead`);
|
|
998
|
+
}
|
|
999
|
+
if (n.placement !== void 0) {
|
|
1000
|
+
problems.push(`${where}.placement is not supported \u2014 a loop's body nodes always run in node topics`);
|
|
1001
|
+
}
|
|
974
1002
|
let maxIterations;
|
|
975
1003
|
if (typeof n.maxIterations !== "number" || !Number.isInteger(n.maxIterations) || n.maxIterations < 1) {
|
|
976
1004
|
problems.push(`${where}.maxIterations must be a positive integer`);
|
|
@@ -1017,6 +1045,12 @@ function normLoopFields(n, id, problems, schemaVersion) {
|
|
|
1017
1045
|
if (b.triggerRule !== void 0) {
|
|
1018
1046
|
problems.push(`${where}.body node "${bid}".triggerRule is not supported inside a loop body (first cut)`);
|
|
1019
1047
|
}
|
|
1048
|
+
if (b.checkpoint !== void 0) {
|
|
1049
|
+
problems.push(`${where}.body node "${bid}".checkpoint is not supported inside a loop body (first cut)`);
|
|
1050
|
+
}
|
|
1051
|
+
if (b.placement !== void 0) {
|
|
1052
|
+
problems.push(`${where}.body node "${bid}".placement is not supported \u2014 loop body nodes always run in node topics`);
|
|
1053
|
+
}
|
|
1020
1054
|
const bdepends = normDepends(b.depends, `${where}.body node "${bid}"`, problems);
|
|
1021
1055
|
const bFromList = bdepends.map((d) => d.from);
|
|
1022
1056
|
if (bFromList.includes(bid)) problems.push(`${where}.body node "${bid}" depends on itself`);
|
|
@@ -2606,6 +2640,7 @@ function materialize(events) {
|
|
|
2606
2640
|
const attempts = /* @__PURE__ */ new Map();
|
|
2607
2641
|
const loops = /* @__PURE__ */ new Map();
|
|
2608
2642
|
const edges = /* @__PURE__ */ new Map();
|
|
2643
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
2609
2644
|
let runStatus = "running";
|
|
2610
2645
|
let cancelRequestId;
|
|
2611
2646
|
let cancelRequestedBy;
|
|
@@ -2615,6 +2650,7 @@ function materialize(events) {
|
|
|
2615
2650
|
let blockedNodeId;
|
|
2616
2651
|
let uncertainHostEffects;
|
|
2617
2652
|
const cancelledAttempts = /* @__PURE__ */ new Map();
|
|
2653
|
+
const regateInstances = /* @__PURE__ */ new Set();
|
|
2618
2654
|
const resolvedWaitIds = /* @__PURE__ */ new Set();
|
|
2619
2655
|
const set = (id, status, gateCleared) => {
|
|
2620
2656
|
const prev = nodes.get(id);
|
|
@@ -2635,6 +2671,16 @@ function materialize(events) {
|
|
|
2635
2671
|
nodes.set(nodeId, { ...prev, status });
|
|
2636
2672
|
}
|
|
2637
2673
|
};
|
|
2674
|
+
const supersedeInstance = (nodeId, instanceId) => {
|
|
2675
|
+
instances.set(instanceId, { status: "superseded" });
|
|
2676
|
+
if (nodes.get(nodeId)?.effectiveInstanceId === instanceId) {
|
|
2677
|
+
nodes.set(nodeId, { status: "pending" });
|
|
2678
|
+
}
|
|
2679
|
+
if (runStatus === "blocked" && blockedNodeId === nodeId) {
|
|
2680
|
+
runStatus = "running";
|
|
2681
|
+
blockedNodeId = void 0;
|
|
2682
|
+
}
|
|
2683
|
+
};
|
|
2638
2684
|
const loop = (id) => {
|
|
2639
2685
|
let ls = loops.get(id);
|
|
2640
2686
|
if (!ls) {
|
|
@@ -2667,6 +2713,7 @@ function materialize(events) {
|
|
|
2667
2713
|
attempts.set(e.instanceId ?? e.nodeId, e.attemptId);
|
|
2668
2714
|
break;
|
|
2669
2715
|
case "hostInputPrepared":
|
|
2716
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2670
2717
|
instances.set(e.instanceId, { status: "pending" });
|
|
2671
2718
|
nodes.set(e.nodeId, {
|
|
2672
2719
|
status: "pending",
|
|
@@ -2676,6 +2723,7 @@ function materialize(events) {
|
|
|
2676
2723
|
break;
|
|
2677
2724
|
case "hostEffectIntent":
|
|
2678
2725
|
if (instances.get(e.instanceId)?.status === "cancelled") break;
|
|
2726
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2679
2727
|
recordInstance(e.nodeId, e.instanceId, "running", true);
|
|
2680
2728
|
attempts.set(e.instanceId, e.attemptId);
|
|
2681
2729
|
break;
|
|
@@ -2715,6 +2763,15 @@ function materialize(events) {
|
|
|
2715
2763
|
} else set(e.nodeId, "blocked");
|
|
2716
2764
|
break;
|
|
2717
2765
|
case "nodeRetryRequested":
|
|
2766
|
+
if (e.instanceId && regateInstances.has(e.instanceId)) {
|
|
2767
|
+
regateInstances.delete(e.instanceId);
|
|
2768
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2769
|
+
if (runStatus === "blocked") {
|
|
2770
|
+
runStatus = "running";
|
|
2771
|
+
blockedNodeId = void 0;
|
|
2772
|
+
}
|
|
2773
|
+
break;
|
|
2774
|
+
}
|
|
2718
2775
|
if (e.resetGate) {
|
|
2719
2776
|
nodes.set(e.nodeId, {
|
|
2720
2777
|
status: "pending",
|
|
@@ -2734,15 +2791,40 @@ function materialize(events) {
|
|
|
2734
2791
|
// ── cross-node revisit / instance refresh ──
|
|
2735
2792
|
case "nodeRevisitRequested":
|
|
2736
2793
|
break;
|
|
2737
|
-
case "nodeInstanceSuperseded":
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2794
|
+
case "nodeInstanceSuperseded":
|
|
2795
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2796
|
+
break;
|
|
2797
|
+
case "dagRevised": {
|
|
2798
|
+
const effects = e.effects;
|
|
2799
|
+
for (const item of effects?.supersede ?? []) {
|
|
2800
|
+
if (item.waitId) resolvedWaitIds.add(item.waitId);
|
|
2801
|
+
supersedeInstance(item.nodeId, item.instanceId);
|
|
2741
2802
|
}
|
|
2742
|
-
|
|
2803
|
+
for (const item of effects?.cancel ?? []) {
|
|
2804
|
+
cancelledAttempts.set(item.instanceId ?? item.nodeId, item.attemptId);
|
|
2805
|
+
if (item.instanceId) instances.set(item.instanceId, { status: "cancelled" });
|
|
2806
|
+
const prev = nodes.get(item.nodeId);
|
|
2807
|
+
nodes.set(item.nodeId, {
|
|
2808
|
+
status: "cancelled",
|
|
2809
|
+
...item.instanceId ?? prev?.effectiveInstanceId ? { effectiveInstanceId: item.instanceId ?? prev?.effectiveInstanceId } : {}
|
|
2810
|
+
});
|
|
2811
|
+
}
|
|
2812
|
+
if (runStatus === "blocked" && (effects?.cancel ?? []).length > 0 && ((effects?.cancel ?? []).some((item) => item.nodeId === blockedNodeId) || ![...nodes.values()].some((state) => state.status === "blocked"))) {
|
|
2743
2813
|
runStatus = "running";
|
|
2744
2814
|
blockedNodeId = void 0;
|
|
2745
2815
|
}
|
|
2816
|
+
for (const item of effects?.regate ?? []) {
|
|
2817
|
+
const prev = nodes.get(item.nodeId);
|
|
2818
|
+
if (prev?.effectiveInstanceId !== item.instanceId) continue;
|
|
2819
|
+
const { gateCleared: _dropped, ...rest } = prev;
|
|
2820
|
+
nodes.set(item.nodeId, rest);
|
|
2821
|
+
const instance = instances.get(item.instanceId);
|
|
2822
|
+
if (instance) {
|
|
2823
|
+
const { gateCleared: _instanceDropped, ...instanceRest } = instance;
|
|
2824
|
+
instances.set(item.instanceId, instanceRest);
|
|
2825
|
+
}
|
|
2826
|
+
regateInstances.add(item.instanceId);
|
|
2827
|
+
}
|
|
2746
2828
|
break;
|
|
2747
2829
|
}
|
|
2748
2830
|
case "gateDispatched":
|
|
@@ -2800,6 +2882,33 @@ function materialize(events) {
|
|
|
2800
2882
|
if (e.instanceId) recordInstance(e.nodeId, e.instanceId, "cancelled", false);
|
|
2801
2883
|
else set(e.nodeId, "cancelled");
|
|
2802
2884
|
break;
|
|
2885
|
+
case "nodeHandoffPrepared":
|
|
2886
|
+
case "nodeHandoffSent":
|
|
2887
|
+
case "nodeHandoffAccepted":
|
|
2888
|
+
case "nodeHandoffStarted":
|
|
2889
|
+
case "nodeStalled":
|
|
2890
|
+
break;
|
|
2891
|
+
case "nodeReassigned": {
|
|
2892
|
+
assignments.set(e.nodeId, e.to);
|
|
2893
|
+
if (!e.attemptId || runStatus !== "running" && runStatus !== "blocked") break;
|
|
2894
|
+
const key = e.instanceId ?? e.nodeId;
|
|
2895
|
+
if (attempts.get(key) !== e.attemptId) break;
|
|
2896
|
+
if (e.instanceId) {
|
|
2897
|
+
if (instances.get(e.instanceId)?.status !== "running") break;
|
|
2898
|
+
instances.set(e.instanceId, { status: "pending" });
|
|
2899
|
+
const current = nodes.get(e.nodeId);
|
|
2900
|
+
if (current?.effectiveInstanceId === e.instanceId && current.status === "running") {
|
|
2901
|
+
nodes.set(e.nodeId, {
|
|
2902
|
+
status: "pending",
|
|
2903
|
+
effectiveInstanceId: e.instanceId,
|
|
2904
|
+
...current.gateCleared ? { gateCleared: true } : {}
|
|
2905
|
+
});
|
|
2906
|
+
}
|
|
2907
|
+
} else if (nodes.get(e.nodeId)?.status === "running") {
|
|
2908
|
+
set(e.nodeId, "pending");
|
|
2909
|
+
}
|
|
2910
|
+
break;
|
|
2911
|
+
}
|
|
2803
2912
|
case "nodeAttemptDrained": {
|
|
2804
2913
|
if (runStatus !== "running" && runStatus !== "blocked") break;
|
|
2805
2914
|
const key = e.instanceId ?? e.nodeId;
|
|
@@ -2894,7 +3003,9 @@ function materialize(events) {
|
|
|
2894
3003
|
instances,
|
|
2895
3004
|
attempts,
|
|
2896
3005
|
loops,
|
|
2897
|
-
edges
|
|
3006
|
+
edges,
|
|
3007
|
+
assignments,
|
|
3008
|
+
...regateInstances.size > 0 ? { regatePending: regateInstances } : {}
|
|
2898
3009
|
};
|
|
2899
3010
|
}
|
|
2900
3011
|
function writeState(statePath, snap) {
|
|
@@ -2914,6 +3025,7 @@ function writeState(statePath, snap) {
|
|
|
2914
3025
|
attempts: Object.fromEntries(snap.attempts),
|
|
2915
3026
|
loops: snap.loops.size > 0 ? Object.fromEntries(snap.loops) : void 0,
|
|
2916
3027
|
edges: snap.edges.size > 0 ? Object.fromEntries(snap.edges) : void 0,
|
|
3028
|
+
assignments: snap.assignments.size > 0 ? Object.fromEntries(snap.assignments) : void 0,
|
|
2917
3029
|
updatedAt: Date.now()
|
|
2918
3030
|
};
|
|
2919
3031
|
const tmp = `${statePath}.tmp`;
|
|
@@ -3265,9 +3377,9 @@ function assertOrCreatePortableRunSnapshot(runDir, dag, profiles) {
|
|
|
3265
3377
|
}
|
|
3266
3378
|
|
|
3267
3379
|
// src/workflows/v3/shared-node-runtime.ts
|
|
3268
|
-
import { randomUUID as
|
|
3269
|
-
import { mkdirSync as
|
|
3270
|
-
import { join as
|
|
3380
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
3381
|
+
import { mkdirSync as mkdirSync8, writeFileSync as writeFileSync8, readFileSync as readFileSync12, existsSync as existsSync10, rmSync as rmSync2 } from "node:fs";
|
|
3382
|
+
import { join as join10, dirname as dirname8, relative as relative5, isAbsolute as isAbsolute5, resolve as resolve5, sep as sep3 } from "node:path";
|
|
3271
3383
|
|
|
3272
3384
|
// src/workflows/v3/core-control.ts
|
|
3273
3385
|
function matchLoopExitWhen(when, value) {
|
|
@@ -3289,6 +3401,15 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3289
3401
|
if (event.type === "nodeRevisitRequested") {
|
|
3290
3402
|
runUsed++;
|
|
3291
3403
|
if (event.nodeId === sourceNodeId && event.toNodeId === toNodeId) pairUsed++;
|
|
3404
|
+
} else if (event.type === "dagRevised") {
|
|
3405
|
+
for (const nodeId of event.effects?.rerun ?? []) {
|
|
3406
|
+
runUsed++;
|
|
3407
|
+
if (nodeId === sourceNodeId && nodeId === toNodeId) pairUsed++;
|
|
3408
|
+
}
|
|
3409
|
+
for (const grant of event.effects?.budgetGrants ?? []) {
|
|
3410
|
+
if (grant.tier === "run") runGranted++;
|
|
3411
|
+
else if (grant.nodeId === sourceNodeId && grant.nodeId === toNodeId) pairGranted++;
|
|
3412
|
+
}
|
|
3292
3413
|
} else if (event.type === "revisitBudgetGranted") {
|
|
3293
3414
|
if (event.sourceNodeId === sourceNodeId && event.toNodeId === toNodeId) pairGranted++;
|
|
3294
3415
|
else if (event.sourceNodeId === void 0 && event.toNodeId === void 0) runGranted++;
|
|
@@ -3313,6 +3434,72 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3313
3434
|
return { ok: true };
|
|
3314
3435
|
}
|
|
3315
3436
|
|
|
3437
|
+
// src/workflows/v3/revisit-plan.ts
|
|
3438
|
+
import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "node:fs";
|
|
3439
|
+
import { join as join4, relative as relative2 } from "node:path";
|
|
3440
|
+
function affectedNodesFrom(dag, root) {
|
|
3441
|
+
const reachable = /* @__PURE__ */ new Set([root]);
|
|
3442
|
+
let changed = true;
|
|
3443
|
+
while (changed) {
|
|
3444
|
+
changed = false;
|
|
3445
|
+
for (const n of dag.nodes) {
|
|
3446
|
+
if (reachable.has(n.id)) continue;
|
|
3447
|
+
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
3448
|
+
reachable.add(n.id);
|
|
3449
|
+
changed = true;
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
return [...reachable];
|
|
3454
|
+
}
|
|
3455
|
+
function planV3Revisit(input) {
|
|
3456
|
+
const { runDir, request } = input;
|
|
3457
|
+
const snap = materialize([...input.events]);
|
|
3458
|
+
const targetEff = snap.nodes.get(request.toNodeId)?.effectiveInstanceId;
|
|
3459
|
+
const targetSucc = targetEff ? [...input.events].reverse().find((e) => e.type === "nodeSucceeded" && (e.instanceId ?? e.nodeId) === targetEff) : void 0;
|
|
3460
|
+
let reasonPathRel;
|
|
3461
|
+
if (request.reason) {
|
|
3462
|
+
const dir = join4(runDir, "revisits");
|
|
3463
|
+
mkdirSync4(dir, { recursive: true });
|
|
3464
|
+
const abs = join4(dir, `${input.instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
3465
|
+
writeFileSync4(abs, `# Revisit reason (from ${input.nodeId} / ${input.instanceId})
|
|
3466
|
+
|
|
3467
|
+
${request.reason}
|
|
3468
|
+
`);
|
|
3469
|
+
reasonPathRel = relative2(runDir, abs);
|
|
3470
|
+
}
|
|
3471
|
+
const events = [{
|
|
3472
|
+
type: "nodeRevisitRequested",
|
|
3473
|
+
nodeId: input.nodeId,
|
|
3474
|
+
instanceId: input.instanceId,
|
|
3475
|
+
attemptId: input.attemptId,
|
|
3476
|
+
toNodeId: request.toNodeId,
|
|
3477
|
+
...request.reason ? { reason: request.reason } : {},
|
|
3478
|
+
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
3479
|
+
sourceManifestPath: relative2(runDir, input.sourceManifestPath),
|
|
3480
|
+
...targetSucc ? { targetPreviousManifestPath: relative2(runDir, targetSucc.manifestPath) } : {}
|
|
3481
|
+
}];
|
|
3482
|
+
const supersededInstanceIds = [];
|
|
3483
|
+
for (const affectedNodeId of affectedNodesFrom(input.dag, request.toNodeId)) {
|
|
3484
|
+
const eff = snap.nodes.get(affectedNodeId)?.effectiveInstanceId;
|
|
3485
|
+
if (!eff) continue;
|
|
3486
|
+
events.push({
|
|
3487
|
+
type: "nodeInstanceSuperseded",
|
|
3488
|
+
nodeId: affectedNodeId,
|
|
3489
|
+
instanceId: eff,
|
|
3490
|
+
byNodeId: request.toNodeId,
|
|
3491
|
+
reason: "refresh"
|
|
3492
|
+
});
|
|
3493
|
+
supersededInstanceIds.push(eff);
|
|
3494
|
+
}
|
|
3495
|
+
return { events, supersededInstanceIds };
|
|
3496
|
+
}
|
|
3497
|
+
|
|
3498
|
+
// src/workflows/v3/dag-revision.ts
|
|
3499
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
3500
|
+
import { existsSync as existsSync5, lstatSync as lstatSync4, readFileSync as readFileSync7 } from "node:fs";
|
|
3501
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
3502
|
+
|
|
3316
3503
|
// src/workflows/v3/gate-policy.ts
|
|
3317
3504
|
function normalizeGateWaitInput(gate) {
|
|
3318
3505
|
const options = gate.options ?? [...DEFAULT_HUMAN_GATE_OPTIONS];
|
|
@@ -3324,29 +3511,528 @@ function normalizeGateWaitInput(gate) {
|
|
|
3324
3511
|
};
|
|
3325
3512
|
}
|
|
3326
3513
|
|
|
3514
|
+
// src/workflows/v3/attempt-numbering.ts
|
|
3515
|
+
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
3516
|
+
function attemptNumber(attemptId) {
|
|
3517
|
+
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
3518
|
+
return m ? parseInt(m[1], 10) : void 0;
|
|
3519
|
+
}
|
|
3520
|
+
function nextAttemptIdFor(events, key) {
|
|
3521
|
+
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
3522
|
+
let maxSeen = 0;
|
|
3523
|
+
let reserved;
|
|
3524
|
+
let prepared;
|
|
3525
|
+
for (const e of events) {
|
|
3526
|
+
if (e.type === "nodeDispatched" && matches(e)) {
|
|
3527
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3528
|
+
if (n2 === void 0) continue;
|
|
3529
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3530
|
+
if (reserved === n2) reserved = void 0;
|
|
3531
|
+
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
3532
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3533
|
+
if (n2 === void 0) continue;
|
|
3534
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3535
|
+
if (reserved === n2) reserved = void 0;
|
|
3536
|
+
prepared = e.attemptId;
|
|
3537
|
+
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
3538
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3539
|
+
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
3540
|
+
if (prepared === e.attemptId) prepared = void 0;
|
|
3541
|
+
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
3542
|
+
prepared = void 0;
|
|
3543
|
+
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
3544
|
+
const n2 = attemptNumber(e.nextAttemptId);
|
|
3545
|
+
if (n2 === void 0) continue;
|
|
3546
|
+
reserved = n2;
|
|
3547
|
+
prepared = void 0;
|
|
3548
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3549
|
+
}
|
|
3550
|
+
}
|
|
3551
|
+
if (prepared) return prepared;
|
|
3552
|
+
const n = reserved ?? maxSeen + 1;
|
|
3553
|
+
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
3554
|
+
}
|
|
3555
|
+
function latestAttemptIdFor(events, key) {
|
|
3556
|
+
let latest;
|
|
3557
|
+
let latestNumber = -1;
|
|
3558
|
+
for (const e of events) {
|
|
3559
|
+
if ((e.type === "nodeDispatched" || e.type === "hostInputPrepared" || e.type === "hostEffectIntent" || e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && (e.instanceId ?? e.nodeId) === key) {
|
|
3560
|
+
const n = attemptNumber(e.attemptId);
|
|
3561
|
+
if (n === void 0) {
|
|
3562
|
+
if (latestNumber < 0) latest = e.attemptId;
|
|
3563
|
+
} else if (n >= latestNumber) {
|
|
3564
|
+
latest = e.attemptId;
|
|
3565
|
+
latestNumber = n;
|
|
3566
|
+
}
|
|
3567
|
+
}
|
|
3568
|
+
}
|
|
3569
|
+
return latest;
|
|
3570
|
+
}
|
|
3571
|
+
|
|
3572
|
+
// src/workflows/v3/dag-revision.ts
|
|
3573
|
+
var V3_DAG_REVISIONS_DIR = "dag-revisions";
|
|
3574
|
+
function v3RevisionStem(revision) {
|
|
3575
|
+
if (!Number.isSafeInteger(revision) || revision < 2) {
|
|
3576
|
+
throw new Error(`v3 revision: invalid revision number ${revision}`);
|
|
3577
|
+
}
|
|
3578
|
+
return String(revision).padStart(3, "0");
|
|
3579
|
+
}
|
|
3580
|
+
function v3RevisionFilePaths(runDir, revision) {
|
|
3581
|
+
const stem = v3RevisionStem(revision);
|
|
3582
|
+
const dir = join5(runDir, V3_DAG_REVISIONS_DIR);
|
|
3583
|
+
return {
|
|
3584
|
+
dag: join5(dir, `${stem}.json`),
|
|
3585
|
+
patch: join5(dir, `${stem}.patch.json`),
|
|
3586
|
+
bots: join5(dir, `${stem}.bots.json`)
|
|
3587
|
+
};
|
|
3588
|
+
}
|
|
3589
|
+
function v3RevisionDigest(bytes) {
|
|
3590
|
+
return `sha256:${createHash4("sha256").update(bytes).digest("hex")}`;
|
|
3591
|
+
}
|
|
3592
|
+
var V3RevisionIntegrityError = class extends Error {
|
|
3593
|
+
constructor(message) {
|
|
3594
|
+
super(message);
|
|
3595
|
+
this.name = "V3RevisionIntegrityError";
|
|
3596
|
+
}
|
|
3597
|
+
};
|
|
3598
|
+
function readPinnedFile(path, expected, label) {
|
|
3599
|
+
let stat;
|
|
3600
|
+
try {
|
|
3601
|
+
stat = lstatSync4(path);
|
|
3602
|
+
} catch {
|
|
3603
|
+
throw new V3RevisionIntegrityError(`${label} is missing (${path})`);
|
|
3604
|
+
}
|
|
3605
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3606
|
+
throw new V3RevisionIntegrityError(`${label} must be a regular file (${path})`);
|
|
3607
|
+
}
|
|
3608
|
+
const dirStat = lstatSync4(dirname4(path));
|
|
3609
|
+
if (!dirStat.isDirectory() || dirStat.isSymbolicLink()) {
|
|
3610
|
+
throw new V3RevisionIntegrityError(`${label} directory must be a real directory`);
|
|
3611
|
+
}
|
|
3612
|
+
const bytes = readFileSync7(path);
|
|
3613
|
+
const actual = v3RevisionDigest(bytes);
|
|
3614
|
+
if (actual !== expected) {
|
|
3615
|
+
throw new V3RevisionIntegrityError(`${label} digest mismatch (journal ${expected}, file ${actual})`);
|
|
3616
|
+
}
|
|
3617
|
+
return bytes;
|
|
3618
|
+
}
|
|
3619
|
+
var V3_UPDATE_NODE_FIELDS = [
|
|
3620
|
+
"goal",
|
|
3621
|
+
"outputs",
|
|
3622
|
+
"resultSchema",
|
|
3623
|
+
"timeoutSec",
|
|
3624
|
+
"humanGate",
|
|
3625
|
+
"override",
|
|
3626
|
+
"revisitTo",
|
|
3627
|
+
"checkpoint",
|
|
3628
|
+
"placement",
|
|
3629
|
+
"input",
|
|
3630
|
+
"maxIterations",
|
|
3631
|
+
"body",
|
|
3632
|
+
"exit",
|
|
3633
|
+
"feedback",
|
|
3634
|
+
"output"
|
|
3635
|
+
];
|
|
3636
|
+
var UPDATE_FIELDS = new Set(V3_UPDATE_NODE_FIELDS);
|
|
3637
|
+
function isPlainObject(value) {
|
|
3638
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3639
|
+
}
|
|
3640
|
+
function foldV3Revisions(events) {
|
|
3641
|
+
let current = 1;
|
|
3642
|
+
let maxRevision = 1;
|
|
3643
|
+
const records = [];
|
|
3644
|
+
const byRevision = /* @__PURE__ */ new Map();
|
|
3645
|
+
const applied = [];
|
|
3646
|
+
for (const event of events) {
|
|
3647
|
+
switch (event.type) {
|
|
3648
|
+
case "dagRevisionProposed": {
|
|
3649
|
+
if (byRevision.has(event.revision) || event.revision <= maxRevision) {
|
|
3650
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3651
|
+
}
|
|
3652
|
+
for (const prior of records) {
|
|
3653
|
+
if (prior.state !== "pending") continue;
|
|
3654
|
+
prior.state = "rejected";
|
|
3655
|
+
prior.rejected = { by: "system", reason: "superseded" };
|
|
3656
|
+
}
|
|
3657
|
+
const record = {
|
|
3658
|
+
revision: event.revision,
|
|
3659
|
+
baseRevision: event.baseRevision,
|
|
3660
|
+
sha256: event.sha256,
|
|
3661
|
+
patchSha256: event.patchSha256,
|
|
3662
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3663
|
+
summary: event.summary,
|
|
3664
|
+
...event.note ? { note: event.note } : {},
|
|
3665
|
+
by: event.by,
|
|
3666
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3667
|
+
state: "pending",
|
|
3668
|
+
ts: event.ts,
|
|
3669
|
+
reasons: [...event.reasons],
|
|
3670
|
+
touched: [...event.touched],
|
|
3671
|
+
...event.carriedFrom !== void 0 ? { carriedFrom: event.carriedFrom } : {}
|
|
3672
|
+
};
|
|
3673
|
+
records.push(record);
|
|
3674
|
+
byRevision.set(event.revision, record);
|
|
3675
|
+
maxRevision = event.revision;
|
|
3676
|
+
break;
|
|
3677
|
+
}
|
|
3678
|
+
case "dagRevised": {
|
|
3679
|
+
if (event.baseRevision !== current) {
|
|
3680
|
+
throw new V3RevisionIntegrityError(
|
|
3681
|
+
`revision ${event.revision} is based on ${event.baseRevision} but ${current} was in effect`
|
|
3682
|
+
);
|
|
3683
|
+
}
|
|
3684
|
+
let record = byRevision.get(event.revision);
|
|
3685
|
+
if (event.approval) {
|
|
3686
|
+
if (!record || record.state !== "pending" || record.sha256 !== event.sha256 || record.patchSha256 !== event.patchSha256 || record.botsSha256 !== event.botsSha256 || record.baseRevision !== event.baseRevision) {
|
|
3687
|
+
throw new V3RevisionIntegrityError(
|
|
3688
|
+
`approved revision ${event.revision} does not match a pending proposal`
|
|
3689
|
+
);
|
|
3690
|
+
}
|
|
3691
|
+
record.approvedBy = event.approval.by;
|
|
3692
|
+
} else {
|
|
3693
|
+
if (record || event.revision <= maxRevision) {
|
|
3694
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3695
|
+
}
|
|
3696
|
+
record = {
|
|
3697
|
+
revision: event.revision,
|
|
3698
|
+
baseRevision: event.baseRevision,
|
|
3699
|
+
sha256: event.sha256,
|
|
3700
|
+
patchSha256: event.patchSha256,
|
|
3701
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3702
|
+
summary: event.summary,
|
|
3703
|
+
...event.note ? { note: event.note } : {},
|
|
3704
|
+
by: event.by,
|
|
3705
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3706
|
+
state: "pending",
|
|
3707
|
+
ts: event.ts
|
|
3708
|
+
};
|
|
3709
|
+
records.push(record);
|
|
3710
|
+
byRevision.set(event.revision, record);
|
|
3711
|
+
maxRevision = event.revision;
|
|
3712
|
+
}
|
|
3713
|
+
record.state = "applied";
|
|
3714
|
+
record.appliedAt = event.ts;
|
|
3715
|
+
if (event.effects) record.effects = event.effects;
|
|
3716
|
+
applied.push(record);
|
|
3717
|
+
current = event.revision;
|
|
3718
|
+
break;
|
|
3719
|
+
}
|
|
3720
|
+
case "dagRevisionApproved": {
|
|
3721
|
+
const record = byRevision.get(event.revision);
|
|
3722
|
+
if (record && !record.approvedBy) record.approvedBy = event.by;
|
|
3723
|
+
break;
|
|
3724
|
+
}
|
|
3725
|
+
case "dagRevisionRejected": {
|
|
3726
|
+
const record = byRevision.get(event.revision);
|
|
3727
|
+
if (record?.state === "pending") {
|
|
3728
|
+
record.state = "rejected";
|
|
3729
|
+
record.rejected = {
|
|
3730
|
+
by: event.by,
|
|
3731
|
+
reason: event.reason,
|
|
3732
|
+
...event.detail ? { detail: event.detail } : {}
|
|
3733
|
+
};
|
|
3734
|
+
}
|
|
3735
|
+
break;
|
|
3736
|
+
}
|
|
3737
|
+
default:
|
|
3738
|
+
break;
|
|
3739
|
+
}
|
|
3740
|
+
}
|
|
3741
|
+
let pending;
|
|
3742
|
+
for (const record of records) {
|
|
3743
|
+
if (record.state !== "pending") continue;
|
|
3744
|
+
if (record.baseRevision !== current) {
|
|
3745
|
+
record.state = "rejected";
|
|
3746
|
+
record.rejected = { by: "system", reason: "stale" };
|
|
3747
|
+
continue;
|
|
3748
|
+
}
|
|
3749
|
+
pending = record;
|
|
3750
|
+
}
|
|
3751
|
+
return { current, records, applied, ...pending ? { pending } : {}, maxRevision };
|
|
3752
|
+
}
|
|
3753
|
+
function currentV3Revision(events) {
|
|
3754
|
+
let current = 1;
|
|
3755
|
+
for (const event of events) if (event.type === "dagRevised") current = event.revision;
|
|
3756
|
+
return current;
|
|
3757
|
+
}
|
|
3758
|
+
function parseRevisionDag(bytes, base, label) {
|
|
3759
|
+
let raw;
|
|
3760
|
+
try {
|
|
3761
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3762
|
+
} catch (err) {
|
|
3763
|
+
throw new V3RevisionIntegrityError(`${label} is not JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
3764
|
+
}
|
|
3765
|
+
let dag;
|
|
3766
|
+
try {
|
|
3767
|
+
dag = validateDag(raw);
|
|
3768
|
+
} catch (err) {
|
|
3769
|
+
throw new V3RevisionIntegrityError(
|
|
3770
|
+
`${label} failed validation: ${err instanceof DagValidationError ? err.problems.join("; ") : String(err)}`
|
|
3771
|
+
);
|
|
3772
|
+
}
|
|
3773
|
+
if (dag.runId !== base.runId) throw new V3RevisionIntegrityError(`${label} belongs to run ${dag.runId}`);
|
|
3774
|
+
if ((dag.schemaVersion ?? 1) !== (base.schemaVersion ?? 1)) {
|
|
3775
|
+
throw new V3RevisionIntegrityError(`${label} changed the DAG schemaVersion`);
|
|
3776
|
+
}
|
|
3777
|
+
return dag;
|
|
3778
|
+
}
|
|
3779
|
+
function parseRevisionBotSnapshots(raw, label = "revision bot snapshots") {
|
|
3780
|
+
if (!isPlainObject(raw)) throw new V3RevisionIntegrityError(`${label} must be an object`);
|
|
3781
|
+
const out = /* @__PURE__ */ new Map();
|
|
3782
|
+
for (const [selector, value] of Object.entries(raw)) {
|
|
3783
|
+
if (!isPlainObject(value) || typeof value.larkAppId !== "string" || value.larkAppId.length === 0 || typeof value.cliId !== "string" || typeof value.workingDir !== "string") {
|
|
3784
|
+
throw new V3RevisionIntegrityError(`${label}[${JSON.stringify(selector)}] is not a bot snapshot`);
|
|
3785
|
+
}
|
|
3786
|
+
out.set(selector, value);
|
|
3787
|
+
}
|
|
3788
|
+
return out;
|
|
3789
|
+
}
|
|
3790
|
+
function readVerifiedV3Revision(runDir, record, base) {
|
|
3791
|
+
const paths = v3RevisionFilePaths(runDir, record.revision);
|
|
3792
|
+
const label = `dag-revisions/${v3RevisionStem(record.revision)}`;
|
|
3793
|
+
const dag = parseRevisionDag(readPinnedFile(paths.dag, record.sha256, `${label}.json`), base, `${label}.json`);
|
|
3794
|
+
readPinnedFile(paths.patch, record.patchSha256, `${label}.patch.json`);
|
|
3795
|
+
let bots = /* @__PURE__ */ new Map();
|
|
3796
|
+
if (record.botsSha256) {
|
|
3797
|
+
const bytes = readPinnedFile(paths.bots, record.botsSha256, `${label}.bots.json`);
|
|
3798
|
+
let raw;
|
|
3799
|
+
try {
|
|
3800
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3801
|
+
} catch {
|
|
3802
|
+
throw new V3RevisionIntegrityError(`${label}.bots.json is not JSON`);
|
|
3803
|
+
}
|
|
3804
|
+
bots = parseRevisionBotSnapshots(raw, `${label}.bots.json`);
|
|
3805
|
+
}
|
|
3806
|
+
return { dag, bots };
|
|
3807
|
+
}
|
|
3808
|
+
function loadEffectiveV3Dag(runDir, base, events) {
|
|
3809
|
+
const history = foldV3Revisions(events);
|
|
3810
|
+
const botAdditions = /* @__PURE__ */ new Map();
|
|
3811
|
+
if (history.applied.length === 0) return { dag: base, revision: 1, history, botAdditions };
|
|
3812
|
+
let dag = base;
|
|
3813
|
+
for (const record of history.applied) {
|
|
3814
|
+
const verified = readVerifiedV3Revision(runDir, record, base);
|
|
3815
|
+
dag = verified.dag;
|
|
3816
|
+
for (const [selector, snapshot] of verified.bots) botAdditions.set(selector, snapshot);
|
|
3817
|
+
}
|
|
3818
|
+
return { dag, revision: history.current, history, botAdditions };
|
|
3819
|
+
}
|
|
3820
|
+
function snapshotForSelector(snapshots, selector) {
|
|
3821
|
+
const key = selector ?? "";
|
|
3822
|
+
const exact = snapshots.get(key);
|
|
3823
|
+
if (exact) return exact;
|
|
3824
|
+
if (!key) return void 0;
|
|
3825
|
+
for (const snapshot of snapshots.values()) if (snapshot.larkAppId === key) return snapshot;
|
|
3826
|
+
return void 0;
|
|
3827
|
+
}
|
|
3828
|
+
function latestBlockOf(events, snap, nodeId) {
|
|
3829
|
+
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
3830
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
3831
|
+
const event = events[i];
|
|
3832
|
+
if (event.type === "hostEffectUncertain" && (event.instanceId ?? event.nodeId) === key) return { kind: "hostUncertain" };
|
|
3833
|
+
if (event.type === "nodeBlocked" && (event.instanceId ?? event.nodeId) === key) {
|
|
3834
|
+
return {
|
|
3835
|
+
kind: "nodeBlocked",
|
|
3836
|
+
errorClass: event.errorClass,
|
|
3837
|
+
...event.errorCode ? { errorCode: event.errorCode } : {},
|
|
3838
|
+
reviseRequired: event.recovery === "reviseWorkflow"
|
|
3839
|
+
};
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
return void 0;
|
|
3843
|
+
}
|
|
3844
|
+
function v3ApprovedBotScope(frozen, botAdditions) {
|
|
3845
|
+
const scope = new Map(frozen);
|
|
3846
|
+
for (const [selector, snapshot] of botAdditions) if (!scope.has(selector)) scope.set(selector, snapshot);
|
|
3847
|
+
return scope;
|
|
3848
|
+
}
|
|
3849
|
+
function v3NodeHeldByProposal(pendingTouched, nodeId, loopId) {
|
|
3850
|
+
if (!pendingTouched || pendingTouched.size === 0) return false;
|
|
3851
|
+
return pendingTouched.has(nodeId) || loopId !== void 0 && pendingTouched.has(loopId);
|
|
3852
|
+
}
|
|
3853
|
+
function v3RevisionFollowUps(events, input) {
|
|
3854
|
+
const out = [];
|
|
3855
|
+
const provenance = { by: input.by, ...input.auto ? { auto: input.auto } : {} };
|
|
3856
|
+
const snap = materialize([...events]);
|
|
3857
|
+
for (const item of input.assign ?? []) {
|
|
3858
|
+
if (snap.nodes.get(item.nodeId)?.status === "cancelled") continue;
|
|
3859
|
+
if (input.liveNodeIds && !input.liveNodeIds.has(item.nodeId)) continue;
|
|
3860
|
+
out.push({
|
|
3861
|
+
type: "nodeReassigned",
|
|
3862
|
+
nodeId: item.nodeId,
|
|
3863
|
+
...snap.nodes.get(item.nodeId)?.effectiveInstanceId ? { instanceId: snap.nodes.get(item.nodeId).effectiveInstanceId } : {},
|
|
3864
|
+
from: item.from ?? snap.assignments.get(item.nodeId) ?? "",
|
|
3865
|
+
to: item.to,
|
|
3866
|
+
...provenance
|
|
3867
|
+
});
|
|
3868
|
+
}
|
|
3869
|
+
for (const item of input.retry ?? []) {
|
|
3870
|
+
const state = snap.nodes.get(item.nodeId);
|
|
3871
|
+
if (state?.status !== "blocked" || state.effectiveInstanceId !== item.instanceId) continue;
|
|
3872
|
+
const key = item.instanceId ?? item.nodeId;
|
|
3873
|
+
const previousAttemptId = latestAttemptIdFor([...events], key);
|
|
3874
|
+
if (!previousAttemptId) continue;
|
|
3875
|
+
const blocked = latestBlockOf(events, snap, item.nodeId);
|
|
3876
|
+
out.push({
|
|
3877
|
+
type: "nodeRetryRequested",
|
|
3878
|
+
nodeId: item.nodeId,
|
|
3879
|
+
...item.instanceId ? { instanceId: item.instanceId } : {},
|
|
3880
|
+
previousAttemptId,
|
|
3881
|
+
nextAttemptId: nextAttemptIdFor([...events], key),
|
|
3882
|
+
reason: "blockedRetry",
|
|
3883
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorClass ? { previousErrorClass: blocked.errorClass } : {},
|
|
3884
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorCode ? { previousErrorCode: blocked.errorCode } : {},
|
|
3885
|
+
...provenance
|
|
3886
|
+
});
|
|
3887
|
+
}
|
|
3888
|
+
return out;
|
|
3889
|
+
}
|
|
3890
|
+
function missingV3RevisionFollowUps(events, liveNodeIds) {
|
|
3891
|
+
if (!events.some((event) => event.type === "dagRevised" && (event.effects?.retry || event.effects?.assign))) return [];
|
|
3892
|
+
const snap = materialize([...events]);
|
|
3893
|
+
if (snap.runStatus !== "running" && snap.runStatus !== "blocked") return [];
|
|
3894
|
+
const missing = [];
|
|
3895
|
+
events.forEach((event, index) => {
|
|
3896
|
+
if (event.type !== "dagRevised" || !event.effects?.retry && !event.effects?.assign) return;
|
|
3897
|
+
const later = events.slice(index + 1);
|
|
3898
|
+
const assign = (event.effects.assign ?? []).filter((item) => !later.some((after) => after.type === "nodeReassigned" && after.nodeId === item.nodeId || after.type === "dagRevised" && after.effects?.assign?.some((other) => other.nodeId === item.nodeId)));
|
|
3899
|
+
const retry = (event.effects.retry ?? []).filter((item) => !later.some((after) => after.type === "nodeRetryRequested" && after.nodeId === item.nodeId));
|
|
3900
|
+
if (assign.length === 0 && retry.length === 0) return;
|
|
3901
|
+
missing.push(...v3RevisionFollowUps(events, {
|
|
3902
|
+
assign,
|
|
3903
|
+
retry,
|
|
3904
|
+
by: event.by,
|
|
3905
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3906
|
+
...liveNodeIds ? { liveNodeIds } : {}
|
|
3907
|
+
}));
|
|
3908
|
+
});
|
|
3909
|
+
return missing;
|
|
3910
|
+
}
|
|
3911
|
+
function appendMissingV3RevisionFollowUps(journalPath, scope) {
|
|
3912
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
3913
|
+
if (currentV3Revision(events) !== scope.revision) return { kind: "stale" };
|
|
3914
|
+
const missing = missingV3RevisionFollowUps(events, scope.liveNodeIds);
|
|
3915
|
+
for (const event of missing) append(event, { durable: true });
|
|
3916
|
+
return { kind: "appended", count: missing.length };
|
|
3917
|
+
});
|
|
3918
|
+
}
|
|
3919
|
+
|
|
3920
|
+
// src/workflows/v3/checkpoint-hold.ts
|
|
3921
|
+
function marked(dag, nodeId) {
|
|
3922
|
+
const node = dag.nodes.find((candidate) => candidate.id === nodeId);
|
|
3923
|
+
return !!node && isGoalNode(node) && node.checkpoint === true;
|
|
3924
|
+
}
|
|
3925
|
+
function v3CheckpointMarkedSince(runDir, authorizedDag, events) {
|
|
3926
|
+
const cache = /* @__PURE__ */ new Map();
|
|
3927
|
+
let revisions;
|
|
3928
|
+
const loadRevisions = () => {
|
|
3929
|
+
if (revisions !== void 0) return revisions;
|
|
3930
|
+
try {
|
|
3931
|
+
const positions = /* @__PURE__ */ new Map();
|
|
3932
|
+
events.forEach((event, index) => {
|
|
3933
|
+
if (event.type === "dagRevised") positions.set(event.revision, index + 1);
|
|
3934
|
+
});
|
|
3935
|
+
revisions = foldV3Revisions(events).applied.map((record) => ({
|
|
3936
|
+
dag: readVerifiedV3Revision(runDir, record, authorizedDag).dag,
|
|
3937
|
+
position: positions.get(record.revision) ?? 0
|
|
3938
|
+
}));
|
|
3939
|
+
} catch {
|
|
3940
|
+
revisions = null;
|
|
3941
|
+
}
|
|
3942
|
+
return revisions;
|
|
3943
|
+
};
|
|
3944
|
+
return (nodeId) => {
|
|
3945
|
+
if (cache.has(nodeId)) return cache.get(nodeId);
|
|
3946
|
+
let since = marked(authorizedDag, nodeId) ? 0 : void 0;
|
|
3947
|
+
if (events.some((event) => event.type === "dagRevised")) {
|
|
3948
|
+
const list = loadRevisions();
|
|
3949
|
+
if (list === null) since = void 0;
|
|
3950
|
+
else {
|
|
3951
|
+
for (const revision of list) {
|
|
3952
|
+
if (!marked(revision.dag, nodeId)) since = void 0;
|
|
3953
|
+
else if (since === void 0) since = revision.position;
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3956
|
+
}
|
|
3957
|
+
cache.set(nodeId, since);
|
|
3958
|
+
return since;
|
|
3959
|
+
};
|
|
3960
|
+
}
|
|
3961
|
+
function v3CheckpointSuccesses(input) {
|
|
3962
|
+
const events = [...input.events];
|
|
3963
|
+
const snap = materialize(events);
|
|
3964
|
+
const live = snap.runStatus === "running" || snap.runStatus === "blocked";
|
|
3965
|
+
const releases = /* @__PURE__ */ new Map();
|
|
3966
|
+
for (const event of events) {
|
|
3967
|
+
if (event.type === "checkpointReleased" && !releases.has(event.attemptId)) {
|
|
3968
|
+
releases.set(event.attemptId, { reason: event.reason, at: event.ts });
|
|
3969
|
+
}
|
|
3970
|
+
}
|
|
3971
|
+
const out = [];
|
|
3972
|
+
for (const node of input.dag.nodes) {
|
|
3973
|
+
if (!isGoalNode(node) || node.checkpoint !== true) continue;
|
|
3974
|
+
const state = snap.nodes.get(node.id);
|
|
3975
|
+
if (state?.status !== "done") continue;
|
|
3976
|
+
const key = state.effectiveInstanceId ?? node.id;
|
|
3977
|
+
const attemptId = latestAttemptIdFor(events, key);
|
|
3978
|
+
if (!attemptId) continue;
|
|
3979
|
+
let index = -1;
|
|
3980
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
3981
|
+
const event = events[i];
|
|
3982
|
+
if (event.type === "nodeSucceeded" && event.attemptId === attemptId && (event.instanceId ?? event.nodeId) === key) {
|
|
3983
|
+
index = i;
|
|
3984
|
+
break;
|
|
3985
|
+
}
|
|
3986
|
+
}
|
|
3987
|
+
if (index < 0) continue;
|
|
3988
|
+
const since = input.markedSince(node.id);
|
|
3989
|
+
if (since === void 0 || index < since) continue;
|
|
3990
|
+
const released = releases.get(attemptId);
|
|
3991
|
+
if (!released && !live) continue;
|
|
3992
|
+
out.push({
|
|
3993
|
+
nodeId: node.id,
|
|
3994
|
+
...state.effectiveInstanceId ? { instanceId: state.effectiveInstanceId } : {},
|
|
3995
|
+
attemptId,
|
|
3996
|
+
eventSeq: index + 1,
|
|
3997
|
+
at: events[index].ts,
|
|
3998
|
+
...released ? { released } : {}
|
|
3999
|
+
});
|
|
4000
|
+
}
|
|
4001
|
+
return out;
|
|
4002
|
+
}
|
|
4003
|
+
function v3CheckpointHeldNodes(dag, successes) {
|
|
4004
|
+
const unreleased = new Set(successes.filter((success) => !success.released).map((success) => success.nodeId));
|
|
4005
|
+
const held = /* @__PURE__ */ new Set();
|
|
4006
|
+
if (unreleased.size === 0) return held;
|
|
4007
|
+
for (const node of dag.nodes) {
|
|
4008
|
+
if (node.depends.some((dep) => unreleased.has(dep.from))) held.add(node.id);
|
|
4009
|
+
}
|
|
4010
|
+
return held;
|
|
4011
|
+
}
|
|
4012
|
+
|
|
3327
4013
|
// src/workflows/v3/gate-wait-store.ts
|
|
3328
4014
|
import { randomBytes } from "node:crypto";
|
|
3329
4015
|
import {
|
|
3330
4016
|
chmodSync,
|
|
3331
4017
|
closeSync as closeSync5,
|
|
3332
4018
|
constants as constants3,
|
|
3333
|
-
writeFileSync as
|
|
3334
|
-
readFileSync as
|
|
3335
|
-
existsSync as
|
|
4019
|
+
writeFileSync as writeFileSync5,
|
|
4020
|
+
readFileSync as readFileSync8,
|
|
4021
|
+
existsSync as existsSync6,
|
|
3336
4022
|
fsyncSync as fsyncSync3,
|
|
3337
|
-
lstatSync as
|
|
3338
|
-
mkdirSync as
|
|
4023
|
+
lstatSync as lstatSync5,
|
|
4024
|
+
mkdirSync as mkdirSync5,
|
|
3339
4025
|
openSync as openSync5,
|
|
3340
4026
|
renameSync as renameSync2,
|
|
3341
4027
|
readdirSync as readdirSync2,
|
|
3342
4028
|
unlinkSync as unlinkSync2
|
|
3343
4029
|
} from "node:fs";
|
|
3344
|
-
import { dirname as
|
|
4030
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
3345
4031
|
function waitsDir(runDir) {
|
|
3346
|
-
return
|
|
4032
|
+
return join6(runDir, "waits");
|
|
3347
4033
|
}
|
|
3348
4034
|
function waitPath(runDir, waitId) {
|
|
3349
|
-
return
|
|
4035
|
+
return join6(waitsDir(runDir), `${waitId}.json`);
|
|
3350
4036
|
}
|
|
3351
4037
|
function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
3352
4038
|
const base = instanceId ?? nodeId;
|
|
@@ -3358,18 +4044,18 @@ function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
|
3358
4044
|
return `${base}-host-${attempt}-gate`;
|
|
3359
4045
|
}
|
|
3360
4046
|
function atomicWriteJson(path, value) {
|
|
3361
|
-
const dir =
|
|
3362
|
-
const runDir =
|
|
4047
|
+
const dir = dirname5(path);
|
|
4048
|
+
const runDir = dirname5(dir);
|
|
3363
4049
|
assertRealDirectory(runDir, "run directory");
|
|
3364
|
-
if (!
|
|
4050
|
+
if (!existsSync6(dir)) {
|
|
3365
4051
|
try {
|
|
3366
|
-
|
|
4052
|
+
mkdirSync5(dir, { mode: 448 });
|
|
3367
4053
|
} catch {
|
|
3368
4054
|
}
|
|
3369
4055
|
}
|
|
3370
4056
|
assertRealDirectory(dir, "waits directory");
|
|
3371
4057
|
chmodSync(dir, 448);
|
|
3372
|
-
if (
|
|
4058
|
+
if (existsSync6(path)) assertPrivateRegularWaitFile(path, false);
|
|
3373
4059
|
const tmp = `${path}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
|
|
3374
4060
|
let fd;
|
|
3375
4061
|
try {
|
|
@@ -3378,7 +4064,7 @@ function atomicWriteJson(path, value) {
|
|
|
3378
4064
|
constants3.O_WRONLY | constants3.O_CREAT | constants3.O_EXCL | (constants3.O_NOFOLLOW ?? 0),
|
|
3379
4065
|
384
|
|
3380
4066
|
);
|
|
3381
|
-
|
|
4067
|
+
writeFileSync5(fd, JSON.stringify(value, null, 2), "utf-8");
|
|
3382
4068
|
fsyncSync3(fd);
|
|
3383
4069
|
closeSync5(fd);
|
|
3384
4070
|
fd = void 0;
|
|
@@ -3401,13 +4087,13 @@ function atomicWriteJson(path, value) {
|
|
|
3401
4087
|
fsyncDirectorySyncPortable(dir);
|
|
3402
4088
|
}
|
|
3403
4089
|
function assertRealDirectory(path, label) {
|
|
3404
|
-
const stat =
|
|
4090
|
+
const stat = lstatSync5(path);
|
|
3405
4091
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
3406
4092
|
throw new Error(`v3 human-gate: ${label} must be a real directory`);
|
|
3407
4093
|
}
|
|
3408
4094
|
}
|
|
3409
4095
|
function assertPrivateRegularWaitFile(path, requirePrivate = true) {
|
|
3410
|
-
const stat =
|
|
4096
|
+
const stat = lstatSync5(path);
|
|
3411
4097
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3412
4098
|
throw new Error("v3 human-gate: wait file must be a regular file");
|
|
3413
4099
|
}
|
|
@@ -3521,6 +4207,7 @@ function openV3WorkerAttempts(events) {
|
|
|
3521
4207
|
const opened = /* @__PURE__ */ new Map();
|
|
3522
4208
|
const closed = /* @__PURE__ */ new Set();
|
|
3523
4209
|
for (const event of events) {
|
|
4210
|
+
if (event.type === "nodeDispatched" && event.nodeExecutor === "bot") continue;
|
|
3524
4211
|
if (isAttemptOpened(event)) {
|
|
3525
4212
|
const prior = opened.get(event.attemptId);
|
|
3526
4213
|
if (prior && (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId)) {
|
|
@@ -3628,25 +4315,577 @@ function sameIdentity(left, right) {
|
|
|
3628
4315
|
return left.nodeId === right.nodeId && left.instanceId === right.instanceId && left.attemptId === right.attemptId && left.executor === right.executor && left.provider === right.provider && left.inputRef.path === right.inputRef.path && left.inputRef.sha256 === right.inputRef.sha256 && left.inputRef.bytes === right.inputRef.bytes && left.inputHash === right.inputHash && left.idempotencyKey === right.idempotencyKey && left.idempotencyTtlMs === right.idempotencyTtlMs && left.approvalDigest === right.approvalDigest && left.attemptedAtMs === right.ts;
|
|
3629
4316
|
}
|
|
3630
4317
|
|
|
3631
|
-
// src/workflows/v3/
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
4318
|
+
// src/workflows/v3/external-attempt-ledger.ts
|
|
4319
|
+
function closureOf(event) {
|
|
4320
|
+
switch (event.type) {
|
|
4321
|
+
case "nodeSucceeded":
|
|
4322
|
+
return { kind: "succeeded" };
|
|
4323
|
+
case "nodeBlocked":
|
|
4324
|
+
return { kind: "blocked", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4325
|
+
case "nodeFailed":
|
|
4326
|
+
return { kind: "failed", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4327
|
+
case "nodeCancelled":
|
|
4328
|
+
return event.reason === "runCancelled" && event.attemptId ? { kind: "cancelled" } : void 0;
|
|
4329
|
+
case "nodeAttemptDrained":
|
|
4330
|
+
return { kind: "drained", reason: event.reason };
|
|
4331
|
+
case "nodeReassigned":
|
|
4332
|
+
return event.attemptId ? { kind: "reassigned", to: event.to } : void 0;
|
|
4333
|
+
default:
|
|
4334
|
+
return void 0;
|
|
4335
|
+
}
|
|
4336
|
+
}
|
|
4337
|
+
function attemptIdOf(event) {
|
|
4338
|
+
return "attemptId" in event && typeof event.attemptId === "string" ? event.attemptId : void 0;
|
|
4339
|
+
}
|
|
4340
|
+
function identityOf(event) {
|
|
4341
|
+
const record = event;
|
|
4342
|
+
return {
|
|
4343
|
+
...typeof record.nodeId === "string" ? { nodeId: record.nodeId } : {},
|
|
4344
|
+
...typeof record.instanceId === "string" ? { instanceId: record.instanceId } : {}
|
|
4345
|
+
};
|
|
4346
|
+
}
|
|
4347
|
+
function foldV3ExternalAttempts(events) {
|
|
4348
|
+
const attempts = /* @__PURE__ */ new Map();
|
|
4349
|
+
for (const event of events) {
|
|
4350
|
+
if (event.type === "nodeDispatched") {
|
|
4351
|
+
if (event.nodeExecutor !== "bot") continue;
|
|
4352
|
+
const prior = attempts.get(event.attemptId);
|
|
4353
|
+
if (prior) {
|
|
4354
|
+
if (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId) {
|
|
4355
|
+
throw new Error(`v3 external ledger: identity changed for ${event.attemptId}`);
|
|
4356
|
+
}
|
|
4357
|
+
continue;
|
|
4358
|
+
}
|
|
4359
|
+
attempts.set(event.attemptId, {
|
|
4360
|
+
nodeId: event.nodeId,
|
|
4361
|
+
...event.instanceId ? { instanceId: event.instanceId } : {},
|
|
4362
|
+
attemptId: event.attemptId,
|
|
4363
|
+
...event.loop ? { loop: event.loop } : {},
|
|
4364
|
+
phase: "dispatched"
|
|
4365
|
+
});
|
|
4366
|
+
continue;
|
|
4367
|
+
}
|
|
4368
|
+
const attemptId = attemptIdOf(event);
|
|
4369
|
+
if (!attemptId) continue;
|
|
4370
|
+
const attempt = attempts.get(attemptId);
|
|
4371
|
+
if (!attempt) continue;
|
|
4372
|
+
if (attempt.closed) continue;
|
|
4373
|
+
const identity = identityOf(event);
|
|
4374
|
+
if (identity.nodeId !== attempt.nodeId || identity.instanceId !== attempt.instanceId) {
|
|
4375
|
+
throw new Error(`v3 external ledger: event identity changed for ${attemptId}`);
|
|
4376
|
+
}
|
|
4377
|
+
switch (event.type) {
|
|
4378
|
+
case "nodeHandoffPrepared":
|
|
4379
|
+
if (attempt.phase !== "dispatched") break;
|
|
4380
|
+
attempt.phase = "prepared";
|
|
4381
|
+
attempt.targetLarkAppId = event.targetLarkAppId;
|
|
4382
|
+
attempt.placement = event.placement;
|
|
4383
|
+
attempt.revision = event.revision;
|
|
4384
|
+
attempt.idempotency = event.idempotency;
|
|
4385
|
+
attempt.preparedAt = event.preparedAt;
|
|
4386
|
+
break;
|
|
4387
|
+
case "nodeHandoffSent":
|
|
4388
|
+
if (attempt.phase === "dispatched" || attempt.handoffMessageId !== void 0) break;
|
|
4389
|
+
if (attempt.phase === "prepared") attempt.phase = "sent";
|
|
4390
|
+
attempt.handoffMessageId = event.handoffMessageId;
|
|
4391
|
+
if (event.nodeCardMessageId) attempt.nodeCardMessageId = event.nodeCardMessageId;
|
|
4392
|
+
if (event.threadId) attempt.threadId = event.threadId;
|
|
4393
|
+
attempt.sentAt = event.sentAt;
|
|
4394
|
+
attempt.acceptDeadlineAt = event.acceptDeadlineAt;
|
|
4395
|
+
break;
|
|
4396
|
+
case "nodeHandoffAccepted":
|
|
4397
|
+
if (attempt.phase !== "prepared" && attempt.phase !== "sent") break;
|
|
4398
|
+
attempt.phase = "accepted";
|
|
4399
|
+
attempt.sessionId = event.sessionId;
|
|
4400
|
+
attempt.acceptedAt = event.acceptedAt;
|
|
4401
|
+
attempt.deadlineAt = event.deadlineAt;
|
|
4402
|
+
if (event.startPending) attempt.startPending = true;
|
|
4403
|
+
break;
|
|
4404
|
+
case "nodeHandoffStarted":
|
|
4405
|
+
if (attempt.phase !== "accepted" || !attempt.startPending) break;
|
|
4406
|
+
delete attempt.startPending;
|
|
4407
|
+
attempt.startedAt = event.startedAt;
|
|
4408
|
+
attempt.deadlineAt = event.deadlineAt;
|
|
4409
|
+
break;
|
|
4410
|
+
case "nodeStalled":
|
|
4411
|
+
attempt.stalledAt = event.at;
|
|
4412
|
+
break;
|
|
4413
|
+
case "nodeHandedToHuman":
|
|
4414
|
+
attempt.handedToHuman = { reason: event.reason };
|
|
4415
|
+
break;
|
|
4416
|
+
default: {
|
|
4417
|
+
const closure = closureOf(event);
|
|
4418
|
+
if (closure) attempt.closed = closure;
|
|
4419
|
+
}
|
|
4420
|
+
}
|
|
4421
|
+
}
|
|
4422
|
+
return attempts;
|
|
4423
|
+
}
|
|
4424
|
+
function openV3ExternalAttempts(events) {
|
|
4425
|
+
return [...foldV3ExternalAttempts(events).values()].filter((attempt) => !attempt.closed);
|
|
4426
|
+
}
|
|
4427
|
+
|
|
4428
|
+
// src/workflows/v3/external-executor.ts
|
|
4429
|
+
var V3_EXTERNAL_ERROR_CODES = {
|
|
4430
|
+
BOT_NOT_IN_CHAT: "BOT_NOT_IN_CHAT",
|
|
4431
|
+
BOT_OFFLINE: "BOT_OFFLINE",
|
|
4432
|
+
COLLABORATION_DENIED: "COLLABORATION_DENIED",
|
|
4433
|
+
IDENTITY_UNAVAILABLE: "IDENTITY_UNAVAILABLE",
|
|
4434
|
+
HANDOFF_NOT_ACCEPTED: "HANDOFF_NOT_ACCEPTED",
|
|
4435
|
+
HANDOFF_UNCERTAIN: "HANDOFF_UNCERTAIN",
|
|
4436
|
+
BOT_REPORTED_BLOCKED: "BOT_REPORTED_BLOCKED",
|
|
4437
|
+
/** The executing bot's daemon would not start the node session in the
|
|
4438
|
+
* run's Gate-2 working dir: not inside its configured range (design §15.4). */
|
|
4439
|
+
WORKING_DIR_REFUSED: "WORKING_DIR_REFUSED",
|
|
4440
|
+
/** Main-topic placement (design §5.5): the executing bot's session in the
|
|
4441
|
+
* main topic is executing another workflow node right now (another run in
|
|
4442
|
+
* the same topic); one session cannot carry two nodes at once. */
|
|
4443
|
+
MAIN_TOPIC_BUSY: "MAIN_TOPIC_BUSY",
|
|
4444
|
+
NODE_TIMEOUT: "NODE_TIMEOUT",
|
|
4445
|
+
/** The main bot (or the initiator) handed the node to a human (§13.6). */
|
|
4446
|
+
HANDED_TO_HUMAN: "HANDED_TO_HUMAN"
|
|
4447
|
+
};
|
|
4448
|
+
var V3_HANDOFF_IDEMPOTENCY_WINDOW_MS = 50 * 60 * 1e3;
|
|
4449
|
+
var V3_EXTERNAL_CALL_TIMEOUT_MS = 3e4;
|
|
4450
|
+
var V3_MAIN_TOPIC_QUEUE_GRACE_MIN_MS = 60 * 60 * 1e3;
|
|
4451
|
+
function v3MainTopicQueueGraceMs(nodeTimeoutMs2) {
|
|
4452
|
+
return Math.max(nodeTimeoutMs2, V3_MAIN_TOPIC_QUEUE_GRACE_MIN_MS);
|
|
4453
|
+
}
|
|
4454
|
+
var V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS = 15e3;
|
|
4455
|
+
|
|
4456
|
+
// src/workflows/v3/external-control.ts
|
|
4457
|
+
import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
|
|
4458
|
+
import {
|
|
4459
|
+
existsSync as existsSync8,
|
|
4460
|
+
lstatSync as lstatSync6,
|
|
4461
|
+
mkdirSync as mkdirSync6,
|
|
4462
|
+
readFileSync as readFileSync10,
|
|
4463
|
+
renameSync as renameSync3,
|
|
4464
|
+
rmdirSync,
|
|
4465
|
+
rmSync,
|
|
4466
|
+
writeFileSync as writeFileSync6
|
|
4467
|
+
} from "node:fs";
|
|
4468
|
+
import { basename as basename2, dirname as dirname6, extname, isAbsolute as isAbsolute3, join as join8, relative as relative3, resolve as resolve3, sep } from "node:path";
|
|
4469
|
+
|
|
4470
|
+
// src/workflows/v3/result-contract.ts
|
|
4471
|
+
import { readFileSync as readFileSync9, existsSync as existsSync7 } from "node:fs";
|
|
4472
|
+
import { join as join7 } from "node:path";
|
|
4473
|
+
function readRevisitRequest(manifest, outputDir) {
|
|
4474
|
+
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4475
|
+
if (!entry) return { ok: true };
|
|
4476
|
+
let value;
|
|
4477
|
+
try {
|
|
4478
|
+
value = JSON.parse(readFileSync9(join7(outputDir, entry.path), "utf-8"));
|
|
4479
|
+
} catch {
|
|
4480
|
+
return { ok: true };
|
|
4481
|
+
}
|
|
4482
|
+
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4483
|
+
return { ok: true };
|
|
4484
|
+
}
|
|
4485
|
+
const v = value;
|
|
4486
|
+
const problems = [];
|
|
4487
|
+
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4488
|
+
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4489
|
+
}
|
|
4490
|
+
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4491
|
+
problems.push('result.json "reason" must be a string when present');
|
|
4492
|
+
}
|
|
4493
|
+
if (problems.length > 0) return { ok: false, problems };
|
|
4494
|
+
return {
|
|
4495
|
+
ok: true,
|
|
4496
|
+
request: {
|
|
4497
|
+
toNodeId: v.revisitTo,
|
|
4498
|
+
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4499
|
+
}
|
|
4500
|
+
};
|
|
4501
|
+
}
|
|
4502
|
+
function validateResult(filePath, schema) {
|
|
4503
|
+
if (!existsSync7(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4504
|
+
let parsed;
|
|
4505
|
+
try {
|
|
4506
|
+
parsed = JSON.parse(readFileSync9(filePath, "utf-8"));
|
|
4507
|
+
} catch (err) {
|
|
4508
|
+
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4509
|
+
}
|
|
4510
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4511
|
+
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4512
|
+
}
|
|
4513
|
+
const obj = parsed;
|
|
4514
|
+
const problems = [];
|
|
4515
|
+
for (const field of schema.required ?? []) {
|
|
4516
|
+
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4517
|
+
}
|
|
4518
|
+
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4519
|
+
if (!(name in obj)) continue;
|
|
4520
|
+
const v = obj[name];
|
|
4521
|
+
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4522
|
+
if (!okType) {
|
|
4523
|
+
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4524
|
+
continue;
|
|
4525
|
+
}
|
|
4526
|
+
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4527
|
+
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4528
|
+
}
|
|
4529
|
+
}
|
|
4530
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4531
|
+
}
|
|
4532
|
+
|
|
4533
|
+
// src/workflows/v3/external-control.ts
|
|
4534
|
+
function classifyExternalAttempt(events, attemptId, expect = {}) {
|
|
4535
|
+
const attempt = foldV3ExternalAttempts(events).get(attemptId);
|
|
4536
|
+
if (!attempt) return { ok: false, reason: "unknown-attempt" };
|
|
4537
|
+
if (expect.instanceId !== void 0 && expect.instanceId !== attempt.instanceId) {
|
|
4538
|
+
return { ok: false, reason: "unknown-attempt" };
|
|
4539
|
+
}
|
|
4540
|
+
const snap = materialize([...events]);
|
|
4541
|
+
const newerAttempt = snap.attempts.get(attempt.instanceId ?? attempt.nodeId) !== attemptId;
|
|
4542
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
4543
|
+
const closed = attempt.closed;
|
|
4544
|
+
if (closed) {
|
|
4545
|
+
const reason = closed.kind === "succeeded" ? "already-succeeded" : closed.kind === "reassigned" ? "reassigned" : closed.kind === "cancelled" ? "cancelled" : closed.kind === "blocked" && closed.errorCode === V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT ? "timed-out" : closed.kind === "drained" && closed.reason === "runCancellation" ? "cancelled" : closed.kind === "drained" && closed.reason === "obsoleteAttempt" ? state?.status === "cancelled" ? "cancelled" : "superseded" : newerAttempt ? "retried" : "closed";
|
|
4546
|
+
return { ok: false, reason, attempt };
|
|
4547
|
+
}
|
|
4548
|
+
if (snap.cancelRequestId !== void 0) return { ok: false, reason: "cancelled", attempt };
|
|
4549
|
+
if (snap.runStatus !== "running") return { ok: false, reason: "run-not-running", attempt };
|
|
4550
|
+
if (state?.status === "superseded") return { ok: false, reason: "superseded", attempt };
|
|
4551
|
+
if (state?.status === "cancelled") return { ok: false, reason: "cancelled", attempt };
|
|
4552
|
+
if (newerAttempt) return { ok: false, reason: "retried", attempt };
|
|
4553
|
+
if (state?.status !== "running") return { ok: false, reason: "closed", attempt };
|
|
4554
|
+
if (expect.sessionId !== void 0 && attempt.sessionId !== void 0 && expect.sessionId !== attempt.sessionId) {
|
|
4555
|
+
return { ok: false, reason: "session-mismatch", attempt };
|
|
4556
|
+
}
|
|
4557
|
+
if (attempt.handedToHuman && expect.sessionId !== void 0) {
|
|
4558
|
+
return { ok: false, reason: "handed-to-human", attempt };
|
|
4559
|
+
}
|
|
4560
|
+
return { ok: true, attempt, snap };
|
|
4561
|
+
}
|
|
4562
|
+
function externalAttemptRef(runId, attempt) {
|
|
4563
|
+
if (!attempt.targetLarkAppId || !attempt.placement || attempt.revision === void 0) {
|
|
4564
|
+
return void 0;
|
|
4565
|
+
}
|
|
4566
|
+
return {
|
|
4567
|
+
runId,
|
|
4568
|
+
nodeId: attempt.nodeId,
|
|
4569
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4570
|
+
attemptId: attempt.attemptId,
|
|
4571
|
+
revision: attempt.revision,
|
|
4572
|
+
targetLarkAppId: attempt.targetLarkAppId,
|
|
4573
|
+
placement: attempt.placement
|
|
4574
|
+
};
|
|
4575
|
+
}
|
|
4576
|
+
async function withExternalCallTimeout(call, timeoutMs, label) {
|
|
4577
|
+
let timer;
|
|
4578
|
+
try {
|
|
4579
|
+
return await Promise.race([
|
|
4580
|
+
call,
|
|
4581
|
+
new Promise((_resolve, reject) => {
|
|
4582
|
+
timer = setTimeout(() => reject(new Error(`external ${label} did not settle within ${timeoutMs}ms`)), timeoutMs);
|
|
4583
|
+
timer.unref?.();
|
|
4584
|
+
})
|
|
4585
|
+
]);
|
|
4586
|
+
} finally {
|
|
4587
|
+
if (timer) clearTimeout(timer);
|
|
4588
|
+
}
|
|
4589
|
+
}
|
|
4590
|
+
async function stopExternalAttempt(executor, runId, attempt, reason, timeoutMs = V3_EXTERNAL_CALL_TIMEOUT_MS) {
|
|
4591
|
+
const ref = externalAttemptRef(runId, attempt);
|
|
4592
|
+
if (!executor || !ref) return { ok: true };
|
|
4593
|
+
try {
|
|
4594
|
+
await withExternalCallTimeout(Promise.resolve().then(() => executor.stop({
|
|
4595
|
+
...ref,
|
|
4596
|
+
reason,
|
|
4597
|
+
...attempt.idempotency ? { idempotency: attempt.idempotency } : {},
|
|
4598
|
+
...attempt.handoffMessageId ? { handoffMessageId: attempt.handoffMessageId } : {},
|
|
4599
|
+
...attempt.nodeCardMessageId ? { nodeCardMessageId: attempt.nodeCardMessageId } : {},
|
|
4600
|
+
...attempt.threadId ? { threadId: attempt.threadId } : {},
|
|
4601
|
+
...attempt.sessionId ? { sessionId: attempt.sessionId } : {}
|
|
4602
|
+
})), timeoutMs, "stop");
|
|
4603
|
+
return { ok: true };
|
|
4604
|
+
} catch (error) {
|
|
4605
|
+
return { ok: false, error };
|
|
4606
|
+
}
|
|
4607
|
+
}
|
|
4608
|
+
function journalPathOf(runDir) {
|
|
4609
|
+
return join8(runDir, "journal.ndjson");
|
|
4610
|
+
}
|
|
4611
|
+
function externalAttemptNode(dag, attempt) {
|
|
4612
|
+
if (!attempt.loop) return dag.nodes.find((node) => node.id === attempt.nodeId);
|
|
4613
|
+
const loopNode = dag.nodes.find((node) => node.id === attempt.loop.loopId);
|
|
4614
|
+
if (!loopNode || !isLoopNode(loopNode)) return void 0;
|
|
4615
|
+
const body = loopNode.body.nodes.find((node) => node.id === attempt.loop.bodyNodeId);
|
|
4616
|
+
if (!body) return void 0;
|
|
4617
|
+
const ref = attempt.loop;
|
|
4618
|
+
return {
|
|
4619
|
+
...body,
|
|
4620
|
+
id: loopInstanceId(ref.loopId, ref.iteration, ref.bodyNodeId),
|
|
4621
|
+
bot: body.bot ?? loopNode.bot,
|
|
4622
|
+
depends: body.depends.map((d) => ({ from: loopInstanceId(ref.loopId, ref.iteration, d.from) })),
|
|
4623
|
+
inputs: body.inputs.map((r) => ({
|
|
4624
|
+
from: loopInstanceId(ref.loopId, ref.iteration, r.from),
|
|
4625
|
+
...r.select ? { select: r.select } : {},
|
|
4626
|
+
...r.output !== void 0 ? { output: r.output } : {}
|
|
4627
|
+
}))
|
|
4628
|
+
};
|
|
4629
|
+
}
|
|
4630
|
+
function nodeTimeoutMs(node) {
|
|
4631
|
+
return (node?.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC) * 1e3;
|
|
4632
|
+
}
|
|
4633
|
+
function safeExternalAttemptDir(runDir, attemptId) {
|
|
4634
|
+
const root = resolve3(runDir);
|
|
4635
|
+
const candidate = resolve3(root, attemptId);
|
|
4636
|
+
const rel = relative3(root, candidate);
|
|
4637
|
+
if (!rel || isAbsolute3(rel) || rel === ".." || rel.startsWith(`..${sep}`)) {
|
|
4638
|
+
throw new Error(`v3 external: attempt path escapes run dir (${attemptId})`);
|
|
4639
|
+
}
|
|
4640
|
+
return candidate;
|
|
4641
|
+
}
|
|
4642
|
+
function appendAcceptedLocked(events, append, dag, input, now) {
|
|
4643
|
+
const current = classifyExternalAttempt(events, input.attemptId, {
|
|
4644
|
+
...input.instanceId !== void 0 ? { instanceId: input.instanceId } : {},
|
|
4645
|
+
sessionId: input.sessionId
|
|
4646
|
+
});
|
|
4647
|
+
if (!current.ok) return { ok: false, reason: current.reason };
|
|
4648
|
+
const { attempt } = current;
|
|
4649
|
+
const queued = attempt.placement === "main";
|
|
4650
|
+
const startedAt = queued && input.startedAt !== void 0 && Number.isFinite(input.startedAt) ? Math.min(input.startedAt, now) : void 0;
|
|
4651
|
+
if (attempt.phase === "accepted") {
|
|
4652
|
+
if (attempt.startPending && startedAt !== void 0) {
|
|
4653
|
+
const started = Math.max(startedAt, attempt.acceptedAt ?? startedAt);
|
|
4654
|
+
const deadlineAt2 = started + nodeTimeoutMs(externalAttemptNode(dag, attempt));
|
|
4655
|
+
append({
|
|
4656
|
+
type: "nodeHandoffStarted",
|
|
4657
|
+
nodeId: attempt.nodeId,
|
|
4658
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4659
|
+
attemptId: attempt.attemptId,
|
|
4660
|
+
startedAt: started,
|
|
4661
|
+
deadlineAt: deadlineAt2
|
|
4662
|
+
}, { durable: true });
|
|
4663
|
+
return { ok: true, deadlineAt: deadlineAt2, alreadyAccepted: false };
|
|
4664
|
+
}
|
|
4665
|
+
return { ok: true, deadlineAt: attempt.deadlineAt, alreadyAccepted: true };
|
|
4666
|
+
}
|
|
4667
|
+
if (attempt.phase === "dispatched") return { ok: false, reason: "unknown-attempt" };
|
|
4668
|
+
const acceptedAt = Math.min(input.acceptedAt ?? now, now);
|
|
4669
|
+
const timeoutMs = nodeTimeoutMs(externalAttemptNode(dag, attempt));
|
|
4670
|
+
const pending = queued && startedAt === void 0;
|
|
4671
|
+
const countFrom = startedAt !== void 0 ? Math.max(startedAt, acceptedAt) : acceptedAt;
|
|
4672
|
+
const deadlineAt = countFrom + timeoutMs + (pending ? v3MainTopicQueueGraceMs(timeoutMs) : 0);
|
|
4673
|
+
append({
|
|
4674
|
+
type: "nodeHandoffAccepted",
|
|
4675
|
+
nodeId: attempt.nodeId,
|
|
4676
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4677
|
+
attemptId: attempt.attemptId,
|
|
4678
|
+
sessionId: input.sessionId,
|
|
4679
|
+
acceptedAt,
|
|
4680
|
+
deadlineAt,
|
|
4681
|
+
...pending ? { startPending: true } : {}
|
|
4682
|
+
}, { durable: true });
|
|
4683
|
+
return { ok: true, deadlineAt, alreadyAccepted: false };
|
|
4684
|
+
}
|
|
4685
|
+
var RESULT_FILE = "result.json";
|
|
4686
|
+
function judgeStagedSubmission(node, manifest, workDir) {
|
|
4687
|
+
const revisit = readRevisitRequest(manifest, workDir);
|
|
4688
|
+
if (!revisit.ok) return { kind: "invalid", problems: revisit.problems };
|
|
4689
|
+
if (revisit.request) {
|
|
4690
|
+
if (!node.revisitTo?.includes(revisit.request.toNodeId)) {
|
|
4691
|
+
return {
|
|
4692
|
+
kind: "invalid",
|
|
4693
|
+
problems: [`result.json requests revisit to "${revisit.request.toNodeId}", not in node "${node.id}".revisitTo`]
|
|
4694
|
+
};
|
|
4695
|
+
}
|
|
4696
|
+
return { kind: "revisit", request: revisit.request };
|
|
4697
|
+
}
|
|
4698
|
+
if (node.resultSchema) {
|
|
4699
|
+
const entry = manifest.files.find((f) => f.path === RESULT_FILE);
|
|
4700
|
+
if (!entry) {
|
|
4701
|
+
return { kind: "invalid", problems: ["node declares resultSchema but the submission has no result.json"] };
|
|
4702
|
+
}
|
|
4703
|
+
const res = validateResult(join8(workDir, entry.path), node.resultSchema);
|
|
4704
|
+
if (!res.ok) return { kind: "invalid", problems: res.problems ?? ["result.json failed schema validation"] };
|
|
4705
|
+
}
|
|
4706
|
+
const contract = validateManifestArtifactContract(node.outputs, manifest);
|
|
4707
|
+
if (!contract.ok) return { kind: "invalid", problems: contract.problems };
|
|
4708
|
+
return { kind: "success" };
|
|
4709
|
+
}
|
|
4710
|
+
function moveAsideIfPresent(path, tag) {
|
|
4711
|
+
if (!existsSync8(path)) return;
|
|
4712
|
+
renameSync3(path, `${path}.uncommitted-${tag}`);
|
|
4713
|
+
}
|
|
4714
|
+
var V3_SUBMISSION_INTENT_FILE = "submission.intent.json";
|
|
4715
|
+
function submissionIntentPath(attemptDir) {
|
|
4716
|
+
return join8(attemptDir, V3_SUBMISSION_INTENT_FILE);
|
|
4717
|
+
}
|
|
4718
|
+
function removeSubmissionIntent(attemptDir, stageTag) {
|
|
4719
|
+
rmSync(submissionIntentPath(attemptDir), { force: true });
|
|
4720
|
+
rmSync(join8(attemptDir, "submissions", stageTag), { recursive: true, force: true });
|
|
4721
|
+
try {
|
|
4722
|
+
rmdirSync(join8(attemptDir, "submissions"));
|
|
4723
|
+
} catch {
|
|
4724
|
+
}
|
|
4725
|
+
}
|
|
4726
|
+
function readSubmissionIntent(attemptDir) {
|
|
4727
|
+
const path = submissionIntentPath(attemptDir);
|
|
4728
|
+
if (!existsSync8(path)) return void 0;
|
|
4729
|
+
try {
|
|
4730
|
+
const raw = JSON.parse(readFileSync10(path, "utf-8"));
|
|
4731
|
+
if (raw.schemaVersion !== 1 || typeof raw.attemptId !== "string" || typeof raw.stageTag !== "string") {
|
|
4732
|
+
return "corrupt";
|
|
4733
|
+
}
|
|
4734
|
+
if (!/^[0-9a-f-]{36}$/.test(raw.stageTag)) return "corrupt";
|
|
4735
|
+
return { schemaVersion: 1, attemptId: raw.attemptId, stageTag: raw.stageTag };
|
|
4736
|
+
} catch {
|
|
4737
|
+
return "corrupt";
|
|
4738
|
+
}
|
|
4739
|
+
}
|
|
4740
|
+
function verifyInstalledSubmissionSync(manifestPath, workDir) {
|
|
4741
|
+
let manifest;
|
|
4742
|
+
try {
|
|
4743
|
+
manifest = JSON.parse(readFileSync10(manifestPath, "utf-8"));
|
|
4744
|
+
} catch (err) {
|
|
4745
|
+
return { ok: false, problems: [`manifest unreadable: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4746
|
+
}
|
|
4747
|
+
if (manifest?.schemaVersion !== MANIFEST_SCHEMA_VERSION || manifest.status !== "ok" || !Array.isArray(manifest.files) || manifest.files.length === 0) {
|
|
4748
|
+
return { ok: false, problems: ["manifest is not an engine-generated success manifest"] };
|
|
4749
|
+
}
|
|
4750
|
+
const root = resolve3(workDir);
|
|
4751
|
+
const problems = [];
|
|
4752
|
+
for (const file of manifest.files) {
|
|
4753
|
+
const target = resolve3(root, file.path);
|
|
4754
|
+
const rel = relative3(root, target);
|
|
4755
|
+
if (!rel || isAbsolute3(rel) || rel.startsWith("..")) {
|
|
4756
|
+
problems.push(`${file.path}: escapes the output directory`);
|
|
4757
|
+
continue;
|
|
4758
|
+
}
|
|
4759
|
+
try {
|
|
4760
|
+
if (!lstatSync6(target).isFile()) {
|
|
4761
|
+
problems.push(`${file.path}: not a regular file`);
|
|
4762
|
+
continue;
|
|
4763
|
+
}
|
|
4764
|
+
const bytes = readFileSync10(target);
|
|
4765
|
+
if (bytes.length !== file.bytes || createHash5("sha256").update(bytes).digest("hex") !== file.sha256) {
|
|
4766
|
+
problems.push(`${file.path}: bytes do not match the manifest`);
|
|
4767
|
+
}
|
|
4768
|
+
} catch (err) {
|
|
4769
|
+
problems.push(`${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
4770
|
+
}
|
|
4771
|
+
}
|
|
4772
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true, manifest };
|
|
4773
|
+
}
|
|
4774
|
+
function appendInstalledVerdictLocked(input) {
|
|
4775
|
+
const { attempt, append, events, verdict, manifestPath } = input;
|
|
4776
|
+
const identity = {
|
|
4777
|
+
nodeId: attempt.nodeId,
|
|
4778
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4779
|
+
attemptId: attempt.attemptId
|
|
4780
|
+
};
|
|
4781
|
+
if (verdict.kind === "revisit") {
|
|
4782
|
+
const budget = revisitBudgetStatus([...events], attempt.nodeId, verdict.request.toNodeId);
|
|
4783
|
+
if (!budget.ok) {
|
|
4784
|
+
append({
|
|
4785
|
+
type: "nodeBlocked",
|
|
4786
|
+
...identity,
|
|
4787
|
+
errorClass: "resultInvalid",
|
|
4788
|
+
errorCode: "REVISIT_BUDGET_EXHAUSTED",
|
|
4789
|
+
message: budget.detail,
|
|
4790
|
+
revisitTo: verdict.request.toNodeId
|
|
4791
|
+
}, { durable: true });
|
|
4792
|
+
return { ok: true, kind: "blocked", errorCode: "REVISIT_BUDGET_EXHAUSTED" };
|
|
4793
|
+
}
|
|
4794
|
+
const plan = planV3Revisit({
|
|
4795
|
+
runDir: input.runDir,
|
|
4796
|
+
dag: input.dag,
|
|
4797
|
+
events,
|
|
4798
|
+
nodeId: attempt.nodeId,
|
|
4799
|
+
instanceId: attempt.instanceId ?? attempt.nodeId,
|
|
4800
|
+
attemptId: attempt.attemptId,
|
|
4801
|
+
request: verdict.request,
|
|
4802
|
+
sourceManifestPath: manifestPath
|
|
4803
|
+
});
|
|
4804
|
+
for (const event of plan.events) append(event);
|
|
4805
|
+
append({ type: "nodeAttemptDrained", ...identity, reason: "obsoleteAttempt" }, { durable: true });
|
|
4806
|
+
return { ok: true, kind: "revisit", toNodeId: verdict.request.toNodeId };
|
|
4807
|
+
}
|
|
4808
|
+
append({ type: "nodeSucceeded", ...identity, manifestPath }, { durable: true });
|
|
4809
|
+
return { ok: true, kind: "succeeded", manifestPath };
|
|
4810
|
+
}
|
|
4811
|
+
function installVerdictCommitted(events, attemptId) {
|
|
4812
|
+
return events.some((event) => event.type === "nodeSucceeded" && event.attemptId === attemptId || event.type === "nodeRevisitRequested" && event.attemptId === attemptId || event.type === "nodeBlocked" && event.attemptId === attemptId && event.errorCode === "REVISIT_BUDGET_EXHAUSTED");
|
|
4813
|
+
}
|
|
4814
|
+
function settleInstallIntentLocked(runDir, dag, events, append, attemptId) {
|
|
4815
|
+
const attemptDir = safeExternalAttemptDir(runDir, attemptId);
|
|
4816
|
+
const intent = readSubmissionIntent(attemptDir);
|
|
4817
|
+
if (intent === void 0) return { kind: "none" };
|
|
4818
|
+
if (intent === "corrupt" || intent.attemptId !== attemptId) {
|
|
4819
|
+
rmSync(submissionIntentPath(attemptDir), { force: true });
|
|
4820
|
+
return { kind: "discarded" };
|
|
4821
|
+
}
|
|
4822
|
+
const workDir = join8(attemptDir, "work");
|
|
4823
|
+
const manifestPath = join8(attemptDir, "manifest.json");
|
|
4824
|
+
const stageDir = join8(attemptDir, "submissions", intent.stageTag);
|
|
4825
|
+
const discard = () => {
|
|
4826
|
+
if (!installVerdictCommitted(events, attemptId)) {
|
|
4827
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4828
|
+
moveAsideIfPresent(manifestPath, intent.stageTag);
|
|
4829
|
+
}
|
|
4830
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4831
|
+
return { kind: "discarded" };
|
|
4832
|
+
};
|
|
4833
|
+
if (installVerdictCommitted(events, attemptId)) {
|
|
4834
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4835
|
+
return { kind: "cleaned" };
|
|
4836
|
+
}
|
|
4837
|
+
const current = classifyExternalAttempt(events, attemptId);
|
|
4838
|
+
if (!current.ok) return discard();
|
|
4839
|
+
const node = externalAttemptNode(dag, current.attempt);
|
|
4840
|
+
if (!node || !isGoalNode(node)) return discard();
|
|
4841
|
+
if (existsSync8(join8(stageDir, "work"))) {
|
|
4842
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4843
|
+
renameSync3(join8(stageDir, "work"), workDir);
|
|
4844
|
+
}
|
|
4845
|
+
if (existsSync8(join8(stageDir, "manifest.json"))) renameSync3(join8(stageDir, "manifest.json"), manifestPath);
|
|
4846
|
+
const verified = verifyInstalledSubmissionSync(manifestPath, workDir);
|
|
4847
|
+
if (!verified.ok) return discard();
|
|
4848
|
+
const verdict = judgeStagedSubmission(node, verified.manifest, workDir);
|
|
4849
|
+
if (verdict.kind === "invalid") return discard();
|
|
4850
|
+
const outcome = appendInstalledVerdictLocked({
|
|
4851
|
+
runDir,
|
|
4852
|
+
dag,
|
|
4853
|
+
events,
|
|
4854
|
+
append,
|
|
4855
|
+
attempt: current.attempt,
|
|
4856
|
+
verdict,
|
|
4857
|
+
manifestPath
|
|
4858
|
+
});
|
|
4859
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4860
|
+
return { kind: "committed", outcome };
|
|
4861
|
+
}
|
|
4862
|
+
function settleExternalInstallIntent(runDir, dag, attemptId) {
|
|
4863
|
+
if (!existsSync8(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)))) return { kind: "none" };
|
|
4864
|
+
return withJournalMutationSync(journalPathOf(runDir), ({ events, append }) => settleInstallIntentLocked(runDir, dag, events, append, attemptId));
|
|
4865
|
+
}
|
|
4866
|
+
function hasExternalInstallIntent(runDir, attemptId) {
|
|
4867
|
+
return existsSync8(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)));
|
|
4868
|
+
}
|
|
4869
|
+
|
|
4870
|
+
// src/workflows/v3/host-execution.ts
|
|
4871
|
+
import { createHash as createHash7, randomUUID as randomUUID3 } from "node:crypto";
|
|
4872
|
+
import {
|
|
4873
|
+
closeSync as closeSync6,
|
|
4874
|
+
constants as constants4,
|
|
4875
|
+
existsSync as existsSync9,
|
|
4876
|
+
fsyncSync as fsyncSync4,
|
|
4877
|
+
lstatSync as lstatSync7,
|
|
4878
|
+
linkSync as linkSync2,
|
|
4879
|
+
mkdirSync as mkdirSync7,
|
|
4880
|
+
openSync as openSync6,
|
|
4881
|
+
readFileSync as readFileSync11,
|
|
4882
|
+
unlinkSync as unlinkSync3,
|
|
4883
|
+
writeFileSync as writeFileSync7
|
|
4884
|
+
} from "node:fs";
|
|
4885
|
+
import { dirname as dirname7, isAbsolute as isAbsolute4, join as join9, relative as relative4, resolve as resolve4, sep as sep2 } from "node:path";
|
|
4886
|
+
|
|
4887
|
+
// src/workflows/shared/idempotency-key.ts
|
|
4888
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
3650
4889
|
function deriveIdempotencyKey(tuple, opts = {}) {
|
|
3651
4890
|
const namespace = opts.namespace ?? "wf_";
|
|
3652
4891
|
const maxLength = opts.maxLength ?? 50;
|
|
@@ -3663,7 +4902,7 @@ function deriveIdempotencyKey(tuple, opts = {}) {
|
|
|
3663
4902
|
}
|
|
3664
4903
|
}
|
|
3665
4904
|
const seed = canonicalJson(tuple);
|
|
3666
|
-
const hash =
|
|
4905
|
+
const hash = createHash6("sha256").update(seed, "utf-8").digest("hex");
|
|
3667
4906
|
return namespace + hash.substring(0, maxLength - namespace.length);
|
|
3668
4907
|
}
|
|
3669
4908
|
|
|
@@ -3724,23 +4963,23 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3724
4963
|
idempotencyTtlMs: input.registered.executor.idempotencyTtlMs,
|
|
3725
4964
|
approvalDigest
|
|
3726
4965
|
};
|
|
3727
|
-
const absolutePath =
|
|
4966
|
+
const absolutePath = join9(input.attemptDir, "host-input.json");
|
|
3728
4967
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3729
4968
|
ensureSecureDirectoryTree(input.runDir, input.attemptDir);
|
|
3730
4969
|
const content = `${JSON.stringify(prepared, null, 2)}
|
|
3731
4970
|
`;
|
|
3732
4971
|
writeCreateOrVerify(absolutePath, content, 384, "host input");
|
|
3733
|
-
const existing =
|
|
4972
|
+
const existing = readFileSync11(absolutePath, "utf-8");
|
|
3734
4973
|
if (canonicalJson(parsePreparedHostInput(JSON.parse(existing))) !== canonicalJson(prepared)) {
|
|
3735
4974
|
throw new Error(`v3 host input conflict at ${absolutePath}; refusing to overwrite prepared bytes`);
|
|
3736
4975
|
}
|
|
3737
4976
|
fsyncRegularFileSync(absolutePath);
|
|
3738
4977
|
fsyncDirectorySyncPortable(input.attemptDir);
|
|
3739
|
-
const bytes =
|
|
4978
|
+
const bytes = readFileSync11(absolutePath);
|
|
3740
4979
|
return {
|
|
3741
4980
|
prepared,
|
|
3742
4981
|
inputRef: {
|
|
3743
|
-
path:
|
|
4982
|
+
path: relative4(input.runDir, absolutePath),
|
|
3744
4983
|
sha256: sha256(bytes),
|
|
3745
4984
|
bytes: bytes.length
|
|
3746
4985
|
},
|
|
@@ -3748,12 +4987,12 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3748
4987
|
};
|
|
3749
4988
|
}
|
|
3750
4989
|
function readCrashLeftV3PreparedHostInput(input) {
|
|
3751
|
-
const absolutePath =
|
|
3752
|
-
if (!
|
|
4990
|
+
const absolutePath = join9(input.attemptDir, "host-input.json");
|
|
4991
|
+
if (!existsSync9(absolutePath)) return void 0;
|
|
3753
4992
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3754
4993
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3755
4994
|
assertRegularFile(absolutePath, "host input");
|
|
3756
|
-
const bytes =
|
|
4995
|
+
const bytes = readFileSync11(absolutePath);
|
|
3757
4996
|
let prepared;
|
|
3758
4997
|
try {
|
|
3759
4998
|
prepared = parsePreparedHostInput(JSON.parse(bytes.toString("utf-8")));
|
|
@@ -3772,7 +5011,7 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3772
5011
|
inputHash
|
|
3773
5012
|
});
|
|
3774
5013
|
const inputRef = {
|
|
3775
|
-
path:
|
|
5014
|
+
path: relative4(input.runDir, absolutePath),
|
|
3776
5015
|
sha256: sha256(bytes),
|
|
3777
5016
|
bytes: bytes.length
|
|
3778
5017
|
};
|
|
@@ -3795,14 +5034,14 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3795
5034
|
});
|
|
3796
5035
|
}
|
|
3797
5036
|
function readAndVerifyV3PreparedHostInput(input) {
|
|
3798
|
-
if (
|
|
5037
|
+
if (isAbsolute4(input.inputRef.path) || input.inputRef.path.includes("\0")) {
|
|
3799
5038
|
throw new Error("v3 host inputRef.path must be run-relative");
|
|
3800
5039
|
}
|
|
3801
|
-
const absolutePath =
|
|
5040
|
+
const absolutePath = resolve4(input.runDir, input.inputRef.path);
|
|
3802
5041
|
assertPathInside(input.runDir, absolutePath);
|
|
3803
5042
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3804
5043
|
assertRegularFile(absolutePath, "host input");
|
|
3805
|
-
const bytes =
|
|
5044
|
+
const bytes = readFileSync11(absolutePath);
|
|
3806
5045
|
if (bytes.length !== input.inputRef.bytes || sha256(bytes) !== input.inputRef.sha256) {
|
|
3807
5046
|
throw new Error("v3 host input sidecar bytes/hash mismatch");
|
|
3808
5047
|
}
|
|
@@ -3845,10 +5084,10 @@ function readAndVerifyV3PreparedHostInput(input) {
|
|
|
3845
5084
|
function writeV3HostSuccessArtifacts(input) {
|
|
3846
5085
|
canonicalJson(input.output);
|
|
3847
5086
|
canonicalJson(input.externalRefs);
|
|
3848
|
-
const outputDir =
|
|
5087
|
+
const outputDir = join9(input.attemptDir, "work");
|
|
3849
5088
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3850
5089
|
ensureSecureDirectoryTree(input.runDir, outputDir);
|
|
3851
|
-
const resultPath =
|
|
5090
|
+
const resultPath = join9(outputDir, "result.json");
|
|
3852
5091
|
const resultContent = `${JSON.stringify({
|
|
3853
5092
|
schemaVersion: 1,
|
|
3854
5093
|
runId: input.runId,
|
|
@@ -3879,7 +5118,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3879
5118
|
mime: "application/json"
|
|
3880
5119
|
}]
|
|
3881
5120
|
};
|
|
3882
|
-
const manifestPath =
|
|
5121
|
+
const manifestPath = join9(input.attemptDir, "manifest.json");
|
|
3883
5122
|
writeCreateOrVerify(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
3884
5123
|
`, 384, "host manifest");
|
|
3885
5124
|
fsyncRegularFileSync(resultPath);
|
|
@@ -3889,7 +5128,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3889
5128
|
return { manifestPath, resultPath };
|
|
3890
5129
|
}
|
|
3891
5130
|
function readAndVerifyV3HostSuccessResult(input) {
|
|
3892
|
-
assertRegularFile(
|
|
5131
|
+
assertRegularFile(join9(input.attemptDir, "manifest.json"), "host manifest");
|
|
3893
5132
|
if (input.manifest.status !== "ok") {
|
|
3894
5133
|
throw new Error("v3 host success manifest status must be ok");
|
|
3895
5134
|
}
|
|
@@ -3900,11 +5139,11 @@ function readAndVerifyV3HostSuccessResult(input) {
|
|
|
3900
5139
|
if (file.name !== "result" || file.path !== "result.json" || file.kind !== "json") {
|
|
3901
5140
|
throw new Error("v3 host success manifest must expose exactly result.json as JSON");
|
|
3902
5141
|
}
|
|
3903
|
-
const resultPath =
|
|
5142
|
+
const resultPath = join9(input.attemptDir, "work", "result.json");
|
|
3904
5143
|
assertPathInside(input.runDir, resultPath);
|
|
3905
5144
|
assertSecureExistingPath(input.runDir, resultPath, "host result");
|
|
3906
5145
|
assertRegularFile(resultPath, "host result");
|
|
3907
|
-
const bytes =
|
|
5146
|
+
const bytes = readFileSync11(resultPath);
|
|
3908
5147
|
if (bytes.length !== file.bytes || sha256(bytes) !== file.sha256) {
|
|
3909
5148
|
throw new Error("v3 host success result bytes/hash mismatch");
|
|
3910
5149
|
}
|
|
@@ -3991,7 +5230,7 @@ function parsePreparedHostInput(raw) {
|
|
|
3991
5230
|
return raw;
|
|
3992
5231
|
}
|
|
3993
5232
|
function writeCreateOrVerify(path, content, mode, label) {
|
|
3994
|
-
const tempPath = `${path}.tmp-${process.pid}-${
|
|
5233
|
+
const tempPath = `${path}.tmp-${process.pid}-${randomUUID3()}`;
|
|
3995
5234
|
let fd;
|
|
3996
5235
|
try {
|
|
3997
5236
|
fd = openSync6(
|
|
@@ -3999,17 +5238,17 @@ function writeCreateOrVerify(path, content, mode, label) {
|
|
|
3999
5238
|
constants4.O_WRONLY | constants4.O_CREAT | constants4.O_EXCL | (constants4.O_NOFOLLOW ?? 0),
|
|
4000
5239
|
mode
|
|
4001
5240
|
);
|
|
4002
|
-
|
|
5241
|
+
writeFileSync7(fd, content, "utf-8");
|
|
4003
5242
|
fsyncSync4(fd);
|
|
4004
5243
|
closeSync6(fd);
|
|
4005
5244
|
fd = void 0;
|
|
4006
5245
|
try {
|
|
4007
5246
|
linkSync2(tempPath, path);
|
|
4008
|
-
fsyncDirectorySyncPortable(
|
|
5247
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4009
5248
|
} catch (err) {
|
|
4010
5249
|
if (err.code !== "EEXIST") throw err;
|
|
4011
5250
|
assertRegularFile(path, label);
|
|
4012
|
-
if (
|
|
5251
|
+
if (readFileSync11(path, "utf-8") !== content) {
|
|
4013
5252
|
throw new Error(`v3 ${label} conflict at ${path}; refusing to overwrite committed bytes`);
|
|
4014
5253
|
}
|
|
4015
5254
|
}
|
|
@@ -4017,15 +5256,15 @@ function writeCreateOrVerify(path, content, mode, label) {
|
|
|
4017
5256
|
if (fd !== void 0) closeSync6(fd);
|
|
4018
5257
|
try {
|
|
4019
5258
|
unlinkSync3(tempPath);
|
|
4020
|
-
fsyncDirectorySyncPortable(
|
|
5259
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4021
5260
|
} catch {
|
|
4022
5261
|
}
|
|
4023
5262
|
}
|
|
4024
5263
|
fsyncRegularFileSync(path);
|
|
4025
|
-
fsyncDirectorySyncPortable(
|
|
5264
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4026
5265
|
}
|
|
4027
5266
|
function assertRegularFile(path, label) {
|
|
4028
|
-
const stat =
|
|
5267
|
+
const stat = lstatSync7(path);
|
|
4029
5268
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
4030
5269
|
throw new Error(`v3 ${label} must be a regular file: ${path}`);
|
|
4031
5270
|
}
|
|
@@ -4034,47 +5273,47 @@ function assertRegularFile(path, label) {
|
|
|
4034
5273
|
}
|
|
4035
5274
|
}
|
|
4036
5275
|
function assertPathInside(parent, child) {
|
|
4037
|
-
const root =
|
|
4038
|
-
const rel =
|
|
4039
|
-
if (rel === "" || rel === ".." || rel.startsWith(`..${
|
|
5276
|
+
const root = resolve4(parent);
|
|
5277
|
+
const rel = relative4(root, resolve4(child));
|
|
5278
|
+
if (rel === "" || rel === ".." || rel.startsWith(`..${sep2}`) || isAbsolute4(rel)) {
|
|
4040
5279
|
throw new Error("v3 host path escapes runDir");
|
|
4041
5280
|
}
|
|
4042
5281
|
}
|
|
4043
5282
|
function ensureSecureDirectoryTree(rootPath, targetPath) {
|
|
4044
|
-
const root =
|
|
4045
|
-
const target =
|
|
4046
|
-
const rel =
|
|
4047
|
-
if (rel === ".." || rel.startsWith(`..${
|
|
5283
|
+
const root = resolve4(rootPath);
|
|
5284
|
+
const target = resolve4(targetPath);
|
|
5285
|
+
const rel = relative4(root, target);
|
|
5286
|
+
if (rel === ".." || rel.startsWith(`..${sep2}`) || isAbsolute4(rel)) {
|
|
4048
5287
|
throw new Error("v3 host directory escapes runDir");
|
|
4049
5288
|
}
|
|
4050
5289
|
assertDirectory(root, "run directory");
|
|
4051
5290
|
let current = root;
|
|
4052
5291
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4053
|
-
current =
|
|
4054
|
-
if (!
|
|
5292
|
+
current = join9(current, segment);
|
|
5293
|
+
if (!existsSync9(current)) mkdirSync7(current, { mode: 448 });
|
|
4055
5294
|
assertDirectory(current, "host artifact directory");
|
|
4056
5295
|
}
|
|
4057
5296
|
}
|
|
4058
5297
|
function assertSecureExistingPath(rootPath, filePath, label) {
|
|
4059
|
-
const root =
|
|
4060
|
-
const target =
|
|
5298
|
+
const root = resolve4(rootPath);
|
|
5299
|
+
const target = resolve4(filePath);
|
|
4061
5300
|
assertPathInside(root, target);
|
|
4062
5301
|
assertDirectory(root, "run directory");
|
|
4063
|
-
const rel =
|
|
5302
|
+
const rel = relative4(root, dirname7(target));
|
|
4064
5303
|
let current = root;
|
|
4065
5304
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4066
|
-
current =
|
|
5305
|
+
current = join9(current, segment);
|
|
4067
5306
|
assertDirectory(current, `${label} parent`);
|
|
4068
5307
|
}
|
|
4069
5308
|
}
|
|
4070
5309
|
function assertDirectory(path, label) {
|
|
4071
|
-
const stat =
|
|
5310
|
+
const stat = lstatSync7(path);
|
|
4072
5311
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
4073
5312
|
throw new Error(`v3 ${label} must be a real directory: ${path}`);
|
|
4074
5313
|
}
|
|
4075
5314
|
}
|
|
4076
5315
|
function sha256(value) {
|
|
4077
|
-
return
|
|
5316
|
+
return createHash7("sha256").update(value).digest("hex");
|
|
4078
5317
|
}
|
|
4079
5318
|
function isRecord3(value) {
|
|
4080
5319
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -4136,7 +5375,7 @@ function renderGoalFile(goal, resultSchema, loopCtx, nodeInstructions, hasWorkfl
|
|
|
4136
5375
|
...artifactSection,
|
|
4137
5376
|
"## How to complete this node",
|
|
4138
5377
|
"You are an autonomous agent completing exactly ONE botmux v3 workflow node.",
|
|
4139
|
-
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (
|
|
5378
|
+
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (the same protocol is available via `botmux skill show botmux-goal-ask`).",
|
|
4140
5379
|
"",
|
|
4141
5380
|
`- Upstream inputs: the file at $${E.INPUTS_PATH} is a JSON object \`{ "inputs": [...] }\` listing upstream products, each with an absolute \`path\`. Read only the ones the goal needs (it may be empty). If it includes an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`, read that JSON file before continuing. If an \`omitted\` array is present, those declared inputs were intentionally not produced (their workflow branch was not taken) \u2014 treat their absence as by-design, do NOT invent their content.`,
|
|
4142
5381
|
`- Revisit feedback: if any input has \`"from": "revisit"\`, a DOWNSTREAM node sent this node back because its product was inadequate. You MUST read these before doing anything else: \`reason\` (why you were sent back), \`source:*\` (the downstream node's output \u2014 the evidence of what was wrong), and \`previous:*\` (YOUR OWN previous output \u2014 edit/fix it, do not rewrite from scratch). Address the reason; do not just reproduce the prior output.`,
|
|
@@ -4190,10 +5429,10 @@ function classifyTerminal(errorClass, opts) {
|
|
|
4190
5429
|
}
|
|
4191
5430
|
}
|
|
4192
5431
|
function readGoalAsk(askPath) {
|
|
4193
|
-
if (!
|
|
5432
|
+
if (!existsSync10(askPath)) return void 0;
|
|
4194
5433
|
let parsed;
|
|
4195
5434
|
try {
|
|
4196
|
-
parsed = JSON.parse(
|
|
5435
|
+
parsed = JSON.parse(readFileSync12(askPath, "utf-8"));
|
|
4197
5436
|
} catch {
|
|
4198
5437
|
return void 0;
|
|
4199
5438
|
}
|
|
@@ -4217,126 +5456,12 @@ function mergeNodeCapability(snap, override) {
|
|
|
4217
5456
|
...override.model ? { model: override.model } : {}
|
|
4218
5457
|
};
|
|
4219
5458
|
}
|
|
4220
|
-
|
|
4221
|
-
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4222
|
-
if (!entry) return { ok: true };
|
|
4223
|
-
let value;
|
|
4224
|
-
try {
|
|
4225
|
-
value = JSON.parse(readFileSync9(join6(outputDir, entry.path), "utf-8"));
|
|
4226
|
-
} catch {
|
|
4227
|
-
return { ok: true };
|
|
4228
|
-
}
|
|
4229
|
-
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4230
|
-
return { ok: true };
|
|
4231
|
-
}
|
|
4232
|
-
const v = value;
|
|
4233
|
-
const problems = [];
|
|
4234
|
-
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4235
|
-
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4236
|
-
}
|
|
4237
|
-
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4238
|
-
problems.push('result.json "reason" must be a string when present');
|
|
4239
|
-
}
|
|
4240
|
-
if (problems.length > 0) return { ok: false, problems };
|
|
4241
|
-
return {
|
|
4242
|
-
ok: true,
|
|
4243
|
-
request: {
|
|
4244
|
-
toNodeId: v.revisitTo,
|
|
4245
|
-
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4246
|
-
}
|
|
4247
|
-
};
|
|
4248
|
-
}
|
|
4249
|
-
function validateResult(filePath, schema) {
|
|
4250
|
-
if (!existsSync7(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4251
|
-
let parsed;
|
|
4252
|
-
try {
|
|
4253
|
-
parsed = JSON.parse(readFileSync9(filePath, "utf-8"));
|
|
4254
|
-
} catch (err) {
|
|
4255
|
-
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4256
|
-
}
|
|
4257
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4258
|
-
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4259
|
-
}
|
|
4260
|
-
const obj = parsed;
|
|
4261
|
-
const problems = [];
|
|
4262
|
-
for (const field of schema.required ?? []) {
|
|
4263
|
-
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4264
|
-
}
|
|
4265
|
-
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4266
|
-
if (!(name in obj)) continue;
|
|
4267
|
-
const v = obj[name];
|
|
4268
|
-
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4269
|
-
if (!okType) {
|
|
4270
|
-
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4271
|
-
continue;
|
|
4272
|
-
}
|
|
4273
|
-
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4274
|
-
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4275
|
-
}
|
|
4276
|
-
}
|
|
4277
|
-
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4278
|
-
}
|
|
4279
|
-
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
4280
|
-
function attemptNumber(attemptId) {
|
|
4281
|
-
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
4282
|
-
return m ? parseInt(m[1], 10) : void 0;
|
|
4283
|
-
}
|
|
4284
|
-
function nextAttemptIdFor(events, key) {
|
|
4285
|
-
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
4286
|
-
let maxSeen = 0;
|
|
4287
|
-
let reserved;
|
|
4288
|
-
let prepared;
|
|
4289
|
-
for (const e of events) {
|
|
4290
|
-
if (e.type === "nodeDispatched" && matches(e)) {
|
|
4291
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4292
|
-
if (n2 === void 0) continue;
|
|
4293
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4294
|
-
if (reserved === n2) reserved = void 0;
|
|
4295
|
-
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
4296
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4297
|
-
if (n2 === void 0) continue;
|
|
4298
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4299
|
-
if (reserved === n2) reserved = void 0;
|
|
4300
|
-
prepared = e.attemptId;
|
|
4301
|
-
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
4302
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4303
|
-
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
4304
|
-
if (prepared === e.attemptId) prepared = void 0;
|
|
4305
|
-
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
4306
|
-
prepared = void 0;
|
|
4307
|
-
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
4308
|
-
const n2 = attemptNumber(e.nextAttemptId);
|
|
4309
|
-
if (n2 === void 0) continue;
|
|
4310
|
-
reserved = n2;
|
|
4311
|
-
prepared = void 0;
|
|
4312
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4313
|
-
}
|
|
4314
|
-
}
|
|
4315
|
-
if (prepared) return prepared;
|
|
4316
|
-
const n = reserved ?? maxSeen + 1;
|
|
4317
|
-
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
4318
|
-
}
|
|
4319
|
-
function latestAttemptIdFor(events, key) {
|
|
4320
|
-
let latest;
|
|
4321
|
-
let latestNumber = -1;
|
|
4322
|
-
for (const e of events) {
|
|
4323
|
-
if ((e.type === "nodeDispatched" || e.type === "hostInputPrepared" || e.type === "hostEffectIntent" || e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && (e.instanceId ?? e.nodeId) === key) {
|
|
4324
|
-
const n = attemptNumber(e.attemptId);
|
|
4325
|
-
if (n === void 0) {
|
|
4326
|
-
if (latestNumber < 0) latest = e.attemptId;
|
|
4327
|
-
} else if (n >= latestNumber) {
|
|
4328
|
-
latest = e.attemptId;
|
|
4329
|
-
latestNumber = n;
|
|
4330
|
-
}
|
|
4331
|
-
}
|
|
4332
|
-
}
|
|
4333
|
-
return latest;
|
|
4334
|
-
}
|
|
5459
|
+
var V3_DEFAULT_CONCURRENCY = Object.freeze({ global: 4, perBot: 1, perCli: 2 });
|
|
4335
5460
|
function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) => candidate) {
|
|
4336
5461
|
const expectedHash = computeInputHash(normalize(value));
|
|
4337
|
-
if (!
|
|
5462
|
+
if (!existsSync10(path)) {
|
|
4338
5463
|
try {
|
|
4339
|
-
|
|
5464
|
+
writeFileSync8(path, `${JSON.stringify(value, null, 2)}
|
|
4340
5465
|
`, {
|
|
4341
5466
|
encoding: "utf-8",
|
|
4342
5467
|
flag: "wx",
|
|
@@ -4350,7 +5475,7 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4350
5475
|
}
|
|
4351
5476
|
let stored;
|
|
4352
5477
|
try {
|
|
4353
|
-
stored = normalize(JSON.parse(
|
|
5478
|
+
stored = normalize(JSON.parse(readFileSync12(path, "utf-8")));
|
|
4354
5479
|
} catch (error) {
|
|
4355
5480
|
throw new Error(
|
|
4356
5481
|
`v3 runtime: frozen ${label} is unreadable: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -4362,14 +5487,27 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4362
5487
|
);
|
|
4363
5488
|
}
|
|
4364
5489
|
}
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
const
|
|
4371
|
-
const
|
|
4372
|
-
|
|
5490
|
+
var HANDOFF_RESEND_BACKOFF_MS = 3e4;
|
|
5491
|
+
var PROBE_RETRY_BACKOFF_MS = 3e4;
|
|
5492
|
+
var PROBE_UNCERTAIN_AFTER_MS = 10 * 6e4;
|
|
5493
|
+
var probeFailures = /* @__PURE__ */ new Map();
|
|
5494
|
+
function forgetProbeFailuresOf(runDir) {
|
|
5495
|
+
const prefix = `${runDir}\0`;
|
|
5496
|
+
for (const key of probeFailures.keys()) {
|
|
5497
|
+
if (key.startsWith(prefix)) probeFailures.delete(key);
|
|
5498
|
+
}
|
|
5499
|
+
}
|
|
5500
|
+
async function runWorkflow(authorizedDag, deps, opts) {
|
|
5501
|
+
let dag = authorizedDag;
|
|
5502
|
+
const runDir = join10(opts.baseDir, dag.runId);
|
|
5503
|
+
mkdirSync8(runDir, { recursive: true });
|
|
5504
|
+
const journalPath = join10(runDir, "journal.ndjson");
|
|
5505
|
+
const statePath = join10(runDir, "STATE");
|
|
5506
|
+
const globalCap = opts.globalConcurrency ?? V3_DEFAULT_CONCURRENCY.global;
|
|
5507
|
+
const perBotCap = opts.perBotConcurrency ?? V3_DEFAULT_CONCURRENCY.perBot;
|
|
5508
|
+
const perCliDefaultCap = opts.perCliConcurrency ?? V3_DEFAULT_CONCURRENCY.perCli;
|
|
5509
|
+
const perCliById = opts.perCliConcurrencyById ?? {};
|
|
5510
|
+
const perCliCap = (cliId) => Object.prototype.hasOwnProperty.call(perCliById, cliId) ? perCliById[cliId] : perCliDefaultCap;
|
|
4373
5511
|
const gateMode = opts.gateMode ?? "blocking";
|
|
4374
5512
|
const hostResponseWaitMs = opts.hostResponseWaitMs ?? 15e3;
|
|
4375
5513
|
const executionContext = opts.executionContext ?? opts.resolvedWorkflowData;
|
|
@@ -4379,7 +5517,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4379
5517
|
if (!Number.isSafeInteger(hostResponseWaitMs) || hostResponseWaitMs < 1) {
|
|
4380
5518
|
throw new Error("v3 runtime: hostResponseWaitMs must be a positive safe integer");
|
|
4381
5519
|
}
|
|
4382
|
-
const
|
|
5520
|
+
const externalMode = opts.nodeExecutor === "bot";
|
|
5521
|
+
const externalExecutor = deps.externalExecutor;
|
|
5522
|
+
if (externalMode && !externalExecutor) {
|
|
5523
|
+
throw new Error('v3 runtime: nodeExecutor "bot" requires an external node executor');
|
|
5524
|
+
}
|
|
5525
|
+
const externalAcceptTimeoutMs = opts.externalAcceptTimeoutMs ?? V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS;
|
|
5526
|
+
if (!Number.isSafeInteger(externalAcceptTimeoutMs) || externalAcceptTimeoutMs < 0) {
|
|
5527
|
+
throw new Error("v3 runtime: externalAcceptTimeoutMs must be a non-negative safe integer");
|
|
5528
|
+
}
|
|
5529
|
+
const clock = () => deps.now?.() ?? Date.now();
|
|
5530
|
+
const externalCallTimeoutMs = opts.externalCallTimeoutMs ?? V3_EXTERNAL_CALL_TIMEOUT_MS;
|
|
5531
|
+
if (!Number.isSafeInteger(externalCallTimeoutMs) || externalCallTimeoutMs < 1) {
|
|
5532
|
+
throw new Error("v3 runtime: externalCallTimeoutMs must be a positive safe integer");
|
|
5533
|
+
}
|
|
5534
|
+
const probeKey = (attemptId) => `${runDir}\0${attemptId}`;
|
|
5535
|
+
let nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5536
|
+
let dagRevision = 1;
|
|
5537
|
+
let pendingTouched;
|
|
5538
|
+
let pendingRevision;
|
|
4383
5539
|
const botSnapshots = opts.frozenBotSnapshots ? new Map(opts.frozenBotSnapshots) : /* @__PURE__ */ new Map();
|
|
4384
5540
|
const freezeBot = (bot) => {
|
|
4385
5541
|
const key = bot ?? "";
|
|
@@ -4400,7 +5556,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4400
5556
|
for (const b of node.body.nodes) freezeBot(b.bot ?? node.bot);
|
|
4401
5557
|
}
|
|
4402
5558
|
}
|
|
4403
|
-
|
|
5559
|
+
const validateSnapshot = (key, snap) => {
|
|
4404
5560
|
if (deps.validateExecutionSnapshot) {
|
|
4405
5561
|
deps.validateExecutionSnapshot(snap, key);
|
|
4406
5562
|
} else if (!isV3SupportedCli(snap.cliId)) {
|
|
@@ -4408,22 +5564,83 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4408
5564
|
`v3 runtime: bot "${key || "<default>"}" resolves to CLI "${snap.cliId}", which is not supported by v3 goal-mode (supported: ${V3_SUPPORTED_CLIS.join(", ")})`
|
|
4409
5565
|
);
|
|
4410
5566
|
}
|
|
4411
|
-
}
|
|
5567
|
+
};
|
|
5568
|
+
for (const [key, snap] of botSnapshots) validateSnapshot(key, snap);
|
|
5569
|
+
const botSnapshotFor2 = (selector) => {
|
|
5570
|
+
const snap = snapshotForSelector(botSnapshots, selector);
|
|
5571
|
+
if (!snap) {
|
|
5572
|
+
throw new Error(`v3 runtime: no frozen bot snapshot for selector "${selector || "<default>"}"`);
|
|
5573
|
+
}
|
|
5574
|
+
return snap;
|
|
5575
|
+
};
|
|
5576
|
+
const refreshEffectiveDag = (events) => {
|
|
5577
|
+
const history = foldV3Revisions(events);
|
|
5578
|
+
pendingTouched = history.pending ? new Set(history.pending.touched) : void 0;
|
|
5579
|
+
pendingRevision = history.pending?.revision;
|
|
5580
|
+
if (history.current === dagRevision) return;
|
|
5581
|
+
const effective = loadEffectiveV3Dag(runDir, authorizedDag, events);
|
|
5582
|
+
for (const [selector, snap] of v3ApprovedBotScope(botSnapshots, effective.botAdditions)) {
|
|
5583
|
+
if (botSnapshots.has(selector)) continue;
|
|
5584
|
+
validateSnapshot(selector, snap);
|
|
5585
|
+
botSnapshots.set(selector, snap);
|
|
5586
|
+
}
|
|
5587
|
+
for (const node of effective.dag.nodes) {
|
|
5588
|
+
if (isHostNode(node) && !deps.hostExecutors?.has(node.executor)) {
|
|
5589
|
+
throw new Error(`v3 runtime: host executor "${node.executor}" is not registered`);
|
|
5590
|
+
}
|
|
5591
|
+
if (isGoalNode(node)) botSnapshotFor2(node.bot);
|
|
5592
|
+
if (isLoopNode(node)) for (const body of node.body.nodes) botSnapshotFor2(body.bot ?? node.bot);
|
|
5593
|
+
}
|
|
5594
|
+
dag = effective.dag;
|
|
5595
|
+
nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5596
|
+
dagRevision = effective.revision;
|
|
5597
|
+
};
|
|
5598
|
+
const decisionStillCurrent = (latest, nodeId, loopId) => {
|
|
5599
|
+
if (currentV3Revision(latest) !== dagRevision) return false;
|
|
5600
|
+
if (nodeId === void 0) return true;
|
|
5601
|
+
const pending = foldV3Revisions(latest).pending;
|
|
5602
|
+
return !pending || !v3NodeHeldByProposal(new Set(pending.touched), nodeId, loopId);
|
|
5603
|
+
};
|
|
5604
|
+
const appendControlEvent = (event) => withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5605
|
+
if (!decisionStillCurrent(events)) return false;
|
|
5606
|
+
append(event);
|
|
5607
|
+
return true;
|
|
5608
|
+
});
|
|
5609
|
+
let checkpointHeld = /* @__PURE__ */ new Set();
|
|
5610
|
+
const refreshCheckpointHolds = (events) => {
|
|
5611
|
+
if (!externalMode) return;
|
|
5612
|
+
checkpointHeld = v3CheckpointHeldNodes(dag, v3CheckpointSuccesses({
|
|
5613
|
+
events,
|
|
5614
|
+
dag,
|
|
5615
|
+
markedSince: v3CheckpointMarkedSince(runDir, authorizedDag, events)
|
|
5616
|
+
}));
|
|
5617
|
+
};
|
|
5618
|
+
const heldByProposal = (nodeId, loopId) => v3NodeHeldByProposal(pendingTouched, nodeId, loopId) || checkpointHeld.has(nodeId) || loopId !== void 0 && checkpointHeld.has(loopId);
|
|
5619
|
+
const reconcileRevisionFollowUps = (events) => {
|
|
5620
|
+
const live = new Set(dag.nodes.map((node) => node.id));
|
|
5621
|
+
if (missingV3RevisionFollowUps(events, live).length === 0) return false;
|
|
5622
|
+
const result = appendMissingV3RevisionFollowUps(journalPath, { revision: dagRevision, liveNodeIds: live });
|
|
5623
|
+
return result.kind === "stale" || result.count > 0;
|
|
5624
|
+
};
|
|
4412
5625
|
if (!opts.authorizedArtifacts) {
|
|
4413
5626
|
assertOrCreateFrozenJson(
|
|
4414
|
-
|
|
5627
|
+
join10(runDir, "bots.snapshot.json"),
|
|
4415
5628
|
Object.fromEntries(botSnapshots),
|
|
4416
5629
|
"execution profile snapshot"
|
|
4417
5630
|
);
|
|
4418
5631
|
assertOrCreateFrozenJson(
|
|
4419
|
-
|
|
5632
|
+
join10(runDir, "dag.json"),
|
|
4420
5633
|
dag,
|
|
4421
5634
|
"workflow definition",
|
|
4422
5635
|
validateDag
|
|
4423
5636
|
);
|
|
4424
5637
|
}
|
|
4425
5638
|
if (readJournal(journalPath).length === 0) {
|
|
4426
|
-
appendEvent(journalPath, {
|
|
5639
|
+
appendEvent(journalPath, {
|
|
5640
|
+
type: "runStarted",
|
|
5641
|
+
runId: dag.runId,
|
|
5642
|
+
...externalMode ? { nodeExecutor: "bot" } : {}
|
|
5643
|
+
});
|
|
4427
5644
|
}
|
|
4428
5645
|
const inFlight = /* @__PURE__ */ new Map();
|
|
4429
5646
|
const botInFlight = /* @__PURE__ */ new Map();
|
|
@@ -4434,6 +5651,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4434
5651
|
const hostPromiseAttempts = /* @__PURE__ */ new Map();
|
|
4435
5652
|
const hostDeadlineCancels = /* @__PURE__ */ new Map();
|
|
4436
5653
|
const blockingGateOwners = /* @__PURE__ */ new Map();
|
|
5654
|
+
const handoffControllers = /* @__PURE__ */ new Map();
|
|
5655
|
+
const handoffRetryAt = /* @__PURE__ */ new Map();
|
|
5656
|
+
const acceptanceObserved = /* @__PURE__ */ new Set();
|
|
4437
5657
|
class HostProviderResponseTimeoutError extends Error {
|
|
4438
5658
|
constructor() {
|
|
4439
5659
|
super(`host provider did not settle within ${hostResponseWaitMs}ms`);
|
|
@@ -4480,6 +5700,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4480
5700
|
};
|
|
4481
5701
|
while (true) {
|
|
4482
5702
|
const events = readJournal(journalPath);
|
|
5703
|
+
refreshEffectiveDag(events);
|
|
5704
|
+
refreshCheckpointHolds(events);
|
|
5705
|
+
if (reconcileRevisionFollowUps(events)) continue;
|
|
4483
5706
|
const snap = materialize(events);
|
|
4484
5707
|
writeState(statePath, snap);
|
|
4485
5708
|
if (opts.cancelSignal?.aborted && snap.runStatus !== "cancelling" && !isTrueRunTerminal(snap.runStatus)) {
|
|
@@ -4490,15 +5713,21 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4490
5713
|
if (openHostEffects.length > 0) {
|
|
4491
5714
|
const settled = await reconcileOpenHostEffects(openHostEffects);
|
|
4492
5715
|
detachClosedHostInvocations();
|
|
4493
|
-
if (!settled) await new Promise((
|
|
5716
|
+
if (!settled) await new Promise((resolve6) => setTimeout(resolve6, 1e3));
|
|
4494
5717
|
continue;
|
|
4495
5718
|
}
|
|
5719
|
+
if (settleExternalLeftovers(events)) continue;
|
|
4496
5720
|
if (snap.runStatus === "cancelling" && snap.cancelRequestId) {
|
|
4497
5721
|
for (const controller of nodeControllers.values()) {
|
|
4498
5722
|
if (!controller.signal.aborted) {
|
|
4499
5723
|
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
4500
5724
|
}
|
|
4501
5725
|
}
|
|
5726
|
+
for (const controller of handoffControllers.values()) {
|
|
5727
|
+
if (!controller.signal.aborted) {
|
|
5728
|
+
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
5729
|
+
}
|
|
5730
|
+
}
|
|
4502
5731
|
cancelUnfinishedNodes(snap.cancelRequestId, { includeRunning: false });
|
|
4503
5732
|
for (const key of [...inFlight.keys()]) {
|
|
4504
5733
|
if (!key.endsWith("::gate")) continue;
|
|
@@ -4509,6 +5738,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4509
5738
|
await Promise.race(inFlight.values());
|
|
4510
5739
|
continue;
|
|
4511
5740
|
}
|
|
5741
|
+
if (openV3ExternalAttempts(events).length > 0) {
|
|
5742
|
+
await closeExternalAttemptsForCancellation(snap.cancelRequestId);
|
|
5743
|
+
continue;
|
|
5744
|
+
}
|
|
4512
5745
|
let externalDrained = false;
|
|
4513
5746
|
try {
|
|
4514
5747
|
externalDrained = drainExternallyOwnedAttempts({
|
|
@@ -4518,7 +5751,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4518
5751
|
} catch {
|
|
4519
5752
|
}
|
|
4520
5753
|
if (!externalDrained) {
|
|
4521
|
-
await new Promise((
|
|
5754
|
+
await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4522
5755
|
continue;
|
|
4523
5756
|
}
|
|
4524
5757
|
finalizeRuntimeCancellation(snap.cancelRequestId);
|
|
@@ -4533,7 +5766,16 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4533
5766
|
open,
|
|
4534
5767
|
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
4535
5768
|
);
|
|
4536
|
-
if (!drained) await new Promise((
|
|
5769
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5770
|
+
continue;
|
|
5771
|
+
}
|
|
5772
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5773
|
+
if (openExternal.length > 0) {
|
|
5774
|
+
await drainExternalAttempts(
|
|
5775
|
+
openExternal,
|
|
5776
|
+
"runBoundary",
|
|
5777
|
+
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
5778
|
+
);
|
|
4537
5779
|
continue;
|
|
4538
5780
|
}
|
|
4539
5781
|
break;
|
|
@@ -4543,7 +5785,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4543
5785
|
const obsolete = obsoleteOpenAttempts(events, snap);
|
|
4544
5786
|
if (obsolete.length > 0) {
|
|
4545
5787
|
const drained = await quiesceAttempts(obsolete, "obsoleteAttempt");
|
|
4546
|
-
if (!drained) await new Promise((
|
|
5788
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4547
5789
|
continue;
|
|
4548
5790
|
}
|
|
4549
5791
|
const orphaned = openV3WorkerAttempts(events).filter((attempt) => {
|
|
@@ -4552,34 +5794,56 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4552
5794
|
});
|
|
4553
5795
|
if (orphaned.length > 0) {
|
|
4554
5796
|
const drained = await quiesceAttempts(orphaned, "orphanRecovery");
|
|
4555
|
-
if (!drained) await new Promise((
|
|
5797
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4556
5798
|
continue;
|
|
4557
5799
|
}
|
|
5800
|
+
const obsoleteExternal = obsoleteExternalAttempts(events, snap);
|
|
5801
|
+
if (obsoleteExternal.length > 0) {
|
|
5802
|
+
await drainExternalAttempts(obsoleteExternal, "obsolete", "obsoleteAttempt");
|
|
5803
|
+
continue;
|
|
5804
|
+
}
|
|
5805
|
+
if (await reconcileExternalAttempts(events)) continue;
|
|
4558
5806
|
const actions = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances);
|
|
4559
5807
|
const terminal = actions.find(
|
|
4560
5808
|
(a) => a.kind === "completeRunSucceeded" || a.kind === "completeRunFailed" || a.kind === "completeRunBlocked"
|
|
4561
5809
|
);
|
|
4562
5810
|
if (terminal) {
|
|
5811
|
+
if (terminal.kind === "completeRunSucceeded" && pendingRevision !== void 0) {
|
|
5812
|
+
if (inFlight.size > 0) {
|
|
5813
|
+
await waitForInFlightOrDurableCancellation();
|
|
5814
|
+
continue;
|
|
5815
|
+
}
|
|
5816
|
+
return { reason: "awaitingGate", pendingWaits: [], pendingRevision, runDir };
|
|
5817
|
+
}
|
|
4563
5818
|
dropAllBlockingGatePromises();
|
|
4564
5819
|
const open = openV3WorkerAttempts(events);
|
|
4565
5820
|
if (open.length > 0) {
|
|
4566
5821
|
const drained = await quiesceAttempts(open, "terminalPeer");
|
|
4567
|
-
if (!drained) await new Promise((
|
|
5822
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5823
|
+
continue;
|
|
5824
|
+
}
|
|
5825
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5826
|
+
if (openExternal.length > 0) {
|
|
5827
|
+
await drainExternalAttempts(openExternal, "runBoundary", "terminalPeer");
|
|
4568
5828
|
continue;
|
|
4569
5829
|
}
|
|
4570
5830
|
appendRunTerminalIfQuiescent();
|
|
4571
5831
|
continue;
|
|
4572
5832
|
}
|
|
4573
5833
|
if (gateMode === "blocking") reattachBlockingGates(snap);
|
|
4574
|
-
const
|
|
5834
|
+
const allControls = actions.filter(
|
|
4575
5835
|
(a) => a.kind === "startLoop" || a.kind === "startLoopIteration" || a.kind === "evaluateLoopIteration" || a.kind === "completeLoop" || a.kind === "resolveEdge" || a.kind === "skipNode"
|
|
4576
5836
|
);
|
|
5837
|
+
const controls = allControls.filter((a) => !heldByProposal(
|
|
5838
|
+
a.kind === "resolveEdge" ? a.to : a.kind === "skipNode" ? a.nodeId : a.loopId
|
|
5839
|
+
));
|
|
5840
|
+
const heldControls = allControls.length - controls.length;
|
|
4577
5841
|
if (controls.length > 0) {
|
|
4578
5842
|
const eventsForControl = readJournal(journalPath);
|
|
4579
5843
|
for (const a of controls) {
|
|
4580
5844
|
if (a.kind === "resolveEdge") applyResolveEdge(a, eventsForControl);
|
|
4581
5845
|
else if (a.kind === "skipNode") {
|
|
4582
|
-
|
|
5846
|
+
appendControlEvent({
|
|
4583
5847
|
type: "nodeSkipped",
|
|
4584
5848
|
nodeId: a.nodeId,
|
|
4585
5849
|
reason: "triggerRuleUnsatisfied",
|
|
@@ -4589,19 +5853,42 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4589
5853
|
}
|
|
4590
5854
|
continue;
|
|
4591
5855
|
}
|
|
5856
|
+
const openExternalNow = openV3ExternalAttempts(events);
|
|
5857
|
+
let externalOccupancy = openExternalNow.length;
|
|
5858
|
+
const externalByBot = /* @__PURE__ */ new Map();
|
|
5859
|
+
for (const attempt of openExternalNow) {
|
|
5860
|
+
if (!attempt.targetLarkAppId) continue;
|
|
5861
|
+
externalByBot.set(attempt.targetLarkAppId, (externalByBot.get(attempt.targetLarkAppId) ?? 0) + 1);
|
|
5862
|
+
}
|
|
5863
|
+
const occupancy = () => inFlight.size - [...inFlight.keys()].filter((key) => key.endsWith("::handoff")).length + externalOccupancy;
|
|
4592
5864
|
let startedThisTick = 0;
|
|
5865
|
+
let heldThisTick = heldControls;
|
|
4593
5866
|
for (const a of actions) {
|
|
4594
|
-
if (
|
|
5867
|
+
if (occupancy() >= globalCap) break;
|
|
5868
|
+
if ((a.kind === "dispatchWork" || a.kind === "dispatchGate") && heldByProposal(a.nodeId, a.kind === "dispatchWork" ? a.loop?.loopId : void 0)) {
|
|
5869
|
+
heldThisTick++;
|
|
5870
|
+
continue;
|
|
5871
|
+
}
|
|
4595
5872
|
if (a.kind === "dispatchWork") {
|
|
4596
5873
|
const node = a.loop ? instanceNodeFor(a.loop) : nodesById.get(a.nodeId);
|
|
4597
5874
|
if (isHostNode(node)) {
|
|
4598
5875
|
if (!a.instanceId) throw new Error(`v3 runtime: host node "${node.id}" has no runtime instance`);
|
|
4599
5876
|
if (startHost(node, events, a.instanceId)) startedThisTick++;
|
|
4600
5877
|
} else {
|
|
4601
|
-
const
|
|
4602
|
-
|
|
5878
|
+
const botSnap = botSnapshotFor2(node.bot);
|
|
5879
|
+
if (externalMode) {
|
|
5880
|
+
const target = snap.assignments.get(node.id) ?? botSnap.larkAppId;
|
|
5881
|
+
if ((botInFlight.get(target) ?? 0) + (externalByBot.get(target) ?? 0) >= perBotCap) continue;
|
|
5882
|
+
if (startExternal(node, botSnap, events, a.loop, a.omitted, a.instanceId)) {
|
|
5883
|
+
startedThisTick++;
|
|
5884
|
+
externalOccupancy++;
|
|
5885
|
+
externalByBot.set(target, (externalByBot.get(target) ?? 0) + 1);
|
|
5886
|
+
}
|
|
5887
|
+
continue;
|
|
5888
|
+
}
|
|
5889
|
+
const botKey = botSnap.larkAppId;
|
|
4603
5890
|
if ((botInFlight.get(botKey) ?? 0) >= perBotCap) continue;
|
|
4604
|
-
if ((cliInFlight.get(botSnap.cliId) ?? 0) >= perCliCap) continue;
|
|
5891
|
+
if ((cliInFlight.get(botSnap.cliId) ?? 0) >= perCliCap(botSnap.cliId)) continue;
|
|
4605
5892
|
if (startWork(node, botSnap, botKey, events, a.loop, a.omitted, a.instanceId)) {
|
|
4606
5893
|
startedThisTick++;
|
|
4607
5894
|
}
|
|
@@ -4610,7 +5897,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4610
5897
|
if (startGate(nodesById.get(a.nodeId), a.instanceId)) startedThisTick++;
|
|
4611
5898
|
}
|
|
4612
5899
|
}
|
|
4613
|
-
const cancels = actions.filter((a) => a.kind === "cancelNode");
|
|
5900
|
+
const cancels = actions.filter((a) => a.kind === "cancelNode" && !heldByProposal(a.nodeId) && !checkpointHeld.has(a.byNodeId));
|
|
4614
5901
|
let cancelledThisTick = false;
|
|
4615
5902
|
for (const a of cancels) {
|
|
4616
5903
|
cancelledThisTick = applyCancelNode(a) || cancelledThisTick;
|
|
@@ -4619,20 +5906,31 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4619
5906
|
if (inFlight.size === 0) {
|
|
4620
5907
|
if (startedThisTick === 0) {
|
|
4621
5908
|
const pendingWaits = gateMode === "suspend" ? pendingGateWaits(snap.nodes) : [];
|
|
4622
|
-
|
|
5909
|
+
const latestEvents = readJournal(journalPath);
|
|
5910
|
+
if (externalMode && latestEvents.length !== events.length) continue;
|
|
5911
|
+
const waitingExternal = openV3ExternalAttempts(latestEvents);
|
|
5912
|
+
if (latestEvents.length !== events.length && currentV3Revision(latestEvents) !== dagRevision) continue;
|
|
5913
|
+
if (pendingWaits.length > 0 || waitingExternal.length > 0 || heldThisTick > 0) {
|
|
4623
5914
|
const open = openV3WorkerAttempts(events);
|
|
4624
5915
|
if (open.length > 0) {
|
|
4625
5916
|
const drained = await quiesceAttempts(open, "orphanRecovery");
|
|
4626
|
-
if (!drained) await new Promise((
|
|
5917
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4627
5918
|
continue;
|
|
4628
5919
|
}
|
|
4629
|
-
|
|
5920
|
+
if (waitingExternal.length > 0) return awaitingExternalOutcome(waitingExternal, pendingWaits);
|
|
5921
|
+
return {
|
|
5922
|
+
reason: "awaitingGate",
|
|
5923
|
+
pendingWaits,
|
|
5924
|
+
...heldThisTick > 0 && pendingRevision !== void 0 ? { pendingRevision } : {},
|
|
5925
|
+
runDir
|
|
5926
|
+
};
|
|
4630
5927
|
}
|
|
4631
5928
|
throw new Error("v3 runtime: no progress possible and run is not terminal");
|
|
4632
5929
|
}
|
|
4633
5930
|
}
|
|
4634
5931
|
if (inFlight.size > 0) await waitForInFlightOrDurableCancellation();
|
|
4635
5932
|
}
|
|
5933
|
+
forgetProbeFailuresOf(runDir);
|
|
4636
5934
|
const finalSnap = materialize(readJournal(journalPath));
|
|
4637
5935
|
return {
|
|
4638
5936
|
reason: "terminal",
|
|
@@ -4655,21 +5953,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4655
5953
|
if (isTrueRunTerminal(snap.runStatus) || snap.runStatus === "cancelling") return;
|
|
4656
5954
|
append({
|
|
4657
5955
|
type: "runCancelRequested",
|
|
4658
|
-
cancelRequestId: `cancel-${
|
|
5956
|
+
cancelRequestId: `cancel-${randomUUID4()}`,
|
|
4659
5957
|
by: "runtime-signal"
|
|
4660
5958
|
}, { durable: true });
|
|
4661
5959
|
});
|
|
4662
5960
|
}
|
|
4663
5961
|
function appendRunTerminalIfQuiescent() {
|
|
4664
5962
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5963
|
+
if (!decisionStillCurrent(events)) return false;
|
|
4665
5964
|
const snap = materialize([...events]);
|
|
4666
5965
|
if (snap.runStatus !== "running") return false;
|
|
4667
5966
|
if (openV3WorkerAttempts(events).length > 0) return false;
|
|
5967
|
+
const pendingProposal = foldV3Revisions(events).pending;
|
|
4668
5968
|
if (openV3HostEffects(events).length > 0) return false;
|
|
5969
|
+
if (openV3ExternalAttempts(events).length > 0) return false;
|
|
4669
5970
|
const terminal = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances).find(
|
|
4670
5971
|
(action) => action.kind === "completeRunSucceeded" || action.kind === "completeRunFailed" || action.kind === "completeRunBlocked"
|
|
4671
5972
|
);
|
|
4672
5973
|
if (!terminal) return false;
|
|
5974
|
+
if (terminal.kind === "completeRunSucceeded" && pendingProposal) return false;
|
|
4673
5975
|
const event = terminal.kind === "completeRunSucceeded" ? { type: "runSucceeded" } : terminal.kind === "completeRunFailed" ? {
|
|
4674
5976
|
type: "runFailed",
|
|
4675
5977
|
failedNodeId: terminal.failedNodeId,
|
|
@@ -4682,21 +5984,42 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4682
5984
|
}
|
|
4683
5985
|
function claimDispatch(event) {
|
|
4684
5986
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5987
|
+
if (!decisionStillCurrent(
|
|
5988
|
+
events,
|
|
5989
|
+
event.nodeId,
|
|
5990
|
+
event.type === "nodeDispatched" ? event.loop?.loopId : void 0
|
|
5991
|
+
)) return false;
|
|
4685
5992
|
const snap = materialize([...events]);
|
|
4686
5993
|
if (snap.runStatus !== "running") return false;
|
|
4687
5994
|
const status = snap.nodes.get(event.nodeId)?.status ?? "pending";
|
|
4688
5995
|
if (status !== "pending") return false;
|
|
5996
|
+
if (event.type === "gateDispatched" && gateWaitAlreadyDecided(events, event.waitId)) {
|
|
5997
|
+
throw new Error(
|
|
5998
|
+
`v3 runtime: gate ${event.waitId} of node "${event.nodeId}" was already decided, but the node has no approval to run on; refusing to re-dispatch the same gate`
|
|
5999
|
+
);
|
|
6000
|
+
}
|
|
4689
6001
|
append(event, { durable: true });
|
|
4690
6002
|
return true;
|
|
4691
6003
|
});
|
|
4692
6004
|
}
|
|
6005
|
+
function gateWaitAlreadyDecided(events, waitId) {
|
|
6006
|
+
return events.some((event) => event.type === "gateResolved" && event.waitId === waitId || event.type === "dagRevised" && (event.effects?.supersede?.some((item) => item.waitId === waitId) === true || event.effects?.cancel?.some((item) => item.waitId === waitId) === true));
|
|
6007
|
+
}
|
|
6008
|
+
function revisionSignature(events) {
|
|
6009
|
+
let count = 0;
|
|
6010
|
+
for (const event of events) {
|
|
6011
|
+
if (event.type === "dagRevised" || event.type === "dagRevisionProposed" || event.type === "dagRevisionRejected") count++;
|
|
6012
|
+
}
|
|
6013
|
+
return count;
|
|
6014
|
+
}
|
|
4693
6015
|
function isRunCancelling() {
|
|
4694
6016
|
return materialize(readJournal(journalPath)).runStatus === "cancelling";
|
|
4695
6017
|
}
|
|
4696
6018
|
async function waitForInFlightOrDurableCancellation() {
|
|
4697
6019
|
const active = [...inFlight.values()];
|
|
4698
6020
|
if (active.length === 0) return;
|
|
4699
|
-
|
|
6021
|
+
const revisionSignatureAtStart = revisionSignature(readJournal(journalPath));
|
|
6022
|
+
await new Promise((resolve6, reject) => {
|
|
4700
6023
|
let done = false;
|
|
4701
6024
|
let timer;
|
|
4702
6025
|
const signal = opts.cancelSignal;
|
|
@@ -4705,13 +6028,14 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4705
6028
|
done = true;
|
|
4706
6029
|
if (timer) clearTimeout(timer);
|
|
4707
6030
|
signal?.removeEventListener("abort", finish);
|
|
4708
|
-
|
|
6031
|
+
resolve6();
|
|
4709
6032
|
};
|
|
4710
6033
|
for (const promise of active) promise.then(finish, finish);
|
|
4711
6034
|
const poll = () => {
|
|
4712
6035
|
if (done) return;
|
|
4713
6036
|
try {
|
|
4714
|
-
|
|
6037
|
+
const latest = readJournal(journalPath);
|
|
6038
|
+
if (signal?.aborted || materialize(latest).runStatus === "cancelling" || revisionSignature(latest) !== revisionSignatureAtStart) {
|
|
4715
6039
|
finish();
|
|
4716
6040
|
return;
|
|
4717
6041
|
}
|
|
@@ -4874,10 +6198,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4874
6198
|
return allDrained;
|
|
4875
6199
|
}
|
|
4876
6200
|
function safeAttemptDirFromId(attemptId) {
|
|
4877
|
-
const root =
|
|
4878
|
-
const candidate =
|
|
4879
|
-
const rel =
|
|
4880
|
-
if (!rel ||
|
|
6201
|
+
const root = resolve5(runDir);
|
|
6202
|
+
const candidate = resolve5(root, attemptId);
|
|
6203
|
+
const rel = relative5(root, candidate);
|
|
6204
|
+
if (!rel || isAbsolute5(rel) || rel === ".." || rel.startsWith(`..${sep3}`)) {
|
|
4881
6205
|
throw new Error(`v3 runtime: attempt path escapes run dir (${attemptId})`);
|
|
4882
6206
|
}
|
|
4883
6207
|
return candidate;
|
|
@@ -4914,6 +6238,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4914
6238
|
const snap = materialize([...events]);
|
|
4915
6239
|
if (snap.runStatus === "cancelled") return;
|
|
4916
6240
|
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
6241
|
+
if (openV3ExternalAttempts(events).length > 0) return;
|
|
4917
6242
|
const ids = /* @__PURE__ */ new Set([
|
|
4918
6243
|
...dag.nodes.map((node) => node.id),
|
|
4919
6244
|
...snap.nodes.keys()
|
|
@@ -5000,7 +6325,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5000
6325
|
(event) => event.type === "nodeSucceeded" && (event.instanceId ?? event.nodeId) === key
|
|
5001
6326
|
);
|
|
5002
6327
|
if (!success) throw new Error(`v3 host input source "${nodeId}" has no successful current instance`);
|
|
5003
|
-
const outputDir =
|
|
6328
|
+
const outputDir = join10(dirname8(success.manifestPath), "work");
|
|
5004
6329
|
const verdict = await deps.validateManifest(success.manifestPath, outputDir);
|
|
5005
6330
|
if (!verdict.ok || verdict.manifest?.status !== "ok") {
|
|
5006
6331
|
throw new Error(
|
|
@@ -5012,7 +6337,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5012
6337
|
throw new Error(`v3 host input source "${nodeId}" must expose exactly one result.json (found ${matches.length})`);
|
|
5013
6338
|
}
|
|
5014
6339
|
try {
|
|
5015
|
-
return JSON.parse(
|
|
6340
|
+
return JSON.parse(readFileSync12(join10(outputDir, matches[0].path), "utf-8"));
|
|
5016
6341
|
} catch (err) {
|
|
5017
6342
|
throw new Error(
|
|
5018
6343
|
`v3 host input source "${nodeId}" result.json is unreadable: ${err instanceof Error ? err.message : String(err)}`
|
|
@@ -5068,6 +6393,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5068
6393
|
executionContext
|
|
5069
6394
|
});
|
|
5070
6395
|
const committed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6396
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5071
6397
|
const current = materialize([...latest]);
|
|
5072
6398
|
if (current.runStatus !== "running") return false;
|
|
5073
6399
|
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
@@ -5140,6 +6466,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5140
6466
|
executionContext
|
|
5141
6467
|
});
|
|
5142
6468
|
withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6469
|
+
if (!decisionStillCurrent(latest, node.id)) return;
|
|
5143
6470
|
const current = materialize([...latest]);
|
|
5144
6471
|
if (current.runStatus !== "running") return;
|
|
5145
6472
|
const status = current.nodes.get(node.id)?.status ?? "pending";
|
|
@@ -5238,6 +6565,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5238
6565
|
return blocked;
|
|
5239
6566
|
}
|
|
5240
6567
|
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6568
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5241
6569
|
const current = materialize([...latest]);
|
|
5242
6570
|
if (current.runStatus !== "running") return false;
|
|
5243
6571
|
const state = current.nodes.get(node.id);
|
|
@@ -5355,9 +6683,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5355
6683
|
continue;
|
|
5356
6684
|
}
|
|
5357
6685
|
const attemptDir = safeAttemptDirFromId(effect.attemptId);
|
|
5358
|
-
const existingManifest =
|
|
5359
|
-
if (
|
|
5360
|
-
const validated = await deps.validateManifest(existingManifest,
|
|
6686
|
+
const existingManifest = join10(attemptDir, "manifest.json");
|
|
6687
|
+
if (existsSync10(existingManifest)) {
|
|
6688
|
+
const validated = await deps.validateManifest(existingManifest, join10(attemptDir, "work"));
|
|
5361
6689
|
if (validated.ok && validated.manifest?.status === "ok") {
|
|
5362
6690
|
try {
|
|
5363
6691
|
readAndVerifyV3HostSuccessResult({
|
|
@@ -5634,47 +6962,434 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5634
6962
|
function computeHostCanonicalHash(value) {
|
|
5635
6963
|
return computeInputHash(value);
|
|
5636
6964
|
}
|
|
5637
|
-
function
|
|
6965
|
+
function writeWorkflowDataFile(node, attemptDir) {
|
|
6966
|
+
if (!executionContext) return void 0;
|
|
6967
|
+
const bindings = savedWorkflowBindingsForNode(node);
|
|
6968
|
+
if (bindings.params.length === 0 && bindings.context.length === 0) return void 0;
|
|
6969
|
+
const params = /* @__PURE__ */ Object.create(null);
|
|
6970
|
+
const context = /* @__PURE__ */ Object.create(null);
|
|
6971
|
+
for (const name of bindings.params) {
|
|
6972
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.params, name)) {
|
|
6973
|
+
throw new Error(`v3 runtime: resolved params missing referenced key ${name}`);
|
|
6974
|
+
}
|
|
6975
|
+
params[name] = executionContext.params[name];
|
|
6976
|
+
}
|
|
6977
|
+
for (const name of bindings.context) {
|
|
6978
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.context, name)) {
|
|
6979
|
+
throw new Error(`v3 runtime: resolved context missing referenced key ${name}`);
|
|
6980
|
+
}
|
|
6981
|
+
context[name] = executionContext.context[name];
|
|
6982
|
+
}
|
|
6983
|
+
const workflowDataPath = join10(attemptDir, "workflow-inputs.json");
|
|
6984
|
+
writeFileSync8(
|
|
6985
|
+
workflowDataPath,
|
|
6986
|
+
`${JSON.stringify({ params, context }, null, 2)}
|
|
6987
|
+
`,
|
|
6988
|
+
{ mode: 384 }
|
|
6989
|
+
);
|
|
6990
|
+
return workflowDataPath;
|
|
6991
|
+
}
|
|
6992
|
+
function reportExternalError(error, attemptId, operation) {
|
|
6993
|
+
try {
|
|
6994
|
+
deps.onExternalError?.(error, { attemptId, operation });
|
|
6995
|
+
} catch {
|
|
6996
|
+
}
|
|
6997
|
+
}
|
|
6998
|
+
async function stopExternal(attempt, reason) {
|
|
6999
|
+
const stopped = await stopExternalAttempt(externalExecutor, dag.runId, attempt, reason, externalCallTimeoutMs);
|
|
7000
|
+
if (!stopped.ok) reportExternalError(stopped.error, attempt.attemptId, "stop");
|
|
7001
|
+
}
|
|
7002
|
+
function externalNodeFor(attempt) {
|
|
7003
|
+
return attempt.loop ? instanceNodeFor(attempt.loop) : nodesById.get(attempt.nodeId);
|
|
7004
|
+
}
|
|
7005
|
+
function startExternal(node, botSnap, events, loopRef, omitted, instanceId) {
|
|
7006
|
+
if (!isGoalNode(node)) throw new Error(`v3 runtime: node "${node.id}" is not a goal node`);
|
|
5638
7007
|
const dispatchKey = instanceId ?? node.id;
|
|
5639
7008
|
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
5640
7009
|
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
5641
|
-
const attemptDir =
|
|
5642
|
-
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
7010
|
+
const attemptDir = join10(runDir, dispatchKey, "attempts", attemptNNN);
|
|
7011
|
+
mkdirSync8(attemptDir, { recursive: true });
|
|
7012
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
7013
|
+
writeFileSync8(
|
|
7014
|
+
join10(attemptDir, "inputs.json"),
|
|
7015
|
+
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
7016
|
+
);
|
|
7017
|
+
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
7018
|
+
if (!decisionStillCurrent(latest, node.id, loopRef?.loopId)) return false;
|
|
7019
|
+
const current = materialize([...latest]);
|
|
7020
|
+
if (current.runStatus !== "running") return false;
|
|
7021
|
+
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
7022
|
+
const identity = { nodeId: node.id, ...instanceId ? { instanceId } : {}, attemptId };
|
|
7023
|
+
append({
|
|
7024
|
+
type: "nodeDispatched",
|
|
7025
|
+
...identity,
|
|
7026
|
+
loop: loopRef,
|
|
7027
|
+
nodeExecutor: "bot",
|
|
7028
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
7029
|
+
}, { durable: true });
|
|
7030
|
+
append({
|
|
7031
|
+
type: "nodeHandoffPrepared",
|
|
7032
|
+
...identity,
|
|
7033
|
+
targetLarkAppId: current.assignments.get(node.id) ?? botSnap.larkAppId,
|
|
7034
|
+
...externalPlacementIntent(loopRef ? void 0 : node),
|
|
7035
|
+
revision: dagRevision,
|
|
7036
|
+
preparedAt: clock()
|
|
7037
|
+
}, { durable: true });
|
|
7038
|
+
return true;
|
|
7039
|
+
});
|
|
7040
|
+
if (!claimed) return false;
|
|
7041
|
+
const attempt = foldV3ExternalAttempts(readJournal(journalPath)).get(attemptId);
|
|
7042
|
+
if (attempt && !attempt.closed) startHandoffSend(attempt, false);
|
|
7043
|
+
return true;
|
|
7044
|
+
}
|
|
7045
|
+
function externalPlacementIntent(node) {
|
|
7046
|
+
return node && isGoalNode(node) && node.placement === "main" ? { placement: "main", idempotency: { handoff: randomUUID4() } } : { placement: "topic", idempotency: { nodeCard: randomUUID4(), handoff: randomUUID4() } };
|
|
7047
|
+
}
|
|
7048
|
+
function appendMissingHandoffIntent(attempt) {
|
|
7049
|
+
const node = externalNodeFor(attempt);
|
|
7050
|
+
const frozen = node ? snapshotForSelector(botSnapshots, node.bot) : void 0;
|
|
7051
|
+
if (!frozen) return false;
|
|
7052
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7053
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
7054
|
+
if (!current.ok || current.attempt.phase !== "dispatched") return false;
|
|
7055
|
+
const dispatched = events.find((event) => event.type === "nodeDispatched" && event.attemptId === attempt.attemptId);
|
|
7056
|
+
append({
|
|
7057
|
+
type: "nodeHandoffPrepared",
|
|
7058
|
+
nodeId: attempt.nodeId,
|
|
7059
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7060
|
+
attemptId: attempt.attemptId,
|
|
7061
|
+
targetLarkAppId: current.snap.assignments.get(attempt.nodeId) ?? frozen.larkAppId,
|
|
7062
|
+
...externalPlacementIntent(attempt.loop ? void 0 : node),
|
|
7063
|
+
revision: dispatched?.revision ?? 1,
|
|
7064
|
+
preparedAt: clock()
|
|
7065
|
+
}, { durable: true });
|
|
7066
|
+
return true;
|
|
7067
|
+
});
|
|
7068
|
+
}
|
|
7069
|
+
function buildExternalDispatchRequest(attempt, redispatch, signal) {
|
|
7070
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
7071
|
+
const node = externalNodeFor(attempt);
|
|
7072
|
+
if (!ref || !attempt.idempotency || !node || !isGoalNode(node)) {
|
|
7073
|
+
throw new Error(`v3 runtime: external attempt ${attempt.attemptId} has no prepared goal handoff`);
|
|
7074
|
+
}
|
|
7075
|
+
const attemptDir = safeAttemptDirFromId(attempt.attemptId);
|
|
7076
|
+
const inputsPath = join10(attemptDir, "inputs.json");
|
|
7077
|
+
const inputs = JSON.parse(readFileSync12(inputsPath, "utf-8"));
|
|
7078
|
+
const events = readJournal(journalPath);
|
|
7079
|
+
const key = attempt.instanceId ?? attempt.nodeId;
|
|
7080
|
+
const retry = events.find(
|
|
7081
|
+
(event) => event.type === "nodeRetryRequested" && (event.instanceId ?? event.nodeId) === key && event.nextAttemptId === attempt.attemptId
|
|
7082
|
+
);
|
|
7083
|
+
const previousBlock = retry ? [...events].reverse().find(
|
|
7084
|
+
(event) => event.type === "nodeBlocked" && event.attemptId === retry.previousAttemptId
|
|
7085
|
+
) : void 0;
|
|
7086
|
+
const reassigned = [...events].reverse().find(
|
|
7087
|
+
(event) => event.type === "nodeReassigned" && event.nodeId === attempt.nodeId && event.to === ref.targetLarkAppId
|
|
7088
|
+
);
|
|
7089
|
+
const revised = [...events].reverse().find(
|
|
7090
|
+
(event) => event.type === "dagRevised" && event.revision <= ref.revision && (event.effects?.supersede?.some((item) => item.nodeId === attempt.nodeId) === true || event.effects?.assign?.some((item) => item.nodeId === attempt.nodeId) === true)
|
|
7091
|
+
);
|
|
7092
|
+
const revisedAssign = revised?.effects?.assign?.find((item) => item.nodeId === attempt.nodeId && item.to === ref.targetLarkAppId);
|
|
7093
|
+
return {
|
|
7094
|
+
...ref,
|
|
7095
|
+
idempotency: attempt.idempotency,
|
|
7096
|
+
redispatch,
|
|
7097
|
+
node,
|
|
7098
|
+
attemptNumber: attemptNumber(attempt.attemptId) ?? 1,
|
|
7099
|
+
...attempt.loop ? { loop: attempt.loop } : {},
|
|
7100
|
+
inputs,
|
|
7101
|
+
inputsPath,
|
|
7102
|
+
attemptDir,
|
|
7103
|
+
runDir,
|
|
7104
|
+
timeoutSec: node.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC,
|
|
7105
|
+
...opts.chatBinding ? { chatBinding: opts.chatBinding } : {},
|
|
7106
|
+
...retry ? {
|
|
7107
|
+
retryOf: {
|
|
7108
|
+
attemptId: retry.previousAttemptId,
|
|
7109
|
+
...retry.previousErrorCode ? { errorCode: retry.previousErrorCode } : {},
|
|
7110
|
+
...previousBlock?.message ? { message: previousBlock.message } : {}
|
|
7111
|
+
}
|
|
7112
|
+
} : {},
|
|
7113
|
+
...reassigned ? { reassignedFrom: reassigned.from } : revisedAssign?.from ? { reassignedFrom: revisedAssign.from } : {},
|
|
7114
|
+
...revised ? { revisedBy: { revision: revised.revision, summary: revised.summary } } : {},
|
|
7115
|
+
signal
|
|
7116
|
+
};
|
|
7117
|
+
}
|
|
7118
|
+
function startHandoffSend(attempt, redispatch) {
|
|
7119
|
+
const executor = externalExecutor;
|
|
7120
|
+
const key = `${attempt.instanceId ?? attempt.nodeId}::handoff`;
|
|
7121
|
+
if (!executor || inFlight.has(key) || handoffControllers.has(attempt.attemptId)) return;
|
|
7122
|
+
const controller = new AbortController();
|
|
7123
|
+
handoffControllers.set(attempt.attemptId, controller);
|
|
7124
|
+
const p = Promise.resolve().then(() => withExternalCallTimeout(
|
|
7125
|
+
Promise.resolve().then(() => executor.dispatch(buildExternalDispatchRequest(attempt, redispatch, controller.signal))),
|
|
7126
|
+
externalCallTimeoutMs,
|
|
7127
|
+
"dispatch"
|
|
7128
|
+
).catch((err) => {
|
|
7129
|
+
if (!controller.signal.aborted) controller.abort(err);
|
|
7130
|
+
throw err;
|
|
7131
|
+
})).then(async (outcome) => {
|
|
7132
|
+
handoffRetryAt.delete(attempt.attemptId);
|
|
7133
|
+
if (outcome.kind === "rejected") {
|
|
7134
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7135
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
7136
|
+
if (!current.ok || current.attempt.phase !== "prepared") return;
|
|
7137
|
+
append({
|
|
7138
|
+
type: "nodeBlocked",
|
|
7139
|
+
nodeId: attempt.nodeId,
|
|
7140
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7141
|
+
attemptId: attempt.attemptId,
|
|
7142
|
+
errorClass: "workerError",
|
|
7143
|
+
errorCode: outcome.code,
|
|
7144
|
+
message: outcome.message ?? `handoff was not sent (${outcome.code})`
|
|
7145
|
+
}, { durable: true });
|
|
7146
|
+
});
|
|
7147
|
+
return;
|
|
7148
|
+
}
|
|
7149
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
7150
|
+
const sentAt = clock();
|
|
7151
|
+
const recorded = withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7152
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7153
|
+
if (!latest || latest.closed) return false;
|
|
7154
|
+
if (latest.handoffMessageId !== void 0) return true;
|
|
7155
|
+
append({
|
|
7156
|
+
type: "nodeHandoffSent",
|
|
7157
|
+
nodeId: attempt.nodeId,
|
|
7158
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7159
|
+
attemptId: attempt.attemptId,
|
|
7160
|
+
handoffMessageId: outcome.handoffMessageId,
|
|
7161
|
+
...outcome.nodeCardMessageId ? { nodeCardMessageId: outcome.nodeCardMessageId } : {},
|
|
7162
|
+
...outcome.threadId ? { threadId: outcome.threadId } : {},
|
|
7163
|
+
sentAt,
|
|
7164
|
+
acceptDeadlineAt: sentAt + externalAcceptTimeoutMs
|
|
7165
|
+
}, { durable: true });
|
|
7166
|
+
return true;
|
|
7167
|
+
});
|
|
7168
|
+
if (!recorded) {
|
|
7169
|
+
await stopExternal({ ...attempt, handoffMessageId: outcome.handoffMessageId }, "obsolete");
|
|
7170
|
+
}
|
|
7171
|
+
}).catch((err) => {
|
|
7172
|
+
reportExternalError(err, attempt.attemptId, "dispatch");
|
|
7173
|
+
handoffRetryAt.set(attempt.attemptId, clock() + HANDOFF_RESEND_BACKOFF_MS);
|
|
7174
|
+
}).finally(() => {
|
|
7175
|
+
if (handoffControllers.get(attempt.attemptId) === controller) {
|
|
7176
|
+
handoffControllers.delete(attempt.attemptId);
|
|
7177
|
+
}
|
|
7178
|
+
if (inFlight.get(key) === p) inFlight.delete(key);
|
|
7179
|
+
});
|
|
7180
|
+
inFlight.set(key, p);
|
|
7181
|
+
}
|
|
7182
|
+
async function blockExternalAttempt(attempt, stopReason, verdict) {
|
|
7183
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
7184
|
+
await stopExternal(attempt, stopReason);
|
|
7185
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7186
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
7187
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
7188
|
+
if (!current.ok) return false;
|
|
7189
|
+
append({
|
|
7190
|
+
type: "nodeBlocked",
|
|
7191
|
+
nodeId: attempt.nodeId,
|
|
7192
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7193
|
+
attemptId: attempt.attemptId,
|
|
7194
|
+
...verdict
|
|
7195
|
+
}, { durable: true });
|
|
7196
|
+
return true;
|
|
7197
|
+
});
|
|
7198
|
+
}
|
|
7199
|
+
async function probeExternalAcceptance(attempt) {
|
|
7200
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
7201
|
+
const executor = externalExecutor;
|
|
7202
|
+
if (!executor || !ref || !attempt.handoffMessageId) return { status: "unknown" };
|
|
7203
|
+
const handoffMessageId = attempt.handoffMessageId;
|
|
7204
|
+
try {
|
|
7205
|
+
const probe = await withExternalCallTimeout(
|
|
7206
|
+
Promise.resolve().then(() => executor.probeAcceptance({ ...ref, handoffMessageId })),
|
|
7207
|
+
externalCallTimeoutMs,
|
|
7208
|
+
"probe"
|
|
7209
|
+
);
|
|
7210
|
+
return probe.status === "accepted" ? {
|
|
7211
|
+
status: "accepted",
|
|
7212
|
+
sessionId: probe.sessionId,
|
|
7213
|
+
...probe.acceptedAt !== void 0 ? { acceptedAt: probe.acceptedAt } : {}
|
|
7214
|
+
} : { status: "notAccepted", ...typeof probe.detail === "string" && probe.detail ? { detail: probe.detail.slice(0, 2e3) } : {} };
|
|
7215
|
+
} catch (err) {
|
|
7216
|
+
reportExternalError(err, attempt.attemptId, "probe");
|
|
7217
|
+
return { status: "unknown" };
|
|
7218
|
+
}
|
|
7219
|
+
}
|
|
7220
|
+
async function reconcileExternalAttempts(events) {
|
|
7221
|
+
if (!externalMode) return false;
|
|
7222
|
+
let progressed = false;
|
|
7223
|
+
for (const attempt of openV3ExternalAttempts(events)) {
|
|
7224
|
+
if (handoffControllers.has(attempt.attemptId)) continue;
|
|
7225
|
+
if (!classifyExternalAttempt(events, attempt.attemptId).ok) continue;
|
|
7226
|
+
const now = clock();
|
|
7227
|
+
if (attempt.handedToHuman) {
|
|
7228
|
+
await blockExternalAttempt(attempt, "handedToHuman", {
|
|
7229
|
+
errorClass: "workerError",
|
|
7230
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDED_TO_HUMAN,
|
|
7231
|
+
message: attempt.handedToHuman.reason,
|
|
7232
|
+
recovery: "retry"
|
|
7233
|
+
});
|
|
7234
|
+
progressed = true;
|
|
7235
|
+
continue;
|
|
7236
|
+
}
|
|
7237
|
+
if (attempt.phase === "dispatched") {
|
|
7238
|
+
progressed = appendMissingHandoffIntent(attempt) || progressed;
|
|
7239
|
+
} else if (attempt.phase === "prepared") {
|
|
7240
|
+
if (now >= attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) {
|
|
7241
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7242
|
+
errorClass: "workerError",
|
|
7243
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7244
|
+
message: "handoff delivery could not be confirmed within the idempotency window; check the chat before retrying"
|
|
7245
|
+
});
|
|
7246
|
+
progressed = true;
|
|
7247
|
+
} else if (now >= (handoffRetryAt.get(attempt.attemptId) ?? 0)) {
|
|
7248
|
+
startHandoffSend(attempt, true);
|
|
5655
7249
|
}
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
7250
|
+
} else if (attempt.phase === "sent") {
|
|
7251
|
+
const due = now >= attempt.acceptDeadlineAt;
|
|
7252
|
+
const key = probeKey(attempt.attemptId);
|
|
7253
|
+
const failure = probeFailures.get(key);
|
|
7254
|
+
const probeDue = failure ? now >= failure.retryAt : !acceptanceObserved.has(attempt.attemptId) || due;
|
|
7255
|
+
if (probeDue) {
|
|
7256
|
+
const probe = await probeExternalAcceptance(attempt);
|
|
7257
|
+
progressed = true;
|
|
7258
|
+
if (probe.status === "unknown") {
|
|
7259
|
+
const firstFailedAt = failure?.firstFailedAt ?? now;
|
|
7260
|
+
if (due && now - firstFailedAt >= PROBE_UNCERTAIN_AFTER_MS) {
|
|
7261
|
+
probeFailures.delete(key);
|
|
7262
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7263
|
+
errorClass: "workerError",
|
|
7264
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7265
|
+
message: "whether the target bot accepted the handoff could not be confirmed; check the chat before retrying"
|
|
7266
|
+
});
|
|
7267
|
+
} else {
|
|
7268
|
+
probeFailures.set(key, { firstFailedAt, retryAt: now + PROBE_RETRY_BACKOFF_MS });
|
|
7269
|
+
}
|
|
7270
|
+
} else {
|
|
7271
|
+
probeFailures.delete(key);
|
|
7272
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
7273
|
+
if (probe.status === "accepted") {
|
|
7274
|
+
const { sessionId, acceptedAt } = probe;
|
|
7275
|
+
withJournalMutationSync(journalPath, ({ events: latest, append }) => appendAcceptedLocked(latest, append, dag, {
|
|
7276
|
+
attemptId: attempt.attemptId,
|
|
7277
|
+
sessionId,
|
|
7278
|
+
...acceptedAt !== void 0 ? { acceptedAt } : {}
|
|
7279
|
+
}, clock()));
|
|
7280
|
+
} else if (due) {
|
|
7281
|
+
await blockExternalAttempt(attempt, "notAccepted", {
|
|
7282
|
+
errorClass: "workerError",
|
|
7283
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_NOT_ACCEPTED,
|
|
7284
|
+
message: probe.detail ? `the target bot did not accept the handoff before the acceptance deadline: ${probe.detail}` : "the target bot did not accept the handoff before the acceptance deadline"
|
|
7285
|
+
});
|
|
7286
|
+
}
|
|
5659
7287
|
}
|
|
5660
|
-
context[name] = executionContext.context[name];
|
|
5661
7288
|
}
|
|
5662
|
-
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
5668
|
-
|
|
7289
|
+
} else if (now >= attempt.deadlineAt) {
|
|
7290
|
+
await blockExternalAttempt(attempt, "timeout", {
|
|
7291
|
+
errorClass: "timeout",
|
|
7292
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT,
|
|
7293
|
+
message: attempt.startPending ? "\u5230\u671F\u9650\u8FD8\u6CA1\u6536\u5230\u8FD9\u4E00\u8F6E\u5F00\u59CB\u7684\u4FE1\u53F7\uFF1A\u591A\u534A\u662F\u4E00\u76F4\u5728\u6267\u884C Bot \u7684\u4E3B\u8BDD\u9898\u4F1A\u8BDD\u91CC\u6392\u961F\uFF08\u5B83\u524D\u9762\u7684\u56DE\u5408\u4E00\u76F4\u6CA1\u7ED3\u675F\uFF09\uFF0C\u4E5F\u53EF\u80FD\u662F\u5F00\u59CB\u4FE1\u53F7\u6CA1\u9001\u5230\uFF1B\u53EF\u4EE5\u91CD\u8BD5\uFF0C\u6216\u628A\u8282\u70B9\u6539\u6210\u8282\u70B9\u8BDD\u9898" : "the node did not submit a valid result before its timeout"
|
|
7294
|
+
});
|
|
7295
|
+
progressed = true;
|
|
5669
7296
|
}
|
|
5670
7297
|
}
|
|
5671
|
-
|
|
7298
|
+
return progressed;
|
|
7299
|
+
}
|
|
7300
|
+
function settleExternalLeftovers(events) {
|
|
7301
|
+
let progressed = false;
|
|
7302
|
+
for (const attempt of foldV3ExternalAttempts(events).values()) {
|
|
7303
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) {
|
|
7304
|
+
if (settleExternalInstallIntent(runDir, dag, attempt.attemptId).kind !== "none") progressed = true;
|
|
7305
|
+
} else if (attempt.closed) {
|
|
7306
|
+
rmSync2(join10(safeAttemptDirFromId(attempt.attemptId), "submissions"), { recursive: true, force: true });
|
|
7307
|
+
}
|
|
7308
|
+
}
|
|
7309
|
+
return progressed;
|
|
7310
|
+
}
|
|
7311
|
+
function obsoleteExternalAttempts(events, snap) {
|
|
7312
|
+
return openV3ExternalAttempts(events).filter((attempt) => {
|
|
7313
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
7314
|
+
return state?.status === "cancelled" || state?.status === "superseded";
|
|
7315
|
+
});
|
|
7316
|
+
}
|
|
7317
|
+
async function drainExternalAttempts(attempts, stopReason, drainReason) {
|
|
7318
|
+
for (const attempt of attempts) {
|
|
7319
|
+
await stopExternal(attempt, stopReason);
|
|
7320
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7321
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7322
|
+
if (!latest || latest.closed) return;
|
|
7323
|
+
append({
|
|
7324
|
+
type: "nodeAttemptDrained",
|
|
7325
|
+
nodeId: attempt.nodeId,
|
|
7326
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7327
|
+
attemptId: attempt.attemptId,
|
|
7328
|
+
reason: drainReason
|
|
7329
|
+
}, { durable: true });
|
|
7330
|
+
});
|
|
7331
|
+
}
|
|
7332
|
+
}
|
|
7333
|
+
async function closeExternalAttemptsForCancellation(cancelRequestId) {
|
|
7334
|
+
for (const attempt of openV3ExternalAttempts(readJournal(journalPath))) {
|
|
7335
|
+
await stopExternal(attempt, "runCancelled");
|
|
7336
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7337
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7338
|
+
if (!latest || latest.closed) return;
|
|
7339
|
+
const snap = materialize([...events]);
|
|
7340
|
+
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
7341
|
+
append({
|
|
7342
|
+
type: "nodeAttemptDrained",
|
|
7343
|
+
nodeId: attempt.nodeId,
|
|
7344
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7345
|
+
attemptId: attempt.attemptId,
|
|
7346
|
+
reason: "runCancellation"
|
|
7347
|
+
}, { durable: true });
|
|
7348
|
+
});
|
|
7349
|
+
}
|
|
7350
|
+
}
|
|
7351
|
+
function sentWakeAt(attempt, now) {
|
|
7352
|
+
const failure = probeFailures.get(probeKey(attempt.attemptId));
|
|
7353
|
+
if (!failure) return attempt.acceptDeadlineAt;
|
|
7354
|
+
return attempt.acceptDeadlineAt > now ? Math.min(attempt.acceptDeadlineAt, failure.retryAt) : failure.retryAt;
|
|
7355
|
+
}
|
|
7356
|
+
function awaitingExternalOutcome(open, pendingWaits) {
|
|
7357
|
+
const now = clock();
|
|
7358
|
+
const externalAttempts = open.map((attempt) => {
|
|
7359
|
+
const wakeAt = attempt.phase === "dispatched" ? now : attempt.phase === "prepared" ? Math.min(handoffRetryAt.get(attempt.attemptId) ?? now, attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) : attempt.phase === "sent" ? sentWakeAt(attempt, now) : attempt.deadlineAt;
|
|
7360
|
+
return {
|
|
7361
|
+
nodeId: attempt.nodeId,
|
|
7362
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7363
|
+
attemptId: attempt.attemptId,
|
|
7364
|
+
phase: attempt.phase,
|
|
7365
|
+
...attempt.targetLarkAppId ? { targetLarkAppId: attempt.targetLarkAppId } : {},
|
|
7366
|
+
...wakeAt !== void 0 ? { wakeAt } : {}
|
|
7367
|
+
};
|
|
7368
|
+
});
|
|
7369
|
+
const wakes = externalAttempts.flatMap((attempt) => attempt.wakeAt !== void 0 ? [attempt.wakeAt] : []);
|
|
7370
|
+
return {
|
|
7371
|
+
reason: "awaitingExternal",
|
|
7372
|
+
pendingWaits,
|
|
7373
|
+
externalAttempts,
|
|
7374
|
+
...wakes.length > 0 ? { wakeAt: Math.min(...wakes) } : {},
|
|
7375
|
+
runDir
|
|
7376
|
+
};
|
|
7377
|
+
}
|
|
7378
|
+
function startWork(node, botSnap, botKey, events, loopRef, omitted, instanceId) {
|
|
7379
|
+
const dispatchKey = instanceId ?? node.id;
|
|
7380
|
+
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
7381
|
+
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
7382
|
+
const attemptDir = join10(runDir, dispatchKey, "attempts", attemptNNN);
|
|
7383
|
+
const outputDir = join10(attemptDir, "work");
|
|
7384
|
+
mkdirSync8(outputDir, { recursive: true });
|
|
7385
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
7386
|
+
const goalPath = join10(attemptDir, "goal.txt");
|
|
5672
7387
|
const loopCtx = loopRef ? {
|
|
5673
7388
|
loopId: loopRef.loopId,
|
|
5674
7389
|
iteration: loopRef.iteration,
|
|
5675
7390
|
maxIterations: nodesById.get(loopRef.loopId).maxIterations
|
|
5676
7391
|
} : void 0;
|
|
5677
|
-
|
|
7392
|
+
writeFileSync8(
|
|
5678
7393
|
goalPath,
|
|
5679
7394
|
renderGoalFile(
|
|
5680
7395
|
node.goal ?? "",
|
|
@@ -5686,12 +7401,12 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5686
7401
|
)
|
|
5687
7402
|
);
|
|
5688
7403
|
const effSnap = mergeNodeCapability(botSnap, node.override);
|
|
5689
|
-
const inputsPath =
|
|
5690
|
-
|
|
7404
|
+
const inputsPath = join10(attemptDir, "inputs.json");
|
|
7405
|
+
writeFileSync8(
|
|
5691
7406
|
inputsPath,
|
|
5692
7407
|
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
5693
7408
|
);
|
|
5694
|
-
const manifestPath =
|
|
7409
|
+
const manifestPath = join10(attemptDir, "manifest.json");
|
|
5695
7410
|
const env = {
|
|
5696
7411
|
[GOAL_ENV.GOAL_PATH]: goalPath,
|
|
5697
7412
|
[GOAL_ENV.INPUTS_PATH]: inputsPath,
|
|
@@ -5705,7 +7420,8 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5705
7420
|
nodeId: node.id,
|
|
5706
7421
|
...instanceId ? { instanceId } : {},
|
|
5707
7422
|
attemptId,
|
|
5708
|
-
loop: loopRef
|
|
7423
|
+
loop: loopRef,
|
|
7424
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
5709
7425
|
})) return false;
|
|
5710
7426
|
const leaseBinding = {
|
|
5711
7427
|
attemptDir,
|
|
@@ -5911,7 +7627,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5911
7627
|
});
|
|
5912
7628
|
return;
|
|
5913
7629
|
}
|
|
5914
|
-
const res = validateResult(
|
|
7630
|
+
const res = validateResult(join10(outputDir, entry.path), node.resultSchema);
|
|
5915
7631
|
if (!res.ok) {
|
|
5916
7632
|
appendWorkerOutcome({
|
|
5917
7633
|
type: "nodeBlocked",
|
|
@@ -5971,7 +7687,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5971
7687
|
}
|
|
5972
7688
|
const kind = classifyTerminal(errorClass, { selfReportedFail, retryable });
|
|
5973
7689
|
if (kind === "blocked") {
|
|
5974
|
-
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk(
|
|
7690
|
+
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk(join10(attemptDir, GOAL_ASK_FILE)) : void 0;
|
|
5975
7691
|
appendWorkerOutcome({
|
|
5976
7692
|
type: "nodeBlocked",
|
|
5977
7693
|
nodeId: node.id,
|
|
@@ -6028,58 +7744,19 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
6028
7744
|
inFlight.set(dispatchKey, p);
|
|
6029
7745
|
return true;
|
|
6030
7746
|
}
|
|
6031
|
-
function affectedNodesFrom(root) {
|
|
6032
|
-
const reachable = /* @__PURE__ */ new Set([root]);
|
|
6033
|
-
let changed = true;
|
|
6034
|
-
while (changed) {
|
|
6035
|
-
changed = false;
|
|
6036
|
-
for (const n of dag.nodes) {
|
|
6037
|
-
if (reachable.has(n.id)) continue;
|
|
6038
|
-
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
6039
|
-
reachable.add(n.id);
|
|
6040
|
-
changed = true;
|
|
6041
|
-
}
|
|
6042
|
-
}
|
|
6043
|
-
}
|
|
6044
|
-
return [...reachable];
|
|
6045
|
-
}
|
|
6046
7747
|
function appendRevisitEvents(nodeId, instanceId, attemptId, request, sourceManifestPath) {
|
|
6047
|
-
const
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
let reasonPathRel;
|
|
6052
|
-
if (request.reason) {
|
|
6053
|
-
const dir = join6(runDir, "revisits");
|
|
6054
|
-
mkdirSync6(dir, { recursive: true });
|
|
6055
|
-
const abs = join6(dir, `${instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
6056
|
-
writeFileSync6(abs, `# Revisit reason (from ${nodeId} / ${instanceId})
|
|
6057
|
-
|
|
6058
|
-
${request.reason}
|
|
6059
|
-
`);
|
|
6060
|
-
reasonPathRel = relative3(runDir, abs);
|
|
6061
|
-
}
|
|
6062
|
-
appendEvent(journalPath, {
|
|
6063
|
-
type: "nodeRevisitRequested",
|
|
7748
|
+
const plan = planV3Revisit({
|
|
7749
|
+
runDir,
|
|
7750
|
+
dag,
|
|
7751
|
+
events: readJournal(journalPath),
|
|
6064
7752
|
nodeId,
|
|
6065
7753
|
instanceId,
|
|
6066
7754
|
attemptId,
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
6070
|
-
sourceManifestPath: relative3(runDir, sourceManifestPath),
|
|
6071
|
-
...targetSucc ? { targetPreviousManifestPath: relative3(runDir, targetSucc.manifestPath) } : {}
|
|
7755
|
+
request,
|
|
7756
|
+
sourceManifestPath
|
|
6072
7757
|
});
|
|
6073
|
-
for (const
|
|
6074
|
-
|
|
6075
|
-
if (!eff) continue;
|
|
6076
|
-
appendEvent(journalPath, {
|
|
6077
|
-
type: "nodeInstanceSuperseded",
|
|
6078
|
-
nodeId: affectedNodeId,
|
|
6079
|
-
instanceId: eff,
|
|
6080
|
-
byNodeId: request.toNodeId,
|
|
6081
|
-
reason: "refresh"
|
|
6082
|
-
});
|
|
7758
|
+
for (const event of plan.events) appendEvent(journalPath, event);
|
|
7759
|
+
for (const eff of plan.supersededInstanceIds) {
|
|
6083
7760
|
const controller = nodeControllers.get(eff);
|
|
6084
7761
|
if (controller && !controller.signal.aborted) {
|
|
6085
7762
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6201,11 +7878,11 @@ ${request.reason}
|
|
|
6201
7878
|
}
|
|
6202
7879
|
function applyLoopControl(a) {
|
|
6203
7880
|
if (a.kind === "startLoop") {
|
|
6204
|
-
|
|
7881
|
+
appendControlEvent({ type: "loopStarted", loopId: a.loopId });
|
|
6205
7882
|
return;
|
|
6206
7883
|
}
|
|
6207
7884
|
if (a.kind === "startLoopIteration") {
|
|
6208
|
-
|
|
7885
|
+
appendControlEvent({ type: "loopIterationStarted", loopId: a.loopId, iteration: a.iteration });
|
|
6209
7886
|
return;
|
|
6210
7887
|
}
|
|
6211
7888
|
const loopNode = nodesById.get(a.loopId);
|
|
@@ -6214,7 +7891,7 @@ ${request.reason}
|
|
|
6214
7891
|
const outInstId = loopInstanceId(a.loopId, a.iteration, loopNode.output.from);
|
|
6215
7892
|
const succ2 = [...events2].reverse().find((e) => e.type === "nodeSucceeded" && e.nodeId === outInstId);
|
|
6216
7893
|
if (!succ2) {
|
|
6217
|
-
|
|
7894
|
+
appendControlEvent({
|
|
6218
7895
|
type: "nodeFailed",
|
|
6219
7896
|
nodeId: a.loopId,
|
|
6220
7897
|
attemptId: `${a.loopId}/iterations/${String(a.iteration).padStart(3, "0")}`,
|
|
@@ -6223,7 +7900,7 @@ ${request.reason}
|
|
|
6223
7900
|
});
|
|
6224
7901
|
return;
|
|
6225
7902
|
}
|
|
6226
|
-
|
|
7903
|
+
appendControlEvent({
|
|
6227
7904
|
type: "nodeSucceeded",
|
|
6228
7905
|
nodeId: a.loopId,
|
|
6229
7906
|
attemptId: succ2.attemptId,
|
|
@@ -6239,11 +7916,11 @@ ${request.reason}
|
|
|
6239
7916
|
let observed = `${loopNode.exit.when.path}=<unreadable>`;
|
|
6240
7917
|
if (succ) {
|
|
6241
7918
|
try {
|
|
6242
|
-
const manifest = JSON.parse(
|
|
7919
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6243
7920
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6244
7921
|
if (entry) {
|
|
6245
7922
|
const result = JSON.parse(
|
|
6246
|
-
|
|
7923
|
+
readFileSync12(join10(dirname8(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6247
7924
|
);
|
|
6248
7925
|
observed = `${loopNode.exit.when.path}=${JSON.stringify(result[key])}`;
|
|
6249
7926
|
matched = matchLoopExitWhen(loopNode.exit.when, result[key]);
|
|
@@ -6257,7 +7934,7 @@ ${request.reason}
|
|
|
6257
7934
|
const effectiveMax = loopNode.maxIterations + granted;
|
|
6258
7935
|
const anomalous = observed.endsWith("<unreadable>");
|
|
6259
7936
|
const decision = matched ? "exit" : !anomalous && a.iteration < effectiveMax ? "continue" : "exhausted";
|
|
6260
|
-
|
|
7937
|
+
appendControlEvent({
|
|
6261
7938
|
type: "loopIterationDecision",
|
|
6262
7939
|
loopId: a.loopId,
|
|
6263
7940
|
iteration: a.iteration,
|
|
@@ -6274,7 +7951,7 @@ ${request.reason}
|
|
|
6274
7951
|
const target = nodesById.get(a.to);
|
|
6275
7952
|
const dep = target?.depends.find((d) => d.from === a.from);
|
|
6276
7953
|
if (!target || !dep?.when) {
|
|
6277
|
-
|
|
7954
|
+
appendControlEvent({
|
|
6278
7955
|
type: "edgeResolved",
|
|
6279
7956
|
from: a.from,
|
|
6280
7957
|
to: a.to,
|
|
@@ -6292,11 +7969,11 @@ ${request.reason}
|
|
|
6292
7969
|
let detail = `${dep.when.path}=<unreadable>`;
|
|
6293
7970
|
if (succ) {
|
|
6294
7971
|
try {
|
|
6295
|
-
const manifest = JSON.parse(
|
|
7972
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6296
7973
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6297
7974
|
if (entry) {
|
|
6298
7975
|
const result = JSON.parse(
|
|
6299
|
-
|
|
7976
|
+
readFileSync12(join10(dirname8(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6300
7977
|
);
|
|
6301
7978
|
detail = `${dep.when.path}=${JSON.stringify(result[key])}`;
|
|
6302
7979
|
active = matchLoopExitWhen(dep.when, result[key]);
|
|
@@ -6304,7 +7981,7 @@ ${request.reason}
|
|
|
6304
7981
|
} catch {
|
|
6305
7982
|
}
|
|
6306
7983
|
}
|
|
6307
|
-
|
|
7984
|
+
appendControlEvent({
|
|
6308
7985
|
type: "edgeResolved",
|
|
6309
7986
|
from: a.from,
|
|
6310
7987
|
to: a.to,
|
|
@@ -6321,7 +7998,7 @@ ${request.reason}
|
|
|
6321
7998
|
if (status !== "pending" && status !== "gateWaiting" && status !== "running") return false;
|
|
6322
7999
|
const instanceId = snap.nodes.get(a.nodeId)?.effectiveInstanceId;
|
|
6323
8000
|
const attemptId = latestAttemptIdFor(events, instanceId ?? a.nodeId);
|
|
6324
|
-
|
|
8001
|
+
const appended = appendControlEvent({
|
|
6325
8002
|
type: "nodeCancelled",
|
|
6326
8003
|
nodeId: a.nodeId,
|
|
6327
8004
|
...instanceId ? { instanceId } : {},
|
|
@@ -6330,6 +8007,7 @@ ${request.reason}
|
|
|
6330
8007
|
byNodeId: a.byNodeId,
|
|
6331
8008
|
detail: a.detail
|
|
6332
8009
|
});
|
|
8010
|
+
if (!appended) return true;
|
|
6333
8011
|
const controller = nodeControllers.get(instanceId ?? a.nodeId);
|
|
6334
8012
|
if (controller && !controller.signal.aborted) {
|
|
6335
8013
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6399,8 +8077,8 @@ ${request.reason}
|
|
|
6399
8077
|
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
6400
8078
|
const succ = latestSuccess(key);
|
|
6401
8079
|
if (!succ) return;
|
|
6402
|
-
const upstreamOutputDir =
|
|
6403
|
-
const manifest = JSON.parse(
|
|
8080
|
+
const upstreamOutputDir = join10(dirname8(succ.manifestPath), "work");
|
|
8081
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6404
8082
|
for (const f of manifest.files) {
|
|
6405
8083
|
if (filter && !filter(f)) continue;
|
|
6406
8084
|
inputs.push({
|
|
@@ -6408,29 +8086,29 @@ ${request.reason}
|
|
|
6408
8086
|
...output ? { output } : {},
|
|
6409
8087
|
...succ.instanceId ? { instanceId: succ.instanceId } : {},
|
|
6410
8088
|
name: f.name,
|
|
6411
|
-
path:
|
|
8089
|
+
path: join10(upstreamOutputDir, f.path),
|
|
6412
8090
|
kind: f.kind,
|
|
6413
8091
|
preview: f.preview
|
|
6414
8092
|
});
|
|
6415
8093
|
}
|
|
6416
8094
|
};
|
|
6417
|
-
const resolveRunPath = (p) =>
|
|
8095
|
+
const resolveRunPath = (p) => isAbsolute5(p) ? p : join10(runDir, p);
|
|
6418
8096
|
const pushManifestByPath = (label, manifestPath, namePrefix) => {
|
|
6419
8097
|
if (!manifestPath) return;
|
|
6420
8098
|
const abs = resolveRunPath(manifestPath);
|
|
6421
|
-
if (!
|
|
8099
|
+
if (!existsSync10(abs)) return;
|
|
6422
8100
|
let manifest;
|
|
6423
8101
|
try {
|
|
6424
|
-
manifest = JSON.parse(
|
|
8102
|
+
manifest = JSON.parse(readFileSync12(abs, "utf-8"));
|
|
6425
8103
|
} catch {
|
|
6426
8104
|
return;
|
|
6427
8105
|
}
|
|
6428
|
-
const outDir =
|
|
8106
|
+
const outDir = join10(dirname8(abs), "work");
|
|
6429
8107
|
for (const f of manifest.files) {
|
|
6430
8108
|
inputs.push({
|
|
6431
8109
|
from: label,
|
|
6432
8110
|
name: `${namePrefix}:${f.name}`,
|
|
6433
|
-
path:
|
|
8111
|
+
path: join10(outDir, f.path),
|
|
6434
8112
|
kind: f.kind,
|
|
6435
8113
|
preview: f.preview
|
|
6436
8114
|
});
|
|
@@ -6500,7 +8178,7 @@ ${request.reason}
|
|
|
6500
8178
|
}
|
|
6501
8179
|
|
|
6502
8180
|
// src/workflows/v3/portable-runtime.ts
|
|
6503
|
-
import { join as
|
|
8181
|
+
import { join as join11 } from "node:path";
|
|
6504
8182
|
function defaultValidateProfile(profile, selector) {
|
|
6505
8183
|
if (!profile.profileId || !profile.executorId || !profile.workingDirectory) {
|
|
6506
8184
|
throw new Error(
|
|
@@ -6549,7 +8227,7 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6549
8227
|
validateProfile(executionProfile, selector);
|
|
6550
8228
|
}
|
|
6551
8229
|
const frozenDefinition = assertOrCreatePortableRunSnapshot(
|
|
6552
|
-
|
|
8230
|
+
join11(options.baseDir, normalizedDag.runId),
|
|
6553
8231
|
normalizedDag,
|
|
6554
8232
|
profiles
|
|
6555
8233
|
);
|
|
@@ -6627,6 +8305,9 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6627
8305
|
])
|
|
6628
8306
|
)
|
|
6629
8307
|
});
|
|
8308
|
+
if (outcome.reason === "awaitingExternal") {
|
|
8309
|
+
throw new Error("portable workflow runtime does not drive bot-executed goal nodes");
|
|
8310
|
+
}
|
|
6630
8311
|
if (outcome.reason !== "terminal") return outcome;
|
|
6631
8312
|
return {
|
|
6632
8313
|
...outcome,
|