botmux-workflow-core 3.55.0 → 3.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/control.cjs +9 -0
- package/dist/cjs/control.cjs.map +2 -2
- package/dist/cjs/index.cjs +9 -0
- package/dist/cjs/index.cjs.map +2 -2
- package/dist/cjs/runtime.cjs +1895 -419
- package/dist/cjs/runtime.cjs.map +4 -4
- package/dist/esm/control.js +9 -0
- package/dist/esm/control.js.map +2 -2
- package/dist/esm/index.js +9 -0
- package/dist/esm/index.js.map +2 -2
- package/dist/esm/runtime.js +1900 -415
- package/dist/esm/runtime.js.map +4 -4
- package/dist/types/src/workflows/v3/event-contract.d.ts +205 -0
- package/package.json +1 -1
package/dist/esm/runtime.js
CHANGED
|
@@ -2606,6 +2606,7 @@ function materialize(events) {
|
|
|
2606
2606
|
const attempts = /* @__PURE__ */ new Map();
|
|
2607
2607
|
const loops = /* @__PURE__ */ new Map();
|
|
2608
2608
|
const edges = /* @__PURE__ */ new Map();
|
|
2609
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
2609
2610
|
let runStatus = "running";
|
|
2610
2611
|
let cancelRequestId;
|
|
2611
2612
|
let cancelRequestedBy;
|
|
@@ -2615,6 +2616,7 @@ function materialize(events) {
|
|
|
2615
2616
|
let blockedNodeId;
|
|
2616
2617
|
let uncertainHostEffects;
|
|
2617
2618
|
const cancelledAttempts = /* @__PURE__ */ new Map();
|
|
2619
|
+
const regateInstances = /* @__PURE__ */ new Set();
|
|
2618
2620
|
const resolvedWaitIds = /* @__PURE__ */ new Set();
|
|
2619
2621
|
const set = (id, status, gateCleared) => {
|
|
2620
2622
|
const prev = nodes.get(id);
|
|
@@ -2635,6 +2637,16 @@ function materialize(events) {
|
|
|
2635
2637
|
nodes.set(nodeId, { ...prev, status });
|
|
2636
2638
|
}
|
|
2637
2639
|
};
|
|
2640
|
+
const supersedeInstance = (nodeId, instanceId) => {
|
|
2641
|
+
instances.set(instanceId, { status: "superseded" });
|
|
2642
|
+
if (nodes.get(nodeId)?.effectiveInstanceId === instanceId) {
|
|
2643
|
+
nodes.set(nodeId, { status: "pending" });
|
|
2644
|
+
}
|
|
2645
|
+
if (runStatus === "blocked" && blockedNodeId === nodeId) {
|
|
2646
|
+
runStatus = "running";
|
|
2647
|
+
blockedNodeId = void 0;
|
|
2648
|
+
}
|
|
2649
|
+
};
|
|
2638
2650
|
const loop = (id) => {
|
|
2639
2651
|
let ls = loops.get(id);
|
|
2640
2652
|
if (!ls) {
|
|
@@ -2667,6 +2679,7 @@ function materialize(events) {
|
|
|
2667
2679
|
attempts.set(e.instanceId ?? e.nodeId, e.attemptId);
|
|
2668
2680
|
break;
|
|
2669
2681
|
case "hostInputPrepared":
|
|
2682
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2670
2683
|
instances.set(e.instanceId, { status: "pending" });
|
|
2671
2684
|
nodes.set(e.nodeId, {
|
|
2672
2685
|
status: "pending",
|
|
@@ -2676,6 +2689,7 @@ function materialize(events) {
|
|
|
2676
2689
|
break;
|
|
2677
2690
|
case "hostEffectIntent":
|
|
2678
2691
|
if (instances.get(e.instanceId)?.status === "cancelled") break;
|
|
2692
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2679
2693
|
recordInstance(e.nodeId, e.instanceId, "running", true);
|
|
2680
2694
|
attempts.set(e.instanceId, e.attemptId);
|
|
2681
2695
|
break;
|
|
@@ -2715,6 +2729,15 @@ function materialize(events) {
|
|
|
2715
2729
|
} else set(e.nodeId, "blocked");
|
|
2716
2730
|
break;
|
|
2717
2731
|
case "nodeRetryRequested":
|
|
2732
|
+
if (e.instanceId && regateInstances.has(e.instanceId)) {
|
|
2733
|
+
regateInstances.delete(e.instanceId);
|
|
2734
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2735
|
+
if (runStatus === "blocked") {
|
|
2736
|
+
runStatus = "running";
|
|
2737
|
+
blockedNodeId = void 0;
|
|
2738
|
+
}
|
|
2739
|
+
break;
|
|
2740
|
+
}
|
|
2718
2741
|
if (e.resetGate) {
|
|
2719
2742
|
nodes.set(e.nodeId, {
|
|
2720
2743
|
status: "pending",
|
|
@@ -2734,15 +2757,40 @@ function materialize(events) {
|
|
|
2734
2757
|
// ── cross-node revisit / instance refresh ──
|
|
2735
2758
|
case "nodeRevisitRequested":
|
|
2736
2759
|
break;
|
|
2737
|
-
case "nodeInstanceSuperseded":
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2760
|
+
case "nodeInstanceSuperseded":
|
|
2761
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2762
|
+
break;
|
|
2763
|
+
case "dagRevised": {
|
|
2764
|
+
const effects = e.effects;
|
|
2765
|
+
for (const item of effects?.supersede ?? []) {
|
|
2766
|
+
if (item.waitId) resolvedWaitIds.add(item.waitId);
|
|
2767
|
+
supersedeInstance(item.nodeId, item.instanceId);
|
|
2768
|
+
}
|
|
2769
|
+
for (const item of effects?.cancel ?? []) {
|
|
2770
|
+
cancelledAttempts.set(item.instanceId ?? item.nodeId, item.attemptId);
|
|
2771
|
+
if (item.instanceId) instances.set(item.instanceId, { status: "cancelled" });
|
|
2772
|
+
const prev = nodes.get(item.nodeId);
|
|
2773
|
+
nodes.set(item.nodeId, {
|
|
2774
|
+
status: "cancelled",
|
|
2775
|
+
...item.instanceId ?? prev?.effectiveInstanceId ? { effectiveInstanceId: item.instanceId ?? prev?.effectiveInstanceId } : {}
|
|
2776
|
+
});
|
|
2741
2777
|
}
|
|
2742
|
-
if (runStatus === "blocked" &&
|
|
2778
|
+
if (runStatus === "blocked" && (effects?.cancel ?? []).length > 0 && ((effects?.cancel ?? []).some((item) => item.nodeId === blockedNodeId) || ![...nodes.values()].some((state) => state.status === "blocked"))) {
|
|
2743
2779
|
runStatus = "running";
|
|
2744
2780
|
blockedNodeId = void 0;
|
|
2745
2781
|
}
|
|
2782
|
+
for (const item of effects?.regate ?? []) {
|
|
2783
|
+
const prev = nodes.get(item.nodeId);
|
|
2784
|
+
if (prev?.effectiveInstanceId !== item.instanceId) continue;
|
|
2785
|
+
const { gateCleared: _dropped, ...rest } = prev;
|
|
2786
|
+
nodes.set(item.nodeId, rest);
|
|
2787
|
+
const instance = instances.get(item.instanceId);
|
|
2788
|
+
if (instance) {
|
|
2789
|
+
const { gateCleared: _instanceDropped, ...instanceRest } = instance;
|
|
2790
|
+
instances.set(item.instanceId, instanceRest);
|
|
2791
|
+
}
|
|
2792
|
+
regateInstances.add(item.instanceId);
|
|
2793
|
+
}
|
|
2746
2794
|
break;
|
|
2747
2795
|
}
|
|
2748
2796
|
case "gateDispatched":
|
|
@@ -2800,6 +2848,32 @@ function materialize(events) {
|
|
|
2800
2848
|
if (e.instanceId) recordInstance(e.nodeId, e.instanceId, "cancelled", false);
|
|
2801
2849
|
else set(e.nodeId, "cancelled");
|
|
2802
2850
|
break;
|
|
2851
|
+
case "nodeHandoffPrepared":
|
|
2852
|
+
case "nodeHandoffSent":
|
|
2853
|
+
case "nodeHandoffAccepted":
|
|
2854
|
+
case "nodeStalled":
|
|
2855
|
+
break;
|
|
2856
|
+
case "nodeReassigned": {
|
|
2857
|
+
assignments.set(e.nodeId, e.to);
|
|
2858
|
+
if (!e.attemptId || runStatus !== "running" && runStatus !== "blocked") break;
|
|
2859
|
+
const key = e.instanceId ?? e.nodeId;
|
|
2860
|
+
if (attempts.get(key) !== e.attemptId) break;
|
|
2861
|
+
if (e.instanceId) {
|
|
2862
|
+
if (instances.get(e.instanceId)?.status !== "running") break;
|
|
2863
|
+
instances.set(e.instanceId, { status: "pending" });
|
|
2864
|
+
const current = nodes.get(e.nodeId);
|
|
2865
|
+
if (current?.effectiveInstanceId === e.instanceId && current.status === "running") {
|
|
2866
|
+
nodes.set(e.nodeId, {
|
|
2867
|
+
status: "pending",
|
|
2868
|
+
effectiveInstanceId: e.instanceId,
|
|
2869
|
+
...current.gateCleared ? { gateCleared: true } : {}
|
|
2870
|
+
});
|
|
2871
|
+
}
|
|
2872
|
+
} else if (nodes.get(e.nodeId)?.status === "running") {
|
|
2873
|
+
set(e.nodeId, "pending");
|
|
2874
|
+
}
|
|
2875
|
+
break;
|
|
2876
|
+
}
|
|
2803
2877
|
case "nodeAttemptDrained": {
|
|
2804
2878
|
if (runStatus !== "running" && runStatus !== "blocked") break;
|
|
2805
2879
|
const key = e.instanceId ?? e.nodeId;
|
|
@@ -2894,7 +2968,9 @@ function materialize(events) {
|
|
|
2894
2968
|
instances,
|
|
2895
2969
|
attempts,
|
|
2896
2970
|
loops,
|
|
2897
|
-
edges
|
|
2971
|
+
edges,
|
|
2972
|
+
assignments,
|
|
2973
|
+
...regateInstances.size > 0 ? { regatePending: regateInstances } : {}
|
|
2898
2974
|
};
|
|
2899
2975
|
}
|
|
2900
2976
|
function writeState(statePath, snap) {
|
|
@@ -2914,6 +2990,7 @@ function writeState(statePath, snap) {
|
|
|
2914
2990
|
attempts: Object.fromEntries(snap.attempts),
|
|
2915
2991
|
loops: snap.loops.size > 0 ? Object.fromEntries(snap.loops) : void 0,
|
|
2916
2992
|
edges: snap.edges.size > 0 ? Object.fromEntries(snap.edges) : void 0,
|
|
2993
|
+
assignments: snap.assignments.size > 0 ? Object.fromEntries(snap.assignments) : void 0,
|
|
2917
2994
|
updatedAt: Date.now()
|
|
2918
2995
|
};
|
|
2919
2996
|
const tmp = `${statePath}.tmp`;
|
|
@@ -3265,9 +3342,9 @@ function assertOrCreatePortableRunSnapshot(runDir, dag, profiles) {
|
|
|
3265
3342
|
}
|
|
3266
3343
|
|
|
3267
3344
|
// src/workflows/v3/shared-node-runtime.ts
|
|
3268
|
-
import { randomUUID as
|
|
3269
|
-
import { mkdirSync as
|
|
3270
|
-
import { join as
|
|
3345
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
3346
|
+
import { mkdirSync as mkdirSync8, writeFileSync as writeFileSync8, readFileSync as readFileSync12, existsSync as existsSync10, rmSync as rmSync2 } from "node:fs";
|
|
3347
|
+
import { join as join10, dirname as dirname8, relative as relative5, isAbsolute as isAbsolute5, resolve as resolve5, sep as sep3 } from "node:path";
|
|
3271
3348
|
|
|
3272
3349
|
// src/workflows/v3/core-control.ts
|
|
3273
3350
|
function matchLoopExitWhen(when, value) {
|
|
@@ -3289,6 +3366,15 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3289
3366
|
if (event.type === "nodeRevisitRequested") {
|
|
3290
3367
|
runUsed++;
|
|
3291
3368
|
if (event.nodeId === sourceNodeId && event.toNodeId === toNodeId) pairUsed++;
|
|
3369
|
+
} else if (event.type === "dagRevised") {
|
|
3370
|
+
for (const nodeId of event.effects?.rerun ?? []) {
|
|
3371
|
+
runUsed++;
|
|
3372
|
+
if (nodeId === sourceNodeId && nodeId === toNodeId) pairUsed++;
|
|
3373
|
+
}
|
|
3374
|
+
for (const grant of event.effects?.budgetGrants ?? []) {
|
|
3375
|
+
if (grant.tier === "run") runGranted++;
|
|
3376
|
+
else if (grant.nodeId === sourceNodeId && grant.nodeId === toNodeId) pairGranted++;
|
|
3377
|
+
}
|
|
3292
3378
|
} else if (event.type === "revisitBudgetGranted") {
|
|
3293
3379
|
if (event.sourceNodeId === sourceNodeId && event.toNodeId === toNodeId) pairGranted++;
|
|
3294
3380
|
else if (event.sourceNodeId === void 0 && event.toNodeId === void 0) runGranted++;
|
|
@@ -3313,6 +3399,72 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3313
3399
|
return { ok: true };
|
|
3314
3400
|
}
|
|
3315
3401
|
|
|
3402
|
+
// src/workflows/v3/revisit-plan.ts
|
|
3403
|
+
import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "node:fs";
|
|
3404
|
+
import { join as join4, relative as relative2 } from "node:path";
|
|
3405
|
+
function affectedNodesFrom(dag, root) {
|
|
3406
|
+
const reachable = /* @__PURE__ */ new Set([root]);
|
|
3407
|
+
let changed = true;
|
|
3408
|
+
while (changed) {
|
|
3409
|
+
changed = false;
|
|
3410
|
+
for (const n of dag.nodes) {
|
|
3411
|
+
if (reachable.has(n.id)) continue;
|
|
3412
|
+
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
3413
|
+
reachable.add(n.id);
|
|
3414
|
+
changed = true;
|
|
3415
|
+
}
|
|
3416
|
+
}
|
|
3417
|
+
}
|
|
3418
|
+
return [...reachable];
|
|
3419
|
+
}
|
|
3420
|
+
function planV3Revisit(input) {
|
|
3421
|
+
const { runDir, request } = input;
|
|
3422
|
+
const snap = materialize([...input.events]);
|
|
3423
|
+
const targetEff = snap.nodes.get(request.toNodeId)?.effectiveInstanceId;
|
|
3424
|
+
const targetSucc = targetEff ? [...input.events].reverse().find((e) => e.type === "nodeSucceeded" && (e.instanceId ?? e.nodeId) === targetEff) : void 0;
|
|
3425
|
+
let reasonPathRel;
|
|
3426
|
+
if (request.reason) {
|
|
3427
|
+
const dir = join4(runDir, "revisits");
|
|
3428
|
+
mkdirSync4(dir, { recursive: true });
|
|
3429
|
+
const abs = join4(dir, `${input.instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
3430
|
+
writeFileSync4(abs, `# Revisit reason (from ${input.nodeId} / ${input.instanceId})
|
|
3431
|
+
|
|
3432
|
+
${request.reason}
|
|
3433
|
+
`);
|
|
3434
|
+
reasonPathRel = relative2(runDir, abs);
|
|
3435
|
+
}
|
|
3436
|
+
const events = [{
|
|
3437
|
+
type: "nodeRevisitRequested",
|
|
3438
|
+
nodeId: input.nodeId,
|
|
3439
|
+
instanceId: input.instanceId,
|
|
3440
|
+
attemptId: input.attemptId,
|
|
3441
|
+
toNodeId: request.toNodeId,
|
|
3442
|
+
...request.reason ? { reason: request.reason } : {},
|
|
3443
|
+
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
3444
|
+
sourceManifestPath: relative2(runDir, input.sourceManifestPath),
|
|
3445
|
+
...targetSucc ? { targetPreviousManifestPath: relative2(runDir, targetSucc.manifestPath) } : {}
|
|
3446
|
+
}];
|
|
3447
|
+
const supersededInstanceIds = [];
|
|
3448
|
+
for (const affectedNodeId of affectedNodesFrom(input.dag, request.toNodeId)) {
|
|
3449
|
+
const eff = snap.nodes.get(affectedNodeId)?.effectiveInstanceId;
|
|
3450
|
+
if (!eff) continue;
|
|
3451
|
+
events.push({
|
|
3452
|
+
type: "nodeInstanceSuperseded",
|
|
3453
|
+
nodeId: affectedNodeId,
|
|
3454
|
+
instanceId: eff,
|
|
3455
|
+
byNodeId: request.toNodeId,
|
|
3456
|
+
reason: "refresh"
|
|
3457
|
+
});
|
|
3458
|
+
supersededInstanceIds.push(eff);
|
|
3459
|
+
}
|
|
3460
|
+
return { events, supersededInstanceIds };
|
|
3461
|
+
}
|
|
3462
|
+
|
|
3463
|
+
// src/workflows/v3/dag-revision.ts
|
|
3464
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
3465
|
+
import { existsSync as existsSync5, lstatSync as lstatSync4, readFileSync as readFileSync7 } from "node:fs";
|
|
3466
|
+
import { dirname as dirname4, join as join5 } from "node:path";
|
|
3467
|
+
|
|
3316
3468
|
// src/workflows/v3/gate-policy.ts
|
|
3317
3469
|
function normalizeGateWaitInput(gate) {
|
|
3318
3470
|
const options = gate.options ?? [...DEFAULT_HUMAN_GATE_OPTIONS];
|
|
@@ -3324,29 +3476,433 @@ function normalizeGateWaitInput(gate) {
|
|
|
3324
3476
|
};
|
|
3325
3477
|
}
|
|
3326
3478
|
|
|
3479
|
+
// src/workflows/v3/attempt-numbering.ts
|
|
3480
|
+
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
3481
|
+
function attemptNumber(attemptId) {
|
|
3482
|
+
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
3483
|
+
return m ? parseInt(m[1], 10) : void 0;
|
|
3484
|
+
}
|
|
3485
|
+
function nextAttemptIdFor(events, key) {
|
|
3486
|
+
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
3487
|
+
let maxSeen = 0;
|
|
3488
|
+
let reserved;
|
|
3489
|
+
let prepared;
|
|
3490
|
+
for (const e of events) {
|
|
3491
|
+
if (e.type === "nodeDispatched" && matches(e)) {
|
|
3492
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3493
|
+
if (n2 === void 0) continue;
|
|
3494
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3495
|
+
if (reserved === n2) reserved = void 0;
|
|
3496
|
+
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
3497
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3498
|
+
if (n2 === void 0) continue;
|
|
3499
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3500
|
+
if (reserved === n2) reserved = void 0;
|
|
3501
|
+
prepared = e.attemptId;
|
|
3502
|
+
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
3503
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3504
|
+
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
3505
|
+
if (prepared === e.attemptId) prepared = void 0;
|
|
3506
|
+
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
3507
|
+
prepared = void 0;
|
|
3508
|
+
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
3509
|
+
const n2 = attemptNumber(e.nextAttemptId);
|
|
3510
|
+
if (n2 === void 0) continue;
|
|
3511
|
+
reserved = n2;
|
|
3512
|
+
prepared = void 0;
|
|
3513
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3514
|
+
}
|
|
3515
|
+
}
|
|
3516
|
+
if (prepared) return prepared;
|
|
3517
|
+
const n = reserved ?? maxSeen + 1;
|
|
3518
|
+
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
3519
|
+
}
|
|
3520
|
+
function latestAttemptIdFor(events, key) {
|
|
3521
|
+
let latest;
|
|
3522
|
+
let latestNumber = -1;
|
|
3523
|
+
for (const e of events) {
|
|
3524
|
+
if ((e.type === "nodeDispatched" || e.type === "hostInputPrepared" || e.type === "hostEffectIntent" || e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && (e.instanceId ?? e.nodeId) === key) {
|
|
3525
|
+
const n = attemptNumber(e.attemptId);
|
|
3526
|
+
if (n === void 0) {
|
|
3527
|
+
if (latestNumber < 0) latest = e.attemptId;
|
|
3528
|
+
} else if (n >= latestNumber) {
|
|
3529
|
+
latest = e.attemptId;
|
|
3530
|
+
latestNumber = n;
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
}
|
|
3534
|
+
return latest;
|
|
3535
|
+
}
|
|
3536
|
+
|
|
3537
|
+
// src/workflows/v3/dag-revision.ts
|
|
3538
|
+
var V3_DAG_REVISIONS_DIR = "dag-revisions";
|
|
3539
|
+
function v3RevisionStem(revision) {
|
|
3540
|
+
if (!Number.isSafeInteger(revision) || revision < 2) {
|
|
3541
|
+
throw new Error(`v3 revision: invalid revision number ${revision}`);
|
|
3542
|
+
}
|
|
3543
|
+
return String(revision).padStart(3, "0");
|
|
3544
|
+
}
|
|
3545
|
+
function v3RevisionFilePaths(runDir, revision) {
|
|
3546
|
+
const stem = v3RevisionStem(revision);
|
|
3547
|
+
const dir = join5(runDir, V3_DAG_REVISIONS_DIR);
|
|
3548
|
+
return {
|
|
3549
|
+
dag: join5(dir, `${stem}.json`),
|
|
3550
|
+
patch: join5(dir, `${stem}.patch.json`),
|
|
3551
|
+
bots: join5(dir, `${stem}.bots.json`)
|
|
3552
|
+
};
|
|
3553
|
+
}
|
|
3554
|
+
function v3RevisionDigest(bytes) {
|
|
3555
|
+
return `sha256:${createHash4("sha256").update(bytes).digest("hex")}`;
|
|
3556
|
+
}
|
|
3557
|
+
var V3RevisionIntegrityError = class extends Error {
|
|
3558
|
+
constructor(message) {
|
|
3559
|
+
super(message);
|
|
3560
|
+
this.name = "V3RevisionIntegrityError";
|
|
3561
|
+
}
|
|
3562
|
+
};
|
|
3563
|
+
function readPinnedFile(path, expected, label) {
|
|
3564
|
+
let stat;
|
|
3565
|
+
try {
|
|
3566
|
+
stat = lstatSync4(path);
|
|
3567
|
+
} catch {
|
|
3568
|
+
throw new V3RevisionIntegrityError(`${label} is missing (${path})`);
|
|
3569
|
+
}
|
|
3570
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3571
|
+
throw new V3RevisionIntegrityError(`${label} must be a regular file (${path})`);
|
|
3572
|
+
}
|
|
3573
|
+
const dirStat = lstatSync4(dirname4(path));
|
|
3574
|
+
if (!dirStat.isDirectory() || dirStat.isSymbolicLink()) {
|
|
3575
|
+
throw new V3RevisionIntegrityError(`${label} directory must be a real directory`);
|
|
3576
|
+
}
|
|
3577
|
+
const bytes = readFileSync7(path);
|
|
3578
|
+
const actual = v3RevisionDigest(bytes);
|
|
3579
|
+
if (actual !== expected) {
|
|
3580
|
+
throw new V3RevisionIntegrityError(`${label} digest mismatch (journal ${expected}, file ${actual})`);
|
|
3581
|
+
}
|
|
3582
|
+
return bytes;
|
|
3583
|
+
}
|
|
3584
|
+
var V3_UPDATE_NODE_FIELDS = [
|
|
3585
|
+
"goal",
|
|
3586
|
+
"outputs",
|
|
3587
|
+
"resultSchema",
|
|
3588
|
+
"timeoutSec",
|
|
3589
|
+
"humanGate",
|
|
3590
|
+
"override",
|
|
3591
|
+
"revisitTo",
|
|
3592
|
+
"input",
|
|
3593
|
+
"maxIterations",
|
|
3594
|
+
"body",
|
|
3595
|
+
"exit",
|
|
3596
|
+
"feedback",
|
|
3597
|
+
"output"
|
|
3598
|
+
];
|
|
3599
|
+
var UPDATE_FIELDS = new Set(V3_UPDATE_NODE_FIELDS);
|
|
3600
|
+
function isPlainObject(value) {
|
|
3601
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3602
|
+
}
|
|
3603
|
+
function foldV3Revisions(events) {
|
|
3604
|
+
let current = 1;
|
|
3605
|
+
let maxRevision = 1;
|
|
3606
|
+
const records = [];
|
|
3607
|
+
const byRevision = /* @__PURE__ */ new Map();
|
|
3608
|
+
const applied = [];
|
|
3609
|
+
for (const event of events) {
|
|
3610
|
+
switch (event.type) {
|
|
3611
|
+
case "dagRevisionProposed": {
|
|
3612
|
+
if (byRevision.has(event.revision) || event.revision <= maxRevision) {
|
|
3613
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3614
|
+
}
|
|
3615
|
+
for (const prior of records) {
|
|
3616
|
+
if (prior.state !== "pending") continue;
|
|
3617
|
+
prior.state = "rejected";
|
|
3618
|
+
prior.rejected = { by: "system", reason: "superseded" };
|
|
3619
|
+
}
|
|
3620
|
+
const record = {
|
|
3621
|
+
revision: event.revision,
|
|
3622
|
+
baseRevision: event.baseRevision,
|
|
3623
|
+
sha256: event.sha256,
|
|
3624
|
+
patchSha256: event.patchSha256,
|
|
3625
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3626
|
+
summary: event.summary,
|
|
3627
|
+
...event.note ? { note: event.note } : {},
|
|
3628
|
+
by: event.by,
|
|
3629
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3630
|
+
state: "pending",
|
|
3631
|
+
ts: event.ts,
|
|
3632
|
+
reasons: [...event.reasons],
|
|
3633
|
+
touched: [...event.touched],
|
|
3634
|
+
...event.carriedFrom !== void 0 ? { carriedFrom: event.carriedFrom } : {}
|
|
3635
|
+
};
|
|
3636
|
+
records.push(record);
|
|
3637
|
+
byRevision.set(event.revision, record);
|
|
3638
|
+
maxRevision = event.revision;
|
|
3639
|
+
break;
|
|
3640
|
+
}
|
|
3641
|
+
case "dagRevised": {
|
|
3642
|
+
if (event.baseRevision !== current) {
|
|
3643
|
+
throw new V3RevisionIntegrityError(
|
|
3644
|
+
`revision ${event.revision} is based on ${event.baseRevision} but ${current} was in effect`
|
|
3645
|
+
);
|
|
3646
|
+
}
|
|
3647
|
+
let record = byRevision.get(event.revision);
|
|
3648
|
+
if (event.approval) {
|
|
3649
|
+
if (!record || record.state !== "pending" || record.sha256 !== event.sha256 || record.patchSha256 !== event.patchSha256 || record.botsSha256 !== event.botsSha256 || record.baseRevision !== event.baseRevision) {
|
|
3650
|
+
throw new V3RevisionIntegrityError(
|
|
3651
|
+
`approved revision ${event.revision} does not match a pending proposal`
|
|
3652
|
+
);
|
|
3653
|
+
}
|
|
3654
|
+
record.approvedBy = event.approval.by;
|
|
3655
|
+
} else {
|
|
3656
|
+
if (record || event.revision <= maxRevision) {
|
|
3657
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3658
|
+
}
|
|
3659
|
+
record = {
|
|
3660
|
+
revision: event.revision,
|
|
3661
|
+
baseRevision: event.baseRevision,
|
|
3662
|
+
sha256: event.sha256,
|
|
3663
|
+
patchSha256: event.patchSha256,
|
|
3664
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3665
|
+
summary: event.summary,
|
|
3666
|
+
...event.note ? { note: event.note } : {},
|
|
3667
|
+
by: event.by,
|
|
3668
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3669
|
+
state: "pending",
|
|
3670
|
+
ts: event.ts
|
|
3671
|
+
};
|
|
3672
|
+
records.push(record);
|
|
3673
|
+
byRevision.set(event.revision, record);
|
|
3674
|
+
maxRevision = event.revision;
|
|
3675
|
+
}
|
|
3676
|
+
record.state = "applied";
|
|
3677
|
+
record.appliedAt = event.ts;
|
|
3678
|
+
if (event.effects) record.effects = event.effects;
|
|
3679
|
+
applied.push(record);
|
|
3680
|
+
current = event.revision;
|
|
3681
|
+
break;
|
|
3682
|
+
}
|
|
3683
|
+
case "dagRevisionApproved": {
|
|
3684
|
+
const record = byRevision.get(event.revision);
|
|
3685
|
+
if (record && !record.approvedBy) record.approvedBy = event.by;
|
|
3686
|
+
break;
|
|
3687
|
+
}
|
|
3688
|
+
case "dagRevisionRejected": {
|
|
3689
|
+
const record = byRevision.get(event.revision);
|
|
3690
|
+
if (record?.state === "pending") {
|
|
3691
|
+
record.state = "rejected";
|
|
3692
|
+
record.rejected = {
|
|
3693
|
+
by: event.by,
|
|
3694
|
+
reason: event.reason,
|
|
3695
|
+
...event.detail ? { detail: event.detail } : {}
|
|
3696
|
+
};
|
|
3697
|
+
}
|
|
3698
|
+
break;
|
|
3699
|
+
}
|
|
3700
|
+
default:
|
|
3701
|
+
break;
|
|
3702
|
+
}
|
|
3703
|
+
}
|
|
3704
|
+
let pending;
|
|
3705
|
+
for (const record of records) {
|
|
3706
|
+
if (record.state !== "pending") continue;
|
|
3707
|
+
if (record.baseRevision !== current) {
|
|
3708
|
+
record.state = "rejected";
|
|
3709
|
+
record.rejected = { by: "system", reason: "stale" };
|
|
3710
|
+
continue;
|
|
3711
|
+
}
|
|
3712
|
+
pending = record;
|
|
3713
|
+
}
|
|
3714
|
+
return { current, records, applied, ...pending ? { pending } : {}, maxRevision };
|
|
3715
|
+
}
|
|
3716
|
+
function currentV3Revision(events) {
|
|
3717
|
+
let current = 1;
|
|
3718
|
+
for (const event of events) if (event.type === "dagRevised") current = event.revision;
|
|
3719
|
+
return current;
|
|
3720
|
+
}
|
|
3721
|
+
function parseRevisionDag(bytes, base, label) {
|
|
3722
|
+
let raw;
|
|
3723
|
+
try {
|
|
3724
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3725
|
+
} catch (err) {
|
|
3726
|
+
throw new V3RevisionIntegrityError(`${label} is not JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
3727
|
+
}
|
|
3728
|
+
let dag;
|
|
3729
|
+
try {
|
|
3730
|
+
dag = validateDag(raw);
|
|
3731
|
+
} catch (err) {
|
|
3732
|
+
throw new V3RevisionIntegrityError(
|
|
3733
|
+
`${label} failed validation: ${err instanceof DagValidationError ? err.problems.join("; ") : String(err)}`
|
|
3734
|
+
);
|
|
3735
|
+
}
|
|
3736
|
+
if (dag.runId !== base.runId) throw new V3RevisionIntegrityError(`${label} belongs to run ${dag.runId}`);
|
|
3737
|
+
if ((dag.schemaVersion ?? 1) !== (base.schemaVersion ?? 1)) {
|
|
3738
|
+
throw new V3RevisionIntegrityError(`${label} changed the DAG schemaVersion`);
|
|
3739
|
+
}
|
|
3740
|
+
return dag;
|
|
3741
|
+
}
|
|
3742
|
+
function parseRevisionBotSnapshots(raw, label = "revision bot snapshots") {
|
|
3743
|
+
if (!isPlainObject(raw)) throw new V3RevisionIntegrityError(`${label} must be an object`);
|
|
3744
|
+
const out = /* @__PURE__ */ new Map();
|
|
3745
|
+
for (const [selector, value] of Object.entries(raw)) {
|
|
3746
|
+
if (!isPlainObject(value) || typeof value.larkAppId !== "string" || value.larkAppId.length === 0 || typeof value.cliId !== "string" || typeof value.workingDir !== "string") {
|
|
3747
|
+
throw new V3RevisionIntegrityError(`${label}[${JSON.stringify(selector)}] is not a bot snapshot`);
|
|
3748
|
+
}
|
|
3749
|
+
out.set(selector, value);
|
|
3750
|
+
}
|
|
3751
|
+
return out;
|
|
3752
|
+
}
|
|
3753
|
+
function readVerifiedV3Revision(runDir, record, base) {
|
|
3754
|
+
const paths = v3RevisionFilePaths(runDir, record.revision);
|
|
3755
|
+
const label = `dag-revisions/${v3RevisionStem(record.revision)}`;
|
|
3756
|
+
const dag = parseRevisionDag(readPinnedFile(paths.dag, record.sha256, `${label}.json`), base, `${label}.json`);
|
|
3757
|
+
readPinnedFile(paths.patch, record.patchSha256, `${label}.patch.json`);
|
|
3758
|
+
let bots = /* @__PURE__ */ new Map();
|
|
3759
|
+
if (record.botsSha256) {
|
|
3760
|
+
const bytes = readPinnedFile(paths.bots, record.botsSha256, `${label}.bots.json`);
|
|
3761
|
+
let raw;
|
|
3762
|
+
try {
|
|
3763
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3764
|
+
} catch {
|
|
3765
|
+
throw new V3RevisionIntegrityError(`${label}.bots.json is not JSON`);
|
|
3766
|
+
}
|
|
3767
|
+
bots = parseRevisionBotSnapshots(raw, `${label}.bots.json`);
|
|
3768
|
+
}
|
|
3769
|
+
return { dag, bots };
|
|
3770
|
+
}
|
|
3771
|
+
function loadEffectiveV3Dag(runDir, base, events) {
|
|
3772
|
+
const history = foldV3Revisions(events);
|
|
3773
|
+
const botAdditions = /* @__PURE__ */ new Map();
|
|
3774
|
+
if (history.applied.length === 0) return { dag: base, revision: 1, history, botAdditions };
|
|
3775
|
+
let dag = base;
|
|
3776
|
+
for (const record of history.applied) {
|
|
3777
|
+
const verified = readVerifiedV3Revision(runDir, record, base);
|
|
3778
|
+
dag = verified.dag;
|
|
3779
|
+
for (const [selector, snapshot] of verified.bots) botAdditions.set(selector, snapshot);
|
|
3780
|
+
}
|
|
3781
|
+
return { dag, revision: history.current, history, botAdditions };
|
|
3782
|
+
}
|
|
3783
|
+
function snapshotForSelector(snapshots, selector) {
|
|
3784
|
+
const key = selector ?? "";
|
|
3785
|
+
const exact = snapshots.get(key);
|
|
3786
|
+
if (exact) return exact;
|
|
3787
|
+
if (!key) return void 0;
|
|
3788
|
+
for (const snapshot of snapshots.values()) if (snapshot.larkAppId === key) return snapshot;
|
|
3789
|
+
return void 0;
|
|
3790
|
+
}
|
|
3791
|
+
function latestBlockOf(events, snap, nodeId) {
|
|
3792
|
+
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
3793
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
3794
|
+
const event = events[i];
|
|
3795
|
+
if (event.type === "hostEffectUncertain" && (event.instanceId ?? event.nodeId) === key) return { kind: "hostUncertain" };
|
|
3796
|
+
if (event.type === "nodeBlocked" && (event.instanceId ?? event.nodeId) === key) {
|
|
3797
|
+
return {
|
|
3798
|
+
kind: "nodeBlocked",
|
|
3799
|
+
errorClass: event.errorClass,
|
|
3800
|
+
...event.errorCode ? { errorCode: event.errorCode } : {},
|
|
3801
|
+
reviseRequired: event.recovery === "reviseWorkflow"
|
|
3802
|
+
};
|
|
3803
|
+
}
|
|
3804
|
+
}
|
|
3805
|
+
return void 0;
|
|
3806
|
+
}
|
|
3807
|
+
function v3ApprovedBotScope(frozen, botAdditions) {
|
|
3808
|
+
const scope = new Map(frozen);
|
|
3809
|
+
for (const [selector, snapshot] of botAdditions) if (!scope.has(selector)) scope.set(selector, snapshot);
|
|
3810
|
+
return scope;
|
|
3811
|
+
}
|
|
3812
|
+
function v3NodeHeldByProposal(pendingTouched, nodeId, loopId) {
|
|
3813
|
+
if (!pendingTouched || pendingTouched.size === 0) return false;
|
|
3814
|
+
return pendingTouched.has(nodeId) || loopId !== void 0 && pendingTouched.has(loopId);
|
|
3815
|
+
}
|
|
3816
|
+
function v3RevisionFollowUps(events, input) {
|
|
3817
|
+
const out = [];
|
|
3818
|
+
const provenance = { by: input.by, ...input.auto ? { auto: input.auto } : {} };
|
|
3819
|
+
const snap = materialize([...events]);
|
|
3820
|
+
for (const item of input.assign ?? []) {
|
|
3821
|
+
if (snap.nodes.get(item.nodeId)?.status === "cancelled") continue;
|
|
3822
|
+
if (input.liveNodeIds && !input.liveNodeIds.has(item.nodeId)) continue;
|
|
3823
|
+
out.push({
|
|
3824
|
+
type: "nodeReassigned",
|
|
3825
|
+
nodeId: item.nodeId,
|
|
3826
|
+
...snap.nodes.get(item.nodeId)?.effectiveInstanceId ? { instanceId: snap.nodes.get(item.nodeId).effectiveInstanceId } : {},
|
|
3827
|
+
from: item.from ?? snap.assignments.get(item.nodeId) ?? "",
|
|
3828
|
+
to: item.to,
|
|
3829
|
+
...provenance
|
|
3830
|
+
});
|
|
3831
|
+
}
|
|
3832
|
+
for (const item of input.retry ?? []) {
|
|
3833
|
+
const state = snap.nodes.get(item.nodeId);
|
|
3834
|
+
if (state?.status !== "blocked" || state.effectiveInstanceId !== item.instanceId) continue;
|
|
3835
|
+
const key = item.instanceId ?? item.nodeId;
|
|
3836
|
+
const previousAttemptId = latestAttemptIdFor([...events], key);
|
|
3837
|
+
if (!previousAttemptId) continue;
|
|
3838
|
+
const blocked = latestBlockOf(events, snap, item.nodeId);
|
|
3839
|
+
out.push({
|
|
3840
|
+
type: "nodeRetryRequested",
|
|
3841
|
+
nodeId: item.nodeId,
|
|
3842
|
+
...item.instanceId ? { instanceId: item.instanceId } : {},
|
|
3843
|
+
previousAttemptId,
|
|
3844
|
+
nextAttemptId: nextAttemptIdFor([...events], key),
|
|
3845
|
+
reason: "blockedRetry",
|
|
3846
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorClass ? { previousErrorClass: blocked.errorClass } : {},
|
|
3847
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorCode ? { previousErrorCode: blocked.errorCode } : {},
|
|
3848
|
+
...provenance
|
|
3849
|
+
});
|
|
3850
|
+
}
|
|
3851
|
+
return out;
|
|
3852
|
+
}
|
|
3853
|
+
function missingV3RevisionFollowUps(events, liveNodeIds) {
|
|
3854
|
+
if (!events.some((event) => event.type === "dagRevised" && (event.effects?.retry || event.effects?.assign))) return [];
|
|
3855
|
+
const snap = materialize([...events]);
|
|
3856
|
+
if (snap.runStatus !== "running" && snap.runStatus !== "blocked") return [];
|
|
3857
|
+
const missing = [];
|
|
3858
|
+
events.forEach((event, index) => {
|
|
3859
|
+
if (event.type !== "dagRevised" || !event.effects?.retry && !event.effects?.assign) return;
|
|
3860
|
+
const later = events.slice(index + 1);
|
|
3861
|
+
const assign = (event.effects.assign ?? []).filter((item) => !later.some((after) => after.type === "nodeReassigned" && after.nodeId === item.nodeId || after.type === "dagRevised" && after.effects?.assign?.some((other) => other.nodeId === item.nodeId)));
|
|
3862
|
+
const retry = (event.effects.retry ?? []).filter((item) => !later.some((after) => after.type === "nodeRetryRequested" && after.nodeId === item.nodeId));
|
|
3863
|
+
if (assign.length === 0 && retry.length === 0) return;
|
|
3864
|
+
missing.push(...v3RevisionFollowUps(events, {
|
|
3865
|
+
assign,
|
|
3866
|
+
retry,
|
|
3867
|
+
by: event.by,
|
|
3868
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3869
|
+
...liveNodeIds ? { liveNodeIds } : {}
|
|
3870
|
+
}));
|
|
3871
|
+
});
|
|
3872
|
+
return missing;
|
|
3873
|
+
}
|
|
3874
|
+
function appendMissingV3RevisionFollowUps(journalPath, scope) {
|
|
3875
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
3876
|
+
if (currentV3Revision(events) !== scope.revision) return { kind: "stale" };
|
|
3877
|
+
const missing = missingV3RevisionFollowUps(events, scope.liveNodeIds);
|
|
3878
|
+
for (const event of missing) append(event, { durable: true });
|
|
3879
|
+
return { kind: "appended", count: missing.length };
|
|
3880
|
+
});
|
|
3881
|
+
}
|
|
3882
|
+
|
|
3327
3883
|
// src/workflows/v3/gate-wait-store.ts
|
|
3328
3884
|
import { randomBytes } from "node:crypto";
|
|
3329
3885
|
import {
|
|
3330
3886
|
chmodSync,
|
|
3331
3887
|
closeSync as closeSync5,
|
|
3332
3888
|
constants as constants3,
|
|
3333
|
-
writeFileSync as
|
|
3334
|
-
readFileSync as
|
|
3335
|
-
existsSync as
|
|
3889
|
+
writeFileSync as writeFileSync5,
|
|
3890
|
+
readFileSync as readFileSync8,
|
|
3891
|
+
existsSync as existsSync6,
|
|
3336
3892
|
fsyncSync as fsyncSync3,
|
|
3337
|
-
lstatSync as
|
|
3338
|
-
mkdirSync as
|
|
3893
|
+
lstatSync as lstatSync5,
|
|
3894
|
+
mkdirSync as mkdirSync5,
|
|
3339
3895
|
openSync as openSync5,
|
|
3340
3896
|
renameSync as renameSync2,
|
|
3341
3897
|
readdirSync as readdirSync2,
|
|
3342
3898
|
unlinkSync as unlinkSync2
|
|
3343
3899
|
} from "node:fs";
|
|
3344
|
-
import { dirname as
|
|
3900
|
+
import { dirname as dirname5, join as join6 } from "node:path";
|
|
3345
3901
|
function waitsDir(runDir) {
|
|
3346
|
-
return
|
|
3902
|
+
return join6(runDir, "waits");
|
|
3347
3903
|
}
|
|
3348
3904
|
function waitPath(runDir, waitId) {
|
|
3349
|
-
return
|
|
3905
|
+
return join6(waitsDir(runDir), `${waitId}.json`);
|
|
3350
3906
|
}
|
|
3351
3907
|
function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
3352
3908
|
const base = instanceId ?? nodeId;
|
|
@@ -3358,18 +3914,18 @@ function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
|
3358
3914
|
return `${base}-host-${attempt}-gate`;
|
|
3359
3915
|
}
|
|
3360
3916
|
function atomicWriteJson(path, value) {
|
|
3361
|
-
const dir =
|
|
3362
|
-
const runDir =
|
|
3917
|
+
const dir = dirname5(path);
|
|
3918
|
+
const runDir = dirname5(dir);
|
|
3363
3919
|
assertRealDirectory(runDir, "run directory");
|
|
3364
|
-
if (!
|
|
3920
|
+
if (!existsSync6(dir)) {
|
|
3365
3921
|
try {
|
|
3366
|
-
|
|
3922
|
+
mkdirSync5(dir, { mode: 448 });
|
|
3367
3923
|
} catch {
|
|
3368
3924
|
}
|
|
3369
3925
|
}
|
|
3370
3926
|
assertRealDirectory(dir, "waits directory");
|
|
3371
3927
|
chmodSync(dir, 448);
|
|
3372
|
-
if (
|
|
3928
|
+
if (existsSync6(path)) assertPrivateRegularWaitFile(path, false);
|
|
3373
3929
|
const tmp = `${path}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
|
|
3374
3930
|
let fd;
|
|
3375
3931
|
try {
|
|
@@ -3378,7 +3934,7 @@ function atomicWriteJson(path, value) {
|
|
|
3378
3934
|
constants3.O_WRONLY | constants3.O_CREAT | constants3.O_EXCL | (constants3.O_NOFOLLOW ?? 0),
|
|
3379
3935
|
384
|
|
3380
3936
|
);
|
|
3381
|
-
|
|
3937
|
+
writeFileSync5(fd, JSON.stringify(value, null, 2), "utf-8");
|
|
3382
3938
|
fsyncSync3(fd);
|
|
3383
3939
|
closeSync5(fd);
|
|
3384
3940
|
fd = void 0;
|
|
@@ -3401,13 +3957,13 @@ function atomicWriteJson(path, value) {
|
|
|
3401
3957
|
fsyncDirectorySyncPortable(dir);
|
|
3402
3958
|
}
|
|
3403
3959
|
function assertRealDirectory(path, label) {
|
|
3404
|
-
const stat =
|
|
3960
|
+
const stat = lstatSync5(path);
|
|
3405
3961
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
3406
3962
|
throw new Error(`v3 human-gate: ${label} must be a real directory`);
|
|
3407
3963
|
}
|
|
3408
3964
|
}
|
|
3409
3965
|
function assertPrivateRegularWaitFile(path, requirePrivate = true) {
|
|
3410
|
-
const stat =
|
|
3966
|
+
const stat = lstatSync5(path);
|
|
3411
3967
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3412
3968
|
throw new Error("v3 human-gate: wait file must be a regular file");
|
|
3413
3969
|
}
|
|
@@ -3521,6 +4077,7 @@ function openV3WorkerAttempts(events) {
|
|
|
3521
4077
|
const opened = /* @__PURE__ */ new Map();
|
|
3522
4078
|
const closed = /* @__PURE__ */ new Set();
|
|
3523
4079
|
for (const event of events) {
|
|
4080
|
+
if (event.type === "nodeDispatched" && event.nodeExecutor === "bot") continue;
|
|
3524
4081
|
if (isAttemptOpened(event)) {
|
|
3525
4082
|
const prior = opened.get(event.attemptId);
|
|
3526
4083
|
if (prior && (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId)) {
|
|
@@ -3628,43 +4185,553 @@ function sameIdentity(left, right) {
|
|
|
3628
4185
|
return left.nodeId === right.nodeId && left.instanceId === right.instanceId && left.attemptId === right.attemptId && left.executor === right.executor && left.provider === right.provider && left.inputRef.path === right.inputRef.path && left.inputRef.sha256 === right.inputRef.sha256 && left.inputRef.bytes === right.inputRef.bytes && left.inputHash === right.inputHash && left.idempotencyKey === right.idempotencyKey && left.idempotencyTtlMs === right.idempotencyTtlMs && left.approvalDigest === right.approvalDigest && left.attemptedAtMs === right.ts;
|
|
3629
4186
|
}
|
|
3630
4187
|
|
|
3631
|
-
// src/workflows/v3/
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
// src/workflows/shared/idempotency-key.ts
|
|
3649
|
-
import { createHash as createHash4 } from "node:crypto";
|
|
3650
|
-
function deriveIdempotencyKey(tuple, opts = {}) {
|
|
3651
|
-
const namespace = opts.namespace ?? "wf_";
|
|
3652
|
-
const maxLength = opts.maxLength ?? 50;
|
|
3653
|
-
if (namespace.length >= maxLength) {
|
|
3654
|
-
throw new Error(
|
|
3655
|
-
`deriveIdempotencyKey: namespace '${namespace}' (${namespace.length} chars) leaves no room for hash in maxLength ${maxLength}`
|
|
3656
|
-
);
|
|
4188
|
+
// src/workflows/v3/external-attempt-ledger.ts
|
|
4189
|
+
function closureOf(event) {
|
|
4190
|
+
switch (event.type) {
|
|
4191
|
+
case "nodeSucceeded":
|
|
4192
|
+
return { kind: "succeeded" };
|
|
4193
|
+
case "nodeBlocked":
|
|
4194
|
+
return { kind: "blocked", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4195
|
+
case "nodeFailed":
|
|
4196
|
+
return { kind: "failed", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4197
|
+
case "nodeCancelled":
|
|
4198
|
+
return event.reason === "runCancelled" && event.attemptId ? { kind: "cancelled" } : void 0;
|
|
4199
|
+
case "nodeAttemptDrained":
|
|
4200
|
+
return { kind: "drained", reason: event.reason };
|
|
4201
|
+
case "nodeReassigned":
|
|
4202
|
+
return event.attemptId ? { kind: "reassigned", to: event.to } : void 0;
|
|
4203
|
+
default:
|
|
4204
|
+
return void 0;
|
|
3657
4205
|
}
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
4206
|
+
}
|
|
4207
|
+
function attemptIdOf(event) {
|
|
4208
|
+
return "attemptId" in event && typeof event.attemptId === "string" ? event.attemptId : void 0;
|
|
4209
|
+
}
|
|
4210
|
+
function identityOf(event) {
|
|
4211
|
+
const record = event;
|
|
4212
|
+
return {
|
|
4213
|
+
...typeof record.nodeId === "string" ? { nodeId: record.nodeId } : {},
|
|
4214
|
+
...typeof record.instanceId === "string" ? { instanceId: record.instanceId } : {}
|
|
4215
|
+
};
|
|
4216
|
+
}
|
|
4217
|
+
function foldV3ExternalAttempts(events) {
|
|
4218
|
+
const attempts = /* @__PURE__ */ new Map();
|
|
4219
|
+
for (const event of events) {
|
|
4220
|
+
if (event.type === "nodeDispatched") {
|
|
4221
|
+
if (event.nodeExecutor !== "bot") continue;
|
|
4222
|
+
const prior = attempts.get(event.attemptId);
|
|
4223
|
+
if (prior) {
|
|
4224
|
+
if (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId) {
|
|
4225
|
+
throw new Error(`v3 external ledger: identity changed for ${event.attemptId}`);
|
|
4226
|
+
}
|
|
4227
|
+
continue;
|
|
4228
|
+
}
|
|
4229
|
+
attempts.set(event.attemptId, {
|
|
4230
|
+
nodeId: event.nodeId,
|
|
4231
|
+
...event.instanceId ? { instanceId: event.instanceId } : {},
|
|
4232
|
+
attemptId: event.attemptId,
|
|
4233
|
+
...event.loop ? { loop: event.loop } : {},
|
|
4234
|
+
phase: "dispatched"
|
|
4235
|
+
});
|
|
4236
|
+
continue;
|
|
4237
|
+
}
|
|
4238
|
+
const attemptId = attemptIdOf(event);
|
|
4239
|
+
if (!attemptId) continue;
|
|
4240
|
+
const attempt = attempts.get(attemptId);
|
|
4241
|
+
if (!attempt) continue;
|
|
4242
|
+
if (attempt.closed) continue;
|
|
4243
|
+
const identity = identityOf(event);
|
|
4244
|
+
if (identity.nodeId !== attempt.nodeId || identity.instanceId !== attempt.instanceId) {
|
|
4245
|
+
throw new Error(`v3 external ledger: event identity changed for ${attemptId}`);
|
|
4246
|
+
}
|
|
4247
|
+
switch (event.type) {
|
|
4248
|
+
case "nodeHandoffPrepared":
|
|
4249
|
+
if (attempt.phase !== "dispatched") break;
|
|
4250
|
+
attempt.phase = "prepared";
|
|
4251
|
+
attempt.targetLarkAppId = event.targetLarkAppId;
|
|
4252
|
+
attempt.placement = event.placement;
|
|
4253
|
+
attempt.revision = event.revision;
|
|
4254
|
+
attempt.idempotency = event.idempotency;
|
|
4255
|
+
attempt.preparedAt = event.preparedAt;
|
|
4256
|
+
break;
|
|
4257
|
+
case "nodeHandoffSent":
|
|
4258
|
+
if (attempt.phase === "dispatched" || attempt.handoffMessageId !== void 0) break;
|
|
4259
|
+
if (attempt.phase === "prepared") attempt.phase = "sent";
|
|
4260
|
+
attempt.handoffMessageId = event.handoffMessageId;
|
|
4261
|
+
if (event.nodeCardMessageId) attempt.nodeCardMessageId = event.nodeCardMessageId;
|
|
4262
|
+
if (event.threadId) attempt.threadId = event.threadId;
|
|
4263
|
+
attempt.sentAt = event.sentAt;
|
|
4264
|
+
attempt.acceptDeadlineAt = event.acceptDeadlineAt;
|
|
4265
|
+
break;
|
|
4266
|
+
case "nodeHandoffAccepted":
|
|
4267
|
+
if (attempt.phase !== "prepared" && attempt.phase !== "sent") break;
|
|
4268
|
+
attempt.phase = "accepted";
|
|
4269
|
+
attempt.sessionId = event.sessionId;
|
|
4270
|
+
attempt.acceptedAt = event.acceptedAt;
|
|
4271
|
+
attempt.deadlineAt = event.deadlineAt;
|
|
4272
|
+
break;
|
|
4273
|
+
case "nodeStalled":
|
|
4274
|
+
attempt.stalledAt = event.at;
|
|
4275
|
+
break;
|
|
4276
|
+
default: {
|
|
4277
|
+
const closure = closureOf(event);
|
|
4278
|
+
if (closure) attempt.closed = closure;
|
|
4279
|
+
}
|
|
3663
4280
|
}
|
|
3664
4281
|
}
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
4282
|
+
return attempts;
|
|
4283
|
+
}
|
|
4284
|
+
function openV3ExternalAttempts(events) {
|
|
4285
|
+
return [...foldV3ExternalAttempts(events).values()].filter((attempt) => !attempt.closed);
|
|
4286
|
+
}
|
|
4287
|
+
|
|
4288
|
+
// src/workflows/v3/external-executor.ts
|
|
4289
|
+
var V3_EXTERNAL_ERROR_CODES = {
|
|
4290
|
+
BOT_NOT_IN_CHAT: "BOT_NOT_IN_CHAT",
|
|
4291
|
+
BOT_OFFLINE: "BOT_OFFLINE",
|
|
4292
|
+
COLLABORATION_DENIED: "COLLABORATION_DENIED",
|
|
4293
|
+
IDENTITY_UNAVAILABLE: "IDENTITY_UNAVAILABLE",
|
|
4294
|
+
HANDOFF_NOT_ACCEPTED: "HANDOFF_NOT_ACCEPTED",
|
|
4295
|
+
HANDOFF_UNCERTAIN: "HANDOFF_UNCERTAIN",
|
|
4296
|
+
BOT_REPORTED_BLOCKED: "BOT_REPORTED_BLOCKED",
|
|
4297
|
+
/** The executing bot's daemon would not start the node session in the
|
|
4298
|
+
* run's Gate-2 working dir: not inside its configured range (design §15.4). */
|
|
4299
|
+
WORKING_DIR_REFUSED: "WORKING_DIR_REFUSED",
|
|
4300
|
+
NODE_TIMEOUT: "NODE_TIMEOUT"
|
|
4301
|
+
};
|
|
4302
|
+
var V3_HANDOFF_IDEMPOTENCY_WINDOW_MS = 50 * 60 * 1e3;
|
|
4303
|
+
var V3_EXTERNAL_CALL_TIMEOUT_MS = 3e4;
|
|
4304
|
+
var V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS = 15e3;
|
|
4305
|
+
|
|
4306
|
+
// src/workflows/v3/external-control.ts
|
|
4307
|
+
import { createHash as createHash5, randomUUID as randomUUID2 } from "node:crypto";
|
|
4308
|
+
import {
|
|
4309
|
+
existsSync as existsSync8,
|
|
4310
|
+
lstatSync as lstatSync6,
|
|
4311
|
+
mkdirSync as mkdirSync6,
|
|
4312
|
+
readFileSync as readFileSync10,
|
|
4313
|
+
renameSync as renameSync3,
|
|
4314
|
+
rmdirSync,
|
|
4315
|
+
rmSync,
|
|
4316
|
+
writeFileSync as writeFileSync6
|
|
4317
|
+
} from "node:fs";
|
|
4318
|
+
import { basename as basename2, dirname as dirname6, extname, isAbsolute as isAbsolute3, join as join8, relative as relative3, resolve as resolve3, sep } from "node:path";
|
|
4319
|
+
|
|
4320
|
+
// src/workflows/v3/result-contract.ts
|
|
4321
|
+
import { readFileSync as readFileSync9, existsSync as existsSync7 } from "node:fs";
|
|
4322
|
+
import { join as join7 } from "node:path";
|
|
4323
|
+
function readRevisitRequest(manifest, outputDir) {
|
|
4324
|
+
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4325
|
+
if (!entry) return { ok: true };
|
|
4326
|
+
let value;
|
|
4327
|
+
try {
|
|
4328
|
+
value = JSON.parse(readFileSync9(join7(outputDir, entry.path), "utf-8"));
|
|
4329
|
+
} catch {
|
|
4330
|
+
return { ok: true };
|
|
4331
|
+
}
|
|
4332
|
+
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4333
|
+
return { ok: true };
|
|
4334
|
+
}
|
|
4335
|
+
const v = value;
|
|
4336
|
+
const problems = [];
|
|
4337
|
+
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4338
|
+
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4339
|
+
}
|
|
4340
|
+
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4341
|
+
problems.push('result.json "reason" must be a string when present');
|
|
4342
|
+
}
|
|
4343
|
+
if (problems.length > 0) return { ok: false, problems };
|
|
4344
|
+
return {
|
|
4345
|
+
ok: true,
|
|
4346
|
+
request: {
|
|
4347
|
+
toNodeId: v.revisitTo,
|
|
4348
|
+
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4349
|
+
}
|
|
4350
|
+
};
|
|
4351
|
+
}
|
|
4352
|
+
function validateResult(filePath, schema) {
|
|
4353
|
+
if (!existsSync7(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4354
|
+
let parsed;
|
|
4355
|
+
try {
|
|
4356
|
+
parsed = JSON.parse(readFileSync9(filePath, "utf-8"));
|
|
4357
|
+
} catch (err) {
|
|
4358
|
+
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4359
|
+
}
|
|
4360
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4361
|
+
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4362
|
+
}
|
|
4363
|
+
const obj = parsed;
|
|
4364
|
+
const problems = [];
|
|
4365
|
+
for (const field of schema.required ?? []) {
|
|
4366
|
+
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4367
|
+
}
|
|
4368
|
+
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4369
|
+
if (!(name in obj)) continue;
|
|
4370
|
+
const v = obj[name];
|
|
4371
|
+
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4372
|
+
if (!okType) {
|
|
4373
|
+
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4374
|
+
continue;
|
|
4375
|
+
}
|
|
4376
|
+
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4377
|
+
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4378
|
+
}
|
|
4379
|
+
}
|
|
4380
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4381
|
+
}
|
|
4382
|
+
|
|
4383
|
+
// src/workflows/v3/external-control.ts
|
|
4384
|
+
function classifyExternalAttempt(events, attemptId, expect = {}) {
|
|
4385
|
+
const attempt = foldV3ExternalAttempts(events).get(attemptId);
|
|
4386
|
+
if (!attempt) return { ok: false, reason: "unknown-attempt" };
|
|
4387
|
+
if (expect.instanceId !== void 0 && expect.instanceId !== attempt.instanceId) {
|
|
4388
|
+
return { ok: false, reason: "unknown-attempt" };
|
|
4389
|
+
}
|
|
4390
|
+
const snap = materialize([...events]);
|
|
4391
|
+
const newerAttempt = snap.attempts.get(attempt.instanceId ?? attempt.nodeId) !== attemptId;
|
|
4392
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
4393
|
+
const closed = attempt.closed;
|
|
4394
|
+
if (closed) {
|
|
4395
|
+
const reason = closed.kind === "succeeded" ? "already-succeeded" : closed.kind === "reassigned" ? "reassigned" : closed.kind === "cancelled" ? "cancelled" : closed.kind === "blocked" && closed.errorCode === V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT ? "timed-out" : closed.kind === "drained" && closed.reason === "runCancellation" ? "cancelled" : closed.kind === "drained" && closed.reason === "obsoleteAttempt" ? state?.status === "cancelled" ? "cancelled" : "superseded" : newerAttempt ? "retried" : "closed";
|
|
4396
|
+
return { ok: false, reason, attempt };
|
|
4397
|
+
}
|
|
4398
|
+
if (snap.cancelRequestId !== void 0) return { ok: false, reason: "cancelled", attempt };
|
|
4399
|
+
if (snap.runStatus !== "running") return { ok: false, reason: "run-not-running", attempt };
|
|
4400
|
+
if (state?.status === "superseded") return { ok: false, reason: "superseded", attempt };
|
|
4401
|
+
if (state?.status === "cancelled") return { ok: false, reason: "cancelled", attempt };
|
|
4402
|
+
if (newerAttempt) return { ok: false, reason: "retried", attempt };
|
|
4403
|
+
if (state?.status !== "running") return { ok: false, reason: "closed", attempt };
|
|
4404
|
+
if (expect.sessionId !== void 0 && attempt.sessionId !== void 0 && expect.sessionId !== attempt.sessionId) {
|
|
4405
|
+
return { ok: false, reason: "session-mismatch", attempt };
|
|
4406
|
+
}
|
|
4407
|
+
return { ok: true, attempt, snap };
|
|
4408
|
+
}
|
|
4409
|
+
function externalAttemptRef(runId, attempt) {
|
|
4410
|
+
if (!attempt.targetLarkAppId || !attempt.placement || attempt.revision === void 0) {
|
|
4411
|
+
return void 0;
|
|
4412
|
+
}
|
|
4413
|
+
return {
|
|
4414
|
+
runId,
|
|
4415
|
+
nodeId: attempt.nodeId,
|
|
4416
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4417
|
+
attemptId: attempt.attemptId,
|
|
4418
|
+
revision: attempt.revision,
|
|
4419
|
+
targetLarkAppId: attempt.targetLarkAppId,
|
|
4420
|
+
placement: attempt.placement
|
|
4421
|
+
};
|
|
4422
|
+
}
|
|
4423
|
+
async function withExternalCallTimeout(call, timeoutMs, label) {
|
|
4424
|
+
let timer;
|
|
4425
|
+
try {
|
|
4426
|
+
return await Promise.race([
|
|
4427
|
+
call,
|
|
4428
|
+
new Promise((_resolve, reject) => {
|
|
4429
|
+
timer = setTimeout(() => reject(new Error(`external ${label} did not settle within ${timeoutMs}ms`)), timeoutMs);
|
|
4430
|
+
timer.unref?.();
|
|
4431
|
+
})
|
|
4432
|
+
]);
|
|
4433
|
+
} finally {
|
|
4434
|
+
if (timer) clearTimeout(timer);
|
|
4435
|
+
}
|
|
4436
|
+
}
|
|
4437
|
+
async function stopExternalAttempt(executor, runId, attempt, reason, timeoutMs = V3_EXTERNAL_CALL_TIMEOUT_MS) {
|
|
4438
|
+
const ref = externalAttemptRef(runId, attempt);
|
|
4439
|
+
if (!executor || !ref) return { ok: true };
|
|
4440
|
+
try {
|
|
4441
|
+
await withExternalCallTimeout(Promise.resolve().then(() => executor.stop({
|
|
4442
|
+
...ref,
|
|
4443
|
+
reason,
|
|
4444
|
+
...attempt.idempotency ? { idempotency: attempt.idempotency } : {},
|
|
4445
|
+
...attempt.handoffMessageId ? { handoffMessageId: attempt.handoffMessageId } : {},
|
|
4446
|
+
...attempt.nodeCardMessageId ? { nodeCardMessageId: attempt.nodeCardMessageId } : {},
|
|
4447
|
+
...attempt.threadId ? { threadId: attempt.threadId } : {},
|
|
4448
|
+
...attempt.sessionId ? { sessionId: attempt.sessionId } : {}
|
|
4449
|
+
})), timeoutMs, "stop");
|
|
4450
|
+
return { ok: true };
|
|
4451
|
+
} catch (error) {
|
|
4452
|
+
return { ok: false, error };
|
|
4453
|
+
}
|
|
4454
|
+
}
|
|
4455
|
+
function journalPathOf(runDir) {
|
|
4456
|
+
return join8(runDir, "journal.ndjson");
|
|
4457
|
+
}
|
|
4458
|
+
function externalAttemptNode(dag, attempt) {
|
|
4459
|
+
if (!attempt.loop) return dag.nodes.find((node) => node.id === attempt.nodeId);
|
|
4460
|
+
const loopNode = dag.nodes.find((node) => node.id === attempt.loop.loopId);
|
|
4461
|
+
if (!loopNode || !isLoopNode(loopNode)) return void 0;
|
|
4462
|
+
const body = loopNode.body.nodes.find((node) => node.id === attempt.loop.bodyNodeId);
|
|
4463
|
+
if (!body) return void 0;
|
|
4464
|
+
const ref = attempt.loop;
|
|
4465
|
+
return {
|
|
4466
|
+
...body,
|
|
4467
|
+
id: loopInstanceId(ref.loopId, ref.iteration, ref.bodyNodeId),
|
|
4468
|
+
bot: body.bot ?? loopNode.bot,
|
|
4469
|
+
depends: body.depends.map((d) => ({ from: loopInstanceId(ref.loopId, ref.iteration, d.from) })),
|
|
4470
|
+
inputs: body.inputs.map((r) => ({
|
|
4471
|
+
from: loopInstanceId(ref.loopId, ref.iteration, r.from),
|
|
4472
|
+
...r.select ? { select: r.select } : {},
|
|
4473
|
+
...r.output !== void 0 ? { output: r.output } : {}
|
|
4474
|
+
}))
|
|
4475
|
+
};
|
|
4476
|
+
}
|
|
4477
|
+
function nodeTimeoutMs(node) {
|
|
4478
|
+
return (node?.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC) * 1e3;
|
|
4479
|
+
}
|
|
4480
|
+
function safeExternalAttemptDir(runDir, attemptId) {
|
|
4481
|
+
const root = resolve3(runDir);
|
|
4482
|
+
const candidate = resolve3(root, attemptId);
|
|
4483
|
+
const rel = relative3(root, candidate);
|
|
4484
|
+
if (!rel || isAbsolute3(rel) || rel === ".." || rel.startsWith(`..${sep}`)) {
|
|
4485
|
+
throw new Error(`v3 external: attempt path escapes run dir (${attemptId})`);
|
|
4486
|
+
}
|
|
4487
|
+
return candidate;
|
|
4488
|
+
}
|
|
4489
|
+
function appendAcceptedLocked(events, append, dag, input, now) {
|
|
4490
|
+
const current = classifyExternalAttempt(events, input.attemptId, {
|
|
4491
|
+
...input.instanceId !== void 0 ? { instanceId: input.instanceId } : {},
|
|
4492
|
+
sessionId: input.sessionId
|
|
4493
|
+
});
|
|
4494
|
+
if (!current.ok) return { ok: false, reason: current.reason };
|
|
4495
|
+
const { attempt } = current;
|
|
4496
|
+
if (attempt.phase === "accepted") {
|
|
4497
|
+
return { ok: true, deadlineAt: attempt.deadlineAt, alreadyAccepted: true };
|
|
4498
|
+
}
|
|
4499
|
+
if (attempt.phase === "dispatched") return { ok: false, reason: "unknown-attempt" };
|
|
4500
|
+
const acceptedAt = Math.min(input.acceptedAt ?? now, now);
|
|
4501
|
+
const deadlineAt = acceptedAt + nodeTimeoutMs(externalAttemptNode(dag, attempt));
|
|
4502
|
+
append({
|
|
4503
|
+
type: "nodeHandoffAccepted",
|
|
4504
|
+
nodeId: attempt.nodeId,
|
|
4505
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4506
|
+
attemptId: attempt.attemptId,
|
|
4507
|
+
sessionId: input.sessionId,
|
|
4508
|
+
acceptedAt,
|
|
4509
|
+
deadlineAt
|
|
4510
|
+
}, { durable: true });
|
|
4511
|
+
return { ok: true, deadlineAt, alreadyAccepted: false };
|
|
4512
|
+
}
|
|
4513
|
+
var RESULT_FILE = "result.json";
|
|
4514
|
+
function judgeStagedSubmission(node, manifest, workDir) {
|
|
4515
|
+
const revisit = readRevisitRequest(manifest, workDir);
|
|
4516
|
+
if (!revisit.ok) return { kind: "invalid", problems: revisit.problems };
|
|
4517
|
+
if (revisit.request) {
|
|
4518
|
+
if (!node.revisitTo?.includes(revisit.request.toNodeId)) {
|
|
4519
|
+
return {
|
|
4520
|
+
kind: "invalid",
|
|
4521
|
+
problems: [`result.json requests revisit to "${revisit.request.toNodeId}", not in node "${node.id}".revisitTo`]
|
|
4522
|
+
};
|
|
4523
|
+
}
|
|
4524
|
+
return { kind: "revisit", request: revisit.request };
|
|
4525
|
+
}
|
|
4526
|
+
if (node.resultSchema) {
|
|
4527
|
+
const entry = manifest.files.find((f) => f.path === RESULT_FILE);
|
|
4528
|
+
if (!entry) {
|
|
4529
|
+
return { kind: "invalid", problems: ["node declares resultSchema but the submission has no result.json"] };
|
|
4530
|
+
}
|
|
4531
|
+
const res = validateResult(join8(workDir, entry.path), node.resultSchema);
|
|
4532
|
+
if (!res.ok) return { kind: "invalid", problems: res.problems ?? ["result.json failed schema validation"] };
|
|
4533
|
+
}
|
|
4534
|
+
const contract = validateManifestArtifactContract(node.outputs, manifest);
|
|
4535
|
+
if (!contract.ok) return { kind: "invalid", problems: contract.problems };
|
|
4536
|
+
return { kind: "success" };
|
|
4537
|
+
}
|
|
4538
|
+
function moveAsideIfPresent(path, tag) {
|
|
4539
|
+
if (!existsSync8(path)) return;
|
|
4540
|
+
renameSync3(path, `${path}.uncommitted-${tag}`);
|
|
4541
|
+
}
|
|
4542
|
+
var V3_SUBMISSION_INTENT_FILE = "submission.intent.json";
|
|
4543
|
+
function submissionIntentPath(attemptDir) {
|
|
4544
|
+
return join8(attemptDir, V3_SUBMISSION_INTENT_FILE);
|
|
4545
|
+
}
|
|
4546
|
+
function removeSubmissionIntent(attemptDir, stageTag) {
|
|
4547
|
+
rmSync(submissionIntentPath(attemptDir), { force: true });
|
|
4548
|
+
rmSync(join8(attemptDir, "submissions", stageTag), { recursive: true, force: true });
|
|
4549
|
+
try {
|
|
4550
|
+
rmdirSync(join8(attemptDir, "submissions"));
|
|
4551
|
+
} catch {
|
|
4552
|
+
}
|
|
4553
|
+
}
|
|
4554
|
+
function readSubmissionIntent(attemptDir) {
|
|
4555
|
+
const path = submissionIntentPath(attemptDir);
|
|
4556
|
+
if (!existsSync8(path)) return void 0;
|
|
4557
|
+
try {
|
|
4558
|
+
const raw = JSON.parse(readFileSync10(path, "utf-8"));
|
|
4559
|
+
if (raw.schemaVersion !== 1 || typeof raw.attemptId !== "string" || typeof raw.stageTag !== "string") {
|
|
4560
|
+
return "corrupt";
|
|
4561
|
+
}
|
|
4562
|
+
if (!/^[0-9a-f-]{36}$/.test(raw.stageTag)) return "corrupt";
|
|
4563
|
+
return { schemaVersion: 1, attemptId: raw.attemptId, stageTag: raw.stageTag };
|
|
4564
|
+
} catch {
|
|
4565
|
+
return "corrupt";
|
|
4566
|
+
}
|
|
4567
|
+
}
|
|
4568
|
+
function verifyInstalledSubmissionSync(manifestPath, workDir) {
|
|
4569
|
+
let manifest;
|
|
4570
|
+
try {
|
|
4571
|
+
manifest = JSON.parse(readFileSync10(manifestPath, "utf-8"));
|
|
4572
|
+
} catch (err) {
|
|
4573
|
+
return { ok: false, problems: [`manifest unreadable: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4574
|
+
}
|
|
4575
|
+
if (manifest?.schemaVersion !== MANIFEST_SCHEMA_VERSION || manifest.status !== "ok" || !Array.isArray(manifest.files) || manifest.files.length === 0) {
|
|
4576
|
+
return { ok: false, problems: ["manifest is not an engine-generated success manifest"] };
|
|
4577
|
+
}
|
|
4578
|
+
const root = resolve3(workDir);
|
|
4579
|
+
const problems = [];
|
|
4580
|
+
for (const file of manifest.files) {
|
|
4581
|
+
const target = resolve3(root, file.path);
|
|
4582
|
+
const rel = relative3(root, target);
|
|
4583
|
+
if (!rel || isAbsolute3(rel) || rel.startsWith("..")) {
|
|
4584
|
+
problems.push(`${file.path}: escapes the output directory`);
|
|
4585
|
+
continue;
|
|
4586
|
+
}
|
|
4587
|
+
try {
|
|
4588
|
+
if (!lstatSync6(target).isFile()) {
|
|
4589
|
+
problems.push(`${file.path}: not a regular file`);
|
|
4590
|
+
continue;
|
|
4591
|
+
}
|
|
4592
|
+
const bytes = readFileSync10(target);
|
|
4593
|
+
if (bytes.length !== file.bytes || createHash5("sha256").update(bytes).digest("hex") !== file.sha256) {
|
|
4594
|
+
problems.push(`${file.path}: bytes do not match the manifest`);
|
|
4595
|
+
}
|
|
4596
|
+
} catch (err) {
|
|
4597
|
+
problems.push(`${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
4598
|
+
}
|
|
4599
|
+
}
|
|
4600
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true, manifest };
|
|
4601
|
+
}
|
|
4602
|
+
function appendInstalledVerdictLocked(input) {
|
|
4603
|
+
const { attempt, append, events, verdict, manifestPath } = input;
|
|
4604
|
+
const identity = {
|
|
4605
|
+
nodeId: attempt.nodeId,
|
|
4606
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4607
|
+
attemptId: attempt.attemptId
|
|
4608
|
+
};
|
|
4609
|
+
if (verdict.kind === "revisit") {
|
|
4610
|
+
const budget = revisitBudgetStatus([...events], attempt.nodeId, verdict.request.toNodeId);
|
|
4611
|
+
if (!budget.ok) {
|
|
4612
|
+
append({
|
|
4613
|
+
type: "nodeBlocked",
|
|
4614
|
+
...identity,
|
|
4615
|
+
errorClass: "resultInvalid",
|
|
4616
|
+
errorCode: "REVISIT_BUDGET_EXHAUSTED",
|
|
4617
|
+
message: budget.detail,
|
|
4618
|
+
revisitTo: verdict.request.toNodeId
|
|
4619
|
+
}, { durable: true });
|
|
4620
|
+
return { ok: true, kind: "blocked", errorCode: "REVISIT_BUDGET_EXHAUSTED" };
|
|
4621
|
+
}
|
|
4622
|
+
const plan = planV3Revisit({
|
|
4623
|
+
runDir: input.runDir,
|
|
4624
|
+
dag: input.dag,
|
|
4625
|
+
events,
|
|
4626
|
+
nodeId: attempt.nodeId,
|
|
4627
|
+
instanceId: attempt.instanceId ?? attempt.nodeId,
|
|
4628
|
+
attemptId: attempt.attemptId,
|
|
4629
|
+
request: verdict.request,
|
|
4630
|
+
sourceManifestPath: manifestPath
|
|
4631
|
+
});
|
|
4632
|
+
for (const event of plan.events) append(event);
|
|
4633
|
+
append({ type: "nodeAttemptDrained", ...identity, reason: "obsoleteAttempt" }, { durable: true });
|
|
4634
|
+
return { ok: true, kind: "revisit", toNodeId: verdict.request.toNodeId };
|
|
4635
|
+
}
|
|
4636
|
+
append({ type: "nodeSucceeded", ...identity, manifestPath }, { durable: true });
|
|
4637
|
+
return { ok: true, kind: "succeeded", manifestPath };
|
|
4638
|
+
}
|
|
4639
|
+
function installVerdictCommitted(events, attemptId) {
|
|
4640
|
+
return events.some((event) => event.type === "nodeSucceeded" && event.attemptId === attemptId || event.type === "nodeRevisitRequested" && event.attemptId === attemptId || event.type === "nodeBlocked" && event.attemptId === attemptId && event.errorCode === "REVISIT_BUDGET_EXHAUSTED");
|
|
4641
|
+
}
|
|
4642
|
+
function settleInstallIntentLocked(runDir, dag, events, append, attemptId) {
|
|
4643
|
+
const attemptDir = safeExternalAttemptDir(runDir, attemptId);
|
|
4644
|
+
const intent = readSubmissionIntent(attemptDir);
|
|
4645
|
+
if (intent === void 0) return { kind: "none" };
|
|
4646
|
+
if (intent === "corrupt" || intent.attemptId !== attemptId) {
|
|
4647
|
+
rmSync(submissionIntentPath(attemptDir), { force: true });
|
|
4648
|
+
return { kind: "discarded" };
|
|
4649
|
+
}
|
|
4650
|
+
const workDir = join8(attemptDir, "work");
|
|
4651
|
+
const manifestPath = join8(attemptDir, "manifest.json");
|
|
4652
|
+
const stageDir = join8(attemptDir, "submissions", intent.stageTag);
|
|
4653
|
+
const discard = () => {
|
|
4654
|
+
if (!installVerdictCommitted(events, attemptId)) {
|
|
4655
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4656
|
+
moveAsideIfPresent(manifestPath, intent.stageTag);
|
|
4657
|
+
}
|
|
4658
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4659
|
+
return { kind: "discarded" };
|
|
4660
|
+
};
|
|
4661
|
+
if (installVerdictCommitted(events, attemptId)) {
|
|
4662
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4663
|
+
return { kind: "cleaned" };
|
|
4664
|
+
}
|
|
4665
|
+
const current = classifyExternalAttempt(events, attemptId);
|
|
4666
|
+
if (!current.ok) return discard();
|
|
4667
|
+
const node = externalAttemptNode(dag, current.attempt);
|
|
4668
|
+
if (!node || !isGoalNode(node)) return discard();
|
|
4669
|
+
if (existsSync8(join8(stageDir, "work"))) {
|
|
4670
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4671
|
+
renameSync3(join8(stageDir, "work"), workDir);
|
|
4672
|
+
}
|
|
4673
|
+
if (existsSync8(join8(stageDir, "manifest.json"))) renameSync3(join8(stageDir, "manifest.json"), manifestPath);
|
|
4674
|
+
const verified = verifyInstalledSubmissionSync(manifestPath, workDir);
|
|
4675
|
+
if (!verified.ok) return discard();
|
|
4676
|
+
const verdict = judgeStagedSubmission(node, verified.manifest, workDir);
|
|
4677
|
+
if (verdict.kind === "invalid") return discard();
|
|
4678
|
+
const outcome = appendInstalledVerdictLocked({
|
|
4679
|
+
runDir,
|
|
4680
|
+
dag,
|
|
4681
|
+
events,
|
|
4682
|
+
append,
|
|
4683
|
+
attempt: current.attempt,
|
|
4684
|
+
verdict,
|
|
4685
|
+
manifestPath
|
|
4686
|
+
});
|
|
4687
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4688
|
+
return { kind: "committed", outcome };
|
|
4689
|
+
}
|
|
4690
|
+
function settleExternalInstallIntent(runDir, dag, attemptId) {
|
|
4691
|
+
if (!existsSync8(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)))) return { kind: "none" };
|
|
4692
|
+
return withJournalMutationSync(journalPathOf(runDir), ({ events, append }) => settleInstallIntentLocked(runDir, dag, events, append, attemptId));
|
|
4693
|
+
}
|
|
4694
|
+
function hasExternalInstallIntent(runDir, attemptId) {
|
|
4695
|
+
return existsSync8(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)));
|
|
4696
|
+
}
|
|
4697
|
+
|
|
4698
|
+
// src/workflows/v3/host-execution.ts
|
|
4699
|
+
import { createHash as createHash7, randomUUID as randomUUID3 } from "node:crypto";
|
|
4700
|
+
import {
|
|
4701
|
+
closeSync as closeSync6,
|
|
4702
|
+
constants as constants4,
|
|
4703
|
+
existsSync as existsSync9,
|
|
4704
|
+
fsyncSync as fsyncSync4,
|
|
4705
|
+
lstatSync as lstatSync7,
|
|
4706
|
+
linkSync as linkSync2,
|
|
4707
|
+
mkdirSync as mkdirSync7,
|
|
4708
|
+
openSync as openSync6,
|
|
4709
|
+
readFileSync as readFileSync11,
|
|
4710
|
+
unlinkSync as unlinkSync3,
|
|
4711
|
+
writeFileSync as writeFileSync7
|
|
4712
|
+
} from "node:fs";
|
|
4713
|
+
import { dirname as dirname7, isAbsolute as isAbsolute4, join as join9, relative as relative4, resolve as resolve4, sep as sep2 } from "node:path";
|
|
4714
|
+
|
|
4715
|
+
// src/workflows/shared/idempotency-key.ts
|
|
4716
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
4717
|
+
function deriveIdempotencyKey(tuple, opts = {}) {
|
|
4718
|
+
const namespace = opts.namespace ?? "wf_";
|
|
4719
|
+
const maxLength = opts.maxLength ?? 50;
|
|
4720
|
+
if (namespace.length >= maxLength) {
|
|
4721
|
+
throw new Error(
|
|
4722
|
+
`deriveIdempotencyKey: namespace '${namespace}' (${namespace.length} chars) leaves no room for hash in maxLength ${maxLength}`
|
|
4723
|
+
);
|
|
4724
|
+
}
|
|
4725
|
+
for (const [key, value] of Object.entries(tuple)) {
|
|
4726
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
4727
|
+
throw new Error(
|
|
4728
|
+
`deriveIdempotencyKey: tuple.${key} must be non-empty string, got ${String(value)}`
|
|
4729
|
+
);
|
|
4730
|
+
}
|
|
4731
|
+
}
|
|
4732
|
+
const seed = canonicalJson(tuple);
|
|
4733
|
+
const hash = createHash6("sha256").update(seed, "utf-8").digest("hex");
|
|
4734
|
+
return namespace + hash.substring(0, maxLength - namespace.length);
|
|
3668
4735
|
}
|
|
3669
4736
|
|
|
3670
4737
|
// src/workflows/v3/host-execution.ts
|
|
@@ -3724,23 +4791,23 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3724
4791
|
idempotencyTtlMs: input.registered.executor.idempotencyTtlMs,
|
|
3725
4792
|
approvalDigest
|
|
3726
4793
|
};
|
|
3727
|
-
const absolutePath =
|
|
4794
|
+
const absolutePath = join9(input.attemptDir, "host-input.json");
|
|
3728
4795
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3729
4796
|
ensureSecureDirectoryTree(input.runDir, input.attemptDir);
|
|
3730
4797
|
const content = `${JSON.stringify(prepared, null, 2)}
|
|
3731
4798
|
`;
|
|
3732
4799
|
writeCreateOrVerify(absolutePath, content, 384, "host input");
|
|
3733
|
-
const existing =
|
|
4800
|
+
const existing = readFileSync11(absolutePath, "utf-8");
|
|
3734
4801
|
if (canonicalJson(parsePreparedHostInput(JSON.parse(existing))) !== canonicalJson(prepared)) {
|
|
3735
4802
|
throw new Error(`v3 host input conflict at ${absolutePath}; refusing to overwrite prepared bytes`);
|
|
3736
4803
|
}
|
|
3737
4804
|
fsyncRegularFileSync(absolutePath);
|
|
3738
4805
|
fsyncDirectorySyncPortable(input.attemptDir);
|
|
3739
|
-
const bytes =
|
|
4806
|
+
const bytes = readFileSync11(absolutePath);
|
|
3740
4807
|
return {
|
|
3741
4808
|
prepared,
|
|
3742
4809
|
inputRef: {
|
|
3743
|
-
path:
|
|
4810
|
+
path: relative4(input.runDir, absolutePath),
|
|
3744
4811
|
sha256: sha256(bytes),
|
|
3745
4812
|
bytes: bytes.length
|
|
3746
4813
|
},
|
|
@@ -3748,12 +4815,12 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3748
4815
|
};
|
|
3749
4816
|
}
|
|
3750
4817
|
function readCrashLeftV3PreparedHostInput(input) {
|
|
3751
|
-
const absolutePath =
|
|
3752
|
-
if (!
|
|
4818
|
+
const absolutePath = join9(input.attemptDir, "host-input.json");
|
|
4819
|
+
if (!existsSync9(absolutePath)) return void 0;
|
|
3753
4820
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3754
4821
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3755
4822
|
assertRegularFile(absolutePath, "host input");
|
|
3756
|
-
const bytes =
|
|
4823
|
+
const bytes = readFileSync11(absolutePath);
|
|
3757
4824
|
let prepared;
|
|
3758
4825
|
try {
|
|
3759
4826
|
prepared = parsePreparedHostInput(JSON.parse(bytes.toString("utf-8")));
|
|
@@ -3772,7 +4839,7 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3772
4839
|
inputHash
|
|
3773
4840
|
});
|
|
3774
4841
|
const inputRef = {
|
|
3775
|
-
path:
|
|
4842
|
+
path: relative4(input.runDir, absolutePath),
|
|
3776
4843
|
sha256: sha256(bytes),
|
|
3777
4844
|
bytes: bytes.length
|
|
3778
4845
|
};
|
|
@@ -3795,14 +4862,14 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3795
4862
|
});
|
|
3796
4863
|
}
|
|
3797
4864
|
function readAndVerifyV3PreparedHostInput(input) {
|
|
3798
|
-
if (
|
|
4865
|
+
if (isAbsolute4(input.inputRef.path) || input.inputRef.path.includes("\0")) {
|
|
3799
4866
|
throw new Error("v3 host inputRef.path must be run-relative");
|
|
3800
4867
|
}
|
|
3801
|
-
const absolutePath =
|
|
4868
|
+
const absolutePath = resolve4(input.runDir, input.inputRef.path);
|
|
3802
4869
|
assertPathInside(input.runDir, absolutePath);
|
|
3803
4870
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3804
4871
|
assertRegularFile(absolutePath, "host input");
|
|
3805
|
-
const bytes =
|
|
4872
|
+
const bytes = readFileSync11(absolutePath);
|
|
3806
4873
|
if (bytes.length !== input.inputRef.bytes || sha256(bytes) !== input.inputRef.sha256) {
|
|
3807
4874
|
throw new Error("v3 host input sidecar bytes/hash mismatch");
|
|
3808
4875
|
}
|
|
@@ -3845,10 +4912,10 @@ function readAndVerifyV3PreparedHostInput(input) {
|
|
|
3845
4912
|
function writeV3HostSuccessArtifacts(input) {
|
|
3846
4913
|
canonicalJson(input.output);
|
|
3847
4914
|
canonicalJson(input.externalRefs);
|
|
3848
|
-
const outputDir =
|
|
4915
|
+
const outputDir = join9(input.attemptDir, "work");
|
|
3849
4916
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3850
4917
|
ensureSecureDirectoryTree(input.runDir, outputDir);
|
|
3851
|
-
const resultPath =
|
|
4918
|
+
const resultPath = join9(outputDir, "result.json");
|
|
3852
4919
|
const resultContent = `${JSON.stringify({
|
|
3853
4920
|
schemaVersion: 1,
|
|
3854
4921
|
runId: input.runId,
|
|
@@ -3879,7 +4946,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3879
4946
|
mime: "application/json"
|
|
3880
4947
|
}]
|
|
3881
4948
|
};
|
|
3882
|
-
const manifestPath =
|
|
4949
|
+
const manifestPath = join9(input.attemptDir, "manifest.json");
|
|
3883
4950
|
writeCreateOrVerify(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
3884
4951
|
`, 384, "host manifest");
|
|
3885
4952
|
fsyncRegularFileSync(resultPath);
|
|
@@ -3889,7 +4956,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3889
4956
|
return { manifestPath, resultPath };
|
|
3890
4957
|
}
|
|
3891
4958
|
function readAndVerifyV3HostSuccessResult(input) {
|
|
3892
|
-
assertRegularFile(
|
|
4959
|
+
assertRegularFile(join9(input.attemptDir, "manifest.json"), "host manifest");
|
|
3893
4960
|
if (input.manifest.status !== "ok") {
|
|
3894
4961
|
throw new Error("v3 host success manifest status must be ok");
|
|
3895
4962
|
}
|
|
@@ -3900,11 +4967,11 @@ function readAndVerifyV3HostSuccessResult(input) {
|
|
|
3900
4967
|
if (file.name !== "result" || file.path !== "result.json" || file.kind !== "json") {
|
|
3901
4968
|
throw new Error("v3 host success manifest must expose exactly result.json as JSON");
|
|
3902
4969
|
}
|
|
3903
|
-
const resultPath =
|
|
4970
|
+
const resultPath = join9(input.attemptDir, "work", "result.json");
|
|
3904
4971
|
assertPathInside(input.runDir, resultPath);
|
|
3905
4972
|
assertSecureExistingPath(input.runDir, resultPath, "host result");
|
|
3906
4973
|
assertRegularFile(resultPath, "host result");
|
|
3907
|
-
const bytes =
|
|
4974
|
+
const bytes = readFileSync11(resultPath);
|
|
3908
4975
|
if (bytes.length !== file.bytes || sha256(bytes) !== file.sha256) {
|
|
3909
4976
|
throw new Error("v3 host success result bytes/hash mismatch");
|
|
3910
4977
|
}
|
|
@@ -3991,7 +5058,7 @@ function parsePreparedHostInput(raw) {
|
|
|
3991
5058
|
return raw;
|
|
3992
5059
|
}
|
|
3993
5060
|
function writeCreateOrVerify(path, content, mode, label) {
|
|
3994
|
-
const tempPath = `${path}.tmp-${process.pid}-${
|
|
5061
|
+
const tempPath = `${path}.tmp-${process.pid}-${randomUUID3()}`;
|
|
3995
5062
|
let fd;
|
|
3996
5063
|
try {
|
|
3997
5064
|
fd = openSync6(
|
|
@@ -3999,17 +5066,17 @@ function writeCreateOrVerify(path, content, mode, label) {
|
|
|
3999
5066
|
constants4.O_WRONLY | constants4.O_CREAT | constants4.O_EXCL | (constants4.O_NOFOLLOW ?? 0),
|
|
4000
5067
|
mode
|
|
4001
5068
|
);
|
|
4002
|
-
|
|
5069
|
+
writeFileSync7(fd, content, "utf-8");
|
|
4003
5070
|
fsyncSync4(fd);
|
|
4004
5071
|
closeSync6(fd);
|
|
4005
5072
|
fd = void 0;
|
|
4006
5073
|
try {
|
|
4007
5074
|
linkSync2(tempPath, path);
|
|
4008
|
-
fsyncDirectorySyncPortable(
|
|
5075
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4009
5076
|
} catch (err) {
|
|
4010
5077
|
if (err.code !== "EEXIST") throw err;
|
|
4011
5078
|
assertRegularFile(path, label);
|
|
4012
|
-
if (
|
|
5079
|
+
if (readFileSync11(path, "utf-8") !== content) {
|
|
4013
5080
|
throw new Error(`v3 ${label} conflict at ${path}; refusing to overwrite committed bytes`);
|
|
4014
5081
|
}
|
|
4015
5082
|
}
|
|
@@ -4017,15 +5084,15 @@ function writeCreateOrVerify(path, content, mode, label) {
|
|
|
4017
5084
|
if (fd !== void 0) closeSync6(fd);
|
|
4018
5085
|
try {
|
|
4019
5086
|
unlinkSync3(tempPath);
|
|
4020
|
-
fsyncDirectorySyncPortable(
|
|
5087
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4021
5088
|
} catch {
|
|
4022
5089
|
}
|
|
4023
5090
|
}
|
|
4024
5091
|
fsyncRegularFileSync(path);
|
|
4025
|
-
fsyncDirectorySyncPortable(
|
|
5092
|
+
fsyncDirectorySyncPortable(dirname7(path));
|
|
4026
5093
|
}
|
|
4027
5094
|
function assertRegularFile(path, label) {
|
|
4028
|
-
const stat =
|
|
5095
|
+
const stat = lstatSync7(path);
|
|
4029
5096
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
4030
5097
|
throw new Error(`v3 ${label} must be a regular file: ${path}`);
|
|
4031
5098
|
}
|
|
@@ -4034,47 +5101,47 @@ function assertRegularFile(path, label) {
|
|
|
4034
5101
|
}
|
|
4035
5102
|
}
|
|
4036
5103
|
function assertPathInside(parent, child) {
|
|
4037
|
-
const root =
|
|
4038
|
-
const rel =
|
|
4039
|
-
if (rel === "" || rel === ".." || rel.startsWith(`..${
|
|
5104
|
+
const root = resolve4(parent);
|
|
5105
|
+
const rel = relative4(root, resolve4(child));
|
|
5106
|
+
if (rel === "" || rel === ".." || rel.startsWith(`..${sep2}`) || isAbsolute4(rel)) {
|
|
4040
5107
|
throw new Error("v3 host path escapes runDir");
|
|
4041
5108
|
}
|
|
4042
5109
|
}
|
|
4043
5110
|
function ensureSecureDirectoryTree(rootPath, targetPath) {
|
|
4044
|
-
const root =
|
|
4045
|
-
const target =
|
|
4046
|
-
const rel =
|
|
4047
|
-
if (rel === ".." || rel.startsWith(`..${
|
|
5111
|
+
const root = resolve4(rootPath);
|
|
5112
|
+
const target = resolve4(targetPath);
|
|
5113
|
+
const rel = relative4(root, target);
|
|
5114
|
+
if (rel === ".." || rel.startsWith(`..${sep2}`) || isAbsolute4(rel)) {
|
|
4048
5115
|
throw new Error("v3 host directory escapes runDir");
|
|
4049
5116
|
}
|
|
4050
5117
|
assertDirectory(root, "run directory");
|
|
4051
5118
|
let current = root;
|
|
4052
5119
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4053
|
-
current =
|
|
4054
|
-
if (!
|
|
5120
|
+
current = join9(current, segment);
|
|
5121
|
+
if (!existsSync9(current)) mkdirSync7(current, { mode: 448 });
|
|
4055
5122
|
assertDirectory(current, "host artifact directory");
|
|
4056
5123
|
}
|
|
4057
5124
|
}
|
|
4058
5125
|
function assertSecureExistingPath(rootPath, filePath, label) {
|
|
4059
|
-
const root =
|
|
4060
|
-
const target =
|
|
5126
|
+
const root = resolve4(rootPath);
|
|
5127
|
+
const target = resolve4(filePath);
|
|
4061
5128
|
assertPathInside(root, target);
|
|
4062
5129
|
assertDirectory(root, "run directory");
|
|
4063
|
-
const rel =
|
|
5130
|
+
const rel = relative4(root, dirname7(target));
|
|
4064
5131
|
let current = root;
|
|
4065
5132
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4066
|
-
current =
|
|
5133
|
+
current = join9(current, segment);
|
|
4067
5134
|
assertDirectory(current, `${label} parent`);
|
|
4068
5135
|
}
|
|
4069
5136
|
}
|
|
4070
5137
|
function assertDirectory(path, label) {
|
|
4071
|
-
const stat =
|
|
5138
|
+
const stat = lstatSync7(path);
|
|
4072
5139
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
4073
5140
|
throw new Error(`v3 ${label} must be a real directory: ${path}`);
|
|
4074
5141
|
}
|
|
4075
5142
|
}
|
|
4076
5143
|
function sha256(value) {
|
|
4077
|
-
return
|
|
5144
|
+
return createHash7("sha256").update(value).digest("hex");
|
|
4078
5145
|
}
|
|
4079
5146
|
function isRecord3(value) {
|
|
4080
5147
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -4136,7 +5203,7 @@ function renderGoalFile(goal, resultSchema, loopCtx, nodeInstructions, hasWorkfl
|
|
|
4136
5203
|
...artifactSection,
|
|
4137
5204
|
"## How to complete this node",
|
|
4138
5205
|
"You are an autonomous agent completing exactly ONE botmux v3 workflow node.",
|
|
4139
|
-
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (
|
|
5206
|
+
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (the same protocol is available via `botmux skill show botmux-goal-ask`).",
|
|
4140
5207
|
"",
|
|
4141
5208
|
`- Upstream inputs: the file at $${E.INPUTS_PATH} is a JSON object \`{ "inputs": [...] }\` listing upstream products, each with an absolute \`path\`. Read only the ones the goal needs (it may be empty). If it includes an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`, read that JSON file before continuing. If an \`omitted\` array is present, those declared inputs were intentionally not produced (their workflow branch was not taken) \u2014 treat their absence as by-design, do NOT invent their content.`,
|
|
4142
5209
|
`- Revisit feedback: if any input has \`"from": "revisit"\`, a DOWNSTREAM node sent this node back because its product was inadequate. You MUST read these before doing anything else: \`reason\` (why you were sent back), \`source:*\` (the downstream node's output \u2014 the evidence of what was wrong), and \`previous:*\` (YOUR OWN previous output \u2014 edit/fix it, do not rewrite from scratch). Address the reason; do not just reproduce the prior output.`,
|
|
@@ -4157,186 +5224,71 @@ function renderGoalFile(goal, resultSchema, loopCtx, nodeInstructions, hasWorkfl
|
|
|
4157
5224
|
` - On failure: status "${failStatus}", \`error\` required, \`files\` may be empty. Set \`error.retryable\` honestly: \`true\` when a human can unblock you and a fresh attempt could then succeed; \`false\` when retrying cannot help.`,
|
|
4158
5225
|
` - Every file \`path\` is relative to $${E.OUTPUT_DIR} ITSELF. A file you wrote directly into that directory has a path that is JUST its filename, e.g. \`"path": "report.md"\`. Do NOT prepend the directory or its folder name (NOT \`"work/report.md"\`) and do NOT use an absolute path \u2014 both are rejected.`,
|
|
4159
5226
|
"",
|
|
4160
|
-
...resultSection,
|
|
4161
|
-
`You are DONE only after the manifest at $${E.MANIFEST_PATH} exists and every file it references exists.`,
|
|
4162
|
-
"If you cannot complete the goal, write a failure manifest and stop.",
|
|
4163
|
-
'If you hit an authentication / authorization / interactive-confirmation wall (a login prompt, an expired token, a permission you cannot grant yourself): do NOT wait for a human and do NOT keep retrying. Immediately write a failure manifest with an `error.code` like "AUTH_REQUIRED" and `error.retryable: true`, then stop \u2014 a human will unblock and retry this node.',
|
|
4164
|
-
"",
|
|
4165
|
-
"## Asking a human (only when a DECISION truly needs a person)",
|
|
4166
|
-
`If \u2014 and ONLY if \u2014 you cannot proceed without a human's judgement call (a choice only a person can make; NOT something you can research, infer, or decide yourself), use the runtime human-ask:`,
|
|
4167
|
-
` 1. Write a JSON file to $${E.ATTEMPT_DIR}/${GOAL_ASK_FILE}. Use \`{ "question": "<one clear question>", "options": ["<2-6 concrete choices>"] }\` for a choice, or \`{ "question": "<one clear question>", "freeText": true }\` when the human must provide details in their own words.`,
|
|
4168
|
-
` 2. Write a failure manifest with \`error.code: "${ASK_HUMAN_ERROR_CODE}"\`, \`error.retryable: true\`, and \`summary\` = your question, then STOP.`,
|
|
4169
|
-
`A human answers; this node then RE-RUNS with their answer injected into $${E.INPUTS_PATH} as an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`. Read that JSON file's \`selected\` or \`text\` field and continue from there. Prefer deciding yourself \u2014 every ask pauses the whole workflow on a person.`,
|
|
4170
|
-
""
|
|
4171
|
-
].join("\n");
|
|
4172
|
-
}
|
|
4173
|
-
function classifyTerminal(errorClass, opts) {
|
|
4174
|
-
if (opts?.selfReportedFail) return opts.retryable === false ? "failed" : "blocked";
|
|
4175
|
-
switch (errorClass) {
|
|
4176
|
-
case "manifestInvalid":
|
|
4177
|
-
// agent wrote a bad manifest — a retry may fix it
|
|
4178
|
-
case "artifactContractInvalid":
|
|
4179
|
-
// definition/manifest public product mismatch
|
|
4180
|
-
case "resultInvalid":
|
|
4181
|
-
return "blocked";
|
|
4182
|
-
case "workerError":
|
|
4183
|
-
// process crash = infrastructure
|
|
4184
|
-
case "timeout":
|
|
4185
|
-
// budget exceeded = infrastructure (for now)
|
|
4186
|
-
case "gateRejected":
|
|
4187
|
-
// a human said no — retrying won't change that
|
|
4188
|
-
case "cancelled":
|
|
4189
|
-
return "failed";
|
|
4190
|
-
}
|
|
4191
|
-
}
|
|
4192
|
-
function readGoalAsk(askPath) {
|
|
4193
|
-
if (!existsSync7(askPath)) return void 0;
|
|
4194
|
-
let parsed;
|
|
4195
|
-
try {
|
|
4196
|
-
parsed = JSON.parse(readFileSync9(askPath, "utf-8"));
|
|
4197
|
-
} catch {
|
|
4198
|
-
return void 0;
|
|
4199
|
-
}
|
|
4200
|
-
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
4201
|
-
const o = parsed;
|
|
4202
|
-
const question = typeof o.question === "string" ? o.question.trim() : "";
|
|
4203
|
-
if (!question) return void 0;
|
|
4204
|
-
const hasOptions = Object.prototype.hasOwnProperty.call(o, "options");
|
|
4205
|
-
if (o.freeText === true) {
|
|
4206
|
-
if (hasOptions) return void 0;
|
|
4207
|
-
return { question, freeText: true };
|
|
4208
|
-
}
|
|
4209
|
-
const options = Array.isArray(o.options) ? o.options.filter((x) => typeof x === "string" && x.trim() !== "").map((x) => x.trim()) : [];
|
|
4210
|
-
if (options.length < 2) return void 0;
|
|
4211
|
-
return { question, options: options.slice(0, 6) };
|
|
4212
|
-
}
|
|
4213
|
-
function mergeNodeCapability(snap, override) {
|
|
4214
|
-
if (!override) return snap;
|
|
4215
|
-
return {
|
|
4216
|
-
...snap,
|
|
4217
|
-
...override.model ? { model: override.model } : {}
|
|
4218
|
-
};
|
|
4219
|
-
}
|
|
4220
|
-
function readRevisitRequest(manifest, outputDir) {
|
|
4221
|
-
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4222
|
-
if (!entry) return { ok: true };
|
|
4223
|
-
let value;
|
|
4224
|
-
try {
|
|
4225
|
-
value = JSON.parse(readFileSync9(join6(outputDir, entry.path), "utf-8"));
|
|
4226
|
-
} catch {
|
|
4227
|
-
return { ok: true };
|
|
4228
|
-
}
|
|
4229
|
-
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4230
|
-
return { ok: true };
|
|
4231
|
-
}
|
|
4232
|
-
const v = value;
|
|
4233
|
-
const problems = [];
|
|
4234
|
-
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4235
|
-
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4236
|
-
}
|
|
4237
|
-
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4238
|
-
problems.push('result.json "reason" must be a string when present');
|
|
4239
|
-
}
|
|
4240
|
-
if (problems.length > 0) return { ok: false, problems };
|
|
4241
|
-
return {
|
|
4242
|
-
ok: true,
|
|
4243
|
-
request: {
|
|
4244
|
-
toNodeId: v.revisitTo,
|
|
4245
|
-
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4246
|
-
}
|
|
4247
|
-
};
|
|
4248
|
-
}
|
|
4249
|
-
function validateResult(filePath, schema) {
|
|
4250
|
-
if (!existsSync7(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4251
|
-
let parsed;
|
|
4252
|
-
try {
|
|
4253
|
-
parsed = JSON.parse(readFileSync9(filePath, "utf-8"));
|
|
4254
|
-
} catch (err) {
|
|
4255
|
-
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4256
|
-
}
|
|
4257
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4258
|
-
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4259
|
-
}
|
|
4260
|
-
const obj = parsed;
|
|
4261
|
-
const problems = [];
|
|
4262
|
-
for (const field of schema.required ?? []) {
|
|
4263
|
-
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4264
|
-
}
|
|
4265
|
-
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4266
|
-
if (!(name in obj)) continue;
|
|
4267
|
-
const v = obj[name];
|
|
4268
|
-
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4269
|
-
if (!okType) {
|
|
4270
|
-
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4271
|
-
continue;
|
|
4272
|
-
}
|
|
4273
|
-
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4274
|
-
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4275
|
-
}
|
|
4276
|
-
}
|
|
4277
|
-
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4278
|
-
}
|
|
4279
|
-
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
4280
|
-
function attemptNumber(attemptId) {
|
|
4281
|
-
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
4282
|
-
return m ? parseInt(m[1], 10) : void 0;
|
|
4283
|
-
}
|
|
4284
|
-
function nextAttemptIdFor(events, key) {
|
|
4285
|
-
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
4286
|
-
let maxSeen = 0;
|
|
4287
|
-
let reserved;
|
|
4288
|
-
let prepared;
|
|
4289
|
-
for (const e of events) {
|
|
4290
|
-
if (e.type === "nodeDispatched" && matches(e)) {
|
|
4291
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4292
|
-
if (n2 === void 0) continue;
|
|
4293
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4294
|
-
if (reserved === n2) reserved = void 0;
|
|
4295
|
-
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
4296
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4297
|
-
if (n2 === void 0) continue;
|
|
4298
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4299
|
-
if (reserved === n2) reserved = void 0;
|
|
4300
|
-
prepared = e.attemptId;
|
|
4301
|
-
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
4302
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4303
|
-
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
4304
|
-
if (prepared === e.attemptId) prepared = void 0;
|
|
4305
|
-
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
4306
|
-
prepared = void 0;
|
|
4307
|
-
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
4308
|
-
const n2 = attemptNumber(e.nextAttemptId);
|
|
4309
|
-
if (n2 === void 0) continue;
|
|
4310
|
-
reserved = n2;
|
|
4311
|
-
prepared = void 0;
|
|
4312
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4313
|
-
}
|
|
5227
|
+
...resultSection,
|
|
5228
|
+
`You are DONE only after the manifest at $${E.MANIFEST_PATH} exists and every file it references exists.`,
|
|
5229
|
+
"If you cannot complete the goal, write a failure manifest and stop.",
|
|
5230
|
+
'If you hit an authentication / authorization / interactive-confirmation wall (a login prompt, an expired token, a permission you cannot grant yourself): do NOT wait for a human and do NOT keep retrying. Immediately write a failure manifest with an `error.code` like "AUTH_REQUIRED" and `error.retryable: true`, then stop \u2014 a human will unblock and retry this node.',
|
|
5231
|
+
"",
|
|
5232
|
+
"## Asking a human (only when a DECISION truly needs a person)",
|
|
5233
|
+
`If \u2014 and ONLY if \u2014 you cannot proceed without a human's judgement call (a choice only a person can make; NOT something you can research, infer, or decide yourself), use the runtime human-ask:`,
|
|
5234
|
+
` 1. Write a JSON file to $${E.ATTEMPT_DIR}/${GOAL_ASK_FILE}. Use \`{ "question": "<one clear question>", "options": ["<2-6 concrete choices>"] }\` for a choice, or \`{ "question": "<one clear question>", "freeText": true }\` when the human must provide details in their own words.`,
|
|
5235
|
+
` 2. Write a failure manifest with \`error.code: "${ASK_HUMAN_ERROR_CODE}"\`, \`error.retryable: true\`, and \`summary\` = your question, then STOP.`,
|
|
5236
|
+
`A human answers; this node then RE-RUNS with their answer injected into $${E.INPUTS_PATH} as an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`. Read that JSON file's \`selected\` or \`text\` field and continue from there. Prefer deciding yourself \u2014 every ask pauses the whole workflow on a person.`,
|
|
5237
|
+
""
|
|
5238
|
+
].join("\n");
|
|
5239
|
+
}
|
|
5240
|
+
function classifyTerminal(errorClass, opts) {
|
|
5241
|
+
if (opts?.selfReportedFail) return opts.retryable === false ? "failed" : "blocked";
|
|
5242
|
+
switch (errorClass) {
|
|
5243
|
+
case "manifestInvalid":
|
|
5244
|
+
// agent wrote a bad manifest — a retry may fix it
|
|
5245
|
+
case "artifactContractInvalid":
|
|
5246
|
+
// definition/manifest public product mismatch
|
|
5247
|
+
case "resultInvalid":
|
|
5248
|
+
return "blocked";
|
|
5249
|
+
case "workerError":
|
|
5250
|
+
// process crash = infrastructure
|
|
5251
|
+
case "timeout":
|
|
5252
|
+
// budget exceeded = infrastructure (for now)
|
|
5253
|
+
case "gateRejected":
|
|
5254
|
+
// a human said no — retrying won't change that
|
|
5255
|
+
case "cancelled":
|
|
5256
|
+
return "failed";
|
|
4314
5257
|
}
|
|
4315
|
-
if (prepared) return prepared;
|
|
4316
|
-
const n = reserved ?? maxSeen + 1;
|
|
4317
|
-
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
4318
5258
|
}
|
|
4319
|
-
function
|
|
4320
|
-
|
|
4321
|
-
let
|
|
4322
|
-
|
|
4323
|
-
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
if (latestNumber < 0) latest = e.attemptId;
|
|
4327
|
-
} else if (n >= latestNumber) {
|
|
4328
|
-
latest = e.attemptId;
|
|
4329
|
-
latestNumber = n;
|
|
4330
|
-
}
|
|
4331
|
-
}
|
|
5259
|
+
function readGoalAsk(askPath) {
|
|
5260
|
+
if (!existsSync10(askPath)) return void 0;
|
|
5261
|
+
let parsed;
|
|
5262
|
+
try {
|
|
5263
|
+
parsed = JSON.parse(readFileSync12(askPath, "utf-8"));
|
|
5264
|
+
} catch {
|
|
5265
|
+
return void 0;
|
|
4332
5266
|
}
|
|
4333
|
-
return
|
|
5267
|
+
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
5268
|
+
const o = parsed;
|
|
5269
|
+
const question = typeof o.question === "string" ? o.question.trim() : "";
|
|
5270
|
+
if (!question) return void 0;
|
|
5271
|
+
const hasOptions = Object.prototype.hasOwnProperty.call(o, "options");
|
|
5272
|
+
if (o.freeText === true) {
|
|
5273
|
+
if (hasOptions) return void 0;
|
|
5274
|
+
return { question, freeText: true };
|
|
5275
|
+
}
|
|
5276
|
+
const options = Array.isArray(o.options) ? o.options.filter((x) => typeof x === "string" && x.trim() !== "").map((x) => x.trim()) : [];
|
|
5277
|
+
if (options.length < 2) return void 0;
|
|
5278
|
+
return { question, options: options.slice(0, 6) };
|
|
5279
|
+
}
|
|
5280
|
+
function mergeNodeCapability(snap, override) {
|
|
5281
|
+
if (!override) return snap;
|
|
5282
|
+
return {
|
|
5283
|
+
...snap,
|
|
5284
|
+
...override.model ? { model: override.model } : {}
|
|
5285
|
+
};
|
|
4334
5286
|
}
|
|
4335
5287
|
function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) => candidate) {
|
|
4336
5288
|
const expectedHash = computeInputHash(normalize(value));
|
|
4337
|
-
if (!
|
|
5289
|
+
if (!existsSync10(path)) {
|
|
4338
5290
|
try {
|
|
4339
|
-
|
|
5291
|
+
writeFileSync8(path, `${JSON.stringify(value, null, 2)}
|
|
4340
5292
|
`, {
|
|
4341
5293
|
encoding: "utf-8",
|
|
4342
5294
|
flag: "wx",
|
|
@@ -4350,7 +5302,7 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4350
5302
|
}
|
|
4351
5303
|
let stored;
|
|
4352
5304
|
try {
|
|
4353
|
-
stored = normalize(JSON.parse(
|
|
5305
|
+
stored = normalize(JSON.parse(readFileSync12(path, "utf-8")));
|
|
4354
5306
|
} catch (error) {
|
|
4355
5307
|
throw new Error(
|
|
4356
5308
|
`v3 runtime: frozen ${label} is unreadable: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -4362,11 +5314,22 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4362
5314
|
);
|
|
4363
5315
|
}
|
|
4364
5316
|
}
|
|
4365
|
-
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
5317
|
+
var HANDOFF_RESEND_BACKOFF_MS = 3e4;
|
|
5318
|
+
var PROBE_RETRY_BACKOFF_MS = 3e4;
|
|
5319
|
+
var PROBE_UNCERTAIN_AFTER_MS = 10 * 6e4;
|
|
5320
|
+
var probeFailures = /* @__PURE__ */ new Map();
|
|
5321
|
+
function forgetProbeFailuresOf(runDir) {
|
|
5322
|
+
const prefix = `${runDir}\0`;
|
|
5323
|
+
for (const key of probeFailures.keys()) {
|
|
5324
|
+
if (key.startsWith(prefix)) probeFailures.delete(key);
|
|
5325
|
+
}
|
|
5326
|
+
}
|
|
5327
|
+
async function runWorkflow(authorizedDag, deps, opts) {
|
|
5328
|
+
let dag = authorizedDag;
|
|
5329
|
+
const runDir = join10(opts.baseDir, dag.runId);
|
|
5330
|
+
mkdirSync8(runDir, { recursive: true });
|
|
5331
|
+
const journalPath = join10(runDir, "journal.ndjson");
|
|
5332
|
+
const statePath = join10(runDir, "STATE");
|
|
4370
5333
|
const globalCap = opts.globalConcurrency ?? 4;
|
|
4371
5334
|
const perBotCap = opts.perBotConcurrency ?? 1;
|
|
4372
5335
|
const perCliCap = opts.perCliConcurrency ?? 2;
|
|
@@ -4379,7 +5342,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4379
5342
|
if (!Number.isSafeInteger(hostResponseWaitMs) || hostResponseWaitMs < 1) {
|
|
4380
5343
|
throw new Error("v3 runtime: hostResponseWaitMs must be a positive safe integer");
|
|
4381
5344
|
}
|
|
4382
|
-
const
|
|
5345
|
+
const externalMode = opts.nodeExecutor === "bot";
|
|
5346
|
+
const externalExecutor = deps.externalExecutor;
|
|
5347
|
+
if (externalMode && !externalExecutor) {
|
|
5348
|
+
throw new Error('v3 runtime: nodeExecutor "bot" requires an external node executor');
|
|
5349
|
+
}
|
|
5350
|
+
const externalAcceptTimeoutMs = opts.externalAcceptTimeoutMs ?? V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS;
|
|
5351
|
+
if (!Number.isSafeInteger(externalAcceptTimeoutMs) || externalAcceptTimeoutMs < 0) {
|
|
5352
|
+
throw new Error("v3 runtime: externalAcceptTimeoutMs must be a non-negative safe integer");
|
|
5353
|
+
}
|
|
5354
|
+
const clock = () => deps.now?.() ?? Date.now();
|
|
5355
|
+
const externalCallTimeoutMs = opts.externalCallTimeoutMs ?? V3_EXTERNAL_CALL_TIMEOUT_MS;
|
|
5356
|
+
if (!Number.isSafeInteger(externalCallTimeoutMs) || externalCallTimeoutMs < 1) {
|
|
5357
|
+
throw new Error("v3 runtime: externalCallTimeoutMs must be a positive safe integer");
|
|
5358
|
+
}
|
|
5359
|
+
const probeKey = (attemptId) => `${runDir}\0${attemptId}`;
|
|
5360
|
+
let nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5361
|
+
let dagRevision = 1;
|
|
5362
|
+
let pendingTouched;
|
|
5363
|
+
let pendingRevision;
|
|
4383
5364
|
const botSnapshots = opts.frozenBotSnapshots ? new Map(opts.frozenBotSnapshots) : /* @__PURE__ */ new Map();
|
|
4384
5365
|
const freezeBot = (bot) => {
|
|
4385
5366
|
const key = bot ?? "";
|
|
@@ -4400,7 +5381,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4400
5381
|
for (const b of node.body.nodes) freezeBot(b.bot ?? node.bot);
|
|
4401
5382
|
}
|
|
4402
5383
|
}
|
|
4403
|
-
|
|
5384
|
+
const validateSnapshot = (key, snap) => {
|
|
4404
5385
|
if (deps.validateExecutionSnapshot) {
|
|
4405
5386
|
deps.validateExecutionSnapshot(snap, key);
|
|
4406
5387
|
} else if (!isV3SupportedCli(snap.cliId)) {
|
|
@@ -4408,22 +5389,74 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4408
5389
|
`v3 runtime: bot "${key || "<default>"}" resolves to CLI "${snap.cliId}", which is not supported by v3 goal-mode (supported: ${V3_SUPPORTED_CLIS.join(", ")})`
|
|
4409
5390
|
);
|
|
4410
5391
|
}
|
|
4411
|
-
}
|
|
5392
|
+
};
|
|
5393
|
+
for (const [key, snap] of botSnapshots) validateSnapshot(key, snap);
|
|
5394
|
+
const botSnapshotFor2 = (selector) => {
|
|
5395
|
+
const snap = snapshotForSelector(botSnapshots, selector);
|
|
5396
|
+
if (!snap) {
|
|
5397
|
+
throw new Error(`v3 runtime: no frozen bot snapshot for selector "${selector || "<default>"}"`);
|
|
5398
|
+
}
|
|
5399
|
+
return snap;
|
|
5400
|
+
};
|
|
5401
|
+
const refreshEffectiveDag = (events) => {
|
|
5402
|
+
const history = foldV3Revisions(events);
|
|
5403
|
+
pendingTouched = history.pending ? new Set(history.pending.touched) : void 0;
|
|
5404
|
+
pendingRevision = history.pending?.revision;
|
|
5405
|
+
if (history.current === dagRevision) return;
|
|
5406
|
+
const effective = loadEffectiveV3Dag(runDir, authorizedDag, events);
|
|
5407
|
+
for (const [selector, snap] of v3ApprovedBotScope(botSnapshots, effective.botAdditions)) {
|
|
5408
|
+
if (botSnapshots.has(selector)) continue;
|
|
5409
|
+
validateSnapshot(selector, snap);
|
|
5410
|
+
botSnapshots.set(selector, snap);
|
|
5411
|
+
}
|
|
5412
|
+
for (const node of effective.dag.nodes) {
|
|
5413
|
+
if (isHostNode(node) && !deps.hostExecutors?.has(node.executor)) {
|
|
5414
|
+
throw new Error(`v3 runtime: host executor "${node.executor}" is not registered`);
|
|
5415
|
+
}
|
|
5416
|
+
if (isGoalNode(node)) botSnapshotFor2(node.bot);
|
|
5417
|
+
if (isLoopNode(node)) for (const body of node.body.nodes) botSnapshotFor2(body.bot ?? node.bot);
|
|
5418
|
+
}
|
|
5419
|
+
dag = effective.dag;
|
|
5420
|
+
nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5421
|
+
dagRevision = effective.revision;
|
|
5422
|
+
};
|
|
5423
|
+
const decisionStillCurrent = (latest, nodeId, loopId) => {
|
|
5424
|
+
if (currentV3Revision(latest) !== dagRevision) return false;
|
|
5425
|
+
if (nodeId === void 0) return true;
|
|
5426
|
+
const pending = foldV3Revisions(latest).pending;
|
|
5427
|
+
return !pending || !v3NodeHeldByProposal(new Set(pending.touched), nodeId, loopId);
|
|
5428
|
+
};
|
|
5429
|
+
const appendControlEvent = (event) => withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5430
|
+
if (!decisionStillCurrent(events)) return false;
|
|
5431
|
+
append(event);
|
|
5432
|
+
return true;
|
|
5433
|
+
});
|
|
5434
|
+
const heldByProposal = (nodeId, loopId) => v3NodeHeldByProposal(pendingTouched, nodeId, loopId);
|
|
5435
|
+
const reconcileRevisionFollowUps = (events) => {
|
|
5436
|
+
const live = new Set(dag.nodes.map((node) => node.id));
|
|
5437
|
+
if (missingV3RevisionFollowUps(events, live).length === 0) return false;
|
|
5438
|
+
const result = appendMissingV3RevisionFollowUps(journalPath, { revision: dagRevision, liveNodeIds: live });
|
|
5439
|
+
return result.kind === "stale" || result.count > 0;
|
|
5440
|
+
};
|
|
4412
5441
|
if (!opts.authorizedArtifacts) {
|
|
4413
5442
|
assertOrCreateFrozenJson(
|
|
4414
|
-
|
|
5443
|
+
join10(runDir, "bots.snapshot.json"),
|
|
4415
5444
|
Object.fromEntries(botSnapshots),
|
|
4416
5445
|
"execution profile snapshot"
|
|
4417
5446
|
);
|
|
4418
5447
|
assertOrCreateFrozenJson(
|
|
4419
|
-
|
|
5448
|
+
join10(runDir, "dag.json"),
|
|
4420
5449
|
dag,
|
|
4421
5450
|
"workflow definition",
|
|
4422
5451
|
validateDag
|
|
4423
5452
|
);
|
|
4424
5453
|
}
|
|
4425
5454
|
if (readJournal(journalPath).length === 0) {
|
|
4426
|
-
appendEvent(journalPath, {
|
|
5455
|
+
appendEvent(journalPath, {
|
|
5456
|
+
type: "runStarted",
|
|
5457
|
+
runId: dag.runId,
|
|
5458
|
+
...externalMode ? { nodeExecutor: "bot" } : {}
|
|
5459
|
+
});
|
|
4427
5460
|
}
|
|
4428
5461
|
const inFlight = /* @__PURE__ */ new Map();
|
|
4429
5462
|
const botInFlight = /* @__PURE__ */ new Map();
|
|
@@ -4434,6 +5467,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4434
5467
|
const hostPromiseAttempts = /* @__PURE__ */ new Map();
|
|
4435
5468
|
const hostDeadlineCancels = /* @__PURE__ */ new Map();
|
|
4436
5469
|
const blockingGateOwners = /* @__PURE__ */ new Map();
|
|
5470
|
+
const handoffControllers = /* @__PURE__ */ new Map();
|
|
5471
|
+
const handoffRetryAt = /* @__PURE__ */ new Map();
|
|
5472
|
+
const acceptanceObserved = /* @__PURE__ */ new Set();
|
|
4437
5473
|
class HostProviderResponseTimeoutError extends Error {
|
|
4438
5474
|
constructor() {
|
|
4439
5475
|
super(`host provider did not settle within ${hostResponseWaitMs}ms`);
|
|
@@ -4480,6 +5516,8 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4480
5516
|
};
|
|
4481
5517
|
while (true) {
|
|
4482
5518
|
const events = readJournal(journalPath);
|
|
5519
|
+
refreshEffectiveDag(events);
|
|
5520
|
+
if (reconcileRevisionFollowUps(events)) continue;
|
|
4483
5521
|
const snap = materialize(events);
|
|
4484
5522
|
writeState(statePath, snap);
|
|
4485
5523
|
if (opts.cancelSignal?.aborted && snap.runStatus !== "cancelling" && !isTrueRunTerminal(snap.runStatus)) {
|
|
@@ -4490,15 +5528,21 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4490
5528
|
if (openHostEffects.length > 0) {
|
|
4491
5529
|
const settled = await reconcileOpenHostEffects(openHostEffects);
|
|
4492
5530
|
detachClosedHostInvocations();
|
|
4493
|
-
if (!settled) await new Promise((
|
|
5531
|
+
if (!settled) await new Promise((resolve6) => setTimeout(resolve6, 1e3));
|
|
4494
5532
|
continue;
|
|
4495
5533
|
}
|
|
5534
|
+
if (settleExternalLeftovers(events)) continue;
|
|
4496
5535
|
if (snap.runStatus === "cancelling" && snap.cancelRequestId) {
|
|
4497
5536
|
for (const controller of nodeControllers.values()) {
|
|
4498
5537
|
if (!controller.signal.aborted) {
|
|
4499
5538
|
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
4500
5539
|
}
|
|
4501
5540
|
}
|
|
5541
|
+
for (const controller of handoffControllers.values()) {
|
|
5542
|
+
if (!controller.signal.aborted) {
|
|
5543
|
+
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
5544
|
+
}
|
|
5545
|
+
}
|
|
4502
5546
|
cancelUnfinishedNodes(snap.cancelRequestId, { includeRunning: false });
|
|
4503
5547
|
for (const key of [...inFlight.keys()]) {
|
|
4504
5548
|
if (!key.endsWith("::gate")) continue;
|
|
@@ -4509,6 +5553,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4509
5553
|
await Promise.race(inFlight.values());
|
|
4510
5554
|
continue;
|
|
4511
5555
|
}
|
|
5556
|
+
if (openV3ExternalAttempts(events).length > 0) {
|
|
5557
|
+
await closeExternalAttemptsForCancellation(snap.cancelRequestId);
|
|
5558
|
+
continue;
|
|
5559
|
+
}
|
|
4512
5560
|
let externalDrained = false;
|
|
4513
5561
|
try {
|
|
4514
5562
|
externalDrained = drainExternallyOwnedAttempts({
|
|
@@ -4518,7 +5566,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4518
5566
|
} catch {
|
|
4519
5567
|
}
|
|
4520
5568
|
if (!externalDrained) {
|
|
4521
|
-
await new Promise((
|
|
5569
|
+
await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4522
5570
|
continue;
|
|
4523
5571
|
}
|
|
4524
5572
|
finalizeRuntimeCancellation(snap.cancelRequestId);
|
|
@@ -4533,7 +5581,16 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4533
5581
|
open,
|
|
4534
5582
|
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
4535
5583
|
);
|
|
4536
|
-
if (!drained) await new Promise((
|
|
5584
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5585
|
+
continue;
|
|
5586
|
+
}
|
|
5587
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5588
|
+
if (openExternal.length > 0) {
|
|
5589
|
+
await drainExternalAttempts(
|
|
5590
|
+
openExternal,
|
|
5591
|
+
"runBoundary",
|
|
5592
|
+
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
5593
|
+
);
|
|
4537
5594
|
continue;
|
|
4538
5595
|
}
|
|
4539
5596
|
break;
|
|
@@ -4543,7 +5600,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4543
5600
|
const obsolete = obsoleteOpenAttempts(events, snap);
|
|
4544
5601
|
if (obsolete.length > 0) {
|
|
4545
5602
|
const drained = await quiesceAttempts(obsolete, "obsoleteAttempt");
|
|
4546
|
-
if (!drained) await new Promise((
|
|
5603
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4547
5604
|
continue;
|
|
4548
5605
|
}
|
|
4549
5606
|
const orphaned = openV3WorkerAttempts(events).filter((attempt) => {
|
|
@@ -4552,34 +5609,56 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4552
5609
|
});
|
|
4553
5610
|
if (orphaned.length > 0) {
|
|
4554
5611
|
const drained = await quiesceAttempts(orphaned, "orphanRecovery");
|
|
4555
|
-
if (!drained) await new Promise((
|
|
5612
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5613
|
+
continue;
|
|
5614
|
+
}
|
|
5615
|
+
const obsoleteExternal = obsoleteExternalAttempts(events, snap);
|
|
5616
|
+
if (obsoleteExternal.length > 0) {
|
|
5617
|
+
await drainExternalAttempts(obsoleteExternal, "obsolete", "obsoleteAttempt");
|
|
4556
5618
|
continue;
|
|
4557
5619
|
}
|
|
5620
|
+
if (await reconcileExternalAttempts(events)) continue;
|
|
4558
5621
|
const actions = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances);
|
|
4559
5622
|
const terminal = actions.find(
|
|
4560
5623
|
(a) => a.kind === "completeRunSucceeded" || a.kind === "completeRunFailed" || a.kind === "completeRunBlocked"
|
|
4561
5624
|
);
|
|
4562
5625
|
if (terminal) {
|
|
5626
|
+
if (terminal.kind === "completeRunSucceeded" && pendingRevision !== void 0) {
|
|
5627
|
+
if (inFlight.size > 0) {
|
|
5628
|
+
await waitForInFlightOrDurableCancellation();
|
|
5629
|
+
continue;
|
|
5630
|
+
}
|
|
5631
|
+
return { reason: "awaitingGate", pendingWaits: [], pendingRevision, runDir };
|
|
5632
|
+
}
|
|
4563
5633
|
dropAllBlockingGatePromises();
|
|
4564
5634
|
const open = openV3WorkerAttempts(events);
|
|
4565
5635
|
if (open.length > 0) {
|
|
4566
5636
|
const drained = await quiesceAttempts(open, "terminalPeer");
|
|
4567
|
-
if (!drained) await new Promise((
|
|
5637
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5638
|
+
continue;
|
|
5639
|
+
}
|
|
5640
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5641
|
+
if (openExternal.length > 0) {
|
|
5642
|
+
await drainExternalAttempts(openExternal, "runBoundary", "terminalPeer");
|
|
4568
5643
|
continue;
|
|
4569
5644
|
}
|
|
4570
5645
|
appendRunTerminalIfQuiescent();
|
|
4571
5646
|
continue;
|
|
4572
5647
|
}
|
|
4573
5648
|
if (gateMode === "blocking") reattachBlockingGates(snap);
|
|
4574
|
-
const
|
|
5649
|
+
const allControls = actions.filter(
|
|
4575
5650
|
(a) => a.kind === "startLoop" || a.kind === "startLoopIteration" || a.kind === "evaluateLoopIteration" || a.kind === "completeLoop" || a.kind === "resolveEdge" || a.kind === "skipNode"
|
|
4576
5651
|
);
|
|
5652
|
+
const controls = allControls.filter((a) => !heldByProposal(
|
|
5653
|
+
a.kind === "resolveEdge" ? a.to : a.kind === "skipNode" ? a.nodeId : a.loopId
|
|
5654
|
+
));
|
|
5655
|
+
const heldControls = allControls.length - controls.length;
|
|
4577
5656
|
if (controls.length > 0) {
|
|
4578
5657
|
const eventsForControl = readJournal(journalPath);
|
|
4579
5658
|
for (const a of controls) {
|
|
4580
5659
|
if (a.kind === "resolveEdge") applyResolveEdge(a, eventsForControl);
|
|
4581
5660
|
else if (a.kind === "skipNode") {
|
|
4582
|
-
|
|
5661
|
+
appendControlEvent({
|
|
4583
5662
|
type: "nodeSkipped",
|
|
4584
5663
|
nodeId: a.nodeId,
|
|
4585
5664
|
reason: "triggerRuleUnsatisfied",
|
|
@@ -4589,17 +5668,40 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4589
5668
|
}
|
|
4590
5669
|
continue;
|
|
4591
5670
|
}
|
|
5671
|
+
const openExternalNow = openV3ExternalAttempts(events);
|
|
5672
|
+
let externalOccupancy = openExternalNow.length;
|
|
5673
|
+
const externalByBot = /* @__PURE__ */ new Map();
|
|
5674
|
+
for (const attempt of openExternalNow) {
|
|
5675
|
+
if (!attempt.targetLarkAppId) continue;
|
|
5676
|
+
externalByBot.set(attempt.targetLarkAppId, (externalByBot.get(attempt.targetLarkAppId) ?? 0) + 1);
|
|
5677
|
+
}
|
|
5678
|
+
const occupancy = () => inFlight.size - [...inFlight.keys()].filter((key) => key.endsWith("::handoff")).length + externalOccupancy;
|
|
4592
5679
|
let startedThisTick = 0;
|
|
5680
|
+
let heldThisTick = heldControls;
|
|
4593
5681
|
for (const a of actions) {
|
|
4594
|
-
if (
|
|
5682
|
+
if (occupancy() >= globalCap) break;
|
|
5683
|
+
if ((a.kind === "dispatchWork" || a.kind === "dispatchGate") && heldByProposal(a.nodeId, a.kind === "dispatchWork" ? a.loop?.loopId : void 0)) {
|
|
5684
|
+
heldThisTick++;
|
|
5685
|
+
continue;
|
|
5686
|
+
}
|
|
4595
5687
|
if (a.kind === "dispatchWork") {
|
|
4596
5688
|
const node = a.loop ? instanceNodeFor(a.loop) : nodesById.get(a.nodeId);
|
|
4597
5689
|
if (isHostNode(node)) {
|
|
4598
5690
|
if (!a.instanceId) throw new Error(`v3 runtime: host node "${node.id}" has no runtime instance`);
|
|
4599
5691
|
if (startHost(node, events, a.instanceId)) startedThisTick++;
|
|
4600
5692
|
} else {
|
|
4601
|
-
const
|
|
4602
|
-
|
|
5693
|
+
const botSnap = botSnapshotFor2(node.bot);
|
|
5694
|
+
if (externalMode) {
|
|
5695
|
+
const target = snap.assignments.get(node.id) ?? botSnap.larkAppId;
|
|
5696
|
+
if ((botInFlight.get(target) ?? 0) + (externalByBot.get(target) ?? 0) >= perBotCap) continue;
|
|
5697
|
+
if (startExternal(node, botSnap, events, a.loop, a.omitted, a.instanceId)) {
|
|
5698
|
+
startedThisTick++;
|
|
5699
|
+
externalOccupancy++;
|
|
5700
|
+
externalByBot.set(target, (externalByBot.get(target) ?? 0) + 1);
|
|
5701
|
+
}
|
|
5702
|
+
continue;
|
|
5703
|
+
}
|
|
5704
|
+
const botKey = botSnap.larkAppId;
|
|
4603
5705
|
if ((botInFlight.get(botKey) ?? 0) >= perBotCap) continue;
|
|
4604
5706
|
if ((cliInFlight.get(botSnap.cliId) ?? 0) >= perCliCap) continue;
|
|
4605
5707
|
if (startWork(node, botSnap, botKey, events, a.loop, a.omitted, a.instanceId)) {
|
|
@@ -4610,7 +5712,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4610
5712
|
if (startGate(nodesById.get(a.nodeId), a.instanceId)) startedThisTick++;
|
|
4611
5713
|
}
|
|
4612
5714
|
}
|
|
4613
|
-
const cancels = actions.filter((a) => a.kind === "cancelNode");
|
|
5715
|
+
const cancels = actions.filter((a) => a.kind === "cancelNode" && !heldByProposal(a.nodeId));
|
|
4614
5716
|
let cancelledThisTick = false;
|
|
4615
5717
|
for (const a of cancels) {
|
|
4616
5718
|
cancelledThisTick = applyCancelNode(a) || cancelledThisTick;
|
|
@@ -4619,20 +5721,31 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4619
5721
|
if (inFlight.size === 0) {
|
|
4620
5722
|
if (startedThisTick === 0) {
|
|
4621
5723
|
const pendingWaits = gateMode === "suspend" ? pendingGateWaits(snap.nodes) : [];
|
|
4622
|
-
|
|
5724
|
+
const latestEvents = readJournal(journalPath);
|
|
5725
|
+
if (externalMode && latestEvents.length !== events.length) continue;
|
|
5726
|
+
const waitingExternal = openV3ExternalAttempts(latestEvents);
|
|
5727
|
+
if (latestEvents.length !== events.length && currentV3Revision(latestEvents) !== dagRevision) continue;
|
|
5728
|
+
if (pendingWaits.length > 0 || waitingExternal.length > 0 || heldThisTick > 0) {
|
|
4623
5729
|
const open = openV3WorkerAttempts(events);
|
|
4624
5730
|
if (open.length > 0) {
|
|
4625
5731
|
const drained = await quiesceAttempts(open, "orphanRecovery");
|
|
4626
|
-
if (!drained) await new Promise((
|
|
5732
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4627
5733
|
continue;
|
|
4628
5734
|
}
|
|
4629
|
-
|
|
5735
|
+
if (waitingExternal.length > 0) return awaitingExternalOutcome(waitingExternal, pendingWaits);
|
|
5736
|
+
return {
|
|
5737
|
+
reason: "awaitingGate",
|
|
5738
|
+
pendingWaits,
|
|
5739
|
+
...heldThisTick > 0 && pendingRevision !== void 0 ? { pendingRevision } : {},
|
|
5740
|
+
runDir
|
|
5741
|
+
};
|
|
4630
5742
|
}
|
|
4631
5743
|
throw new Error("v3 runtime: no progress possible and run is not terminal");
|
|
4632
5744
|
}
|
|
4633
5745
|
}
|
|
4634
5746
|
if (inFlight.size > 0) await waitForInFlightOrDurableCancellation();
|
|
4635
5747
|
}
|
|
5748
|
+
forgetProbeFailuresOf(runDir);
|
|
4636
5749
|
const finalSnap = materialize(readJournal(journalPath));
|
|
4637
5750
|
return {
|
|
4638
5751
|
reason: "terminal",
|
|
@@ -4655,21 +5768,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4655
5768
|
if (isTrueRunTerminal(snap.runStatus) || snap.runStatus === "cancelling") return;
|
|
4656
5769
|
append({
|
|
4657
5770
|
type: "runCancelRequested",
|
|
4658
|
-
cancelRequestId: `cancel-${
|
|
5771
|
+
cancelRequestId: `cancel-${randomUUID4()}`,
|
|
4659
5772
|
by: "runtime-signal"
|
|
4660
5773
|
}, { durable: true });
|
|
4661
5774
|
});
|
|
4662
5775
|
}
|
|
4663
5776
|
function appendRunTerminalIfQuiescent() {
|
|
4664
5777
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5778
|
+
if (!decisionStillCurrent(events)) return false;
|
|
4665
5779
|
const snap = materialize([...events]);
|
|
4666
5780
|
if (snap.runStatus !== "running") return false;
|
|
4667
5781
|
if (openV3WorkerAttempts(events).length > 0) return false;
|
|
5782
|
+
const pendingProposal = foldV3Revisions(events).pending;
|
|
4668
5783
|
if (openV3HostEffects(events).length > 0) return false;
|
|
5784
|
+
if (openV3ExternalAttempts(events).length > 0) return false;
|
|
4669
5785
|
const terminal = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances).find(
|
|
4670
5786
|
(action) => action.kind === "completeRunSucceeded" || action.kind === "completeRunFailed" || action.kind === "completeRunBlocked"
|
|
4671
5787
|
);
|
|
4672
5788
|
if (!terminal) return false;
|
|
5789
|
+
if (terminal.kind === "completeRunSucceeded" && pendingProposal) return false;
|
|
4673
5790
|
const event = terminal.kind === "completeRunSucceeded" ? { type: "runSucceeded" } : terminal.kind === "completeRunFailed" ? {
|
|
4674
5791
|
type: "runFailed",
|
|
4675
5792
|
failedNodeId: terminal.failedNodeId,
|
|
@@ -4682,21 +5799,42 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4682
5799
|
}
|
|
4683
5800
|
function claimDispatch(event) {
|
|
4684
5801
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5802
|
+
if (!decisionStillCurrent(
|
|
5803
|
+
events,
|
|
5804
|
+
event.nodeId,
|
|
5805
|
+
event.type === "nodeDispatched" ? event.loop?.loopId : void 0
|
|
5806
|
+
)) return false;
|
|
4685
5807
|
const snap = materialize([...events]);
|
|
4686
5808
|
if (snap.runStatus !== "running") return false;
|
|
4687
5809
|
const status = snap.nodes.get(event.nodeId)?.status ?? "pending";
|
|
4688
5810
|
if (status !== "pending") return false;
|
|
5811
|
+
if (event.type === "gateDispatched" && gateWaitAlreadyDecided(events, event.waitId)) {
|
|
5812
|
+
throw new Error(
|
|
5813
|
+
`v3 runtime: gate ${event.waitId} of node "${event.nodeId}" was already decided, but the node has no approval to run on; refusing to re-dispatch the same gate`
|
|
5814
|
+
);
|
|
5815
|
+
}
|
|
4689
5816
|
append(event, { durable: true });
|
|
4690
5817
|
return true;
|
|
4691
5818
|
});
|
|
4692
5819
|
}
|
|
5820
|
+
function gateWaitAlreadyDecided(events, waitId) {
|
|
5821
|
+
return events.some((event) => event.type === "gateResolved" && event.waitId === waitId || event.type === "dagRevised" && (event.effects?.supersede?.some((item) => item.waitId === waitId) === true || event.effects?.cancel?.some((item) => item.waitId === waitId) === true));
|
|
5822
|
+
}
|
|
5823
|
+
function revisionSignature(events) {
|
|
5824
|
+
let count = 0;
|
|
5825
|
+
for (const event of events) {
|
|
5826
|
+
if (event.type === "dagRevised" || event.type === "dagRevisionProposed" || event.type === "dagRevisionRejected") count++;
|
|
5827
|
+
}
|
|
5828
|
+
return count;
|
|
5829
|
+
}
|
|
4693
5830
|
function isRunCancelling() {
|
|
4694
5831
|
return materialize(readJournal(journalPath)).runStatus === "cancelling";
|
|
4695
5832
|
}
|
|
4696
5833
|
async function waitForInFlightOrDurableCancellation() {
|
|
4697
5834
|
const active = [...inFlight.values()];
|
|
4698
5835
|
if (active.length === 0) return;
|
|
4699
|
-
|
|
5836
|
+
const revisionSignatureAtStart = revisionSignature(readJournal(journalPath));
|
|
5837
|
+
await new Promise((resolve6, reject) => {
|
|
4700
5838
|
let done = false;
|
|
4701
5839
|
let timer;
|
|
4702
5840
|
const signal = opts.cancelSignal;
|
|
@@ -4705,13 +5843,14 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4705
5843
|
done = true;
|
|
4706
5844
|
if (timer) clearTimeout(timer);
|
|
4707
5845
|
signal?.removeEventListener("abort", finish);
|
|
4708
|
-
|
|
5846
|
+
resolve6();
|
|
4709
5847
|
};
|
|
4710
5848
|
for (const promise of active) promise.then(finish, finish);
|
|
4711
5849
|
const poll = () => {
|
|
4712
5850
|
if (done) return;
|
|
4713
5851
|
try {
|
|
4714
|
-
|
|
5852
|
+
const latest = readJournal(journalPath);
|
|
5853
|
+
if (signal?.aborted || materialize(latest).runStatus === "cancelling" || revisionSignature(latest) !== revisionSignatureAtStart) {
|
|
4715
5854
|
finish();
|
|
4716
5855
|
return;
|
|
4717
5856
|
}
|
|
@@ -4874,10 +6013,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4874
6013
|
return allDrained;
|
|
4875
6014
|
}
|
|
4876
6015
|
function safeAttemptDirFromId(attemptId) {
|
|
4877
|
-
const root =
|
|
4878
|
-
const candidate =
|
|
4879
|
-
const rel =
|
|
4880
|
-
if (!rel ||
|
|
6016
|
+
const root = resolve5(runDir);
|
|
6017
|
+
const candidate = resolve5(root, attemptId);
|
|
6018
|
+
const rel = relative5(root, candidate);
|
|
6019
|
+
if (!rel || isAbsolute5(rel) || rel === ".." || rel.startsWith(`..${sep3}`)) {
|
|
4881
6020
|
throw new Error(`v3 runtime: attempt path escapes run dir (${attemptId})`);
|
|
4882
6021
|
}
|
|
4883
6022
|
return candidate;
|
|
@@ -4914,6 +6053,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4914
6053
|
const snap = materialize([...events]);
|
|
4915
6054
|
if (snap.runStatus === "cancelled") return;
|
|
4916
6055
|
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
6056
|
+
if (openV3ExternalAttempts(events).length > 0) return;
|
|
4917
6057
|
const ids = /* @__PURE__ */ new Set([
|
|
4918
6058
|
...dag.nodes.map((node) => node.id),
|
|
4919
6059
|
...snap.nodes.keys()
|
|
@@ -5000,7 +6140,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5000
6140
|
(event) => event.type === "nodeSucceeded" && (event.instanceId ?? event.nodeId) === key
|
|
5001
6141
|
);
|
|
5002
6142
|
if (!success) throw new Error(`v3 host input source "${nodeId}" has no successful current instance`);
|
|
5003
|
-
const outputDir =
|
|
6143
|
+
const outputDir = join10(dirname8(success.manifestPath), "work");
|
|
5004
6144
|
const verdict = await deps.validateManifest(success.manifestPath, outputDir);
|
|
5005
6145
|
if (!verdict.ok || verdict.manifest?.status !== "ok") {
|
|
5006
6146
|
throw new Error(
|
|
@@ -5012,7 +6152,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5012
6152
|
throw new Error(`v3 host input source "${nodeId}" must expose exactly one result.json (found ${matches.length})`);
|
|
5013
6153
|
}
|
|
5014
6154
|
try {
|
|
5015
|
-
return JSON.parse(
|
|
6155
|
+
return JSON.parse(readFileSync12(join10(outputDir, matches[0].path), "utf-8"));
|
|
5016
6156
|
} catch (err) {
|
|
5017
6157
|
throw new Error(
|
|
5018
6158
|
`v3 host input source "${nodeId}" result.json is unreadable: ${err instanceof Error ? err.message : String(err)}`
|
|
@@ -5068,6 +6208,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5068
6208
|
executionContext
|
|
5069
6209
|
});
|
|
5070
6210
|
const committed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6211
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5071
6212
|
const current = materialize([...latest]);
|
|
5072
6213
|
if (current.runStatus !== "running") return false;
|
|
5073
6214
|
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
@@ -5140,6 +6281,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5140
6281
|
executionContext
|
|
5141
6282
|
});
|
|
5142
6283
|
withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6284
|
+
if (!decisionStillCurrent(latest, node.id)) return;
|
|
5143
6285
|
const current = materialize([...latest]);
|
|
5144
6286
|
if (current.runStatus !== "running") return;
|
|
5145
6287
|
const status = current.nodes.get(node.id)?.status ?? "pending";
|
|
@@ -5238,6 +6380,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5238
6380
|
return blocked;
|
|
5239
6381
|
}
|
|
5240
6382
|
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6383
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5241
6384
|
const current = materialize([...latest]);
|
|
5242
6385
|
if (current.runStatus !== "running") return false;
|
|
5243
6386
|
const state = current.nodes.get(node.id);
|
|
@@ -5355,9 +6498,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5355
6498
|
continue;
|
|
5356
6499
|
}
|
|
5357
6500
|
const attemptDir = safeAttemptDirFromId(effect.attemptId);
|
|
5358
|
-
const existingManifest =
|
|
5359
|
-
if (
|
|
5360
|
-
const validated = await deps.validateManifest(existingManifest,
|
|
6501
|
+
const existingManifest = join10(attemptDir, "manifest.json");
|
|
6502
|
+
if (existsSync10(existingManifest)) {
|
|
6503
|
+
const validated = await deps.validateManifest(existingManifest, join10(attemptDir, "work"));
|
|
5361
6504
|
if (validated.ok && validated.manifest?.status === "ok") {
|
|
5362
6505
|
try {
|
|
5363
6506
|
readAndVerifyV3HostSuccessResult({
|
|
@@ -5634,47 +6777,423 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5634
6777
|
function computeHostCanonicalHash(value) {
|
|
5635
6778
|
return computeInputHash(value);
|
|
5636
6779
|
}
|
|
5637
|
-
function
|
|
6780
|
+
function writeWorkflowDataFile(node, attemptDir) {
|
|
6781
|
+
if (!executionContext) return void 0;
|
|
6782
|
+
const bindings = savedWorkflowBindingsForNode(node);
|
|
6783
|
+
if (bindings.params.length === 0 && bindings.context.length === 0) return void 0;
|
|
6784
|
+
const params = /* @__PURE__ */ Object.create(null);
|
|
6785
|
+
const context = /* @__PURE__ */ Object.create(null);
|
|
6786
|
+
for (const name of bindings.params) {
|
|
6787
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.params, name)) {
|
|
6788
|
+
throw new Error(`v3 runtime: resolved params missing referenced key ${name}`);
|
|
6789
|
+
}
|
|
6790
|
+
params[name] = executionContext.params[name];
|
|
6791
|
+
}
|
|
6792
|
+
for (const name of bindings.context) {
|
|
6793
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.context, name)) {
|
|
6794
|
+
throw new Error(`v3 runtime: resolved context missing referenced key ${name}`);
|
|
6795
|
+
}
|
|
6796
|
+
context[name] = executionContext.context[name];
|
|
6797
|
+
}
|
|
6798
|
+
const workflowDataPath = join10(attemptDir, "workflow-inputs.json");
|
|
6799
|
+
writeFileSync8(
|
|
6800
|
+
workflowDataPath,
|
|
6801
|
+
`${JSON.stringify({ params, context }, null, 2)}
|
|
6802
|
+
`,
|
|
6803
|
+
{ mode: 384 }
|
|
6804
|
+
);
|
|
6805
|
+
return workflowDataPath;
|
|
6806
|
+
}
|
|
6807
|
+
function reportExternalError(error, attemptId, operation) {
|
|
6808
|
+
try {
|
|
6809
|
+
deps.onExternalError?.(error, { attemptId, operation });
|
|
6810
|
+
} catch {
|
|
6811
|
+
}
|
|
6812
|
+
}
|
|
6813
|
+
async function stopExternal(attempt, reason) {
|
|
6814
|
+
const stopped = await stopExternalAttempt(externalExecutor, dag.runId, attempt, reason, externalCallTimeoutMs);
|
|
6815
|
+
if (!stopped.ok) reportExternalError(stopped.error, attempt.attemptId, "stop");
|
|
6816
|
+
}
|
|
6817
|
+
function externalNodeFor(attempt) {
|
|
6818
|
+
return attempt.loop ? instanceNodeFor(attempt.loop) : nodesById.get(attempt.nodeId);
|
|
6819
|
+
}
|
|
6820
|
+
function startExternal(node, botSnap, events, loopRef, omitted, instanceId) {
|
|
6821
|
+
if (!isGoalNode(node)) throw new Error(`v3 runtime: node "${node.id}" is not a goal node`);
|
|
5638
6822
|
const dispatchKey = instanceId ?? node.id;
|
|
5639
6823
|
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
5640
6824
|
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
5641
|
-
const attemptDir =
|
|
5642
|
-
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
5650
|
-
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
6825
|
+
const attemptDir = join10(runDir, dispatchKey, "attempts", attemptNNN);
|
|
6826
|
+
mkdirSync8(attemptDir, { recursive: true });
|
|
6827
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
6828
|
+
writeFileSync8(
|
|
6829
|
+
join10(attemptDir, "inputs.json"),
|
|
6830
|
+
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
6831
|
+
);
|
|
6832
|
+
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6833
|
+
if (!decisionStillCurrent(latest, node.id, loopRef?.loopId)) return false;
|
|
6834
|
+
const current = materialize([...latest]);
|
|
6835
|
+
if (current.runStatus !== "running") return false;
|
|
6836
|
+
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
6837
|
+
const identity = { nodeId: node.id, ...instanceId ? { instanceId } : {}, attemptId };
|
|
6838
|
+
append({
|
|
6839
|
+
type: "nodeDispatched",
|
|
6840
|
+
...identity,
|
|
6841
|
+
loop: loopRef,
|
|
6842
|
+
nodeExecutor: "bot",
|
|
6843
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
6844
|
+
}, { durable: true });
|
|
6845
|
+
append({
|
|
6846
|
+
type: "nodeHandoffPrepared",
|
|
6847
|
+
...identity,
|
|
6848
|
+
targetLarkAppId: current.assignments.get(node.id) ?? botSnap.larkAppId,
|
|
6849
|
+
placement: "topic",
|
|
6850
|
+
revision: dagRevision,
|
|
6851
|
+
idempotency: { nodeCard: randomUUID4(), handoff: randomUUID4() },
|
|
6852
|
+
preparedAt: clock()
|
|
6853
|
+
}, { durable: true });
|
|
6854
|
+
return true;
|
|
6855
|
+
});
|
|
6856
|
+
if (!claimed) return false;
|
|
6857
|
+
const attempt = foldV3ExternalAttempts(readJournal(journalPath)).get(attemptId);
|
|
6858
|
+
if (attempt && !attempt.closed) startHandoffSend(attempt, false);
|
|
6859
|
+
return true;
|
|
6860
|
+
}
|
|
6861
|
+
function appendMissingHandoffIntent(attempt) {
|
|
6862
|
+
const node = externalNodeFor(attempt);
|
|
6863
|
+
const frozen = node ? snapshotForSelector(botSnapshots, node.bot) : void 0;
|
|
6864
|
+
if (!frozen) return false;
|
|
6865
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6866
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
6867
|
+
if (!current.ok || current.attempt.phase !== "dispatched") return false;
|
|
6868
|
+
const dispatched = events.find((event) => event.type === "nodeDispatched" && event.attemptId === attempt.attemptId);
|
|
6869
|
+
append({
|
|
6870
|
+
type: "nodeHandoffPrepared",
|
|
6871
|
+
nodeId: attempt.nodeId,
|
|
6872
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6873
|
+
attemptId: attempt.attemptId,
|
|
6874
|
+
targetLarkAppId: current.snap.assignments.get(attempt.nodeId) ?? frozen.larkAppId,
|
|
6875
|
+
placement: "topic",
|
|
6876
|
+
revision: dispatched?.revision ?? 1,
|
|
6877
|
+
idempotency: { nodeCard: randomUUID4(), handoff: randomUUID4() },
|
|
6878
|
+
preparedAt: clock()
|
|
6879
|
+
}, { durable: true });
|
|
6880
|
+
return true;
|
|
6881
|
+
});
|
|
6882
|
+
}
|
|
6883
|
+
function buildExternalDispatchRequest(attempt, redispatch, signal) {
|
|
6884
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
6885
|
+
const node = externalNodeFor(attempt);
|
|
6886
|
+
if (!ref || !attempt.idempotency || !node || !isGoalNode(node)) {
|
|
6887
|
+
throw new Error(`v3 runtime: external attempt ${attempt.attemptId} has no prepared goal handoff`);
|
|
6888
|
+
}
|
|
6889
|
+
const attemptDir = safeAttemptDirFromId(attempt.attemptId);
|
|
6890
|
+
const inputsPath = join10(attemptDir, "inputs.json");
|
|
6891
|
+
const inputs = JSON.parse(readFileSync12(inputsPath, "utf-8"));
|
|
6892
|
+
const events = readJournal(journalPath);
|
|
6893
|
+
const key = attempt.instanceId ?? attempt.nodeId;
|
|
6894
|
+
const retry = events.find(
|
|
6895
|
+
(event) => event.type === "nodeRetryRequested" && (event.instanceId ?? event.nodeId) === key && event.nextAttemptId === attempt.attemptId
|
|
6896
|
+
);
|
|
6897
|
+
const previousBlock = retry ? [...events].reverse().find(
|
|
6898
|
+
(event) => event.type === "nodeBlocked" && event.attemptId === retry.previousAttemptId
|
|
6899
|
+
) : void 0;
|
|
6900
|
+
const reassigned = [...events].reverse().find(
|
|
6901
|
+
(event) => event.type === "nodeReassigned" && event.nodeId === attempt.nodeId && event.to === ref.targetLarkAppId
|
|
6902
|
+
);
|
|
6903
|
+
const revised = [...events].reverse().find(
|
|
6904
|
+
(event) => event.type === "dagRevised" && event.revision <= ref.revision && (event.effects?.supersede?.some((item) => item.nodeId === attempt.nodeId) === true || event.effects?.assign?.some((item) => item.nodeId === attempt.nodeId) === true)
|
|
6905
|
+
);
|
|
6906
|
+
const revisedAssign = revised?.effects?.assign?.find((item) => item.nodeId === attempt.nodeId && item.to === ref.targetLarkAppId);
|
|
6907
|
+
return {
|
|
6908
|
+
...ref,
|
|
6909
|
+
idempotency: attempt.idempotency,
|
|
6910
|
+
redispatch,
|
|
6911
|
+
node,
|
|
6912
|
+
attemptNumber: attemptNumber(attempt.attemptId) ?? 1,
|
|
6913
|
+
...attempt.loop ? { loop: attempt.loop } : {},
|
|
6914
|
+
inputs,
|
|
6915
|
+
inputsPath,
|
|
6916
|
+
attemptDir,
|
|
6917
|
+
runDir,
|
|
6918
|
+
timeoutSec: node.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC,
|
|
6919
|
+
...opts.chatBinding ? { chatBinding: opts.chatBinding } : {},
|
|
6920
|
+
...retry ? {
|
|
6921
|
+
retryOf: {
|
|
6922
|
+
attemptId: retry.previousAttemptId,
|
|
6923
|
+
...retry.previousErrorCode ? { errorCode: retry.previousErrorCode } : {},
|
|
6924
|
+
...previousBlock?.message ? { message: previousBlock.message } : {}
|
|
6925
|
+
}
|
|
6926
|
+
} : {},
|
|
6927
|
+
...reassigned ? { reassignedFrom: reassigned.from } : revisedAssign?.from ? { reassignedFrom: revisedAssign.from } : {},
|
|
6928
|
+
...revised ? { revisedBy: { revision: revised.revision, summary: revised.summary } } : {},
|
|
6929
|
+
signal
|
|
6930
|
+
};
|
|
6931
|
+
}
|
|
6932
|
+
function startHandoffSend(attempt, redispatch) {
|
|
6933
|
+
const executor = externalExecutor;
|
|
6934
|
+
const key = `${attempt.instanceId ?? attempt.nodeId}::handoff`;
|
|
6935
|
+
if (!executor || inFlight.has(key) || handoffControllers.has(attempt.attemptId)) return;
|
|
6936
|
+
const controller = new AbortController();
|
|
6937
|
+
handoffControllers.set(attempt.attemptId, controller);
|
|
6938
|
+
const p = Promise.resolve().then(() => withExternalCallTimeout(
|
|
6939
|
+
Promise.resolve().then(() => executor.dispatch(buildExternalDispatchRequest(attempt, redispatch, controller.signal))),
|
|
6940
|
+
externalCallTimeoutMs,
|
|
6941
|
+
"dispatch"
|
|
6942
|
+
).catch((err) => {
|
|
6943
|
+
if (!controller.signal.aborted) controller.abort(err);
|
|
6944
|
+
throw err;
|
|
6945
|
+
})).then(async (outcome) => {
|
|
6946
|
+
handoffRetryAt.delete(attempt.attemptId);
|
|
6947
|
+
if (outcome.kind === "rejected") {
|
|
6948
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6949
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
6950
|
+
if (!current.ok || current.attempt.phase !== "prepared") return;
|
|
6951
|
+
append({
|
|
6952
|
+
type: "nodeBlocked",
|
|
6953
|
+
nodeId: attempt.nodeId,
|
|
6954
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6955
|
+
attemptId: attempt.attemptId,
|
|
6956
|
+
errorClass: "workerError",
|
|
6957
|
+
errorCode: outcome.code,
|
|
6958
|
+
message: outcome.message ?? `handoff was not sent (${outcome.code})`
|
|
6959
|
+
}, { durable: true });
|
|
6960
|
+
});
|
|
6961
|
+
return;
|
|
6962
|
+
}
|
|
6963
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
6964
|
+
const sentAt = clock();
|
|
6965
|
+
const recorded = withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6966
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
6967
|
+
if (!latest || latest.closed) return false;
|
|
6968
|
+
if (latest.handoffMessageId !== void 0) return true;
|
|
6969
|
+
append({
|
|
6970
|
+
type: "nodeHandoffSent",
|
|
6971
|
+
nodeId: attempt.nodeId,
|
|
6972
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6973
|
+
attemptId: attempt.attemptId,
|
|
6974
|
+
handoffMessageId: outcome.handoffMessageId,
|
|
6975
|
+
...outcome.nodeCardMessageId ? { nodeCardMessageId: outcome.nodeCardMessageId } : {},
|
|
6976
|
+
...outcome.threadId ? { threadId: outcome.threadId } : {},
|
|
6977
|
+
sentAt,
|
|
6978
|
+
acceptDeadlineAt: sentAt + externalAcceptTimeoutMs
|
|
6979
|
+
}, { durable: true });
|
|
6980
|
+
return true;
|
|
6981
|
+
});
|
|
6982
|
+
if (!recorded) {
|
|
6983
|
+
await stopExternal({ ...attempt, handoffMessageId: outcome.handoffMessageId }, "obsolete");
|
|
6984
|
+
}
|
|
6985
|
+
}).catch((err) => {
|
|
6986
|
+
reportExternalError(err, attempt.attemptId, "dispatch");
|
|
6987
|
+
handoffRetryAt.set(attempt.attemptId, clock() + HANDOFF_RESEND_BACKOFF_MS);
|
|
6988
|
+
}).finally(() => {
|
|
6989
|
+
if (handoffControllers.get(attempt.attemptId) === controller) {
|
|
6990
|
+
handoffControllers.delete(attempt.attemptId);
|
|
6991
|
+
}
|
|
6992
|
+
if (inFlight.get(key) === p) inFlight.delete(key);
|
|
6993
|
+
});
|
|
6994
|
+
inFlight.set(key, p);
|
|
6995
|
+
}
|
|
6996
|
+
async function blockExternalAttempt(attempt, stopReason, verdict) {
|
|
6997
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
6998
|
+
await stopExternal(attempt, stopReason);
|
|
6999
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7000
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
7001
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
7002
|
+
if (!current.ok) return false;
|
|
7003
|
+
append({
|
|
7004
|
+
type: "nodeBlocked",
|
|
7005
|
+
nodeId: attempt.nodeId,
|
|
7006
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7007
|
+
attemptId: attempt.attemptId,
|
|
7008
|
+
...verdict
|
|
7009
|
+
}, { durable: true });
|
|
7010
|
+
return true;
|
|
7011
|
+
});
|
|
7012
|
+
}
|
|
7013
|
+
async function probeExternalAcceptance(attempt) {
|
|
7014
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
7015
|
+
const executor = externalExecutor;
|
|
7016
|
+
if (!executor || !ref || !attempt.handoffMessageId) return { status: "unknown" };
|
|
7017
|
+
const handoffMessageId = attempt.handoffMessageId;
|
|
7018
|
+
try {
|
|
7019
|
+
const probe = await withExternalCallTimeout(
|
|
7020
|
+
Promise.resolve().then(() => executor.probeAcceptance({ ...ref, handoffMessageId })),
|
|
7021
|
+
externalCallTimeoutMs,
|
|
7022
|
+
"probe"
|
|
7023
|
+
);
|
|
7024
|
+
return probe.status === "accepted" ? {
|
|
7025
|
+
status: "accepted",
|
|
7026
|
+
sessionId: probe.sessionId,
|
|
7027
|
+
...probe.acceptedAt !== void 0 ? { acceptedAt: probe.acceptedAt } : {}
|
|
7028
|
+
} : { status: "notAccepted", ...typeof probe.detail === "string" && probe.detail ? { detail: probe.detail.slice(0, 2e3) } : {} };
|
|
7029
|
+
} catch (err) {
|
|
7030
|
+
reportExternalError(err, attempt.attemptId, "probe");
|
|
7031
|
+
return { status: "unknown" };
|
|
7032
|
+
}
|
|
7033
|
+
}
|
|
7034
|
+
async function reconcileExternalAttempts(events) {
|
|
7035
|
+
if (!externalMode) return false;
|
|
7036
|
+
let progressed = false;
|
|
7037
|
+
for (const attempt of openV3ExternalAttempts(events)) {
|
|
7038
|
+
if (handoffControllers.has(attempt.attemptId)) continue;
|
|
7039
|
+
if (!classifyExternalAttempt(events, attempt.attemptId).ok) continue;
|
|
7040
|
+
const now = clock();
|
|
7041
|
+
if (attempt.phase === "dispatched") {
|
|
7042
|
+
progressed = appendMissingHandoffIntent(attempt) || progressed;
|
|
7043
|
+
} else if (attempt.phase === "prepared") {
|
|
7044
|
+
if (now >= attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) {
|
|
7045
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7046
|
+
errorClass: "workerError",
|
|
7047
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7048
|
+
message: "handoff delivery could not be confirmed within the idempotency window; check the chat before retrying"
|
|
7049
|
+
});
|
|
7050
|
+
progressed = true;
|
|
7051
|
+
} else if (now >= (handoffRetryAt.get(attempt.attemptId) ?? 0)) {
|
|
7052
|
+
startHandoffSend(attempt, true);
|
|
5655
7053
|
}
|
|
5656
|
-
|
|
5657
|
-
|
|
5658
|
-
|
|
7054
|
+
} else if (attempt.phase === "sent") {
|
|
7055
|
+
const due = now >= attempt.acceptDeadlineAt;
|
|
7056
|
+
const key = probeKey(attempt.attemptId);
|
|
7057
|
+
const failure = probeFailures.get(key);
|
|
7058
|
+
const probeDue = failure ? now >= failure.retryAt : !acceptanceObserved.has(attempt.attemptId) || due;
|
|
7059
|
+
if (probeDue) {
|
|
7060
|
+
const probe = await probeExternalAcceptance(attempt);
|
|
7061
|
+
progressed = true;
|
|
7062
|
+
if (probe.status === "unknown") {
|
|
7063
|
+
const firstFailedAt = failure?.firstFailedAt ?? now;
|
|
7064
|
+
if (due && now - firstFailedAt >= PROBE_UNCERTAIN_AFTER_MS) {
|
|
7065
|
+
probeFailures.delete(key);
|
|
7066
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7067
|
+
errorClass: "workerError",
|
|
7068
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7069
|
+
message: "whether the target bot accepted the handoff could not be confirmed; check the chat before retrying"
|
|
7070
|
+
});
|
|
7071
|
+
} else {
|
|
7072
|
+
probeFailures.set(key, { firstFailedAt, retryAt: now + PROBE_RETRY_BACKOFF_MS });
|
|
7073
|
+
}
|
|
7074
|
+
} else {
|
|
7075
|
+
probeFailures.delete(key);
|
|
7076
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
7077
|
+
if (probe.status === "accepted") {
|
|
7078
|
+
const { sessionId, acceptedAt } = probe;
|
|
7079
|
+
withJournalMutationSync(journalPath, ({ events: latest, append }) => appendAcceptedLocked(latest, append, dag, {
|
|
7080
|
+
attemptId: attempt.attemptId,
|
|
7081
|
+
sessionId,
|
|
7082
|
+
...acceptedAt !== void 0 ? { acceptedAt } : {}
|
|
7083
|
+
}, clock()));
|
|
7084
|
+
} else if (due) {
|
|
7085
|
+
await blockExternalAttempt(attempt, "notAccepted", {
|
|
7086
|
+
errorClass: "workerError",
|
|
7087
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_NOT_ACCEPTED,
|
|
7088
|
+
message: probe.detail ? `the target bot did not accept the handoff before the acceptance deadline: ${probe.detail}` : "the target bot did not accept the handoff before the acceptance deadline"
|
|
7089
|
+
});
|
|
7090
|
+
}
|
|
5659
7091
|
}
|
|
5660
|
-
context[name] = executionContext.context[name];
|
|
5661
7092
|
}
|
|
5662
|
-
|
|
5663
|
-
|
|
5664
|
-
|
|
5665
|
-
|
|
5666
|
-
|
|
5667
|
-
|
|
5668
|
-
|
|
7093
|
+
} else if (now >= attempt.deadlineAt) {
|
|
7094
|
+
await blockExternalAttempt(attempt, "timeout", {
|
|
7095
|
+
errorClass: "timeout",
|
|
7096
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT,
|
|
7097
|
+
message: "the node did not submit a valid result before its timeout"
|
|
7098
|
+
});
|
|
7099
|
+
progressed = true;
|
|
7100
|
+
}
|
|
7101
|
+
}
|
|
7102
|
+
return progressed;
|
|
7103
|
+
}
|
|
7104
|
+
function settleExternalLeftovers(events) {
|
|
7105
|
+
let progressed = false;
|
|
7106
|
+
for (const attempt of foldV3ExternalAttempts(events).values()) {
|
|
7107
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) {
|
|
7108
|
+
if (settleExternalInstallIntent(runDir, dag, attempt.attemptId).kind !== "none") progressed = true;
|
|
7109
|
+
} else if (attempt.closed) {
|
|
7110
|
+
rmSync2(join10(safeAttemptDirFromId(attempt.attemptId), "submissions"), { recursive: true, force: true });
|
|
5669
7111
|
}
|
|
5670
7112
|
}
|
|
5671
|
-
|
|
7113
|
+
return progressed;
|
|
7114
|
+
}
|
|
7115
|
+
function obsoleteExternalAttempts(events, snap) {
|
|
7116
|
+
return openV3ExternalAttempts(events).filter((attempt) => {
|
|
7117
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
7118
|
+
return state?.status === "cancelled" || state?.status === "superseded";
|
|
7119
|
+
});
|
|
7120
|
+
}
|
|
7121
|
+
async function drainExternalAttempts(attempts, stopReason, drainReason) {
|
|
7122
|
+
for (const attempt of attempts) {
|
|
7123
|
+
await stopExternal(attempt, stopReason);
|
|
7124
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7125
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7126
|
+
if (!latest || latest.closed) return;
|
|
7127
|
+
append({
|
|
7128
|
+
type: "nodeAttemptDrained",
|
|
7129
|
+
nodeId: attempt.nodeId,
|
|
7130
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7131
|
+
attemptId: attempt.attemptId,
|
|
7132
|
+
reason: drainReason
|
|
7133
|
+
}, { durable: true });
|
|
7134
|
+
});
|
|
7135
|
+
}
|
|
7136
|
+
}
|
|
7137
|
+
async function closeExternalAttemptsForCancellation(cancelRequestId) {
|
|
7138
|
+
for (const attempt of openV3ExternalAttempts(readJournal(journalPath))) {
|
|
7139
|
+
await stopExternal(attempt, "runCancelled");
|
|
7140
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7141
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7142
|
+
if (!latest || latest.closed) return;
|
|
7143
|
+
const snap = materialize([...events]);
|
|
7144
|
+
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
7145
|
+
append({
|
|
7146
|
+
type: "nodeAttemptDrained",
|
|
7147
|
+
nodeId: attempt.nodeId,
|
|
7148
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7149
|
+
attemptId: attempt.attemptId,
|
|
7150
|
+
reason: "runCancellation"
|
|
7151
|
+
}, { durable: true });
|
|
7152
|
+
});
|
|
7153
|
+
}
|
|
7154
|
+
}
|
|
7155
|
+
function sentWakeAt(attempt, now) {
|
|
7156
|
+
const failure = probeFailures.get(probeKey(attempt.attemptId));
|
|
7157
|
+
if (!failure) return attempt.acceptDeadlineAt;
|
|
7158
|
+
return attempt.acceptDeadlineAt > now ? Math.min(attempt.acceptDeadlineAt, failure.retryAt) : failure.retryAt;
|
|
7159
|
+
}
|
|
7160
|
+
function awaitingExternalOutcome(open, pendingWaits) {
|
|
7161
|
+
const now = clock();
|
|
7162
|
+
const externalAttempts = open.map((attempt) => {
|
|
7163
|
+
const wakeAt = attempt.phase === "dispatched" ? now : attempt.phase === "prepared" ? Math.min(handoffRetryAt.get(attempt.attemptId) ?? now, attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) : attempt.phase === "sent" ? sentWakeAt(attempt, now) : attempt.deadlineAt;
|
|
7164
|
+
return {
|
|
7165
|
+
nodeId: attempt.nodeId,
|
|
7166
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7167
|
+
attemptId: attempt.attemptId,
|
|
7168
|
+
phase: attempt.phase,
|
|
7169
|
+
...attempt.targetLarkAppId ? { targetLarkAppId: attempt.targetLarkAppId } : {},
|
|
7170
|
+
...wakeAt !== void 0 ? { wakeAt } : {}
|
|
7171
|
+
};
|
|
7172
|
+
});
|
|
7173
|
+
const wakes = externalAttempts.flatMap((attempt) => attempt.wakeAt !== void 0 ? [attempt.wakeAt] : []);
|
|
7174
|
+
return {
|
|
7175
|
+
reason: "awaitingExternal",
|
|
7176
|
+
pendingWaits,
|
|
7177
|
+
externalAttempts,
|
|
7178
|
+
...wakes.length > 0 ? { wakeAt: Math.min(...wakes) } : {},
|
|
7179
|
+
runDir
|
|
7180
|
+
};
|
|
7181
|
+
}
|
|
7182
|
+
function startWork(node, botSnap, botKey, events, loopRef, omitted, instanceId) {
|
|
7183
|
+
const dispatchKey = instanceId ?? node.id;
|
|
7184
|
+
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
7185
|
+
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
7186
|
+
const attemptDir = join10(runDir, dispatchKey, "attempts", attemptNNN);
|
|
7187
|
+
const outputDir = join10(attemptDir, "work");
|
|
7188
|
+
mkdirSync8(outputDir, { recursive: true });
|
|
7189
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
7190
|
+
const goalPath = join10(attemptDir, "goal.txt");
|
|
5672
7191
|
const loopCtx = loopRef ? {
|
|
5673
7192
|
loopId: loopRef.loopId,
|
|
5674
7193
|
iteration: loopRef.iteration,
|
|
5675
7194
|
maxIterations: nodesById.get(loopRef.loopId).maxIterations
|
|
5676
7195
|
} : void 0;
|
|
5677
|
-
|
|
7196
|
+
writeFileSync8(
|
|
5678
7197
|
goalPath,
|
|
5679
7198
|
renderGoalFile(
|
|
5680
7199
|
node.goal ?? "",
|
|
@@ -5686,12 +7205,12 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5686
7205
|
)
|
|
5687
7206
|
);
|
|
5688
7207
|
const effSnap = mergeNodeCapability(botSnap, node.override);
|
|
5689
|
-
const inputsPath =
|
|
5690
|
-
|
|
7208
|
+
const inputsPath = join10(attemptDir, "inputs.json");
|
|
7209
|
+
writeFileSync8(
|
|
5691
7210
|
inputsPath,
|
|
5692
7211
|
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
5693
7212
|
);
|
|
5694
|
-
const manifestPath =
|
|
7213
|
+
const manifestPath = join10(attemptDir, "manifest.json");
|
|
5695
7214
|
const env = {
|
|
5696
7215
|
[GOAL_ENV.GOAL_PATH]: goalPath,
|
|
5697
7216
|
[GOAL_ENV.INPUTS_PATH]: inputsPath,
|
|
@@ -5705,7 +7224,8 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5705
7224
|
nodeId: node.id,
|
|
5706
7225
|
...instanceId ? { instanceId } : {},
|
|
5707
7226
|
attemptId,
|
|
5708
|
-
loop: loopRef
|
|
7227
|
+
loop: loopRef,
|
|
7228
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
5709
7229
|
})) return false;
|
|
5710
7230
|
const leaseBinding = {
|
|
5711
7231
|
attemptDir,
|
|
@@ -5911,7 +7431,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5911
7431
|
});
|
|
5912
7432
|
return;
|
|
5913
7433
|
}
|
|
5914
|
-
const res = validateResult(
|
|
7434
|
+
const res = validateResult(join10(outputDir, entry.path), node.resultSchema);
|
|
5915
7435
|
if (!res.ok) {
|
|
5916
7436
|
appendWorkerOutcome({
|
|
5917
7437
|
type: "nodeBlocked",
|
|
@@ -5971,7 +7491,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5971
7491
|
}
|
|
5972
7492
|
const kind = classifyTerminal(errorClass, { selfReportedFail, retryable });
|
|
5973
7493
|
if (kind === "blocked") {
|
|
5974
|
-
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk(
|
|
7494
|
+
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk(join10(attemptDir, GOAL_ASK_FILE)) : void 0;
|
|
5975
7495
|
appendWorkerOutcome({
|
|
5976
7496
|
type: "nodeBlocked",
|
|
5977
7497
|
nodeId: node.id,
|
|
@@ -6028,58 +7548,19 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
6028
7548
|
inFlight.set(dispatchKey, p);
|
|
6029
7549
|
return true;
|
|
6030
7550
|
}
|
|
6031
|
-
function affectedNodesFrom(root) {
|
|
6032
|
-
const reachable = /* @__PURE__ */ new Set([root]);
|
|
6033
|
-
let changed = true;
|
|
6034
|
-
while (changed) {
|
|
6035
|
-
changed = false;
|
|
6036
|
-
for (const n of dag.nodes) {
|
|
6037
|
-
if (reachable.has(n.id)) continue;
|
|
6038
|
-
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
6039
|
-
reachable.add(n.id);
|
|
6040
|
-
changed = true;
|
|
6041
|
-
}
|
|
6042
|
-
}
|
|
6043
|
-
}
|
|
6044
|
-
return [...reachable];
|
|
6045
|
-
}
|
|
6046
7551
|
function appendRevisitEvents(nodeId, instanceId, attemptId, request, sourceManifestPath) {
|
|
6047
|
-
const
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
let reasonPathRel;
|
|
6052
|
-
if (request.reason) {
|
|
6053
|
-
const dir = join6(runDir, "revisits");
|
|
6054
|
-
mkdirSync6(dir, { recursive: true });
|
|
6055
|
-
const abs = join6(dir, `${instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
6056
|
-
writeFileSync6(abs, `# Revisit reason (from ${nodeId} / ${instanceId})
|
|
6057
|
-
|
|
6058
|
-
${request.reason}
|
|
6059
|
-
`);
|
|
6060
|
-
reasonPathRel = relative3(runDir, abs);
|
|
6061
|
-
}
|
|
6062
|
-
appendEvent(journalPath, {
|
|
6063
|
-
type: "nodeRevisitRequested",
|
|
7552
|
+
const plan = planV3Revisit({
|
|
7553
|
+
runDir,
|
|
7554
|
+
dag,
|
|
7555
|
+
events: readJournal(journalPath),
|
|
6064
7556
|
nodeId,
|
|
6065
7557
|
instanceId,
|
|
6066
7558
|
attemptId,
|
|
6067
|
-
|
|
6068
|
-
|
|
6069
|
-
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
6070
|
-
sourceManifestPath: relative3(runDir, sourceManifestPath),
|
|
6071
|
-
...targetSucc ? { targetPreviousManifestPath: relative3(runDir, targetSucc.manifestPath) } : {}
|
|
7559
|
+
request,
|
|
7560
|
+
sourceManifestPath
|
|
6072
7561
|
});
|
|
6073
|
-
for (const
|
|
6074
|
-
|
|
6075
|
-
if (!eff) continue;
|
|
6076
|
-
appendEvent(journalPath, {
|
|
6077
|
-
type: "nodeInstanceSuperseded",
|
|
6078
|
-
nodeId: affectedNodeId,
|
|
6079
|
-
instanceId: eff,
|
|
6080
|
-
byNodeId: request.toNodeId,
|
|
6081
|
-
reason: "refresh"
|
|
6082
|
-
});
|
|
7562
|
+
for (const event of plan.events) appendEvent(journalPath, event);
|
|
7563
|
+
for (const eff of plan.supersededInstanceIds) {
|
|
6083
7564
|
const controller = nodeControllers.get(eff);
|
|
6084
7565
|
if (controller && !controller.signal.aborted) {
|
|
6085
7566
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6201,11 +7682,11 @@ ${request.reason}
|
|
|
6201
7682
|
}
|
|
6202
7683
|
function applyLoopControl(a) {
|
|
6203
7684
|
if (a.kind === "startLoop") {
|
|
6204
|
-
|
|
7685
|
+
appendControlEvent({ type: "loopStarted", loopId: a.loopId });
|
|
6205
7686
|
return;
|
|
6206
7687
|
}
|
|
6207
7688
|
if (a.kind === "startLoopIteration") {
|
|
6208
|
-
|
|
7689
|
+
appendControlEvent({ type: "loopIterationStarted", loopId: a.loopId, iteration: a.iteration });
|
|
6209
7690
|
return;
|
|
6210
7691
|
}
|
|
6211
7692
|
const loopNode = nodesById.get(a.loopId);
|
|
@@ -6214,7 +7695,7 @@ ${request.reason}
|
|
|
6214
7695
|
const outInstId = loopInstanceId(a.loopId, a.iteration, loopNode.output.from);
|
|
6215
7696
|
const succ2 = [...events2].reverse().find((e) => e.type === "nodeSucceeded" && e.nodeId === outInstId);
|
|
6216
7697
|
if (!succ2) {
|
|
6217
|
-
|
|
7698
|
+
appendControlEvent({
|
|
6218
7699
|
type: "nodeFailed",
|
|
6219
7700
|
nodeId: a.loopId,
|
|
6220
7701
|
attemptId: `${a.loopId}/iterations/${String(a.iteration).padStart(3, "0")}`,
|
|
@@ -6223,7 +7704,7 @@ ${request.reason}
|
|
|
6223
7704
|
});
|
|
6224
7705
|
return;
|
|
6225
7706
|
}
|
|
6226
|
-
|
|
7707
|
+
appendControlEvent({
|
|
6227
7708
|
type: "nodeSucceeded",
|
|
6228
7709
|
nodeId: a.loopId,
|
|
6229
7710
|
attemptId: succ2.attemptId,
|
|
@@ -6239,11 +7720,11 @@ ${request.reason}
|
|
|
6239
7720
|
let observed = `${loopNode.exit.when.path}=<unreadable>`;
|
|
6240
7721
|
if (succ) {
|
|
6241
7722
|
try {
|
|
6242
|
-
const manifest = JSON.parse(
|
|
7723
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6243
7724
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6244
7725
|
if (entry) {
|
|
6245
7726
|
const result = JSON.parse(
|
|
6246
|
-
|
|
7727
|
+
readFileSync12(join10(dirname8(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6247
7728
|
);
|
|
6248
7729
|
observed = `${loopNode.exit.when.path}=${JSON.stringify(result[key])}`;
|
|
6249
7730
|
matched = matchLoopExitWhen(loopNode.exit.when, result[key]);
|
|
@@ -6257,7 +7738,7 @@ ${request.reason}
|
|
|
6257
7738
|
const effectiveMax = loopNode.maxIterations + granted;
|
|
6258
7739
|
const anomalous = observed.endsWith("<unreadable>");
|
|
6259
7740
|
const decision = matched ? "exit" : !anomalous && a.iteration < effectiveMax ? "continue" : "exhausted";
|
|
6260
|
-
|
|
7741
|
+
appendControlEvent({
|
|
6261
7742
|
type: "loopIterationDecision",
|
|
6262
7743
|
loopId: a.loopId,
|
|
6263
7744
|
iteration: a.iteration,
|
|
@@ -6274,7 +7755,7 @@ ${request.reason}
|
|
|
6274
7755
|
const target = nodesById.get(a.to);
|
|
6275
7756
|
const dep = target?.depends.find((d) => d.from === a.from);
|
|
6276
7757
|
if (!target || !dep?.when) {
|
|
6277
|
-
|
|
7758
|
+
appendControlEvent({
|
|
6278
7759
|
type: "edgeResolved",
|
|
6279
7760
|
from: a.from,
|
|
6280
7761
|
to: a.to,
|
|
@@ -6292,11 +7773,11 @@ ${request.reason}
|
|
|
6292
7773
|
let detail = `${dep.when.path}=<unreadable>`;
|
|
6293
7774
|
if (succ) {
|
|
6294
7775
|
try {
|
|
6295
|
-
const manifest = JSON.parse(
|
|
7776
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6296
7777
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6297
7778
|
if (entry) {
|
|
6298
7779
|
const result = JSON.parse(
|
|
6299
|
-
|
|
7780
|
+
readFileSync12(join10(dirname8(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6300
7781
|
);
|
|
6301
7782
|
detail = `${dep.when.path}=${JSON.stringify(result[key])}`;
|
|
6302
7783
|
active = matchLoopExitWhen(dep.when, result[key]);
|
|
@@ -6304,7 +7785,7 @@ ${request.reason}
|
|
|
6304
7785
|
} catch {
|
|
6305
7786
|
}
|
|
6306
7787
|
}
|
|
6307
|
-
|
|
7788
|
+
appendControlEvent({
|
|
6308
7789
|
type: "edgeResolved",
|
|
6309
7790
|
from: a.from,
|
|
6310
7791
|
to: a.to,
|
|
@@ -6321,7 +7802,7 @@ ${request.reason}
|
|
|
6321
7802
|
if (status !== "pending" && status !== "gateWaiting" && status !== "running") return false;
|
|
6322
7803
|
const instanceId = snap.nodes.get(a.nodeId)?.effectiveInstanceId;
|
|
6323
7804
|
const attemptId = latestAttemptIdFor(events, instanceId ?? a.nodeId);
|
|
6324
|
-
|
|
7805
|
+
const appended = appendControlEvent({
|
|
6325
7806
|
type: "nodeCancelled",
|
|
6326
7807
|
nodeId: a.nodeId,
|
|
6327
7808
|
...instanceId ? { instanceId } : {},
|
|
@@ -6330,6 +7811,7 @@ ${request.reason}
|
|
|
6330
7811
|
byNodeId: a.byNodeId,
|
|
6331
7812
|
detail: a.detail
|
|
6332
7813
|
});
|
|
7814
|
+
if (!appended) return true;
|
|
6333
7815
|
const controller = nodeControllers.get(instanceId ?? a.nodeId);
|
|
6334
7816
|
if (controller && !controller.signal.aborted) {
|
|
6335
7817
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6399,8 +7881,8 @@ ${request.reason}
|
|
|
6399
7881
|
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
6400
7882
|
const succ = latestSuccess(key);
|
|
6401
7883
|
if (!succ) return;
|
|
6402
|
-
const upstreamOutputDir =
|
|
6403
|
-
const manifest = JSON.parse(
|
|
7884
|
+
const upstreamOutputDir = join10(dirname8(succ.manifestPath), "work");
|
|
7885
|
+
const manifest = JSON.parse(readFileSync12(succ.manifestPath, "utf-8"));
|
|
6404
7886
|
for (const f of manifest.files) {
|
|
6405
7887
|
if (filter && !filter(f)) continue;
|
|
6406
7888
|
inputs.push({
|
|
@@ -6408,29 +7890,29 @@ ${request.reason}
|
|
|
6408
7890
|
...output ? { output } : {},
|
|
6409
7891
|
...succ.instanceId ? { instanceId: succ.instanceId } : {},
|
|
6410
7892
|
name: f.name,
|
|
6411
|
-
path:
|
|
7893
|
+
path: join10(upstreamOutputDir, f.path),
|
|
6412
7894
|
kind: f.kind,
|
|
6413
7895
|
preview: f.preview
|
|
6414
7896
|
});
|
|
6415
7897
|
}
|
|
6416
7898
|
};
|
|
6417
|
-
const resolveRunPath = (p) =>
|
|
7899
|
+
const resolveRunPath = (p) => isAbsolute5(p) ? p : join10(runDir, p);
|
|
6418
7900
|
const pushManifestByPath = (label, manifestPath, namePrefix) => {
|
|
6419
7901
|
if (!manifestPath) return;
|
|
6420
7902
|
const abs = resolveRunPath(manifestPath);
|
|
6421
|
-
if (!
|
|
7903
|
+
if (!existsSync10(abs)) return;
|
|
6422
7904
|
let manifest;
|
|
6423
7905
|
try {
|
|
6424
|
-
manifest = JSON.parse(
|
|
7906
|
+
manifest = JSON.parse(readFileSync12(abs, "utf-8"));
|
|
6425
7907
|
} catch {
|
|
6426
7908
|
return;
|
|
6427
7909
|
}
|
|
6428
|
-
const outDir =
|
|
7910
|
+
const outDir = join10(dirname8(abs), "work");
|
|
6429
7911
|
for (const f of manifest.files) {
|
|
6430
7912
|
inputs.push({
|
|
6431
7913
|
from: label,
|
|
6432
7914
|
name: `${namePrefix}:${f.name}`,
|
|
6433
|
-
path:
|
|
7915
|
+
path: join10(outDir, f.path),
|
|
6434
7916
|
kind: f.kind,
|
|
6435
7917
|
preview: f.preview
|
|
6436
7918
|
});
|
|
@@ -6500,7 +7982,7 @@ ${request.reason}
|
|
|
6500
7982
|
}
|
|
6501
7983
|
|
|
6502
7984
|
// src/workflows/v3/portable-runtime.ts
|
|
6503
|
-
import { join as
|
|
7985
|
+
import { join as join11 } from "node:path";
|
|
6504
7986
|
function defaultValidateProfile(profile, selector) {
|
|
6505
7987
|
if (!profile.profileId || !profile.executorId || !profile.workingDirectory) {
|
|
6506
7988
|
throw new Error(
|
|
@@ -6549,7 +8031,7 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6549
8031
|
validateProfile(executionProfile, selector);
|
|
6550
8032
|
}
|
|
6551
8033
|
const frozenDefinition = assertOrCreatePortableRunSnapshot(
|
|
6552
|
-
|
|
8034
|
+
join11(options.baseDir, normalizedDag.runId),
|
|
6553
8035
|
normalizedDag,
|
|
6554
8036
|
profiles
|
|
6555
8037
|
);
|
|
@@ -6627,6 +8109,9 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6627
8109
|
])
|
|
6628
8110
|
)
|
|
6629
8111
|
});
|
|
8112
|
+
if (outcome.reason === "awaitingExternal") {
|
|
8113
|
+
throw new Error("portable workflow runtime does not drive bot-executed goal nodes");
|
|
8114
|
+
}
|
|
6630
8115
|
if (outcome.reason !== "terminal") return outcome;
|
|
6631
8116
|
return {
|
|
6632
8117
|
...outcome,
|