botmux-workflow-core 3.56.0 → 3.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/control.cjs +9 -0
- package/dist/cjs/control.cjs.map +2 -2
- package/dist/cjs/index.cjs +9 -0
- package/dist/cjs/index.cjs.map +2 -2
- package/dist/cjs/runtime.cjs +1895 -419
- package/dist/cjs/runtime.cjs.map +4 -4
- package/dist/esm/control.js +9 -0
- package/dist/esm/control.js.map +2 -2
- package/dist/esm/index.js +9 -0
- package/dist/esm/index.js.map +2 -2
- package/dist/esm/runtime.js +1900 -415
- package/dist/esm/runtime.js.map +4 -4
- package/dist/types/src/workflows/v3/event-contract.d.ts +205 -0
- package/package.json +1 -1
package/dist/cjs/runtime.cjs
CHANGED
|
@@ -2603,6 +2603,7 @@ function materialize(events) {
|
|
|
2603
2603
|
const attempts = /* @__PURE__ */ new Map();
|
|
2604
2604
|
const loops = /* @__PURE__ */ new Map();
|
|
2605
2605
|
const edges = /* @__PURE__ */ new Map();
|
|
2606
|
+
const assignments = /* @__PURE__ */ new Map();
|
|
2606
2607
|
let runStatus = "running";
|
|
2607
2608
|
let cancelRequestId;
|
|
2608
2609
|
let cancelRequestedBy;
|
|
@@ -2612,6 +2613,7 @@ function materialize(events) {
|
|
|
2612
2613
|
let blockedNodeId;
|
|
2613
2614
|
let uncertainHostEffects;
|
|
2614
2615
|
const cancelledAttempts = /* @__PURE__ */ new Map();
|
|
2616
|
+
const regateInstances = /* @__PURE__ */ new Set();
|
|
2615
2617
|
const resolvedWaitIds = /* @__PURE__ */ new Set();
|
|
2616
2618
|
const set = (id, status, gateCleared) => {
|
|
2617
2619
|
const prev = nodes.get(id);
|
|
@@ -2632,6 +2634,16 @@ function materialize(events) {
|
|
|
2632
2634
|
nodes.set(nodeId, { ...prev, status });
|
|
2633
2635
|
}
|
|
2634
2636
|
};
|
|
2637
|
+
const supersedeInstance = (nodeId, instanceId) => {
|
|
2638
|
+
instances.set(instanceId, { status: "superseded" });
|
|
2639
|
+
if (nodes.get(nodeId)?.effectiveInstanceId === instanceId) {
|
|
2640
|
+
nodes.set(nodeId, { status: "pending" });
|
|
2641
|
+
}
|
|
2642
|
+
if (runStatus === "blocked" && blockedNodeId === nodeId) {
|
|
2643
|
+
runStatus = "running";
|
|
2644
|
+
blockedNodeId = void 0;
|
|
2645
|
+
}
|
|
2646
|
+
};
|
|
2635
2647
|
const loop = (id) => {
|
|
2636
2648
|
let ls = loops.get(id);
|
|
2637
2649
|
if (!ls) {
|
|
@@ -2664,6 +2676,7 @@ function materialize(events) {
|
|
|
2664
2676
|
attempts.set(e.instanceId ?? e.nodeId, e.attemptId);
|
|
2665
2677
|
break;
|
|
2666
2678
|
case "hostInputPrepared":
|
|
2679
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2667
2680
|
instances.set(e.instanceId, { status: "pending" });
|
|
2668
2681
|
nodes.set(e.nodeId, {
|
|
2669
2682
|
status: "pending",
|
|
@@ -2673,6 +2686,7 @@ function materialize(events) {
|
|
|
2673
2686
|
break;
|
|
2674
2687
|
case "hostEffectIntent":
|
|
2675
2688
|
if (instances.get(e.instanceId)?.status === "cancelled") break;
|
|
2689
|
+
if (instances.get(e.instanceId)?.status === "superseded") break;
|
|
2676
2690
|
recordInstance(e.nodeId, e.instanceId, "running", true);
|
|
2677
2691
|
attempts.set(e.instanceId, e.attemptId);
|
|
2678
2692
|
break;
|
|
@@ -2712,6 +2726,15 @@ function materialize(events) {
|
|
|
2712
2726
|
} else set(e.nodeId, "blocked");
|
|
2713
2727
|
break;
|
|
2714
2728
|
case "nodeRetryRequested":
|
|
2729
|
+
if (e.instanceId && regateInstances.has(e.instanceId)) {
|
|
2730
|
+
regateInstances.delete(e.instanceId);
|
|
2731
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2732
|
+
if (runStatus === "blocked") {
|
|
2733
|
+
runStatus = "running";
|
|
2734
|
+
blockedNodeId = void 0;
|
|
2735
|
+
}
|
|
2736
|
+
break;
|
|
2737
|
+
}
|
|
2715
2738
|
if (e.resetGate) {
|
|
2716
2739
|
nodes.set(e.nodeId, {
|
|
2717
2740
|
status: "pending",
|
|
@@ -2731,15 +2754,40 @@ function materialize(events) {
|
|
|
2731
2754
|
// ── cross-node revisit / instance refresh ──
|
|
2732
2755
|
case "nodeRevisitRequested":
|
|
2733
2756
|
break;
|
|
2734
|
-
case "nodeInstanceSuperseded":
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2757
|
+
case "nodeInstanceSuperseded":
|
|
2758
|
+
supersedeInstance(e.nodeId, e.instanceId);
|
|
2759
|
+
break;
|
|
2760
|
+
case "dagRevised": {
|
|
2761
|
+
const effects = e.effects;
|
|
2762
|
+
for (const item of effects?.supersede ?? []) {
|
|
2763
|
+
if (item.waitId) resolvedWaitIds.add(item.waitId);
|
|
2764
|
+
supersedeInstance(item.nodeId, item.instanceId);
|
|
2765
|
+
}
|
|
2766
|
+
for (const item of effects?.cancel ?? []) {
|
|
2767
|
+
cancelledAttempts.set(item.instanceId ?? item.nodeId, item.attemptId);
|
|
2768
|
+
if (item.instanceId) instances.set(item.instanceId, { status: "cancelled" });
|
|
2769
|
+
const prev = nodes.get(item.nodeId);
|
|
2770
|
+
nodes.set(item.nodeId, {
|
|
2771
|
+
status: "cancelled",
|
|
2772
|
+
...item.instanceId ?? prev?.effectiveInstanceId ? { effectiveInstanceId: item.instanceId ?? prev?.effectiveInstanceId } : {}
|
|
2773
|
+
});
|
|
2738
2774
|
}
|
|
2739
|
-
if (runStatus === "blocked" &&
|
|
2775
|
+
if (runStatus === "blocked" && (effects?.cancel ?? []).length > 0 && ((effects?.cancel ?? []).some((item) => item.nodeId === blockedNodeId) || ![...nodes.values()].some((state) => state.status === "blocked"))) {
|
|
2740
2776
|
runStatus = "running";
|
|
2741
2777
|
blockedNodeId = void 0;
|
|
2742
2778
|
}
|
|
2779
|
+
for (const item of effects?.regate ?? []) {
|
|
2780
|
+
const prev = nodes.get(item.nodeId);
|
|
2781
|
+
if (prev?.effectiveInstanceId !== item.instanceId) continue;
|
|
2782
|
+
const { gateCleared: _dropped, ...rest } = prev;
|
|
2783
|
+
nodes.set(item.nodeId, rest);
|
|
2784
|
+
const instance = instances.get(item.instanceId);
|
|
2785
|
+
if (instance) {
|
|
2786
|
+
const { gateCleared: _instanceDropped, ...instanceRest } = instance;
|
|
2787
|
+
instances.set(item.instanceId, instanceRest);
|
|
2788
|
+
}
|
|
2789
|
+
regateInstances.add(item.instanceId);
|
|
2790
|
+
}
|
|
2743
2791
|
break;
|
|
2744
2792
|
}
|
|
2745
2793
|
case "gateDispatched":
|
|
@@ -2797,6 +2845,32 @@ function materialize(events) {
|
|
|
2797
2845
|
if (e.instanceId) recordInstance(e.nodeId, e.instanceId, "cancelled", false);
|
|
2798
2846
|
else set(e.nodeId, "cancelled");
|
|
2799
2847
|
break;
|
|
2848
|
+
case "nodeHandoffPrepared":
|
|
2849
|
+
case "nodeHandoffSent":
|
|
2850
|
+
case "nodeHandoffAccepted":
|
|
2851
|
+
case "nodeStalled":
|
|
2852
|
+
break;
|
|
2853
|
+
case "nodeReassigned": {
|
|
2854
|
+
assignments.set(e.nodeId, e.to);
|
|
2855
|
+
if (!e.attemptId || runStatus !== "running" && runStatus !== "blocked") break;
|
|
2856
|
+
const key = e.instanceId ?? e.nodeId;
|
|
2857
|
+
if (attempts.get(key) !== e.attemptId) break;
|
|
2858
|
+
if (e.instanceId) {
|
|
2859
|
+
if (instances.get(e.instanceId)?.status !== "running") break;
|
|
2860
|
+
instances.set(e.instanceId, { status: "pending" });
|
|
2861
|
+
const current = nodes.get(e.nodeId);
|
|
2862
|
+
if (current?.effectiveInstanceId === e.instanceId && current.status === "running") {
|
|
2863
|
+
nodes.set(e.nodeId, {
|
|
2864
|
+
status: "pending",
|
|
2865
|
+
effectiveInstanceId: e.instanceId,
|
|
2866
|
+
...current.gateCleared ? { gateCleared: true } : {}
|
|
2867
|
+
});
|
|
2868
|
+
}
|
|
2869
|
+
} else if (nodes.get(e.nodeId)?.status === "running") {
|
|
2870
|
+
set(e.nodeId, "pending");
|
|
2871
|
+
}
|
|
2872
|
+
break;
|
|
2873
|
+
}
|
|
2800
2874
|
case "nodeAttemptDrained": {
|
|
2801
2875
|
if (runStatus !== "running" && runStatus !== "blocked") break;
|
|
2802
2876
|
const key = e.instanceId ?? e.nodeId;
|
|
@@ -2891,7 +2965,9 @@ function materialize(events) {
|
|
|
2891
2965
|
instances,
|
|
2892
2966
|
attempts,
|
|
2893
2967
|
loops,
|
|
2894
|
-
edges
|
|
2968
|
+
edges,
|
|
2969
|
+
assignments,
|
|
2970
|
+
...regateInstances.size > 0 ? { regatePending: regateInstances } : {}
|
|
2895
2971
|
};
|
|
2896
2972
|
}
|
|
2897
2973
|
function writeState(statePath, snap) {
|
|
@@ -2911,6 +2987,7 @@ function writeState(statePath, snap) {
|
|
|
2911
2987
|
attempts: Object.fromEntries(snap.attempts),
|
|
2912
2988
|
loops: snap.loops.size > 0 ? Object.fromEntries(snap.loops) : void 0,
|
|
2913
2989
|
edges: snap.edges.size > 0 ? Object.fromEntries(snap.edges) : void 0,
|
|
2990
|
+
assignments: snap.assignments.size > 0 ? Object.fromEntries(snap.assignments) : void 0,
|
|
2914
2991
|
updatedAt: Date.now()
|
|
2915
2992
|
};
|
|
2916
2993
|
const tmp = `${statePath}.tmp`;
|
|
@@ -3257,9 +3334,9 @@ function assertOrCreatePortableRunSnapshot(runDir, dag, profiles) {
|
|
|
3257
3334
|
}
|
|
3258
3335
|
|
|
3259
3336
|
// src/workflows/v3/shared-node-runtime.ts
|
|
3260
|
-
var
|
|
3261
|
-
var
|
|
3262
|
-
var
|
|
3337
|
+
var import_node_crypto9 = require("node:crypto");
|
|
3338
|
+
var import_node_fs15 = require("node:fs");
|
|
3339
|
+
var import_node_path13 = require("node:path");
|
|
3263
3340
|
|
|
3264
3341
|
// src/workflows/v3/core-control.ts
|
|
3265
3342
|
function matchLoopExitWhen(when, value) {
|
|
@@ -3281,6 +3358,15 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3281
3358
|
if (event.type === "nodeRevisitRequested") {
|
|
3282
3359
|
runUsed++;
|
|
3283
3360
|
if (event.nodeId === sourceNodeId && event.toNodeId === toNodeId) pairUsed++;
|
|
3361
|
+
} else if (event.type === "dagRevised") {
|
|
3362
|
+
for (const nodeId of event.effects?.rerun ?? []) {
|
|
3363
|
+
runUsed++;
|
|
3364
|
+
if (nodeId === sourceNodeId && nodeId === toNodeId) pairUsed++;
|
|
3365
|
+
}
|
|
3366
|
+
for (const grant of event.effects?.budgetGrants ?? []) {
|
|
3367
|
+
if (grant.tier === "run") runGranted++;
|
|
3368
|
+
else if (grant.nodeId === sourceNodeId && grant.nodeId === toNodeId) pairGranted++;
|
|
3369
|
+
}
|
|
3284
3370
|
} else if (event.type === "revisitBudgetGranted") {
|
|
3285
3371
|
if (event.sourceNodeId === sourceNodeId && event.toNodeId === toNodeId) pairGranted++;
|
|
3286
3372
|
else if (event.sourceNodeId === void 0 && event.toNodeId === void 0) runGranted++;
|
|
@@ -3305,6 +3391,72 @@ function revisitBudgetStatus(events, sourceNodeId, toNodeId) {
|
|
|
3305
3391
|
return { ok: true };
|
|
3306
3392
|
}
|
|
3307
3393
|
|
|
3394
|
+
// src/workflows/v3/revisit-plan.ts
|
|
3395
|
+
var import_node_fs9 = require("node:fs");
|
|
3396
|
+
var import_node_path7 = require("node:path");
|
|
3397
|
+
function affectedNodesFrom(dag, root) {
|
|
3398
|
+
const reachable = /* @__PURE__ */ new Set([root]);
|
|
3399
|
+
let changed = true;
|
|
3400
|
+
while (changed) {
|
|
3401
|
+
changed = false;
|
|
3402
|
+
for (const n of dag.nodes) {
|
|
3403
|
+
if (reachable.has(n.id)) continue;
|
|
3404
|
+
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
3405
|
+
reachable.add(n.id);
|
|
3406
|
+
changed = true;
|
|
3407
|
+
}
|
|
3408
|
+
}
|
|
3409
|
+
}
|
|
3410
|
+
return [...reachable];
|
|
3411
|
+
}
|
|
3412
|
+
function planV3Revisit(input) {
|
|
3413
|
+
const { runDir, request } = input;
|
|
3414
|
+
const snap = materialize([...input.events]);
|
|
3415
|
+
const targetEff = snap.nodes.get(request.toNodeId)?.effectiveInstanceId;
|
|
3416
|
+
const targetSucc = targetEff ? [...input.events].reverse().find((e) => e.type === "nodeSucceeded" && (e.instanceId ?? e.nodeId) === targetEff) : void 0;
|
|
3417
|
+
let reasonPathRel;
|
|
3418
|
+
if (request.reason) {
|
|
3419
|
+
const dir = (0, import_node_path7.join)(runDir, "revisits");
|
|
3420
|
+
(0, import_node_fs9.mkdirSync)(dir, { recursive: true });
|
|
3421
|
+
const abs = (0, import_node_path7.join)(dir, `${input.instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
3422
|
+
(0, import_node_fs9.writeFileSync)(abs, `# Revisit reason (from ${input.nodeId} / ${input.instanceId})
|
|
3423
|
+
|
|
3424
|
+
${request.reason}
|
|
3425
|
+
`);
|
|
3426
|
+
reasonPathRel = (0, import_node_path7.relative)(runDir, abs);
|
|
3427
|
+
}
|
|
3428
|
+
const events = [{
|
|
3429
|
+
type: "nodeRevisitRequested",
|
|
3430
|
+
nodeId: input.nodeId,
|
|
3431
|
+
instanceId: input.instanceId,
|
|
3432
|
+
attemptId: input.attemptId,
|
|
3433
|
+
toNodeId: request.toNodeId,
|
|
3434
|
+
...request.reason ? { reason: request.reason } : {},
|
|
3435
|
+
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
3436
|
+
sourceManifestPath: (0, import_node_path7.relative)(runDir, input.sourceManifestPath),
|
|
3437
|
+
...targetSucc ? { targetPreviousManifestPath: (0, import_node_path7.relative)(runDir, targetSucc.manifestPath) } : {}
|
|
3438
|
+
}];
|
|
3439
|
+
const supersededInstanceIds = [];
|
|
3440
|
+
for (const affectedNodeId of affectedNodesFrom(input.dag, request.toNodeId)) {
|
|
3441
|
+
const eff = snap.nodes.get(affectedNodeId)?.effectiveInstanceId;
|
|
3442
|
+
if (!eff) continue;
|
|
3443
|
+
events.push({
|
|
3444
|
+
type: "nodeInstanceSuperseded",
|
|
3445
|
+
nodeId: affectedNodeId,
|
|
3446
|
+
instanceId: eff,
|
|
3447
|
+
byNodeId: request.toNodeId,
|
|
3448
|
+
reason: "refresh"
|
|
3449
|
+
});
|
|
3450
|
+
supersededInstanceIds.push(eff);
|
|
3451
|
+
}
|
|
3452
|
+
return { events, supersededInstanceIds };
|
|
3453
|
+
}
|
|
3454
|
+
|
|
3455
|
+
// src/workflows/v3/dag-revision.ts
|
|
3456
|
+
var import_node_crypto4 = require("node:crypto");
|
|
3457
|
+
var import_node_fs10 = require("node:fs");
|
|
3458
|
+
var import_node_path8 = require("node:path");
|
|
3459
|
+
|
|
3308
3460
|
// src/workflows/v3/gate-policy.ts
|
|
3309
3461
|
function normalizeGateWaitInput(gate) {
|
|
3310
3462
|
const options = gate.options ?? [...DEFAULT_HUMAN_GATE_OPTIONS];
|
|
@@ -3316,15 +3468,419 @@ function normalizeGateWaitInput(gate) {
|
|
|
3316
3468
|
};
|
|
3317
3469
|
}
|
|
3318
3470
|
|
|
3471
|
+
// src/workflows/v3/attempt-numbering.ts
|
|
3472
|
+
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
3473
|
+
function attemptNumber(attemptId) {
|
|
3474
|
+
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
3475
|
+
return m ? parseInt(m[1], 10) : void 0;
|
|
3476
|
+
}
|
|
3477
|
+
function nextAttemptIdFor(events, key) {
|
|
3478
|
+
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
3479
|
+
let maxSeen = 0;
|
|
3480
|
+
let reserved;
|
|
3481
|
+
let prepared;
|
|
3482
|
+
for (const e of events) {
|
|
3483
|
+
if (e.type === "nodeDispatched" && matches(e)) {
|
|
3484
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3485
|
+
if (n2 === void 0) continue;
|
|
3486
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3487
|
+
if (reserved === n2) reserved = void 0;
|
|
3488
|
+
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
3489
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3490
|
+
if (n2 === void 0) continue;
|
|
3491
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3492
|
+
if (reserved === n2) reserved = void 0;
|
|
3493
|
+
prepared = e.attemptId;
|
|
3494
|
+
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
3495
|
+
const n2 = attemptNumber(e.attemptId);
|
|
3496
|
+
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
3497
|
+
if (prepared === e.attemptId) prepared = void 0;
|
|
3498
|
+
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
3499
|
+
prepared = void 0;
|
|
3500
|
+
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
3501
|
+
const n2 = attemptNumber(e.nextAttemptId);
|
|
3502
|
+
if (n2 === void 0) continue;
|
|
3503
|
+
reserved = n2;
|
|
3504
|
+
prepared = void 0;
|
|
3505
|
+
maxSeen = Math.max(maxSeen, n2);
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
if (prepared) return prepared;
|
|
3509
|
+
const n = reserved ?? maxSeen + 1;
|
|
3510
|
+
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
3511
|
+
}
|
|
3512
|
+
function latestAttemptIdFor(events, key) {
|
|
3513
|
+
let latest;
|
|
3514
|
+
let latestNumber = -1;
|
|
3515
|
+
for (const e of events) {
|
|
3516
|
+
if ((e.type === "nodeDispatched" || e.type === "hostInputPrepared" || e.type === "hostEffectIntent" || e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && (e.instanceId ?? e.nodeId) === key) {
|
|
3517
|
+
const n = attemptNumber(e.attemptId);
|
|
3518
|
+
if (n === void 0) {
|
|
3519
|
+
if (latestNumber < 0) latest = e.attemptId;
|
|
3520
|
+
} else if (n >= latestNumber) {
|
|
3521
|
+
latest = e.attemptId;
|
|
3522
|
+
latestNumber = n;
|
|
3523
|
+
}
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
return latest;
|
|
3527
|
+
}
|
|
3528
|
+
|
|
3529
|
+
// src/workflows/v3/dag-revision.ts
|
|
3530
|
+
var V3_DAG_REVISIONS_DIR = "dag-revisions";
|
|
3531
|
+
function v3RevisionStem(revision) {
|
|
3532
|
+
if (!Number.isSafeInteger(revision) || revision < 2) {
|
|
3533
|
+
throw new Error(`v3 revision: invalid revision number ${revision}`);
|
|
3534
|
+
}
|
|
3535
|
+
return String(revision).padStart(3, "0");
|
|
3536
|
+
}
|
|
3537
|
+
function v3RevisionFilePaths(runDir, revision) {
|
|
3538
|
+
const stem = v3RevisionStem(revision);
|
|
3539
|
+
const dir = (0, import_node_path8.join)(runDir, V3_DAG_REVISIONS_DIR);
|
|
3540
|
+
return {
|
|
3541
|
+
dag: (0, import_node_path8.join)(dir, `${stem}.json`),
|
|
3542
|
+
patch: (0, import_node_path8.join)(dir, `${stem}.patch.json`),
|
|
3543
|
+
bots: (0, import_node_path8.join)(dir, `${stem}.bots.json`)
|
|
3544
|
+
};
|
|
3545
|
+
}
|
|
3546
|
+
function v3RevisionDigest(bytes) {
|
|
3547
|
+
return `sha256:${(0, import_node_crypto4.createHash)("sha256").update(bytes).digest("hex")}`;
|
|
3548
|
+
}
|
|
3549
|
+
var V3RevisionIntegrityError = class extends Error {
|
|
3550
|
+
constructor(message) {
|
|
3551
|
+
super(message);
|
|
3552
|
+
this.name = "V3RevisionIntegrityError";
|
|
3553
|
+
}
|
|
3554
|
+
};
|
|
3555
|
+
function readPinnedFile(path, expected, label) {
|
|
3556
|
+
let stat;
|
|
3557
|
+
try {
|
|
3558
|
+
stat = (0, import_node_fs10.lstatSync)(path);
|
|
3559
|
+
} catch {
|
|
3560
|
+
throw new V3RevisionIntegrityError(`${label} is missing (${path})`);
|
|
3561
|
+
}
|
|
3562
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3563
|
+
throw new V3RevisionIntegrityError(`${label} must be a regular file (${path})`);
|
|
3564
|
+
}
|
|
3565
|
+
const dirStat = (0, import_node_fs10.lstatSync)((0, import_node_path8.dirname)(path));
|
|
3566
|
+
if (!dirStat.isDirectory() || dirStat.isSymbolicLink()) {
|
|
3567
|
+
throw new V3RevisionIntegrityError(`${label} directory must be a real directory`);
|
|
3568
|
+
}
|
|
3569
|
+
const bytes = (0, import_node_fs10.readFileSync)(path);
|
|
3570
|
+
const actual = v3RevisionDigest(bytes);
|
|
3571
|
+
if (actual !== expected) {
|
|
3572
|
+
throw new V3RevisionIntegrityError(`${label} digest mismatch (journal ${expected}, file ${actual})`);
|
|
3573
|
+
}
|
|
3574
|
+
return bytes;
|
|
3575
|
+
}
|
|
3576
|
+
var V3_UPDATE_NODE_FIELDS = [
|
|
3577
|
+
"goal",
|
|
3578
|
+
"outputs",
|
|
3579
|
+
"resultSchema",
|
|
3580
|
+
"timeoutSec",
|
|
3581
|
+
"humanGate",
|
|
3582
|
+
"override",
|
|
3583
|
+
"revisitTo",
|
|
3584
|
+
"input",
|
|
3585
|
+
"maxIterations",
|
|
3586
|
+
"body",
|
|
3587
|
+
"exit",
|
|
3588
|
+
"feedback",
|
|
3589
|
+
"output"
|
|
3590
|
+
];
|
|
3591
|
+
var UPDATE_FIELDS = new Set(V3_UPDATE_NODE_FIELDS);
|
|
3592
|
+
function isPlainObject(value) {
|
|
3593
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3594
|
+
}
|
|
3595
|
+
function foldV3Revisions(events) {
|
|
3596
|
+
let current = 1;
|
|
3597
|
+
let maxRevision = 1;
|
|
3598
|
+
const records = [];
|
|
3599
|
+
const byRevision = /* @__PURE__ */ new Map();
|
|
3600
|
+
const applied = [];
|
|
3601
|
+
for (const event of events) {
|
|
3602
|
+
switch (event.type) {
|
|
3603
|
+
case "dagRevisionProposed": {
|
|
3604
|
+
if (byRevision.has(event.revision) || event.revision <= maxRevision) {
|
|
3605
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3606
|
+
}
|
|
3607
|
+
for (const prior of records) {
|
|
3608
|
+
if (prior.state !== "pending") continue;
|
|
3609
|
+
prior.state = "rejected";
|
|
3610
|
+
prior.rejected = { by: "system", reason: "superseded" };
|
|
3611
|
+
}
|
|
3612
|
+
const record = {
|
|
3613
|
+
revision: event.revision,
|
|
3614
|
+
baseRevision: event.baseRevision,
|
|
3615
|
+
sha256: event.sha256,
|
|
3616
|
+
patchSha256: event.patchSha256,
|
|
3617
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3618
|
+
summary: event.summary,
|
|
3619
|
+
...event.note ? { note: event.note } : {},
|
|
3620
|
+
by: event.by,
|
|
3621
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3622
|
+
state: "pending",
|
|
3623
|
+
ts: event.ts,
|
|
3624
|
+
reasons: [...event.reasons],
|
|
3625
|
+
touched: [...event.touched],
|
|
3626
|
+
...event.carriedFrom !== void 0 ? { carriedFrom: event.carriedFrom } : {}
|
|
3627
|
+
};
|
|
3628
|
+
records.push(record);
|
|
3629
|
+
byRevision.set(event.revision, record);
|
|
3630
|
+
maxRevision = event.revision;
|
|
3631
|
+
break;
|
|
3632
|
+
}
|
|
3633
|
+
case "dagRevised": {
|
|
3634
|
+
if (event.baseRevision !== current) {
|
|
3635
|
+
throw new V3RevisionIntegrityError(
|
|
3636
|
+
`revision ${event.revision} is based on ${event.baseRevision} but ${current} was in effect`
|
|
3637
|
+
);
|
|
3638
|
+
}
|
|
3639
|
+
let record = byRevision.get(event.revision);
|
|
3640
|
+
if (event.approval) {
|
|
3641
|
+
if (!record || record.state !== "pending" || record.sha256 !== event.sha256 || record.patchSha256 !== event.patchSha256 || record.botsSha256 !== event.botsSha256 || record.baseRevision !== event.baseRevision) {
|
|
3642
|
+
throw new V3RevisionIntegrityError(
|
|
3643
|
+
`approved revision ${event.revision} does not match a pending proposal`
|
|
3644
|
+
);
|
|
3645
|
+
}
|
|
3646
|
+
record.approvedBy = event.approval.by;
|
|
3647
|
+
} else {
|
|
3648
|
+
if (record || event.revision <= maxRevision) {
|
|
3649
|
+
throw new V3RevisionIntegrityError(`revision ${event.revision} was allocated twice`);
|
|
3650
|
+
}
|
|
3651
|
+
record = {
|
|
3652
|
+
revision: event.revision,
|
|
3653
|
+
baseRevision: event.baseRevision,
|
|
3654
|
+
sha256: event.sha256,
|
|
3655
|
+
patchSha256: event.patchSha256,
|
|
3656
|
+
...event.botsSha256 ? { botsSha256: event.botsSha256 } : {},
|
|
3657
|
+
summary: event.summary,
|
|
3658
|
+
...event.note ? { note: event.note } : {},
|
|
3659
|
+
by: event.by,
|
|
3660
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3661
|
+
state: "pending",
|
|
3662
|
+
ts: event.ts
|
|
3663
|
+
};
|
|
3664
|
+
records.push(record);
|
|
3665
|
+
byRevision.set(event.revision, record);
|
|
3666
|
+
maxRevision = event.revision;
|
|
3667
|
+
}
|
|
3668
|
+
record.state = "applied";
|
|
3669
|
+
record.appliedAt = event.ts;
|
|
3670
|
+
if (event.effects) record.effects = event.effects;
|
|
3671
|
+
applied.push(record);
|
|
3672
|
+
current = event.revision;
|
|
3673
|
+
break;
|
|
3674
|
+
}
|
|
3675
|
+
case "dagRevisionApproved": {
|
|
3676
|
+
const record = byRevision.get(event.revision);
|
|
3677
|
+
if (record && !record.approvedBy) record.approvedBy = event.by;
|
|
3678
|
+
break;
|
|
3679
|
+
}
|
|
3680
|
+
case "dagRevisionRejected": {
|
|
3681
|
+
const record = byRevision.get(event.revision);
|
|
3682
|
+
if (record?.state === "pending") {
|
|
3683
|
+
record.state = "rejected";
|
|
3684
|
+
record.rejected = {
|
|
3685
|
+
by: event.by,
|
|
3686
|
+
reason: event.reason,
|
|
3687
|
+
...event.detail ? { detail: event.detail } : {}
|
|
3688
|
+
};
|
|
3689
|
+
}
|
|
3690
|
+
break;
|
|
3691
|
+
}
|
|
3692
|
+
default:
|
|
3693
|
+
break;
|
|
3694
|
+
}
|
|
3695
|
+
}
|
|
3696
|
+
let pending;
|
|
3697
|
+
for (const record of records) {
|
|
3698
|
+
if (record.state !== "pending") continue;
|
|
3699
|
+
if (record.baseRevision !== current) {
|
|
3700
|
+
record.state = "rejected";
|
|
3701
|
+
record.rejected = { by: "system", reason: "stale" };
|
|
3702
|
+
continue;
|
|
3703
|
+
}
|
|
3704
|
+
pending = record;
|
|
3705
|
+
}
|
|
3706
|
+
return { current, records, applied, ...pending ? { pending } : {}, maxRevision };
|
|
3707
|
+
}
|
|
3708
|
+
function currentV3Revision(events) {
|
|
3709
|
+
let current = 1;
|
|
3710
|
+
for (const event of events) if (event.type === "dagRevised") current = event.revision;
|
|
3711
|
+
return current;
|
|
3712
|
+
}
|
|
3713
|
+
function parseRevisionDag(bytes, base, label) {
|
|
3714
|
+
let raw;
|
|
3715
|
+
try {
|
|
3716
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3717
|
+
} catch (err) {
|
|
3718
|
+
throw new V3RevisionIntegrityError(`${label} is not JSON: ${err instanceof Error ? err.message : String(err)}`);
|
|
3719
|
+
}
|
|
3720
|
+
let dag;
|
|
3721
|
+
try {
|
|
3722
|
+
dag = validateDag(raw);
|
|
3723
|
+
} catch (err) {
|
|
3724
|
+
throw new V3RevisionIntegrityError(
|
|
3725
|
+
`${label} failed validation: ${err instanceof DagValidationError ? err.problems.join("; ") : String(err)}`
|
|
3726
|
+
);
|
|
3727
|
+
}
|
|
3728
|
+
if (dag.runId !== base.runId) throw new V3RevisionIntegrityError(`${label} belongs to run ${dag.runId}`);
|
|
3729
|
+
if ((dag.schemaVersion ?? 1) !== (base.schemaVersion ?? 1)) {
|
|
3730
|
+
throw new V3RevisionIntegrityError(`${label} changed the DAG schemaVersion`);
|
|
3731
|
+
}
|
|
3732
|
+
return dag;
|
|
3733
|
+
}
|
|
3734
|
+
function parseRevisionBotSnapshots(raw, label = "revision bot snapshots") {
|
|
3735
|
+
if (!isPlainObject(raw)) throw new V3RevisionIntegrityError(`${label} must be an object`);
|
|
3736
|
+
const out = /* @__PURE__ */ new Map();
|
|
3737
|
+
for (const [selector, value] of Object.entries(raw)) {
|
|
3738
|
+
if (!isPlainObject(value) || typeof value.larkAppId !== "string" || value.larkAppId.length === 0 || typeof value.cliId !== "string" || typeof value.workingDir !== "string") {
|
|
3739
|
+
throw new V3RevisionIntegrityError(`${label}[${JSON.stringify(selector)}] is not a bot snapshot`);
|
|
3740
|
+
}
|
|
3741
|
+
out.set(selector, value);
|
|
3742
|
+
}
|
|
3743
|
+
return out;
|
|
3744
|
+
}
|
|
3745
|
+
function readVerifiedV3Revision(runDir, record, base) {
|
|
3746
|
+
const paths = v3RevisionFilePaths(runDir, record.revision);
|
|
3747
|
+
const label = `dag-revisions/${v3RevisionStem(record.revision)}`;
|
|
3748
|
+
const dag = parseRevisionDag(readPinnedFile(paths.dag, record.sha256, `${label}.json`), base, `${label}.json`);
|
|
3749
|
+
readPinnedFile(paths.patch, record.patchSha256, `${label}.patch.json`);
|
|
3750
|
+
let bots = /* @__PURE__ */ new Map();
|
|
3751
|
+
if (record.botsSha256) {
|
|
3752
|
+
const bytes = readPinnedFile(paths.bots, record.botsSha256, `${label}.bots.json`);
|
|
3753
|
+
let raw;
|
|
3754
|
+
try {
|
|
3755
|
+
raw = JSON.parse(bytes.toString("utf-8"));
|
|
3756
|
+
} catch {
|
|
3757
|
+
throw new V3RevisionIntegrityError(`${label}.bots.json is not JSON`);
|
|
3758
|
+
}
|
|
3759
|
+
bots = parseRevisionBotSnapshots(raw, `${label}.bots.json`);
|
|
3760
|
+
}
|
|
3761
|
+
return { dag, bots };
|
|
3762
|
+
}
|
|
3763
|
+
function loadEffectiveV3Dag(runDir, base, events) {
|
|
3764
|
+
const history = foldV3Revisions(events);
|
|
3765
|
+
const botAdditions = /* @__PURE__ */ new Map();
|
|
3766
|
+
if (history.applied.length === 0) return { dag: base, revision: 1, history, botAdditions };
|
|
3767
|
+
let dag = base;
|
|
3768
|
+
for (const record of history.applied) {
|
|
3769
|
+
const verified = readVerifiedV3Revision(runDir, record, base);
|
|
3770
|
+
dag = verified.dag;
|
|
3771
|
+
for (const [selector, snapshot] of verified.bots) botAdditions.set(selector, snapshot);
|
|
3772
|
+
}
|
|
3773
|
+
return { dag, revision: history.current, history, botAdditions };
|
|
3774
|
+
}
|
|
3775
|
+
function snapshotForSelector(snapshots, selector) {
|
|
3776
|
+
const key = selector ?? "";
|
|
3777
|
+
const exact = snapshots.get(key);
|
|
3778
|
+
if (exact) return exact;
|
|
3779
|
+
if (!key) return void 0;
|
|
3780
|
+
for (const snapshot of snapshots.values()) if (snapshot.larkAppId === key) return snapshot;
|
|
3781
|
+
return void 0;
|
|
3782
|
+
}
|
|
3783
|
+
function latestBlockOf(events, snap, nodeId) {
|
|
3784
|
+
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
3785
|
+
for (let i = events.length - 1; i >= 0; i--) {
|
|
3786
|
+
const event = events[i];
|
|
3787
|
+
if (event.type === "hostEffectUncertain" && (event.instanceId ?? event.nodeId) === key) return { kind: "hostUncertain" };
|
|
3788
|
+
if (event.type === "nodeBlocked" && (event.instanceId ?? event.nodeId) === key) {
|
|
3789
|
+
return {
|
|
3790
|
+
kind: "nodeBlocked",
|
|
3791
|
+
errorClass: event.errorClass,
|
|
3792
|
+
...event.errorCode ? { errorCode: event.errorCode } : {},
|
|
3793
|
+
reviseRequired: event.recovery === "reviseWorkflow"
|
|
3794
|
+
};
|
|
3795
|
+
}
|
|
3796
|
+
}
|
|
3797
|
+
return void 0;
|
|
3798
|
+
}
|
|
3799
|
+
function v3ApprovedBotScope(frozen, botAdditions) {
|
|
3800
|
+
const scope = new Map(frozen);
|
|
3801
|
+
for (const [selector, snapshot] of botAdditions) if (!scope.has(selector)) scope.set(selector, snapshot);
|
|
3802
|
+
return scope;
|
|
3803
|
+
}
|
|
3804
|
+
function v3NodeHeldByProposal(pendingTouched, nodeId, loopId) {
|
|
3805
|
+
if (!pendingTouched || pendingTouched.size === 0) return false;
|
|
3806
|
+
return pendingTouched.has(nodeId) || loopId !== void 0 && pendingTouched.has(loopId);
|
|
3807
|
+
}
|
|
3808
|
+
function v3RevisionFollowUps(events, input) {
|
|
3809
|
+
const out = [];
|
|
3810
|
+
const provenance = { by: input.by, ...input.auto ? { auto: input.auto } : {} };
|
|
3811
|
+
const snap = materialize([...events]);
|
|
3812
|
+
for (const item of input.assign ?? []) {
|
|
3813
|
+
if (snap.nodes.get(item.nodeId)?.status === "cancelled") continue;
|
|
3814
|
+
if (input.liveNodeIds && !input.liveNodeIds.has(item.nodeId)) continue;
|
|
3815
|
+
out.push({
|
|
3816
|
+
type: "nodeReassigned",
|
|
3817
|
+
nodeId: item.nodeId,
|
|
3818
|
+
...snap.nodes.get(item.nodeId)?.effectiveInstanceId ? { instanceId: snap.nodes.get(item.nodeId).effectiveInstanceId } : {},
|
|
3819
|
+
from: item.from ?? snap.assignments.get(item.nodeId) ?? "",
|
|
3820
|
+
to: item.to,
|
|
3821
|
+
...provenance
|
|
3822
|
+
});
|
|
3823
|
+
}
|
|
3824
|
+
for (const item of input.retry ?? []) {
|
|
3825
|
+
const state = snap.nodes.get(item.nodeId);
|
|
3826
|
+
if (state?.status !== "blocked" || state.effectiveInstanceId !== item.instanceId) continue;
|
|
3827
|
+
const key = item.instanceId ?? item.nodeId;
|
|
3828
|
+
const previousAttemptId = latestAttemptIdFor([...events], key);
|
|
3829
|
+
if (!previousAttemptId) continue;
|
|
3830
|
+
const blocked = latestBlockOf(events, snap, item.nodeId);
|
|
3831
|
+
out.push({
|
|
3832
|
+
type: "nodeRetryRequested",
|
|
3833
|
+
nodeId: item.nodeId,
|
|
3834
|
+
...item.instanceId ? { instanceId: item.instanceId } : {},
|
|
3835
|
+
previousAttemptId,
|
|
3836
|
+
nextAttemptId: nextAttemptIdFor([...events], key),
|
|
3837
|
+
reason: "blockedRetry",
|
|
3838
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorClass ? { previousErrorClass: blocked.errorClass } : {},
|
|
3839
|
+
...blocked?.kind === "nodeBlocked" && blocked.errorCode ? { previousErrorCode: blocked.errorCode } : {},
|
|
3840
|
+
...provenance
|
|
3841
|
+
});
|
|
3842
|
+
}
|
|
3843
|
+
return out;
|
|
3844
|
+
}
|
|
3845
|
+
function missingV3RevisionFollowUps(events, liveNodeIds) {
|
|
3846
|
+
if (!events.some((event) => event.type === "dagRevised" && (event.effects?.retry || event.effects?.assign))) return [];
|
|
3847
|
+
const snap = materialize([...events]);
|
|
3848
|
+
if (snap.runStatus !== "running" && snap.runStatus !== "blocked") return [];
|
|
3849
|
+
const missing = [];
|
|
3850
|
+
events.forEach((event, index) => {
|
|
3851
|
+
if (event.type !== "dagRevised" || !event.effects?.retry && !event.effects?.assign) return;
|
|
3852
|
+
const later = events.slice(index + 1);
|
|
3853
|
+
const assign = (event.effects.assign ?? []).filter((item) => !later.some((after) => after.type === "nodeReassigned" && after.nodeId === item.nodeId || after.type === "dagRevised" && after.effects?.assign?.some((other) => other.nodeId === item.nodeId)));
|
|
3854
|
+
const retry = (event.effects.retry ?? []).filter((item) => !later.some((after) => after.type === "nodeRetryRequested" && after.nodeId === item.nodeId));
|
|
3855
|
+
if (assign.length === 0 && retry.length === 0) return;
|
|
3856
|
+
missing.push(...v3RevisionFollowUps(events, {
|
|
3857
|
+
assign,
|
|
3858
|
+
retry,
|
|
3859
|
+
by: event.by,
|
|
3860
|
+
...event.auto ? { auto: event.auto } : {},
|
|
3861
|
+
...liveNodeIds ? { liveNodeIds } : {}
|
|
3862
|
+
}));
|
|
3863
|
+
});
|
|
3864
|
+
return missing;
|
|
3865
|
+
}
|
|
3866
|
+
function appendMissingV3RevisionFollowUps(journalPath, scope) {
|
|
3867
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
3868
|
+
if (currentV3Revision(events) !== scope.revision) return { kind: "stale" };
|
|
3869
|
+
const missing = missingV3RevisionFollowUps(events, scope.liveNodeIds);
|
|
3870
|
+
for (const event of missing) append(event, { durable: true });
|
|
3871
|
+
return { kind: "appended", count: missing.length };
|
|
3872
|
+
});
|
|
3873
|
+
}
|
|
3874
|
+
|
|
3319
3875
|
// src/workflows/v3/gate-wait-store.ts
|
|
3320
|
-
var
|
|
3321
|
-
var
|
|
3322
|
-
var
|
|
3876
|
+
var import_node_crypto5 = require("node:crypto");
|
|
3877
|
+
var import_node_fs11 = require("node:fs");
|
|
3878
|
+
var import_node_path9 = require("node:path");
|
|
3323
3879
|
function waitsDir(runDir) {
|
|
3324
|
-
return (0,
|
|
3880
|
+
return (0, import_node_path9.join)(runDir, "waits");
|
|
3325
3881
|
}
|
|
3326
3882
|
function waitPath(runDir, waitId) {
|
|
3327
|
-
return (0,
|
|
3883
|
+
return (0, import_node_path9.join)(waitsDir(runDir), `${waitId}.json`);
|
|
3328
3884
|
}
|
|
3329
3885
|
function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
3330
3886
|
const base = instanceId ?? nodeId;
|
|
@@ -3336,56 +3892,56 @@ function v3GateWaitId(nodeId, instanceId, hostApproval) {
|
|
|
3336
3892
|
return `${base}-host-${attempt}-gate`;
|
|
3337
3893
|
}
|
|
3338
3894
|
function atomicWriteJson(path, value) {
|
|
3339
|
-
const dir = (0,
|
|
3340
|
-
const runDir = (0,
|
|
3895
|
+
const dir = (0, import_node_path9.dirname)(path);
|
|
3896
|
+
const runDir = (0, import_node_path9.dirname)(dir);
|
|
3341
3897
|
assertRealDirectory(runDir, "run directory");
|
|
3342
|
-
if (!(0,
|
|
3898
|
+
if (!(0, import_node_fs11.existsSync)(dir)) {
|
|
3343
3899
|
try {
|
|
3344
|
-
(0,
|
|
3900
|
+
(0, import_node_fs11.mkdirSync)(dir, { mode: 448 });
|
|
3345
3901
|
} catch {
|
|
3346
3902
|
}
|
|
3347
3903
|
}
|
|
3348
3904
|
assertRealDirectory(dir, "waits directory");
|
|
3349
|
-
(0,
|
|
3350
|
-
if ((0,
|
|
3351
|
-
const tmp = `${path}.${process.pid}.${(0,
|
|
3905
|
+
(0, import_node_fs11.chmodSync)(dir, 448);
|
|
3906
|
+
if ((0, import_node_fs11.existsSync)(path)) assertPrivateRegularWaitFile(path, false);
|
|
3907
|
+
const tmp = `${path}.${process.pid}.${(0, import_node_crypto5.randomBytes)(4).toString("hex")}.tmp`;
|
|
3352
3908
|
let fd;
|
|
3353
3909
|
try {
|
|
3354
|
-
fd = (0,
|
|
3910
|
+
fd = (0, import_node_fs11.openSync)(
|
|
3355
3911
|
tmp,
|
|
3356
|
-
|
|
3912
|
+
import_node_fs11.constants.O_WRONLY | import_node_fs11.constants.O_CREAT | import_node_fs11.constants.O_EXCL | (import_node_fs11.constants.O_NOFOLLOW ?? 0),
|
|
3357
3913
|
384
|
|
3358
3914
|
);
|
|
3359
|
-
(0,
|
|
3360
|
-
(0,
|
|
3361
|
-
(0,
|
|
3915
|
+
(0, import_node_fs11.writeFileSync)(fd, JSON.stringify(value, null, 2), "utf-8");
|
|
3916
|
+
(0, import_node_fs11.fsyncSync)(fd);
|
|
3917
|
+
(0, import_node_fs11.closeSync)(fd);
|
|
3362
3918
|
fd = void 0;
|
|
3363
|
-
(0,
|
|
3919
|
+
(0, import_node_fs11.renameSync)(tmp, path);
|
|
3364
3920
|
} catch (err) {
|
|
3365
3921
|
if (fd !== void 0) {
|
|
3366
3922
|
try {
|
|
3367
|
-
(0,
|
|
3923
|
+
(0, import_node_fs11.closeSync)(fd);
|
|
3368
3924
|
} catch {
|
|
3369
3925
|
}
|
|
3370
3926
|
}
|
|
3371
3927
|
try {
|
|
3372
|
-
(0,
|
|
3928
|
+
(0, import_node_fs11.unlinkSync)(tmp);
|
|
3373
3929
|
} catch {
|
|
3374
3930
|
}
|
|
3375
3931
|
throw err;
|
|
3376
3932
|
}
|
|
3377
|
-
(0,
|
|
3933
|
+
(0, import_node_fs11.chmodSync)(path, 384);
|
|
3378
3934
|
fsyncRegularFileSync(path);
|
|
3379
3935
|
fsyncDirectorySyncPortable(dir);
|
|
3380
3936
|
}
|
|
3381
3937
|
function assertRealDirectory(path, label) {
|
|
3382
|
-
const stat = (0,
|
|
3938
|
+
const stat = (0, import_node_fs11.lstatSync)(path);
|
|
3383
3939
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
3384
3940
|
throw new Error(`v3 human-gate: ${label} must be a real directory`);
|
|
3385
3941
|
}
|
|
3386
3942
|
}
|
|
3387
3943
|
function assertPrivateRegularWaitFile(path, requirePrivate = true) {
|
|
3388
|
-
const stat = (0,
|
|
3944
|
+
const stat = (0, import_node_fs11.lstatSync)(path);
|
|
3389
3945
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3390
3946
|
throw new Error("v3 human-gate: wait file must be a regular file");
|
|
3391
3947
|
}
|
|
@@ -3499,6 +4055,7 @@ function openV3WorkerAttempts(events) {
|
|
|
3499
4055
|
const opened = /* @__PURE__ */ new Map();
|
|
3500
4056
|
const closed = /* @__PURE__ */ new Set();
|
|
3501
4057
|
for (const event of events) {
|
|
4058
|
+
if (event.type === "nodeDispatched" && event.nodeExecutor === "bot") continue;
|
|
3502
4059
|
if (isAttemptOpened(event)) {
|
|
3503
4060
|
const prior = opened.get(event.attemptId);
|
|
3504
4061
|
if (prior && (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId)) {
|
|
@@ -3606,35 +4163,536 @@ function sameIdentity(left, right) {
|
|
|
3606
4163
|
return left.nodeId === right.nodeId && left.instanceId === right.instanceId && left.attemptId === right.attemptId && left.executor === right.executor && left.provider === right.provider && left.inputRef.path === right.inputRef.path && left.inputRef.sha256 === right.inputRef.sha256 && left.inputRef.bytes === right.inputRef.bytes && left.inputHash === right.inputHash && left.idempotencyKey === right.idempotencyKey && left.idempotencyTtlMs === right.idempotencyTtlMs && left.approvalDigest === right.approvalDigest && left.attemptedAtMs === right.ts;
|
|
3607
4164
|
}
|
|
3608
4165
|
|
|
3609
|
-
// src/workflows/v3/
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
4166
|
+
// src/workflows/v3/external-attempt-ledger.ts
|
|
4167
|
+
function closureOf(event) {
|
|
4168
|
+
switch (event.type) {
|
|
4169
|
+
case "nodeSucceeded":
|
|
4170
|
+
return { kind: "succeeded" };
|
|
4171
|
+
case "nodeBlocked":
|
|
4172
|
+
return { kind: "blocked", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4173
|
+
case "nodeFailed":
|
|
4174
|
+
return { kind: "failed", ...event.errorCode ? { errorCode: event.errorCode } : {} };
|
|
4175
|
+
case "nodeCancelled":
|
|
4176
|
+
return event.reason === "runCancelled" && event.attemptId ? { kind: "cancelled" } : void 0;
|
|
4177
|
+
case "nodeAttemptDrained":
|
|
4178
|
+
return { kind: "drained", reason: event.reason };
|
|
4179
|
+
case "nodeReassigned":
|
|
4180
|
+
return event.attemptId ? { kind: "reassigned", to: event.to } : void 0;
|
|
4181
|
+
default:
|
|
4182
|
+
return void 0;
|
|
3623
4183
|
}
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
4184
|
+
}
|
|
4185
|
+
function attemptIdOf(event) {
|
|
4186
|
+
return "attemptId" in event && typeof event.attemptId === "string" ? event.attemptId : void 0;
|
|
4187
|
+
}
|
|
4188
|
+
function identityOf(event) {
|
|
4189
|
+
const record = event;
|
|
4190
|
+
return {
|
|
4191
|
+
...typeof record.nodeId === "string" ? { nodeId: record.nodeId } : {},
|
|
4192
|
+
...typeof record.instanceId === "string" ? { instanceId: record.instanceId } : {}
|
|
4193
|
+
};
|
|
4194
|
+
}
|
|
4195
|
+
function foldV3ExternalAttempts(events) {
|
|
4196
|
+
const attempts = /* @__PURE__ */ new Map();
|
|
4197
|
+
for (const event of events) {
|
|
4198
|
+
if (event.type === "nodeDispatched") {
|
|
4199
|
+
if (event.nodeExecutor !== "bot") continue;
|
|
4200
|
+
const prior = attempts.get(event.attemptId);
|
|
4201
|
+
if (prior) {
|
|
4202
|
+
if (prior.nodeId !== event.nodeId || prior.instanceId !== event.instanceId) {
|
|
4203
|
+
throw new Error(`v3 external ledger: identity changed for ${event.attemptId}`);
|
|
4204
|
+
}
|
|
4205
|
+
continue;
|
|
4206
|
+
}
|
|
4207
|
+
attempts.set(event.attemptId, {
|
|
4208
|
+
nodeId: event.nodeId,
|
|
4209
|
+
...event.instanceId ? { instanceId: event.instanceId } : {},
|
|
4210
|
+
attemptId: event.attemptId,
|
|
4211
|
+
...event.loop ? { loop: event.loop } : {},
|
|
4212
|
+
phase: "dispatched"
|
|
4213
|
+
});
|
|
4214
|
+
continue;
|
|
4215
|
+
}
|
|
4216
|
+
const attemptId = attemptIdOf(event);
|
|
4217
|
+
if (!attemptId) continue;
|
|
4218
|
+
const attempt = attempts.get(attemptId);
|
|
4219
|
+
if (!attempt) continue;
|
|
4220
|
+
if (attempt.closed) continue;
|
|
4221
|
+
const identity = identityOf(event);
|
|
4222
|
+
if (identity.nodeId !== attempt.nodeId || identity.instanceId !== attempt.instanceId) {
|
|
4223
|
+
throw new Error(`v3 external ledger: event identity changed for ${attemptId}`);
|
|
4224
|
+
}
|
|
4225
|
+
switch (event.type) {
|
|
4226
|
+
case "nodeHandoffPrepared":
|
|
4227
|
+
if (attempt.phase !== "dispatched") break;
|
|
4228
|
+
attempt.phase = "prepared";
|
|
4229
|
+
attempt.targetLarkAppId = event.targetLarkAppId;
|
|
4230
|
+
attempt.placement = event.placement;
|
|
4231
|
+
attempt.revision = event.revision;
|
|
4232
|
+
attempt.idempotency = event.idempotency;
|
|
4233
|
+
attempt.preparedAt = event.preparedAt;
|
|
4234
|
+
break;
|
|
4235
|
+
case "nodeHandoffSent":
|
|
4236
|
+
if (attempt.phase === "dispatched" || attempt.handoffMessageId !== void 0) break;
|
|
4237
|
+
if (attempt.phase === "prepared") attempt.phase = "sent";
|
|
4238
|
+
attempt.handoffMessageId = event.handoffMessageId;
|
|
4239
|
+
if (event.nodeCardMessageId) attempt.nodeCardMessageId = event.nodeCardMessageId;
|
|
4240
|
+
if (event.threadId) attempt.threadId = event.threadId;
|
|
4241
|
+
attempt.sentAt = event.sentAt;
|
|
4242
|
+
attempt.acceptDeadlineAt = event.acceptDeadlineAt;
|
|
4243
|
+
break;
|
|
4244
|
+
case "nodeHandoffAccepted":
|
|
4245
|
+
if (attempt.phase !== "prepared" && attempt.phase !== "sent") break;
|
|
4246
|
+
attempt.phase = "accepted";
|
|
4247
|
+
attempt.sessionId = event.sessionId;
|
|
4248
|
+
attempt.acceptedAt = event.acceptedAt;
|
|
4249
|
+
attempt.deadlineAt = event.deadlineAt;
|
|
4250
|
+
break;
|
|
4251
|
+
case "nodeStalled":
|
|
4252
|
+
attempt.stalledAt = event.at;
|
|
4253
|
+
break;
|
|
4254
|
+
default: {
|
|
4255
|
+
const closure = closureOf(event);
|
|
4256
|
+
if (closure) attempt.closed = closure;
|
|
4257
|
+
}
|
|
3629
4258
|
}
|
|
3630
4259
|
}
|
|
3631
|
-
|
|
3632
|
-
|
|
3633
|
-
|
|
4260
|
+
return attempts;
|
|
4261
|
+
}
|
|
4262
|
+
function openV3ExternalAttempts(events) {
|
|
4263
|
+
return [...foldV3ExternalAttempts(events).values()].filter((attempt) => !attempt.closed);
|
|
3634
4264
|
}
|
|
3635
4265
|
|
|
3636
|
-
// src/workflows/v3/
|
|
3637
|
-
var
|
|
4266
|
+
// src/workflows/v3/external-executor.ts
|
|
4267
|
+
var V3_EXTERNAL_ERROR_CODES = {
|
|
4268
|
+
BOT_NOT_IN_CHAT: "BOT_NOT_IN_CHAT",
|
|
4269
|
+
BOT_OFFLINE: "BOT_OFFLINE",
|
|
4270
|
+
COLLABORATION_DENIED: "COLLABORATION_DENIED",
|
|
4271
|
+
IDENTITY_UNAVAILABLE: "IDENTITY_UNAVAILABLE",
|
|
4272
|
+
HANDOFF_NOT_ACCEPTED: "HANDOFF_NOT_ACCEPTED",
|
|
4273
|
+
HANDOFF_UNCERTAIN: "HANDOFF_UNCERTAIN",
|
|
4274
|
+
BOT_REPORTED_BLOCKED: "BOT_REPORTED_BLOCKED",
|
|
4275
|
+
/** The executing bot's daemon would not start the node session in the
|
|
4276
|
+
* run's Gate-2 working dir: not inside its configured range (design §15.4). */
|
|
4277
|
+
WORKING_DIR_REFUSED: "WORKING_DIR_REFUSED",
|
|
4278
|
+
NODE_TIMEOUT: "NODE_TIMEOUT"
|
|
4279
|
+
};
|
|
4280
|
+
var V3_HANDOFF_IDEMPOTENCY_WINDOW_MS = 50 * 60 * 1e3;
|
|
4281
|
+
var V3_EXTERNAL_CALL_TIMEOUT_MS = 3e4;
|
|
4282
|
+
var V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS = 15e3;
|
|
4283
|
+
|
|
4284
|
+
// src/workflows/v3/external-control.ts
|
|
4285
|
+
var import_node_crypto6 = require("node:crypto");
|
|
4286
|
+
var import_node_fs13 = require("node:fs");
|
|
4287
|
+
var import_node_path11 = require("node:path");
|
|
4288
|
+
|
|
4289
|
+
// src/workflows/v3/result-contract.ts
|
|
4290
|
+
var import_node_fs12 = require("node:fs");
|
|
4291
|
+
var import_node_path10 = require("node:path");
|
|
4292
|
+
function readRevisitRequest(manifest, outputDir) {
|
|
4293
|
+
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4294
|
+
if (!entry) return { ok: true };
|
|
4295
|
+
let value;
|
|
4296
|
+
try {
|
|
4297
|
+
value = JSON.parse((0, import_node_fs12.readFileSync)((0, import_node_path10.join)(outputDir, entry.path), "utf-8"));
|
|
4298
|
+
} catch {
|
|
4299
|
+
return { ok: true };
|
|
4300
|
+
}
|
|
4301
|
+
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4302
|
+
return { ok: true };
|
|
4303
|
+
}
|
|
4304
|
+
const v = value;
|
|
4305
|
+
const problems = [];
|
|
4306
|
+
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4307
|
+
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4308
|
+
}
|
|
4309
|
+
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4310
|
+
problems.push('result.json "reason" must be a string when present');
|
|
4311
|
+
}
|
|
4312
|
+
if (problems.length > 0) return { ok: false, problems };
|
|
4313
|
+
return {
|
|
4314
|
+
ok: true,
|
|
4315
|
+
request: {
|
|
4316
|
+
toNodeId: v.revisitTo,
|
|
4317
|
+
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4318
|
+
}
|
|
4319
|
+
};
|
|
4320
|
+
}
|
|
4321
|
+
function validateResult(filePath, schema) {
|
|
4322
|
+
if (!(0, import_node_fs12.existsSync)(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4323
|
+
let parsed;
|
|
4324
|
+
try {
|
|
4325
|
+
parsed = JSON.parse((0, import_node_fs12.readFileSync)(filePath, "utf-8"));
|
|
4326
|
+
} catch (err) {
|
|
4327
|
+
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4328
|
+
}
|
|
4329
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4330
|
+
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4331
|
+
}
|
|
4332
|
+
const obj = parsed;
|
|
4333
|
+
const problems = [];
|
|
4334
|
+
for (const field of schema.required ?? []) {
|
|
4335
|
+
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4336
|
+
}
|
|
4337
|
+
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4338
|
+
if (!(name in obj)) continue;
|
|
4339
|
+
const v = obj[name];
|
|
4340
|
+
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4341
|
+
if (!okType) {
|
|
4342
|
+
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4343
|
+
continue;
|
|
4344
|
+
}
|
|
4345
|
+
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4346
|
+
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4347
|
+
}
|
|
4348
|
+
}
|
|
4349
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4350
|
+
}
|
|
4351
|
+
|
|
4352
|
+
// src/workflows/v3/external-control.ts
|
|
4353
|
+
function classifyExternalAttempt(events, attemptId, expect = {}) {
|
|
4354
|
+
const attempt = foldV3ExternalAttempts(events).get(attemptId);
|
|
4355
|
+
if (!attempt) return { ok: false, reason: "unknown-attempt" };
|
|
4356
|
+
if (expect.instanceId !== void 0 && expect.instanceId !== attempt.instanceId) {
|
|
4357
|
+
return { ok: false, reason: "unknown-attempt" };
|
|
4358
|
+
}
|
|
4359
|
+
const snap = materialize([...events]);
|
|
4360
|
+
const newerAttempt = snap.attempts.get(attempt.instanceId ?? attempt.nodeId) !== attemptId;
|
|
4361
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
4362
|
+
const closed = attempt.closed;
|
|
4363
|
+
if (closed) {
|
|
4364
|
+
const reason = closed.kind === "succeeded" ? "already-succeeded" : closed.kind === "reassigned" ? "reassigned" : closed.kind === "cancelled" ? "cancelled" : closed.kind === "blocked" && closed.errorCode === V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT ? "timed-out" : closed.kind === "drained" && closed.reason === "runCancellation" ? "cancelled" : closed.kind === "drained" && closed.reason === "obsoleteAttempt" ? state?.status === "cancelled" ? "cancelled" : "superseded" : newerAttempt ? "retried" : "closed";
|
|
4365
|
+
return { ok: false, reason, attempt };
|
|
4366
|
+
}
|
|
4367
|
+
if (snap.cancelRequestId !== void 0) return { ok: false, reason: "cancelled", attempt };
|
|
4368
|
+
if (snap.runStatus !== "running") return { ok: false, reason: "run-not-running", attempt };
|
|
4369
|
+
if (state?.status === "superseded") return { ok: false, reason: "superseded", attempt };
|
|
4370
|
+
if (state?.status === "cancelled") return { ok: false, reason: "cancelled", attempt };
|
|
4371
|
+
if (newerAttempt) return { ok: false, reason: "retried", attempt };
|
|
4372
|
+
if (state?.status !== "running") return { ok: false, reason: "closed", attempt };
|
|
4373
|
+
if (expect.sessionId !== void 0 && attempt.sessionId !== void 0 && expect.sessionId !== attempt.sessionId) {
|
|
4374
|
+
return { ok: false, reason: "session-mismatch", attempt };
|
|
4375
|
+
}
|
|
4376
|
+
return { ok: true, attempt, snap };
|
|
4377
|
+
}
|
|
4378
|
+
function externalAttemptRef(runId, attempt) {
|
|
4379
|
+
if (!attempt.targetLarkAppId || !attempt.placement || attempt.revision === void 0) {
|
|
4380
|
+
return void 0;
|
|
4381
|
+
}
|
|
4382
|
+
return {
|
|
4383
|
+
runId,
|
|
4384
|
+
nodeId: attempt.nodeId,
|
|
4385
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4386
|
+
attemptId: attempt.attemptId,
|
|
4387
|
+
revision: attempt.revision,
|
|
4388
|
+
targetLarkAppId: attempt.targetLarkAppId,
|
|
4389
|
+
placement: attempt.placement
|
|
4390
|
+
};
|
|
4391
|
+
}
|
|
4392
|
+
async function withExternalCallTimeout(call, timeoutMs, label) {
|
|
4393
|
+
let timer;
|
|
4394
|
+
try {
|
|
4395
|
+
return await Promise.race([
|
|
4396
|
+
call,
|
|
4397
|
+
new Promise((_resolve, reject) => {
|
|
4398
|
+
timer = setTimeout(() => reject(new Error(`external ${label} did not settle within ${timeoutMs}ms`)), timeoutMs);
|
|
4399
|
+
timer.unref?.();
|
|
4400
|
+
})
|
|
4401
|
+
]);
|
|
4402
|
+
} finally {
|
|
4403
|
+
if (timer) clearTimeout(timer);
|
|
4404
|
+
}
|
|
4405
|
+
}
|
|
4406
|
+
async function stopExternalAttempt(executor, runId, attempt, reason, timeoutMs = V3_EXTERNAL_CALL_TIMEOUT_MS) {
|
|
4407
|
+
const ref = externalAttemptRef(runId, attempt);
|
|
4408
|
+
if (!executor || !ref) return { ok: true };
|
|
4409
|
+
try {
|
|
4410
|
+
await withExternalCallTimeout(Promise.resolve().then(() => executor.stop({
|
|
4411
|
+
...ref,
|
|
4412
|
+
reason,
|
|
4413
|
+
...attempt.idempotency ? { idempotency: attempt.idempotency } : {},
|
|
4414
|
+
...attempt.handoffMessageId ? { handoffMessageId: attempt.handoffMessageId } : {},
|
|
4415
|
+
...attempt.nodeCardMessageId ? { nodeCardMessageId: attempt.nodeCardMessageId } : {},
|
|
4416
|
+
...attempt.threadId ? { threadId: attempt.threadId } : {},
|
|
4417
|
+
...attempt.sessionId ? { sessionId: attempt.sessionId } : {}
|
|
4418
|
+
})), timeoutMs, "stop");
|
|
4419
|
+
return { ok: true };
|
|
4420
|
+
} catch (error) {
|
|
4421
|
+
return { ok: false, error };
|
|
4422
|
+
}
|
|
4423
|
+
}
|
|
4424
|
+
function journalPathOf(runDir) {
|
|
4425
|
+
return (0, import_node_path11.join)(runDir, "journal.ndjson");
|
|
4426
|
+
}
|
|
4427
|
+
function externalAttemptNode(dag, attempt) {
|
|
4428
|
+
if (!attempt.loop) return dag.nodes.find((node) => node.id === attempt.nodeId);
|
|
4429
|
+
const loopNode = dag.nodes.find((node) => node.id === attempt.loop.loopId);
|
|
4430
|
+
if (!loopNode || !isLoopNode(loopNode)) return void 0;
|
|
4431
|
+
const body = loopNode.body.nodes.find((node) => node.id === attempt.loop.bodyNodeId);
|
|
4432
|
+
if (!body) return void 0;
|
|
4433
|
+
const ref = attempt.loop;
|
|
4434
|
+
return {
|
|
4435
|
+
...body,
|
|
4436
|
+
id: loopInstanceId(ref.loopId, ref.iteration, ref.bodyNodeId),
|
|
4437
|
+
bot: body.bot ?? loopNode.bot,
|
|
4438
|
+
depends: body.depends.map((d) => ({ from: loopInstanceId(ref.loopId, ref.iteration, d.from) })),
|
|
4439
|
+
inputs: body.inputs.map((r) => ({
|
|
4440
|
+
from: loopInstanceId(ref.loopId, ref.iteration, r.from),
|
|
4441
|
+
...r.select ? { select: r.select } : {},
|
|
4442
|
+
...r.output !== void 0 ? { output: r.output } : {}
|
|
4443
|
+
}))
|
|
4444
|
+
};
|
|
4445
|
+
}
|
|
4446
|
+
function nodeTimeoutMs(node) {
|
|
4447
|
+
return (node?.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC) * 1e3;
|
|
4448
|
+
}
|
|
4449
|
+
function safeExternalAttemptDir(runDir, attemptId) {
|
|
4450
|
+
const root = (0, import_node_path11.resolve)(runDir);
|
|
4451
|
+
const candidate = (0, import_node_path11.resolve)(root, attemptId);
|
|
4452
|
+
const rel = (0, import_node_path11.relative)(root, candidate);
|
|
4453
|
+
if (!rel || (0, import_node_path11.isAbsolute)(rel) || rel === ".." || rel.startsWith(`..${import_node_path11.sep}`)) {
|
|
4454
|
+
throw new Error(`v3 external: attempt path escapes run dir (${attemptId})`);
|
|
4455
|
+
}
|
|
4456
|
+
return candidate;
|
|
4457
|
+
}
|
|
4458
|
+
function appendAcceptedLocked(events, append, dag, input, now) {
|
|
4459
|
+
const current = classifyExternalAttempt(events, input.attemptId, {
|
|
4460
|
+
...input.instanceId !== void 0 ? { instanceId: input.instanceId } : {},
|
|
4461
|
+
sessionId: input.sessionId
|
|
4462
|
+
});
|
|
4463
|
+
if (!current.ok) return { ok: false, reason: current.reason };
|
|
4464
|
+
const { attempt } = current;
|
|
4465
|
+
if (attempt.phase === "accepted") {
|
|
4466
|
+
return { ok: true, deadlineAt: attempt.deadlineAt, alreadyAccepted: true };
|
|
4467
|
+
}
|
|
4468
|
+
if (attempt.phase === "dispatched") return { ok: false, reason: "unknown-attempt" };
|
|
4469
|
+
const acceptedAt = Math.min(input.acceptedAt ?? now, now);
|
|
4470
|
+
const deadlineAt = acceptedAt + nodeTimeoutMs(externalAttemptNode(dag, attempt));
|
|
4471
|
+
append({
|
|
4472
|
+
type: "nodeHandoffAccepted",
|
|
4473
|
+
nodeId: attempt.nodeId,
|
|
4474
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4475
|
+
attemptId: attempt.attemptId,
|
|
4476
|
+
sessionId: input.sessionId,
|
|
4477
|
+
acceptedAt,
|
|
4478
|
+
deadlineAt
|
|
4479
|
+
}, { durable: true });
|
|
4480
|
+
return { ok: true, deadlineAt, alreadyAccepted: false };
|
|
4481
|
+
}
|
|
4482
|
+
var RESULT_FILE = "result.json";
|
|
4483
|
+
function judgeStagedSubmission(node, manifest, workDir) {
|
|
4484
|
+
const revisit = readRevisitRequest(manifest, workDir);
|
|
4485
|
+
if (!revisit.ok) return { kind: "invalid", problems: revisit.problems };
|
|
4486
|
+
if (revisit.request) {
|
|
4487
|
+
if (!node.revisitTo?.includes(revisit.request.toNodeId)) {
|
|
4488
|
+
return {
|
|
4489
|
+
kind: "invalid",
|
|
4490
|
+
problems: [`result.json requests revisit to "${revisit.request.toNodeId}", not in node "${node.id}".revisitTo`]
|
|
4491
|
+
};
|
|
4492
|
+
}
|
|
4493
|
+
return { kind: "revisit", request: revisit.request };
|
|
4494
|
+
}
|
|
4495
|
+
if (node.resultSchema) {
|
|
4496
|
+
const entry = manifest.files.find((f) => f.path === RESULT_FILE);
|
|
4497
|
+
if (!entry) {
|
|
4498
|
+
return { kind: "invalid", problems: ["node declares resultSchema but the submission has no result.json"] };
|
|
4499
|
+
}
|
|
4500
|
+
const res = validateResult((0, import_node_path11.join)(workDir, entry.path), node.resultSchema);
|
|
4501
|
+
if (!res.ok) return { kind: "invalid", problems: res.problems ?? ["result.json failed schema validation"] };
|
|
4502
|
+
}
|
|
4503
|
+
const contract = validateManifestArtifactContract(node.outputs, manifest);
|
|
4504
|
+
if (!contract.ok) return { kind: "invalid", problems: contract.problems };
|
|
4505
|
+
return { kind: "success" };
|
|
4506
|
+
}
|
|
4507
|
+
function moveAsideIfPresent(path, tag) {
|
|
4508
|
+
if (!(0, import_node_fs13.existsSync)(path)) return;
|
|
4509
|
+
(0, import_node_fs13.renameSync)(path, `${path}.uncommitted-${tag}`);
|
|
4510
|
+
}
|
|
4511
|
+
var V3_SUBMISSION_INTENT_FILE = "submission.intent.json";
|
|
4512
|
+
function submissionIntentPath(attemptDir) {
|
|
4513
|
+
return (0, import_node_path11.join)(attemptDir, V3_SUBMISSION_INTENT_FILE);
|
|
4514
|
+
}
|
|
4515
|
+
function removeSubmissionIntent(attemptDir, stageTag) {
|
|
4516
|
+
(0, import_node_fs13.rmSync)(submissionIntentPath(attemptDir), { force: true });
|
|
4517
|
+
(0, import_node_fs13.rmSync)((0, import_node_path11.join)(attemptDir, "submissions", stageTag), { recursive: true, force: true });
|
|
4518
|
+
try {
|
|
4519
|
+
(0, import_node_fs13.rmdirSync)((0, import_node_path11.join)(attemptDir, "submissions"));
|
|
4520
|
+
} catch {
|
|
4521
|
+
}
|
|
4522
|
+
}
|
|
4523
|
+
function readSubmissionIntent(attemptDir) {
|
|
4524
|
+
const path = submissionIntentPath(attemptDir);
|
|
4525
|
+
if (!(0, import_node_fs13.existsSync)(path)) return void 0;
|
|
4526
|
+
try {
|
|
4527
|
+
const raw = JSON.parse((0, import_node_fs13.readFileSync)(path, "utf-8"));
|
|
4528
|
+
if (raw.schemaVersion !== 1 || typeof raw.attemptId !== "string" || typeof raw.stageTag !== "string") {
|
|
4529
|
+
return "corrupt";
|
|
4530
|
+
}
|
|
4531
|
+
if (!/^[0-9a-f-]{36}$/.test(raw.stageTag)) return "corrupt";
|
|
4532
|
+
return { schemaVersion: 1, attemptId: raw.attemptId, stageTag: raw.stageTag };
|
|
4533
|
+
} catch {
|
|
4534
|
+
return "corrupt";
|
|
4535
|
+
}
|
|
4536
|
+
}
|
|
4537
|
+
function verifyInstalledSubmissionSync(manifestPath, workDir) {
|
|
4538
|
+
let manifest;
|
|
4539
|
+
try {
|
|
4540
|
+
manifest = JSON.parse((0, import_node_fs13.readFileSync)(manifestPath, "utf-8"));
|
|
4541
|
+
} catch (err) {
|
|
4542
|
+
return { ok: false, problems: [`manifest unreadable: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4543
|
+
}
|
|
4544
|
+
if (manifest?.schemaVersion !== MANIFEST_SCHEMA_VERSION || manifest.status !== "ok" || !Array.isArray(manifest.files) || manifest.files.length === 0) {
|
|
4545
|
+
return { ok: false, problems: ["manifest is not an engine-generated success manifest"] };
|
|
4546
|
+
}
|
|
4547
|
+
const root = (0, import_node_path11.resolve)(workDir);
|
|
4548
|
+
const problems = [];
|
|
4549
|
+
for (const file of manifest.files) {
|
|
4550
|
+
const target = (0, import_node_path11.resolve)(root, file.path);
|
|
4551
|
+
const rel = (0, import_node_path11.relative)(root, target);
|
|
4552
|
+
if (!rel || (0, import_node_path11.isAbsolute)(rel) || rel.startsWith("..")) {
|
|
4553
|
+
problems.push(`${file.path}: escapes the output directory`);
|
|
4554
|
+
continue;
|
|
4555
|
+
}
|
|
4556
|
+
try {
|
|
4557
|
+
if (!(0, import_node_fs13.lstatSync)(target).isFile()) {
|
|
4558
|
+
problems.push(`${file.path}: not a regular file`);
|
|
4559
|
+
continue;
|
|
4560
|
+
}
|
|
4561
|
+
const bytes = (0, import_node_fs13.readFileSync)(target);
|
|
4562
|
+
if (bytes.length !== file.bytes || (0, import_node_crypto6.createHash)("sha256").update(bytes).digest("hex") !== file.sha256) {
|
|
4563
|
+
problems.push(`${file.path}: bytes do not match the manifest`);
|
|
4564
|
+
}
|
|
4565
|
+
} catch (err) {
|
|
4566
|
+
problems.push(`${file.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
4567
|
+
}
|
|
4568
|
+
}
|
|
4569
|
+
return problems.length > 0 ? { ok: false, problems } : { ok: true, manifest };
|
|
4570
|
+
}
|
|
4571
|
+
function appendInstalledVerdictLocked(input) {
|
|
4572
|
+
const { attempt, append, events, verdict, manifestPath } = input;
|
|
4573
|
+
const identity = {
|
|
4574
|
+
nodeId: attempt.nodeId,
|
|
4575
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
4576
|
+
attemptId: attempt.attemptId
|
|
4577
|
+
};
|
|
4578
|
+
if (verdict.kind === "revisit") {
|
|
4579
|
+
const budget = revisitBudgetStatus([...events], attempt.nodeId, verdict.request.toNodeId);
|
|
4580
|
+
if (!budget.ok) {
|
|
4581
|
+
append({
|
|
4582
|
+
type: "nodeBlocked",
|
|
4583
|
+
...identity,
|
|
4584
|
+
errorClass: "resultInvalid",
|
|
4585
|
+
errorCode: "REVISIT_BUDGET_EXHAUSTED",
|
|
4586
|
+
message: budget.detail,
|
|
4587
|
+
revisitTo: verdict.request.toNodeId
|
|
4588
|
+
}, { durable: true });
|
|
4589
|
+
return { ok: true, kind: "blocked", errorCode: "REVISIT_BUDGET_EXHAUSTED" };
|
|
4590
|
+
}
|
|
4591
|
+
const plan = planV3Revisit({
|
|
4592
|
+
runDir: input.runDir,
|
|
4593
|
+
dag: input.dag,
|
|
4594
|
+
events,
|
|
4595
|
+
nodeId: attempt.nodeId,
|
|
4596
|
+
instanceId: attempt.instanceId ?? attempt.nodeId,
|
|
4597
|
+
attemptId: attempt.attemptId,
|
|
4598
|
+
request: verdict.request,
|
|
4599
|
+
sourceManifestPath: manifestPath
|
|
4600
|
+
});
|
|
4601
|
+
for (const event of plan.events) append(event);
|
|
4602
|
+
append({ type: "nodeAttemptDrained", ...identity, reason: "obsoleteAttempt" }, { durable: true });
|
|
4603
|
+
return { ok: true, kind: "revisit", toNodeId: verdict.request.toNodeId };
|
|
4604
|
+
}
|
|
4605
|
+
append({ type: "nodeSucceeded", ...identity, manifestPath }, { durable: true });
|
|
4606
|
+
return { ok: true, kind: "succeeded", manifestPath };
|
|
4607
|
+
}
|
|
4608
|
+
function installVerdictCommitted(events, attemptId) {
|
|
4609
|
+
return events.some((event) => event.type === "nodeSucceeded" && event.attemptId === attemptId || event.type === "nodeRevisitRequested" && event.attemptId === attemptId || event.type === "nodeBlocked" && event.attemptId === attemptId && event.errorCode === "REVISIT_BUDGET_EXHAUSTED");
|
|
4610
|
+
}
|
|
4611
|
+
function settleInstallIntentLocked(runDir, dag, events, append, attemptId) {
|
|
4612
|
+
const attemptDir = safeExternalAttemptDir(runDir, attemptId);
|
|
4613
|
+
const intent = readSubmissionIntent(attemptDir);
|
|
4614
|
+
if (intent === void 0) return { kind: "none" };
|
|
4615
|
+
if (intent === "corrupt" || intent.attemptId !== attemptId) {
|
|
4616
|
+
(0, import_node_fs13.rmSync)(submissionIntentPath(attemptDir), { force: true });
|
|
4617
|
+
return { kind: "discarded" };
|
|
4618
|
+
}
|
|
4619
|
+
const workDir = (0, import_node_path11.join)(attemptDir, "work");
|
|
4620
|
+
const manifestPath = (0, import_node_path11.join)(attemptDir, "manifest.json");
|
|
4621
|
+
const stageDir = (0, import_node_path11.join)(attemptDir, "submissions", intent.stageTag);
|
|
4622
|
+
const discard = () => {
|
|
4623
|
+
if (!installVerdictCommitted(events, attemptId)) {
|
|
4624
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4625
|
+
moveAsideIfPresent(manifestPath, intent.stageTag);
|
|
4626
|
+
}
|
|
4627
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4628
|
+
return { kind: "discarded" };
|
|
4629
|
+
};
|
|
4630
|
+
if (installVerdictCommitted(events, attemptId)) {
|
|
4631
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4632
|
+
return { kind: "cleaned" };
|
|
4633
|
+
}
|
|
4634
|
+
const current = classifyExternalAttempt(events, attemptId);
|
|
4635
|
+
if (!current.ok) return discard();
|
|
4636
|
+
const node = externalAttemptNode(dag, current.attempt);
|
|
4637
|
+
if (!node || !isGoalNode(node)) return discard();
|
|
4638
|
+
if ((0, import_node_fs13.existsSync)((0, import_node_path11.join)(stageDir, "work"))) {
|
|
4639
|
+
moveAsideIfPresent(workDir, intent.stageTag);
|
|
4640
|
+
(0, import_node_fs13.renameSync)((0, import_node_path11.join)(stageDir, "work"), workDir);
|
|
4641
|
+
}
|
|
4642
|
+
if ((0, import_node_fs13.existsSync)((0, import_node_path11.join)(stageDir, "manifest.json"))) (0, import_node_fs13.renameSync)((0, import_node_path11.join)(stageDir, "manifest.json"), manifestPath);
|
|
4643
|
+
const verified = verifyInstalledSubmissionSync(manifestPath, workDir);
|
|
4644
|
+
if (!verified.ok) return discard();
|
|
4645
|
+
const verdict = judgeStagedSubmission(node, verified.manifest, workDir);
|
|
4646
|
+
if (verdict.kind === "invalid") return discard();
|
|
4647
|
+
const outcome = appendInstalledVerdictLocked({
|
|
4648
|
+
runDir,
|
|
4649
|
+
dag,
|
|
4650
|
+
events,
|
|
4651
|
+
append,
|
|
4652
|
+
attempt: current.attempt,
|
|
4653
|
+
verdict,
|
|
4654
|
+
manifestPath
|
|
4655
|
+
});
|
|
4656
|
+
removeSubmissionIntent(attemptDir, intent.stageTag);
|
|
4657
|
+
return { kind: "committed", outcome };
|
|
4658
|
+
}
|
|
4659
|
+
function settleExternalInstallIntent(runDir, dag, attemptId) {
|
|
4660
|
+
if (!(0, import_node_fs13.existsSync)(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)))) return { kind: "none" };
|
|
4661
|
+
return withJournalMutationSync(journalPathOf(runDir), ({ events, append }) => settleInstallIntentLocked(runDir, dag, events, append, attemptId));
|
|
4662
|
+
}
|
|
4663
|
+
function hasExternalInstallIntent(runDir, attemptId) {
|
|
4664
|
+
return (0, import_node_fs13.existsSync)(submissionIntentPath(safeExternalAttemptDir(runDir, attemptId)));
|
|
4665
|
+
}
|
|
4666
|
+
|
|
4667
|
+
// src/workflows/v3/host-execution.ts
|
|
4668
|
+
var import_node_crypto8 = require("node:crypto");
|
|
4669
|
+
var import_node_fs14 = require("node:fs");
|
|
4670
|
+
var import_node_path12 = require("node:path");
|
|
4671
|
+
|
|
4672
|
+
// src/workflows/shared/idempotency-key.ts
|
|
4673
|
+
var import_node_crypto7 = require("node:crypto");
|
|
4674
|
+
function deriveIdempotencyKey(tuple, opts = {}) {
|
|
4675
|
+
const namespace = opts.namespace ?? "wf_";
|
|
4676
|
+
const maxLength = opts.maxLength ?? 50;
|
|
4677
|
+
if (namespace.length >= maxLength) {
|
|
4678
|
+
throw new Error(
|
|
4679
|
+
`deriveIdempotencyKey: namespace '${namespace}' (${namespace.length} chars) leaves no room for hash in maxLength ${maxLength}`
|
|
4680
|
+
);
|
|
4681
|
+
}
|
|
4682
|
+
for (const [key, value] of Object.entries(tuple)) {
|
|
4683
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
4684
|
+
throw new Error(
|
|
4685
|
+
`deriveIdempotencyKey: tuple.${key} must be non-empty string, got ${String(value)}`
|
|
4686
|
+
);
|
|
4687
|
+
}
|
|
4688
|
+
}
|
|
4689
|
+
const seed = canonicalJson(tuple);
|
|
4690
|
+
const hash = (0, import_node_crypto7.createHash)("sha256").update(seed, "utf-8").digest("hex");
|
|
4691
|
+
return namespace + hash.substring(0, maxLength - namespace.length);
|
|
4692
|
+
}
|
|
4693
|
+
|
|
4694
|
+
// src/workflows/v3/host-execution.ts
|
|
4695
|
+
var V3_HOST_INPUT_SCHEMA_VERSION = 1;
|
|
3638
4696
|
var V3_MAX_APPROVABLE_HOST_INPUT_BYTES = 8e3;
|
|
3639
4697
|
var INPUT_HASH_RE = /^sha256:[0-9a-f]{64}$/;
|
|
3640
4698
|
function deriveV3HostEffectIdentity(input) {
|
|
@@ -3690,23 +4748,23 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3690
4748
|
idempotencyTtlMs: input.registered.executor.idempotencyTtlMs,
|
|
3691
4749
|
approvalDigest
|
|
3692
4750
|
};
|
|
3693
|
-
const absolutePath = (0,
|
|
4751
|
+
const absolutePath = (0, import_node_path12.join)(input.attemptDir, "host-input.json");
|
|
3694
4752
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3695
4753
|
ensureSecureDirectoryTree(input.runDir, input.attemptDir);
|
|
3696
4754
|
const content = `${JSON.stringify(prepared, null, 2)}
|
|
3697
4755
|
`;
|
|
3698
4756
|
writeCreateOrVerify(absolutePath, content, 384, "host input");
|
|
3699
|
-
const existing = (0,
|
|
4757
|
+
const existing = (0, import_node_fs14.readFileSync)(absolutePath, "utf-8");
|
|
3700
4758
|
if (canonicalJson(parsePreparedHostInput(JSON.parse(existing))) !== canonicalJson(prepared)) {
|
|
3701
4759
|
throw new Error(`v3 host input conflict at ${absolutePath}; refusing to overwrite prepared bytes`);
|
|
3702
4760
|
}
|
|
3703
4761
|
fsyncRegularFileSync(absolutePath);
|
|
3704
4762
|
fsyncDirectorySyncPortable(input.attemptDir);
|
|
3705
|
-
const bytes = (0,
|
|
4763
|
+
const bytes = (0, import_node_fs14.readFileSync)(absolutePath);
|
|
3706
4764
|
return {
|
|
3707
4765
|
prepared,
|
|
3708
4766
|
inputRef: {
|
|
3709
|
-
path: (0,
|
|
4767
|
+
path: (0, import_node_path12.relative)(input.runDir, absolutePath),
|
|
3710
4768
|
sha256: sha256(bytes),
|
|
3711
4769
|
bytes: bytes.length
|
|
3712
4770
|
},
|
|
@@ -3714,12 +4772,12 @@ function prepareV3HostInputArtifact(input) {
|
|
|
3714
4772
|
};
|
|
3715
4773
|
}
|
|
3716
4774
|
function readCrashLeftV3PreparedHostInput(input) {
|
|
3717
|
-
const absolutePath = (0,
|
|
3718
|
-
if (!(0,
|
|
4775
|
+
const absolutePath = (0, import_node_path12.join)(input.attemptDir, "host-input.json");
|
|
4776
|
+
if (!(0, import_node_fs14.existsSync)(absolutePath)) return void 0;
|
|
3719
4777
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3720
4778
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3721
4779
|
assertRegularFile(absolutePath, "host input");
|
|
3722
|
-
const bytes = (0,
|
|
4780
|
+
const bytes = (0, import_node_fs14.readFileSync)(absolutePath);
|
|
3723
4781
|
let prepared;
|
|
3724
4782
|
try {
|
|
3725
4783
|
prepared = parsePreparedHostInput(JSON.parse(bytes.toString("utf-8")));
|
|
@@ -3738,7 +4796,7 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3738
4796
|
inputHash
|
|
3739
4797
|
});
|
|
3740
4798
|
const inputRef = {
|
|
3741
|
-
path: (0,
|
|
4799
|
+
path: (0, import_node_path12.relative)(input.runDir, absolutePath),
|
|
3742
4800
|
sha256: sha256(bytes),
|
|
3743
4801
|
bytes: bytes.length
|
|
3744
4802
|
};
|
|
@@ -3761,14 +4819,14 @@ function readCrashLeftV3PreparedHostInput(input) {
|
|
|
3761
4819
|
});
|
|
3762
4820
|
}
|
|
3763
4821
|
function readAndVerifyV3PreparedHostInput(input) {
|
|
3764
|
-
if ((0,
|
|
4822
|
+
if ((0, import_node_path12.isAbsolute)(input.inputRef.path) || input.inputRef.path.includes("\0")) {
|
|
3765
4823
|
throw new Error("v3 host inputRef.path must be run-relative");
|
|
3766
4824
|
}
|
|
3767
|
-
const absolutePath = (0,
|
|
4825
|
+
const absolutePath = (0, import_node_path12.resolve)(input.runDir, input.inputRef.path);
|
|
3768
4826
|
assertPathInside(input.runDir, absolutePath);
|
|
3769
4827
|
assertSecureExistingPath(input.runDir, absolutePath, "host input");
|
|
3770
4828
|
assertRegularFile(absolutePath, "host input");
|
|
3771
|
-
const bytes = (0,
|
|
4829
|
+
const bytes = (0, import_node_fs14.readFileSync)(absolutePath);
|
|
3772
4830
|
if (bytes.length !== input.inputRef.bytes || sha256(bytes) !== input.inputRef.sha256) {
|
|
3773
4831
|
throw new Error("v3 host input sidecar bytes/hash mismatch");
|
|
3774
4832
|
}
|
|
@@ -3811,10 +4869,10 @@ function readAndVerifyV3PreparedHostInput(input) {
|
|
|
3811
4869
|
function writeV3HostSuccessArtifacts(input) {
|
|
3812
4870
|
canonicalJson(input.output);
|
|
3813
4871
|
canonicalJson(input.externalRefs);
|
|
3814
|
-
const outputDir = (0,
|
|
4872
|
+
const outputDir = (0, import_node_path12.join)(input.attemptDir, "work");
|
|
3815
4873
|
assertPathInside(input.runDir, input.attemptDir);
|
|
3816
4874
|
ensureSecureDirectoryTree(input.runDir, outputDir);
|
|
3817
|
-
const resultPath = (0,
|
|
4875
|
+
const resultPath = (0, import_node_path12.join)(outputDir, "result.json");
|
|
3818
4876
|
const resultContent = `${JSON.stringify({
|
|
3819
4877
|
schemaVersion: 1,
|
|
3820
4878
|
runId: input.runId,
|
|
@@ -3845,7 +4903,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3845
4903
|
mime: "application/json"
|
|
3846
4904
|
}]
|
|
3847
4905
|
};
|
|
3848
|
-
const manifestPath = (0,
|
|
4906
|
+
const manifestPath = (0, import_node_path12.join)(input.attemptDir, "manifest.json");
|
|
3849
4907
|
writeCreateOrVerify(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
3850
4908
|
`, 384, "host manifest");
|
|
3851
4909
|
fsyncRegularFileSync(resultPath);
|
|
@@ -3855,7 +4913,7 @@ function writeV3HostSuccessArtifacts(input) {
|
|
|
3855
4913
|
return { manifestPath, resultPath };
|
|
3856
4914
|
}
|
|
3857
4915
|
function readAndVerifyV3HostSuccessResult(input) {
|
|
3858
|
-
assertRegularFile((0,
|
|
4916
|
+
assertRegularFile((0, import_node_path12.join)(input.attemptDir, "manifest.json"), "host manifest");
|
|
3859
4917
|
if (input.manifest.status !== "ok") {
|
|
3860
4918
|
throw new Error("v3 host success manifest status must be ok");
|
|
3861
4919
|
}
|
|
@@ -3866,11 +4924,11 @@ function readAndVerifyV3HostSuccessResult(input) {
|
|
|
3866
4924
|
if (file.name !== "result" || file.path !== "result.json" || file.kind !== "json") {
|
|
3867
4925
|
throw new Error("v3 host success manifest must expose exactly result.json as JSON");
|
|
3868
4926
|
}
|
|
3869
|
-
const resultPath = (0,
|
|
4927
|
+
const resultPath = (0, import_node_path12.join)(input.attemptDir, "work", "result.json");
|
|
3870
4928
|
assertPathInside(input.runDir, resultPath);
|
|
3871
4929
|
assertSecureExistingPath(input.runDir, resultPath, "host result");
|
|
3872
4930
|
assertRegularFile(resultPath, "host result");
|
|
3873
|
-
const bytes = (0,
|
|
4931
|
+
const bytes = (0, import_node_fs14.readFileSync)(resultPath);
|
|
3874
4932
|
if (bytes.length !== file.bytes || sha256(bytes) !== file.sha256) {
|
|
3875
4933
|
throw new Error("v3 host success result bytes/hash mismatch");
|
|
3876
4934
|
}
|
|
@@ -3957,41 +5015,41 @@ function parsePreparedHostInput(raw) {
|
|
|
3957
5015
|
return raw;
|
|
3958
5016
|
}
|
|
3959
5017
|
function writeCreateOrVerify(path, content, mode, label) {
|
|
3960
|
-
const tempPath = `${path}.tmp-${process.pid}-${(0,
|
|
5018
|
+
const tempPath = `${path}.tmp-${process.pid}-${(0, import_node_crypto8.randomUUID)()}`;
|
|
3961
5019
|
let fd;
|
|
3962
5020
|
try {
|
|
3963
|
-
fd = (0,
|
|
5021
|
+
fd = (0, import_node_fs14.openSync)(
|
|
3964
5022
|
tempPath,
|
|
3965
|
-
|
|
5023
|
+
import_node_fs14.constants.O_WRONLY | import_node_fs14.constants.O_CREAT | import_node_fs14.constants.O_EXCL | (import_node_fs14.constants.O_NOFOLLOW ?? 0),
|
|
3966
5024
|
mode
|
|
3967
5025
|
);
|
|
3968
|
-
(0,
|
|
3969
|
-
(0,
|
|
3970
|
-
(0,
|
|
5026
|
+
(0, import_node_fs14.writeFileSync)(fd, content, "utf-8");
|
|
5027
|
+
(0, import_node_fs14.fsyncSync)(fd);
|
|
5028
|
+
(0, import_node_fs14.closeSync)(fd);
|
|
3971
5029
|
fd = void 0;
|
|
3972
5030
|
try {
|
|
3973
|
-
(0,
|
|
3974
|
-
fsyncDirectorySyncPortable((0,
|
|
5031
|
+
(0, import_node_fs14.linkSync)(tempPath, path);
|
|
5032
|
+
fsyncDirectorySyncPortable((0, import_node_path12.dirname)(path));
|
|
3975
5033
|
} catch (err) {
|
|
3976
5034
|
if (err.code !== "EEXIST") throw err;
|
|
3977
5035
|
assertRegularFile(path, label);
|
|
3978
|
-
if ((0,
|
|
5036
|
+
if ((0, import_node_fs14.readFileSync)(path, "utf-8") !== content) {
|
|
3979
5037
|
throw new Error(`v3 ${label} conflict at ${path}; refusing to overwrite committed bytes`);
|
|
3980
5038
|
}
|
|
3981
5039
|
}
|
|
3982
5040
|
} finally {
|
|
3983
|
-
if (fd !== void 0) (0,
|
|
5041
|
+
if (fd !== void 0) (0, import_node_fs14.closeSync)(fd);
|
|
3984
5042
|
try {
|
|
3985
|
-
(0,
|
|
3986
|
-
fsyncDirectorySyncPortable((0,
|
|
5043
|
+
(0, import_node_fs14.unlinkSync)(tempPath);
|
|
5044
|
+
fsyncDirectorySyncPortable((0, import_node_path12.dirname)(path));
|
|
3987
5045
|
} catch {
|
|
3988
5046
|
}
|
|
3989
5047
|
}
|
|
3990
5048
|
fsyncRegularFileSync(path);
|
|
3991
|
-
fsyncDirectorySyncPortable((0,
|
|
5049
|
+
fsyncDirectorySyncPortable((0, import_node_path12.dirname)(path));
|
|
3992
5050
|
}
|
|
3993
5051
|
function assertRegularFile(path, label) {
|
|
3994
|
-
const stat = (0,
|
|
5052
|
+
const stat = (0, import_node_fs14.lstatSync)(path);
|
|
3995
5053
|
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
3996
5054
|
throw new Error(`v3 ${label} must be a regular file: ${path}`);
|
|
3997
5055
|
}
|
|
@@ -4000,47 +5058,47 @@ function assertRegularFile(path, label) {
|
|
|
4000
5058
|
}
|
|
4001
5059
|
}
|
|
4002
5060
|
function assertPathInside(parent, child) {
|
|
4003
|
-
const root = (0,
|
|
4004
|
-
const rel = (0,
|
|
4005
|
-
if (rel === "" || rel === ".." || rel.startsWith(`..${
|
|
5061
|
+
const root = (0, import_node_path12.resolve)(parent);
|
|
5062
|
+
const rel = (0, import_node_path12.relative)(root, (0, import_node_path12.resolve)(child));
|
|
5063
|
+
if (rel === "" || rel === ".." || rel.startsWith(`..${import_node_path12.sep}`) || (0, import_node_path12.isAbsolute)(rel)) {
|
|
4006
5064
|
throw new Error("v3 host path escapes runDir");
|
|
4007
5065
|
}
|
|
4008
5066
|
}
|
|
4009
5067
|
function ensureSecureDirectoryTree(rootPath, targetPath) {
|
|
4010
|
-
const root = (0,
|
|
4011
|
-
const target = (0,
|
|
4012
|
-
const rel = (0,
|
|
4013
|
-
if (rel === ".." || rel.startsWith(`..${
|
|
5068
|
+
const root = (0, import_node_path12.resolve)(rootPath);
|
|
5069
|
+
const target = (0, import_node_path12.resolve)(targetPath);
|
|
5070
|
+
const rel = (0, import_node_path12.relative)(root, target);
|
|
5071
|
+
if (rel === ".." || rel.startsWith(`..${import_node_path12.sep}`) || (0, import_node_path12.isAbsolute)(rel)) {
|
|
4014
5072
|
throw new Error("v3 host directory escapes runDir");
|
|
4015
5073
|
}
|
|
4016
5074
|
assertDirectory(root, "run directory");
|
|
4017
5075
|
let current = root;
|
|
4018
5076
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4019
|
-
current = (0,
|
|
4020
|
-
if (!(0,
|
|
5077
|
+
current = (0, import_node_path12.join)(current, segment);
|
|
5078
|
+
if (!(0, import_node_fs14.existsSync)(current)) (0, import_node_fs14.mkdirSync)(current, { mode: 448 });
|
|
4021
5079
|
assertDirectory(current, "host artifact directory");
|
|
4022
5080
|
}
|
|
4023
5081
|
}
|
|
4024
5082
|
function assertSecureExistingPath(rootPath, filePath, label) {
|
|
4025
|
-
const root = (0,
|
|
4026
|
-
const target = (0,
|
|
5083
|
+
const root = (0, import_node_path12.resolve)(rootPath);
|
|
5084
|
+
const target = (0, import_node_path12.resolve)(filePath);
|
|
4027
5085
|
assertPathInside(root, target);
|
|
4028
5086
|
assertDirectory(root, "run directory");
|
|
4029
|
-
const rel = (0,
|
|
5087
|
+
const rel = (0, import_node_path12.relative)(root, (0, import_node_path12.dirname)(target));
|
|
4030
5088
|
let current = root;
|
|
4031
5089
|
for (const segment of rel.split(/[\\/]/).filter(Boolean)) {
|
|
4032
|
-
current = (0,
|
|
5090
|
+
current = (0, import_node_path12.join)(current, segment);
|
|
4033
5091
|
assertDirectory(current, `${label} parent`);
|
|
4034
5092
|
}
|
|
4035
5093
|
}
|
|
4036
5094
|
function assertDirectory(path, label) {
|
|
4037
|
-
const stat = (0,
|
|
5095
|
+
const stat = (0, import_node_fs14.lstatSync)(path);
|
|
4038
5096
|
if (!stat.isDirectory() || stat.isSymbolicLink()) {
|
|
4039
5097
|
throw new Error(`v3 ${label} must be a real directory: ${path}`);
|
|
4040
5098
|
}
|
|
4041
5099
|
}
|
|
4042
5100
|
function sha256(value) {
|
|
4043
|
-
return (0,
|
|
5101
|
+
return (0, import_node_crypto8.createHash)("sha256").update(value).digest("hex");
|
|
4044
5102
|
}
|
|
4045
5103
|
function isRecord3(value) {
|
|
4046
5104
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -4102,7 +5160,7 @@ function renderGoalFile(goal, resultSchema, loopCtx, nodeInstructions, hasWorkfl
|
|
|
4102
5160
|
...artifactSection,
|
|
4103
5161
|
"## How to complete this node",
|
|
4104
5162
|
"You are an autonomous agent completing exactly ONE botmux v3 workflow node.",
|
|
4105
|
-
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (
|
|
5163
|
+
"Work toward the goal above until it is done, then stop. Do NOT ask the user with interactive tools (they are disabled in this mode). If you genuinely need a human DECISION to proceed, use the human-ask escape hatch described below (the same protocol is available via `botmux skill show botmux-goal-ask`).",
|
|
4106
5164
|
"",
|
|
4107
5165
|
`- Upstream inputs: the file at $${E.INPUTS_PATH} is a JSON object \`{ "inputs": [...] }\` listing upstream products, each with an absolute \`path\`. Read only the ones the goal needs (it may be empty). If it includes an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`, read that JSON file before continuing. If an \`omitted\` array is present, those declared inputs were intentionally not produced (their workflow branch was not taken) \u2014 treat their absence as by-design, do NOT invent their content.`,
|
|
4108
5166
|
`- Revisit feedback: if any input has \`"from": "revisit"\`, a DOWNSTREAM node sent this node back because its product was inadequate. You MUST read these before doing anything else: \`reason\` (why you were sent back), \`source:*\` (the downstream node's output \u2014 the evidence of what was wrong), and \`previous:*\` (YOUR OWN previous output \u2014 edit/fix it, do not rewrite from scratch). Address the reason; do not just reproduce the prior output.`,
|
|
@@ -4123,186 +5181,71 @@ function renderGoalFile(goal, resultSchema, loopCtx, nodeInstructions, hasWorkfl
|
|
|
4123
5181
|
` - On failure: status "${failStatus}", \`error\` required, \`files\` may be empty. Set \`error.retryable\` honestly: \`true\` when a human can unblock you and a fresh attempt could then succeed; \`false\` when retrying cannot help.`,
|
|
4124
5182
|
` - Every file \`path\` is relative to $${E.OUTPUT_DIR} ITSELF. A file you wrote directly into that directory has a path that is JUST its filename, e.g. \`"path": "report.md"\`. Do NOT prepend the directory or its folder name (NOT \`"work/report.md"\`) and do NOT use an absolute path \u2014 both are rejected.`,
|
|
4125
5183
|
"",
|
|
4126
|
-
...resultSection,
|
|
4127
|
-
`You are DONE only after the manifest at $${E.MANIFEST_PATH} exists and every file it references exists.`,
|
|
4128
|
-
"If you cannot complete the goal, write a failure manifest and stop.",
|
|
4129
|
-
'If you hit an authentication / authorization / interactive-confirmation wall (a login prompt, an expired token, a permission you cannot grant yourself): do NOT wait for a human and do NOT keep retrying. Immediately write a failure manifest with an `error.code` like "AUTH_REQUIRED" and `error.retryable: true`, then stop \u2014 a human will unblock and retry this node.',
|
|
4130
|
-
"",
|
|
4131
|
-
"## Asking a human (only when a DECISION truly needs a person)",
|
|
4132
|
-
`If \u2014 and ONLY if \u2014 you cannot proceed without a human's judgement call (a choice only a person can make; NOT something you can research, infer, or decide yourself), use the runtime human-ask:`,
|
|
4133
|
-
` 1. Write a JSON file to $${E.ATTEMPT_DIR}/${GOAL_ASK_FILE}. Use \`{ "question": "<one clear question>", "options": ["<2-6 concrete choices>"] }\` for a choice, or \`{ "question": "<one clear question>", "freeText": true }\` when the human must provide details in their own words.`,
|
|
4134
|
-
` 2. Write a failure manifest with \`error.code: "${ASK_HUMAN_ERROR_CODE}"\`, \`error.retryable: true\`, and \`summary\` = your question, then STOP.`,
|
|
4135
|
-
`A human answers; this node then RE-RUNS with their answer injected into $${E.INPUTS_PATH} as an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`. Read that JSON file's \`selected\` or \`text\` field and continue from there. Prefer deciding yourself \u2014 every ask pauses the whole workflow on a person.`,
|
|
4136
|
-
""
|
|
4137
|
-
].join("\n");
|
|
4138
|
-
}
|
|
4139
|
-
function classifyTerminal(errorClass, opts) {
|
|
4140
|
-
if (opts?.selfReportedFail) return opts.retryable === false ? "failed" : "blocked";
|
|
4141
|
-
switch (errorClass) {
|
|
4142
|
-
case "manifestInvalid":
|
|
4143
|
-
// agent wrote a bad manifest — a retry may fix it
|
|
4144
|
-
case "artifactContractInvalid":
|
|
4145
|
-
// definition/manifest public product mismatch
|
|
4146
|
-
case "resultInvalid":
|
|
4147
|
-
return "blocked";
|
|
4148
|
-
case "workerError":
|
|
4149
|
-
// process crash = infrastructure
|
|
4150
|
-
case "timeout":
|
|
4151
|
-
// budget exceeded = infrastructure (for now)
|
|
4152
|
-
case "gateRejected":
|
|
4153
|
-
// a human said no — retrying won't change that
|
|
4154
|
-
case "cancelled":
|
|
4155
|
-
return "failed";
|
|
4156
|
-
}
|
|
4157
|
-
}
|
|
4158
|
-
function readGoalAsk(askPath) {
|
|
4159
|
-
if (!(0, import_node_fs11.existsSync)(askPath)) return void 0;
|
|
4160
|
-
let parsed;
|
|
4161
|
-
try {
|
|
4162
|
-
parsed = JSON.parse((0, import_node_fs11.readFileSync)(askPath, "utf-8"));
|
|
4163
|
-
} catch {
|
|
4164
|
-
return void 0;
|
|
4165
|
-
}
|
|
4166
|
-
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
4167
|
-
const o = parsed;
|
|
4168
|
-
const question = typeof o.question === "string" ? o.question.trim() : "";
|
|
4169
|
-
if (!question) return void 0;
|
|
4170
|
-
const hasOptions = Object.prototype.hasOwnProperty.call(o, "options");
|
|
4171
|
-
if (o.freeText === true) {
|
|
4172
|
-
if (hasOptions) return void 0;
|
|
4173
|
-
return { question, freeText: true };
|
|
4174
|
-
}
|
|
4175
|
-
const options = Array.isArray(o.options) ? o.options.filter((x) => typeof x === "string" && x.trim() !== "").map((x) => x.trim()) : [];
|
|
4176
|
-
if (options.length < 2) return void 0;
|
|
4177
|
-
return { question, options: options.slice(0, 6) };
|
|
4178
|
-
}
|
|
4179
|
-
function mergeNodeCapability(snap, override) {
|
|
4180
|
-
if (!override) return snap;
|
|
4181
|
-
return {
|
|
4182
|
-
...snap,
|
|
4183
|
-
...override.model ? { model: override.model } : {}
|
|
4184
|
-
};
|
|
4185
|
-
}
|
|
4186
|
-
function readRevisitRequest(manifest, outputDir) {
|
|
4187
|
-
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
4188
|
-
if (!entry) return { ok: true };
|
|
4189
|
-
let value;
|
|
4190
|
-
try {
|
|
4191
|
-
value = JSON.parse((0, import_node_fs11.readFileSync)((0, import_node_path9.join)(outputDir, entry.path), "utf-8"));
|
|
4192
|
-
} catch {
|
|
4193
|
-
return { ok: true };
|
|
4194
|
-
}
|
|
4195
|
-
if (!value || typeof value !== "object" || value.status !== "revisit") {
|
|
4196
|
-
return { ok: true };
|
|
4197
|
-
}
|
|
4198
|
-
const v = value;
|
|
4199
|
-
const problems = [];
|
|
4200
|
-
if (typeof v.revisitTo !== "string" || v.revisitTo.trim() === "") {
|
|
4201
|
-
problems.push('result.json status "revisit" requires a non-empty string "revisitTo"');
|
|
4202
|
-
}
|
|
4203
|
-
if (v.reason !== void 0 && typeof v.reason !== "string") {
|
|
4204
|
-
problems.push('result.json "reason" must be a string when present');
|
|
4205
|
-
}
|
|
4206
|
-
if (problems.length > 0) return { ok: false, problems };
|
|
4207
|
-
return {
|
|
4208
|
-
ok: true,
|
|
4209
|
-
request: {
|
|
4210
|
-
toNodeId: v.revisitTo,
|
|
4211
|
-
...typeof v.reason === "string" && v.reason ? { reason: v.reason } : {}
|
|
4212
|
-
}
|
|
4213
|
-
};
|
|
4214
|
-
}
|
|
4215
|
-
function validateResult(filePath, schema) {
|
|
4216
|
-
if (!(0, import_node_fs11.existsSync)(filePath)) return { ok: false, problems: [`result.json not found at ${filePath}`] };
|
|
4217
|
-
let parsed;
|
|
4218
|
-
try {
|
|
4219
|
-
parsed = JSON.parse((0, import_node_fs11.readFileSync)(filePath, "utf-8"));
|
|
4220
|
-
} catch (err) {
|
|
4221
|
-
return { ok: false, problems: [`result.json is not valid JSON: ${err instanceof Error ? err.message : String(err)}`] };
|
|
4222
|
-
}
|
|
4223
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
4224
|
-
return { ok: false, problems: ["result.json root must be a JSON object"] };
|
|
4225
|
-
}
|
|
4226
|
-
const obj = parsed;
|
|
4227
|
-
const problems = [];
|
|
4228
|
-
for (const field of schema.required ?? []) {
|
|
4229
|
-
if (!(field in obj)) problems.push(`missing required field "${field}"`);
|
|
4230
|
-
}
|
|
4231
|
-
for (const [name, spec] of Object.entries(schema.properties)) {
|
|
4232
|
-
if (!(name in obj)) continue;
|
|
4233
|
-
const v = obj[name];
|
|
4234
|
-
const okType = spec.type === "string" ? typeof v === "string" : spec.type === "number" ? typeof v === "number" && Number.isFinite(v) : spec.type === "boolean" ? typeof v === "boolean" : spec.type === "array" ? Array.isArray(v) : typeof v === "object" && v !== null && !Array.isArray(v);
|
|
4235
|
-
if (!okType) {
|
|
4236
|
-
problems.push(`field "${name}" must be of type ${spec.type}`);
|
|
4237
|
-
continue;
|
|
4238
|
-
}
|
|
4239
|
-
if (spec.type === "string" && spec.enum && !spec.enum.includes(v)) {
|
|
4240
|
-
problems.push(`field "${name}" must be one of [${spec.enum.join(", ")}] (got ${JSON.stringify(v)})`);
|
|
4241
|
-
}
|
|
4242
|
-
}
|
|
4243
|
-
return problems.length > 0 ? { ok: false, problems } : { ok: true };
|
|
4244
|
-
}
|
|
4245
|
-
var ATTEMPT_NNN_RE = /\/attempts\/(\d{3})$/;
|
|
4246
|
-
function attemptNumber(attemptId) {
|
|
4247
|
-
const m = ATTEMPT_NNN_RE.exec(attemptId);
|
|
4248
|
-
return m ? parseInt(m[1], 10) : void 0;
|
|
4249
|
-
}
|
|
4250
|
-
function nextAttemptIdFor(events, key) {
|
|
4251
|
-
const matches = (e) => (e.instanceId ?? e.nodeId) === key;
|
|
4252
|
-
let maxSeen = 0;
|
|
4253
|
-
let reserved;
|
|
4254
|
-
let prepared;
|
|
4255
|
-
for (const e of events) {
|
|
4256
|
-
if (e.type === "nodeDispatched" && matches(e)) {
|
|
4257
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4258
|
-
if (n2 === void 0) continue;
|
|
4259
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4260
|
-
if (reserved === n2) reserved = void 0;
|
|
4261
|
-
} else if (e.type === "hostInputPrepared" && matches(e)) {
|
|
4262
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4263
|
-
if (n2 === void 0) continue;
|
|
4264
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4265
|
-
if (reserved === n2) reserved = void 0;
|
|
4266
|
-
prepared = e.attemptId;
|
|
4267
|
-
} else if ((e.type === "nodeSucceeded" || e.type === "nodeFailed" || e.type === "nodeBlocked") && matches(e)) {
|
|
4268
|
-
const n2 = attemptNumber(e.attemptId);
|
|
4269
|
-
if (n2 !== void 0) maxSeen = Math.max(maxSeen, n2);
|
|
4270
|
-
if (prepared === e.attemptId) prepared = void 0;
|
|
4271
|
-
} else if (e.type === "hostEffectIntent" && matches(e) && prepared === e.attemptId) {
|
|
4272
|
-
prepared = void 0;
|
|
4273
|
-
} else if (e.type === "nodeRetryRequested" && matches(e)) {
|
|
4274
|
-
const n2 = attemptNumber(e.nextAttemptId);
|
|
4275
|
-
if (n2 === void 0) continue;
|
|
4276
|
-
reserved = n2;
|
|
4277
|
-
prepared = void 0;
|
|
4278
|
-
maxSeen = Math.max(maxSeen, n2);
|
|
4279
|
-
}
|
|
5184
|
+
...resultSection,
|
|
5185
|
+
`You are DONE only after the manifest at $${E.MANIFEST_PATH} exists and every file it references exists.`,
|
|
5186
|
+
"If you cannot complete the goal, write a failure manifest and stop.",
|
|
5187
|
+
'If you hit an authentication / authorization / interactive-confirmation wall (a login prompt, an expired token, a permission you cannot grant yourself): do NOT wait for a human and do NOT keep retrying. Immediately write a failure manifest with an `error.code` like "AUTH_REQUIRED" and `error.retryable: true`, then stop \u2014 a human will unblock and retry this node.',
|
|
5188
|
+
"",
|
|
5189
|
+
"## Asking a human (only when a DECISION truly needs a person)",
|
|
5190
|
+
`If \u2014 and ONLY if \u2014 you cannot proceed without a human's judgement call (a choice only a person can make; NOT something you can research, infer, or decide yourself), use the runtime human-ask:`,
|
|
5191
|
+
` 1. Write a JSON file to $${E.ATTEMPT_DIR}/${GOAL_ASK_FILE}. Use \`{ "question": "<one clear question>", "options": ["<2-6 concrete choices>"] }\` for a choice, or \`{ "question": "<one clear question>", "freeText": true }\` when the human must provide details in their own words.`,
|
|
5192
|
+
` 2. Write a failure manifest with \`error.code: "${ASK_HUMAN_ERROR_CODE}"\`, \`error.retryable: true\`, and \`summary\` = your question, then STOP.`,
|
|
5193
|
+
`A human answers; this node then RE-RUNS with their answer injected into $${E.INPUTS_PATH} as an input entry \`{ "from": "human", "name": "answer", "path": "..." }\`. Read that JSON file's \`selected\` or \`text\` field and continue from there. Prefer deciding yourself \u2014 every ask pauses the whole workflow on a person.`,
|
|
5194
|
+
""
|
|
5195
|
+
].join("\n");
|
|
5196
|
+
}
|
|
5197
|
+
function classifyTerminal(errorClass, opts) {
|
|
5198
|
+
if (opts?.selfReportedFail) return opts.retryable === false ? "failed" : "blocked";
|
|
5199
|
+
switch (errorClass) {
|
|
5200
|
+
case "manifestInvalid":
|
|
5201
|
+
// agent wrote a bad manifest — a retry may fix it
|
|
5202
|
+
case "artifactContractInvalid":
|
|
5203
|
+
// definition/manifest public product mismatch
|
|
5204
|
+
case "resultInvalid":
|
|
5205
|
+
return "blocked";
|
|
5206
|
+
case "workerError":
|
|
5207
|
+
// process crash = infrastructure
|
|
5208
|
+
case "timeout":
|
|
5209
|
+
// budget exceeded = infrastructure (for now)
|
|
5210
|
+
case "gateRejected":
|
|
5211
|
+
// a human said no — retrying won't change that
|
|
5212
|
+
case "cancelled":
|
|
5213
|
+
return "failed";
|
|
4280
5214
|
}
|
|
4281
|
-
if (prepared) return prepared;
|
|
4282
|
-
const n = reserved ?? maxSeen + 1;
|
|
4283
|
-
return `${key}/attempts/${String(n).padStart(3, "0")}`;
|
|
4284
5215
|
}
|
|
4285
|
-
function
|
|
4286
|
-
|
|
4287
|
-
let
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
if (latestNumber < 0) latest = e.attemptId;
|
|
4293
|
-
} else if (n >= latestNumber) {
|
|
4294
|
-
latest = e.attemptId;
|
|
4295
|
-
latestNumber = n;
|
|
4296
|
-
}
|
|
4297
|
-
}
|
|
5216
|
+
function readGoalAsk(askPath) {
|
|
5217
|
+
if (!(0, import_node_fs15.existsSync)(askPath)) return void 0;
|
|
5218
|
+
let parsed;
|
|
5219
|
+
try {
|
|
5220
|
+
parsed = JSON.parse((0, import_node_fs15.readFileSync)(askPath, "utf-8"));
|
|
5221
|
+
} catch {
|
|
5222
|
+
return void 0;
|
|
4298
5223
|
}
|
|
4299
|
-
return
|
|
5224
|
+
if (typeof parsed !== "object" || parsed === null) return void 0;
|
|
5225
|
+
const o = parsed;
|
|
5226
|
+
const question = typeof o.question === "string" ? o.question.trim() : "";
|
|
5227
|
+
if (!question) return void 0;
|
|
5228
|
+
const hasOptions = Object.prototype.hasOwnProperty.call(o, "options");
|
|
5229
|
+
if (o.freeText === true) {
|
|
5230
|
+
if (hasOptions) return void 0;
|
|
5231
|
+
return { question, freeText: true };
|
|
5232
|
+
}
|
|
5233
|
+
const options = Array.isArray(o.options) ? o.options.filter((x) => typeof x === "string" && x.trim() !== "").map((x) => x.trim()) : [];
|
|
5234
|
+
if (options.length < 2) return void 0;
|
|
5235
|
+
return { question, options: options.slice(0, 6) };
|
|
5236
|
+
}
|
|
5237
|
+
function mergeNodeCapability(snap, override) {
|
|
5238
|
+
if (!override) return snap;
|
|
5239
|
+
return {
|
|
5240
|
+
...snap,
|
|
5241
|
+
...override.model ? { model: override.model } : {}
|
|
5242
|
+
};
|
|
4300
5243
|
}
|
|
4301
5244
|
function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) => candidate) {
|
|
4302
5245
|
const expectedHash = computeInputHash(normalize(value));
|
|
4303
|
-
if (!(0,
|
|
5246
|
+
if (!(0, import_node_fs15.existsSync)(path)) {
|
|
4304
5247
|
try {
|
|
4305
|
-
(0,
|
|
5248
|
+
(0, import_node_fs15.writeFileSync)(path, `${JSON.stringify(value, null, 2)}
|
|
4306
5249
|
`, {
|
|
4307
5250
|
encoding: "utf-8",
|
|
4308
5251
|
flag: "wx",
|
|
@@ -4316,7 +5259,7 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4316
5259
|
}
|
|
4317
5260
|
let stored;
|
|
4318
5261
|
try {
|
|
4319
|
-
stored = normalize(JSON.parse((0,
|
|
5262
|
+
stored = normalize(JSON.parse((0, import_node_fs15.readFileSync)(path, "utf-8")));
|
|
4320
5263
|
} catch (error) {
|
|
4321
5264
|
throw new Error(
|
|
4322
5265
|
`v3 runtime: frozen ${label} is unreadable: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -4328,11 +5271,22 @@ function assertOrCreateFrozenJson(path, value, label, normalize = (candidate) =>
|
|
|
4328
5271
|
);
|
|
4329
5272
|
}
|
|
4330
5273
|
}
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
|
|
4334
|
-
|
|
4335
|
-
|
|
5274
|
+
var HANDOFF_RESEND_BACKOFF_MS = 3e4;
|
|
5275
|
+
var PROBE_RETRY_BACKOFF_MS = 3e4;
|
|
5276
|
+
var PROBE_UNCERTAIN_AFTER_MS = 10 * 6e4;
|
|
5277
|
+
var probeFailures = /* @__PURE__ */ new Map();
|
|
5278
|
+
function forgetProbeFailuresOf(runDir) {
|
|
5279
|
+
const prefix = `${runDir}\0`;
|
|
5280
|
+
for (const key of probeFailures.keys()) {
|
|
5281
|
+
if (key.startsWith(prefix)) probeFailures.delete(key);
|
|
5282
|
+
}
|
|
5283
|
+
}
|
|
5284
|
+
async function runWorkflow(authorizedDag, deps, opts) {
|
|
5285
|
+
let dag = authorizedDag;
|
|
5286
|
+
const runDir = (0, import_node_path13.join)(opts.baseDir, dag.runId);
|
|
5287
|
+
(0, import_node_fs15.mkdirSync)(runDir, { recursive: true });
|
|
5288
|
+
const journalPath = (0, import_node_path13.join)(runDir, "journal.ndjson");
|
|
5289
|
+
const statePath = (0, import_node_path13.join)(runDir, "STATE");
|
|
4336
5290
|
const globalCap = opts.globalConcurrency ?? 4;
|
|
4337
5291
|
const perBotCap = opts.perBotConcurrency ?? 1;
|
|
4338
5292
|
const perCliCap = opts.perCliConcurrency ?? 2;
|
|
@@ -4345,7 +5299,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4345
5299
|
if (!Number.isSafeInteger(hostResponseWaitMs) || hostResponseWaitMs < 1) {
|
|
4346
5300
|
throw new Error("v3 runtime: hostResponseWaitMs must be a positive safe integer");
|
|
4347
5301
|
}
|
|
4348
|
-
const
|
|
5302
|
+
const externalMode = opts.nodeExecutor === "bot";
|
|
5303
|
+
const externalExecutor = deps.externalExecutor;
|
|
5304
|
+
if (externalMode && !externalExecutor) {
|
|
5305
|
+
throw new Error('v3 runtime: nodeExecutor "bot" requires an external node executor');
|
|
5306
|
+
}
|
|
5307
|
+
const externalAcceptTimeoutMs = opts.externalAcceptTimeoutMs ?? V3_DEFAULT_HANDOFF_ACCEPT_TIMEOUT_MS;
|
|
5308
|
+
if (!Number.isSafeInteger(externalAcceptTimeoutMs) || externalAcceptTimeoutMs < 0) {
|
|
5309
|
+
throw new Error("v3 runtime: externalAcceptTimeoutMs must be a non-negative safe integer");
|
|
5310
|
+
}
|
|
5311
|
+
const clock = () => deps.now?.() ?? Date.now();
|
|
5312
|
+
const externalCallTimeoutMs = opts.externalCallTimeoutMs ?? V3_EXTERNAL_CALL_TIMEOUT_MS;
|
|
5313
|
+
if (!Number.isSafeInteger(externalCallTimeoutMs) || externalCallTimeoutMs < 1) {
|
|
5314
|
+
throw new Error("v3 runtime: externalCallTimeoutMs must be a positive safe integer");
|
|
5315
|
+
}
|
|
5316
|
+
const probeKey = (attemptId) => `${runDir}\0${attemptId}`;
|
|
5317
|
+
let nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5318
|
+
let dagRevision = 1;
|
|
5319
|
+
let pendingTouched;
|
|
5320
|
+
let pendingRevision;
|
|
4349
5321
|
const botSnapshots = opts.frozenBotSnapshots ? new Map(opts.frozenBotSnapshots) : /* @__PURE__ */ new Map();
|
|
4350
5322
|
const freezeBot = (bot) => {
|
|
4351
5323
|
const key = bot ?? "";
|
|
@@ -4366,7 +5338,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4366
5338
|
for (const b of node.body.nodes) freezeBot(b.bot ?? node.bot);
|
|
4367
5339
|
}
|
|
4368
5340
|
}
|
|
4369
|
-
|
|
5341
|
+
const validateSnapshot = (key, snap) => {
|
|
4370
5342
|
if (deps.validateExecutionSnapshot) {
|
|
4371
5343
|
deps.validateExecutionSnapshot(snap, key);
|
|
4372
5344
|
} else if (!isV3SupportedCli(snap.cliId)) {
|
|
@@ -4374,22 +5346,74 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4374
5346
|
`v3 runtime: bot "${key || "<default>"}" resolves to CLI "${snap.cliId}", which is not supported by v3 goal-mode (supported: ${V3_SUPPORTED_CLIS.join(", ")})`
|
|
4375
5347
|
);
|
|
4376
5348
|
}
|
|
4377
|
-
}
|
|
5349
|
+
};
|
|
5350
|
+
for (const [key, snap] of botSnapshots) validateSnapshot(key, snap);
|
|
5351
|
+
const botSnapshotFor2 = (selector) => {
|
|
5352
|
+
const snap = snapshotForSelector(botSnapshots, selector);
|
|
5353
|
+
if (!snap) {
|
|
5354
|
+
throw new Error(`v3 runtime: no frozen bot snapshot for selector "${selector || "<default>"}"`);
|
|
5355
|
+
}
|
|
5356
|
+
return snap;
|
|
5357
|
+
};
|
|
5358
|
+
const refreshEffectiveDag = (events) => {
|
|
5359
|
+
const history = foldV3Revisions(events);
|
|
5360
|
+
pendingTouched = history.pending ? new Set(history.pending.touched) : void 0;
|
|
5361
|
+
pendingRevision = history.pending?.revision;
|
|
5362
|
+
if (history.current === dagRevision) return;
|
|
5363
|
+
const effective = loadEffectiveV3Dag(runDir, authorizedDag, events);
|
|
5364
|
+
for (const [selector, snap] of v3ApprovedBotScope(botSnapshots, effective.botAdditions)) {
|
|
5365
|
+
if (botSnapshots.has(selector)) continue;
|
|
5366
|
+
validateSnapshot(selector, snap);
|
|
5367
|
+
botSnapshots.set(selector, snap);
|
|
5368
|
+
}
|
|
5369
|
+
for (const node of effective.dag.nodes) {
|
|
5370
|
+
if (isHostNode(node) && !deps.hostExecutors?.has(node.executor)) {
|
|
5371
|
+
throw new Error(`v3 runtime: host executor "${node.executor}" is not registered`);
|
|
5372
|
+
}
|
|
5373
|
+
if (isGoalNode(node)) botSnapshotFor2(node.bot);
|
|
5374
|
+
if (isLoopNode(node)) for (const body of node.body.nodes) botSnapshotFor2(body.bot ?? node.bot);
|
|
5375
|
+
}
|
|
5376
|
+
dag = effective.dag;
|
|
5377
|
+
nodesById = new Map(dag.nodes.map((n) => [n.id, n]));
|
|
5378
|
+
dagRevision = effective.revision;
|
|
5379
|
+
};
|
|
5380
|
+
const decisionStillCurrent = (latest, nodeId, loopId) => {
|
|
5381
|
+
if (currentV3Revision(latest) !== dagRevision) return false;
|
|
5382
|
+
if (nodeId === void 0) return true;
|
|
5383
|
+
const pending = foldV3Revisions(latest).pending;
|
|
5384
|
+
return !pending || !v3NodeHeldByProposal(new Set(pending.touched), nodeId, loopId);
|
|
5385
|
+
};
|
|
5386
|
+
const appendControlEvent = (event) => withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5387
|
+
if (!decisionStillCurrent(events)) return false;
|
|
5388
|
+
append(event);
|
|
5389
|
+
return true;
|
|
5390
|
+
});
|
|
5391
|
+
const heldByProposal = (nodeId, loopId) => v3NodeHeldByProposal(pendingTouched, nodeId, loopId);
|
|
5392
|
+
const reconcileRevisionFollowUps = (events) => {
|
|
5393
|
+
const live = new Set(dag.nodes.map((node) => node.id));
|
|
5394
|
+
if (missingV3RevisionFollowUps(events, live).length === 0) return false;
|
|
5395
|
+
const result = appendMissingV3RevisionFollowUps(journalPath, { revision: dagRevision, liveNodeIds: live });
|
|
5396
|
+
return result.kind === "stale" || result.count > 0;
|
|
5397
|
+
};
|
|
4378
5398
|
if (!opts.authorizedArtifacts) {
|
|
4379
5399
|
assertOrCreateFrozenJson(
|
|
4380
|
-
(0,
|
|
5400
|
+
(0, import_node_path13.join)(runDir, "bots.snapshot.json"),
|
|
4381
5401
|
Object.fromEntries(botSnapshots),
|
|
4382
5402
|
"execution profile snapshot"
|
|
4383
5403
|
);
|
|
4384
5404
|
assertOrCreateFrozenJson(
|
|
4385
|
-
(0,
|
|
5405
|
+
(0, import_node_path13.join)(runDir, "dag.json"),
|
|
4386
5406
|
dag,
|
|
4387
5407
|
"workflow definition",
|
|
4388
5408
|
validateDag
|
|
4389
5409
|
);
|
|
4390
5410
|
}
|
|
4391
5411
|
if (readJournal(journalPath).length === 0) {
|
|
4392
|
-
appendEvent(journalPath, {
|
|
5412
|
+
appendEvent(journalPath, {
|
|
5413
|
+
type: "runStarted",
|
|
5414
|
+
runId: dag.runId,
|
|
5415
|
+
...externalMode ? { nodeExecutor: "bot" } : {}
|
|
5416
|
+
});
|
|
4393
5417
|
}
|
|
4394
5418
|
const inFlight = /* @__PURE__ */ new Map();
|
|
4395
5419
|
const botInFlight = /* @__PURE__ */ new Map();
|
|
@@ -4400,6 +5424,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4400
5424
|
const hostPromiseAttempts = /* @__PURE__ */ new Map();
|
|
4401
5425
|
const hostDeadlineCancels = /* @__PURE__ */ new Map();
|
|
4402
5426
|
const blockingGateOwners = /* @__PURE__ */ new Map();
|
|
5427
|
+
const handoffControllers = /* @__PURE__ */ new Map();
|
|
5428
|
+
const handoffRetryAt = /* @__PURE__ */ new Map();
|
|
5429
|
+
const acceptanceObserved = /* @__PURE__ */ new Set();
|
|
4403
5430
|
class HostProviderResponseTimeoutError extends Error {
|
|
4404
5431
|
constructor() {
|
|
4405
5432
|
super(`host provider did not settle within ${hostResponseWaitMs}ms`);
|
|
@@ -4446,6 +5473,8 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4446
5473
|
};
|
|
4447
5474
|
while (true) {
|
|
4448
5475
|
const events = readJournal(journalPath);
|
|
5476
|
+
refreshEffectiveDag(events);
|
|
5477
|
+
if (reconcileRevisionFollowUps(events)) continue;
|
|
4449
5478
|
const snap = materialize(events);
|
|
4450
5479
|
writeState(statePath, snap);
|
|
4451
5480
|
if (opts.cancelSignal?.aborted && snap.runStatus !== "cancelling" && !isTrueRunTerminal(snap.runStatus)) {
|
|
@@ -4456,15 +5485,21 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4456
5485
|
if (openHostEffects.length > 0) {
|
|
4457
5486
|
const settled = await reconcileOpenHostEffects(openHostEffects);
|
|
4458
5487
|
detachClosedHostInvocations();
|
|
4459
|
-
if (!settled) await new Promise((
|
|
5488
|
+
if (!settled) await new Promise((resolve6) => setTimeout(resolve6, 1e3));
|
|
4460
5489
|
continue;
|
|
4461
5490
|
}
|
|
5491
|
+
if (settleExternalLeftovers(events)) continue;
|
|
4462
5492
|
if (snap.runStatus === "cancelling" && snap.cancelRequestId) {
|
|
4463
5493
|
for (const controller of nodeControllers.values()) {
|
|
4464
5494
|
if (!controller.signal.aborted) {
|
|
4465
5495
|
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
4466
5496
|
}
|
|
4467
5497
|
}
|
|
5498
|
+
for (const controller of handoffControllers.values()) {
|
|
5499
|
+
if (!controller.signal.aborted) {
|
|
5500
|
+
controller.abort({ kind: "run", cancelRequestId: snap.cancelRequestId });
|
|
5501
|
+
}
|
|
5502
|
+
}
|
|
4468
5503
|
cancelUnfinishedNodes(snap.cancelRequestId, { includeRunning: false });
|
|
4469
5504
|
for (const key of [...inFlight.keys()]) {
|
|
4470
5505
|
if (!key.endsWith("::gate")) continue;
|
|
@@ -4475,6 +5510,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4475
5510
|
await Promise.race(inFlight.values());
|
|
4476
5511
|
continue;
|
|
4477
5512
|
}
|
|
5513
|
+
if (openV3ExternalAttempts(events).length > 0) {
|
|
5514
|
+
await closeExternalAttemptsForCancellation(snap.cancelRequestId);
|
|
5515
|
+
continue;
|
|
5516
|
+
}
|
|
4478
5517
|
let externalDrained = false;
|
|
4479
5518
|
try {
|
|
4480
5519
|
externalDrained = drainExternallyOwnedAttempts({
|
|
@@ -4484,7 +5523,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4484
5523
|
} catch {
|
|
4485
5524
|
}
|
|
4486
5525
|
if (!externalDrained) {
|
|
4487
|
-
await new Promise((
|
|
5526
|
+
await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4488
5527
|
continue;
|
|
4489
5528
|
}
|
|
4490
5529
|
finalizeRuntimeCancellation(snap.cancelRequestId);
|
|
@@ -4499,7 +5538,16 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4499
5538
|
open,
|
|
4500
5539
|
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
4501
5540
|
);
|
|
4502
|
-
if (!drained) await new Promise((
|
|
5541
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5542
|
+
continue;
|
|
5543
|
+
}
|
|
5544
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5545
|
+
if (openExternal.length > 0) {
|
|
5546
|
+
await drainExternalAttempts(
|
|
5547
|
+
openExternal,
|
|
5548
|
+
"runBoundary",
|
|
5549
|
+
trueTerminal ? "orphanRecovery" : "terminalPeer"
|
|
5550
|
+
);
|
|
4503
5551
|
continue;
|
|
4504
5552
|
}
|
|
4505
5553
|
break;
|
|
@@ -4509,7 +5557,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4509
5557
|
const obsolete = obsoleteOpenAttempts(events, snap);
|
|
4510
5558
|
if (obsolete.length > 0) {
|
|
4511
5559
|
const drained = await quiesceAttempts(obsolete, "obsoleteAttempt");
|
|
4512
|
-
if (!drained) await new Promise((
|
|
5560
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4513
5561
|
continue;
|
|
4514
5562
|
}
|
|
4515
5563
|
const orphaned = openV3WorkerAttempts(events).filter((attempt) => {
|
|
@@ -4518,34 +5566,56 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4518
5566
|
});
|
|
4519
5567
|
if (orphaned.length > 0) {
|
|
4520
5568
|
const drained = await quiesceAttempts(orphaned, "orphanRecovery");
|
|
4521
|
-
if (!drained) await new Promise((
|
|
5569
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5570
|
+
continue;
|
|
5571
|
+
}
|
|
5572
|
+
const obsoleteExternal = obsoleteExternalAttempts(events, snap);
|
|
5573
|
+
if (obsoleteExternal.length > 0) {
|
|
5574
|
+
await drainExternalAttempts(obsoleteExternal, "obsolete", "obsoleteAttempt");
|
|
4522
5575
|
continue;
|
|
4523
5576
|
}
|
|
5577
|
+
if (await reconcileExternalAttempts(events)) continue;
|
|
4524
5578
|
const actions = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances);
|
|
4525
5579
|
const terminal = actions.find(
|
|
4526
5580
|
(a) => a.kind === "completeRunSucceeded" || a.kind === "completeRunFailed" || a.kind === "completeRunBlocked"
|
|
4527
5581
|
);
|
|
4528
5582
|
if (terminal) {
|
|
5583
|
+
if (terminal.kind === "completeRunSucceeded" && pendingRevision !== void 0) {
|
|
5584
|
+
if (inFlight.size > 0) {
|
|
5585
|
+
await waitForInFlightOrDurableCancellation();
|
|
5586
|
+
continue;
|
|
5587
|
+
}
|
|
5588
|
+
return { reason: "awaitingGate", pendingWaits: [], pendingRevision, runDir };
|
|
5589
|
+
}
|
|
4529
5590
|
dropAllBlockingGatePromises();
|
|
4530
5591
|
const open = openV3WorkerAttempts(events);
|
|
4531
5592
|
if (open.length > 0) {
|
|
4532
5593
|
const drained = await quiesceAttempts(open, "terminalPeer");
|
|
4533
|
-
if (!drained) await new Promise((
|
|
5594
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
5595
|
+
continue;
|
|
5596
|
+
}
|
|
5597
|
+
const openExternal = openV3ExternalAttempts(events);
|
|
5598
|
+
if (openExternal.length > 0) {
|
|
5599
|
+
await drainExternalAttempts(openExternal, "runBoundary", "terminalPeer");
|
|
4534
5600
|
continue;
|
|
4535
5601
|
}
|
|
4536
5602
|
appendRunTerminalIfQuiescent();
|
|
4537
5603
|
continue;
|
|
4538
5604
|
}
|
|
4539
5605
|
if (gateMode === "blocking") reattachBlockingGates(snap);
|
|
4540
|
-
const
|
|
5606
|
+
const allControls = actions.filter(
|
|
4541
5607
|
(a) => a.kind === "startLoop" || a.kind === "startLoopIteration" || a.kind === "evaluateLoopIteration" || a.kind === "completeLoop" || a.kind === "resolveEdge" || a.kind === "skipNode"
|
|
4542
5608
|
);
|
|
5609
|
+
const controls = allControls.filter((a) => !heldByProposal(
|
|
5610
|
+
a.kind === "resolveEdge" ? a.to : a.kind === "skipNode" ? a.nodeId : a.loopId
|
|
5611
|
+
));
|
|
5612
|
+
const heldControls = allControls.length - controls.length;
|
|
4543
5613
|
if (controls.length > 0) {
|
|
4544
5614
|
const eventsForControl = readJournal(journalPath);
|
|
4545
5615
|
for (const a of controls) {
|
|
4546
5616
|
if (a.kind === "resolveEdge") applyResolveEdge(a, eventsForControl);
|
|
4547
5617
|
else if (a.kind === "skipNode") {
|
|
4548
|
-
|
|
5618
|
+
appendControlEvent({
|
|
4549
5619
|
type: "nodeSkipped",
|
|
4550
5620
|
nodeId: a.nodeId,
|
|
4551
5621
|
reason: "triggerRuleUnsatisfied",
|
|
@@ -4555,17 +5625,40 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4555
5625
|
}
|
|
4556
5626
|
continue;
|
|
4557
5627
|
}
|
|
5628
|
+
const openExternalNow = openV3ExternalAttempts(events);
|
|
5629
|
+
let externalOccupancy = openExternalNow.length;
|
|
5630
|
+
const externalByBot = /* @__PURE__ */ new Map();
|
|
5631
|
+
for (const attempt of openExternalNow) {
|
|
5632
|
+
if (!attempt.targetLarkAppId) continue;
|
|
5633
|
+
externalByBot.set(attempt.targetLarkAppId, (externalByBot.get(attempt.targetLarkAppId) ?? 0) + 1);
|
|
5634
|
+
}
|
|
5635
|
+
const occupancy = () => inFlight.size - [...inFlight.keys()].filter((key) => key.endsWith("::handoff")).length + externalOccupancy;
|
|
4558
5636
|
let startedThisTick = 0;
|
|
5637
|
+
let heldThisTick = heldControls;
|
|
4559
5638
|
for (const a of actions) {
|
|
4560
|
-
if (
|
|
5639
|
+
if (occupancy() >= globalCap) break;
|
|
5640
|
+
if ((a.kind === "dispatchWork" || a.kind === "dispatchGate") && heldByProposal(a.nodeId, a.kind === "dispatchWork" ? a.loop?.loopId : void 0)) {
|
|
5641
|
+
heldThisTick++;
|
|
5642
|
+
continue;
|
|
5643
|
+
}
|
|
4561
5644
|
if (a.kind === "dispatchWork") {
|
|
4562
5645
|
const node = a.loop ? instanceNodeFor(a.loop) : nodesById.get(a.nodeId);
|
|
4563
5646
|
if (isHostNode(node)) {
|
|
4564
5647
|
if (!a.instanceId) throw new Error(`v3 runtime: host node "${node.id}" has no runtime instance`);
|
|
4565
5648
|
if (startHost(node, events, a.instanceId)) startedThisTick++;
|
|
4566
5649
|
} else {
|
|
4567
|
-
const
|
|
4568
|
-
|
|
5650
|
+
const botSnap = botSnapshotFor2(node.bot);
|
|
5651
|
+
if (externalMode) {
|
|
5652
|
+
const target = snap.assignments.get(node.id) ?? botSnap.larkAppId;
|
|
5653
|
+
if ((botInFlight.get(target) ?? 0) + (externalByBot.get(target) ?? 0) >= perBotCap) continue;
|
|
5654
|
+
if (startExternal(node, botSnap, events, a.loop, a.omitted, a.instanceId)) {
|
|
5655
|
+
startedThisTick++;
|
|
5656
|
+
externalOccupancy++;
|
|
5657
|
+
externalByBot.set(target, (externalByBot.get(target) ?? 0) + 1);
|
|
5658
|
+
}
|
|
5659
|
+
continue;
|
|
5660
|
+
}
|
|
5661
|
+
const botKey = botSnap.larkAppId;
|
|
4569
5662
|
if ((botInFlight.get(botKey) ?? 0) >= perBotCap) continue;
|
|
4570
5663
|
if ((cliInFlight.get(botSnap.cliId) ?? 0) >= perCliCap) continue;
|
|
4571
5664
|
if (startWork(node, botSnap, botKey, events, a.loop, a.omitted, a.instanceId)) {
|
|
@@ -4576,7 +5669,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4576
5669
|
if (startGate(nodesById.get(a.nodeId), a.instanceId)) startedThisTick++;
|
|
4577
5670
|
}
|
|
4578
5671
|
}
|
|
4579
|
-
const cancels = actions.filter((a) => a.kind === "cancelNode");
|
|
5672
|
+
const cancels = actions.filter((a) => a.kind === "cancelNode" && !heldByProposal(a.nodeId));
|
|
4580
5673
|
let cancelledThisTick = false;
|
|
4581
5674
|
for (const a of cancels) {
|
|
4582
5675
|
cancelledThisTick = applyCancelNode(a) || cancelledThisTick;
|
|
@@ -4585,20 +5678,31 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4585
5678
|
if (inFlight.size === 0) {
|
|
4586
5679
|
if (startedThisTick === 0) {
|
|
4587
5680
|
const pendingWaits = gateMode === "suspend" ? pendingGateWaits(snap.nodes) : [];
|
|
4588
|
-
|
|
5681
|
+
const latestEvents = readJournal(journalPath);
|
|
5682
|
+
if (externalMode && latestEvents.length !== events.length) continue;
|
|
5683
|
+
const waitingExternal = openV3ExternalAttempts(latestEvents);
|
|
5684
|
+
if (latestEvents.length !== events.length && currentV3Revision(latestEvents) !== dagRevision) continue;
|
|
5685
|
+
if (pendingWaits.length > 0 || waitingExternal.length > 0 || heldThisTick > 0) {
|
|
4589
5686
|
const open = openV3WorkerAttempts(events);
|
|
4590
5687
|
if (open.length > 0) {
|
|
4591
5688
|
const drained = await quiesceAttempts(open, "orphanRecovery");
|
|
4592
|
-
if (!drained) await new Promise((
|
|
5689
|
+
if (!drained) await new Promise((resolve6) => setTimeout(resolve6, 250));
|
|
4593
5690
|
continue;
|
|
4594
5691
|
}
|
|
4595
|
-
|
|
5692
|
+
if (waitingExternal.length > 0) return awaitingExternalOutcome(waitingExternal, pendingWaits);
|
|
5693
|
+
return {
|
|
5694
|
+
reason: "awaitingGate",
|
|
5695
|
+
pendingWaits,
|
|
5696
|
+
...heldThisTick > 0 && pendingRevision !== void 0 ? { pendingRevision } : {},
|
|
5697
|
+
runDir
|
|
5698
|
+
};
|
|
4596
5699
|
}
|
|
4597
5700
|
throw new Error("v3 runtime: no progress possible and run is not terminal");
|
|
4598
5701
|
}
|
|
4599
5702
|
}
|
|
4600
5703
|
if (inFlight.size > 0) await waitForInFlightOrDurableCancellation();
|
|
4601
5704
|
}
|
|
5705
|
+
forgetProbeFailuresOf(runDir);
|
|
4602
5706
|
const finalSnap = materialize(readJournal(journalPath));
|
|
4603
5707
|
return {
|
|
4604
5708
|
reason: "terminal",
|
|
@@ -4621,21 +5725,25 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4621
5725
|
if (isTrueRunTerminal(snap.runStatus) || snap.runStatus === "cancelling") return;
|
|
4622
5726
|
append({
|
|
4623
5727
|
type: "runCancelRequested",
|
|
4624
|
-
cancelRequestId: `cancel-${(0,
|
|
5728
|
+
cancelRequestId: `cancel-${(0, import_node_crypto9.randomUUID)()}`,
|
|
4625
5729
|
by: "runtime-signal"
|
|
4626
5730
|
}, { durable: true });
|
|
4627
5731
|
});
|
|
4628
5732
|
}
|
|
4629
5733
|
function appendRunTerminalIfQuiescent() {
|
|
4630
5734
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5735
|
+
if (!decisionStillCurrent(events)) return false;
|
|
4631
5736
|
const snap = materialize([...events]);
|
|
4632
5737
|
if (snap.runStatus !== "running") return false;
|
|
4633
5738
|
if (openV3WorkerAttempts(events).length > 0) return false;
|
|
5739
|
+
const pendingProposal = foldV3Revisions(events).pending;
|
|
4634
5740
|
if (openV3HostEffects(events).length > 0) return false;
|
|
5741
|
+
if (openV3ExternalAttempts(events).length > 0) return false;
|
|
4635
5742
|
const terminal = decideNext(dag, snap.nodes, snap.loops, snap.edges, snap.instances).find(
|
|
4636
5743
|
(action) => action.kind === "completeRunSucceeded" || action.kind === "completeRunFailed" || action.kind === "completeRunBlocked"
|
|
4637
5744
|
);
|
|
4638
5745
|
if (!terminal) return false;
|
|
5746
|
+
if (terminal.kind === "completeRunSucceeded" && pendingProposal) return false;
|
|
4639
5747
|
const event = terminal.kind === "completeRunSucceeded" ? { type: "runSucceeded" } : terminal.kind === "completeRunFailed" ? {
|
|
4640
5748
|
type: "runFailed",
|
|
4641
5749
|
failedNodeId: terminal.failedNodeId,
|
|
@@ -4648,21 +5756,42 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4648
5756
|
}
|
|
4649
5757
|
function claimDispatch(event) {
|
|
4650
5758
|
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
5759
|
+
if (!decisionStillCurrent(
|
|
5760
|
+
events,
|
|
5761
|
+
event.nodeId,
|
|
5762
|
+
event.type === "nodeDispatched" ? event.loop?.loopId : void 0
|
|
5763
|
+
)) return false;
|
|
4651
5764
|
const snap = materialize([...events]);
|
|
4652
5765
|
if (snap.runStatus !== "running") return false;
|
|
4653
5766
|
const status = snap.nodes.get(event.nodeId)?.status ?? "pending";
|
|
4654
5767
|
if (status !== "pending") return false;
|
|
5768
|
+
if (event.type === "gateDispatched" && gateWaitAlreadyDecided(events, event.waitId)) {
|
|
5769
|
+
throw new Error(
|
|
5770
|
+
`v3 runtime: gate ${event.waitId} of node "${event.nodeId}" was already decided, but the node has no approval to run on; refusing to re-dispatch the same gate`
|
|
5771
|
+
);
|
|
5772
|
+
}
|
|
4655
5773
|
append(event, { durable: true });
|
|
4656
5774
|
return true;
|
|
4657
5775
|
});
|
|
4658
5776
|
}
|
|
5777
|
+
function gateWaitAlreadyDecided(events, waitId) {
|
|
5778
|
+
return events.some((event) => event.type === "gateResolved" && event.waitId === waitId || event.type === "dagRevised" && (event.effects?.supersede?.some((item) => item.waitId === waitId) === true || event.effects?.cancel?.some((item) => item.waitId === waitId) === true));
|
|
5779
|
+
}
|
|
5780
|
+
function revisionSignature(events) {
|
|
5781
|
+
let count = 0;
|
|
5782
|
+
for (const event of events) {
|
|
5783
|
+
if (event.type === "dagRevised" || event.type === "dagRevisionProposed" || event.type === "dagRevisionRejected") count++;
|
|
5784
|
+
}
|
|
5785
|
+
return count;
|
|
5786
|
+
}
|
|
4659
5787
|
function isRunCancelling() {
|
|
4660
5788
|
return materialize(readJournal(journalPath)).runStatus === "cancelling";
|
|
4661
5789
|
}
|
|
4662
5790
|
async function waitForInFlightOrDurableCancellation() {
|
|
4663
5791
|
const active = [...inFlight.values()];
|
|
4664
5792
|
if (active.length === 0) return;
|
|
4665
|
-
|
|
5793
|
+
const revisionSignatureAtStart = revisionSignature(readJournal(journalPath));
|
|
5794
|
+
await new Promise((resolve6, reject) => {
|
|
4666
5795
|
let done = false;
|
|
4667
5796
|
let timer;
|
|
4668
5797
|
const signal = opts.cancelSignal;
|
|
@@ -4671,13 +5800,14 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4671
5800
|
done = true;
|
|
4672
5801
|
if (timer) clearTimeout(timer);
|
|
4673
5802
|
signal?.removeEventListener("abort", finish);
|
|
4674
|
-
|
|
5803
|
+
resolve6();
|
|
4675
5804
|
};
|
|
4676
5805
|
for (const promise of active) promise.then(finish, finish);
|
|
4677
5806
|
const poll = () => {
|
|
4678
5807
|
if (done) return;
|
|
4679
5808
|
try {
|
|
4680
|
-
|
|
5809
|
+
const latest = readJournal(journalPath);
|
|
5810
|
+
if (signal?.aborted || materialize(latest).runStatus === "cancelling" || revisionSignature(latest) !== revisionSignatureAtStart) {
|
|
4681
5811
|
finish();
|
|
4682
5812
|
return;
|
|
4683
5813
|
}
|
|
@@ -4840,10 +5970,10 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4840
5970
|
return allDrained;
|
|
4841
5971
|
}
|
|
4842
5972
|
function safeAttemptDirFromId(attemptId) {
|
|
4843
|
-
const root = (0,
|
|
4844
|
-
const candidate = (0,
|
|
4845
|
-
const rel = (0,
|
|
4846
|
-
if (!rel || (0,
|
|
5973
|
+
const root = (0, import_node_path13.resolve)(runDir);
|
|
5974
|
+
const candidate = (0, import_node_path13.resolve)(root, attemptId);
|
|
5975
|
+
const rel = (0, import_node_path13.relative)(root, candidate);
|
|
5976
|
+
if (!rel || (0, import_node_path13.isAbsolute)(rel) || rel === ".." || rel.startsWith(`..${import_node_path13.sep}`)) {
|
|
4847
5977
|
throw new Error(`v3 runtime: attempt path escapes run dir (${attemptId})`);
|
|
4848
5978
|
}
|
|
4849
5979
|
return candidate;
|
|
@@ -4880,6 +6010,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4880
6010
|
const snap = materialize([...events]);
|
|
4881
6011
|
if (snap.runStatus === "cancelled") return;
|
|
4882
6012
|
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
6013
|
+
if (openV3ExternalAttempts(events).length > 0) return;
|
|
4883
6014
|
const ids = /* @__PURE__ */ new Set([
|
|
4884
6015
|
...dag.nodes.map((node) => node.id),
|
|
4885
6016
|
...snap.nodes.keys()
|
|
@@ -4966,7 +6097,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4966
6097
|
(event) => event.type === "nodeSucceeded" && (event.instanceId ?? event.nodeId) === key
|
|
4967
6098
|
);
|
|
4968
6099
|
if (!success) throw new Error(`v3 host input source "${nodeId}" has no successful current instance`);
|
|
4969
|
-
const outputDir = (0,
|
|
6100
|
+
const outputDir = (0, import_node_path13.join)((0, import_node_path13.dirname)(success.manifestPath), "work");
|
|
4970
6101
|
const verdict = await deps.validateManifest(success.manifestPath, outputDir);
|
|
4971
6102
|
if (!verdict.ok || verdict.manifest?.status !== "ok") {
|
|
4972
6103
|
throw new Error(
|
|
@@ -4978,7 +6109,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
4978
6109
|
throw new Error(`v3 host input source "${nodeId}" must expose exactly one result.json (found ${matches.length})`);
|
|
4979
6110
|
}
|
|
4980
6111
|
try {
|
|
4981
|
-
return JSON.parse((0,
|
|
6112
|
+
return JSON.parse((0, import_node_fs15.readFileSync)((0, import_node_path13.join)(outputDir, matches[0].path), "utf-8"));
|
|
4982
6113
|
} catch (err) {
|
|
4983
6114
|
throw new Error(
|
|
4984
6115
|
`v3 host input source "${nodeId}" result.json is unreadable: ${err instanceof Error ? err.message : String(err)}`
|
|
@@ -5034,6 +6165,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5034
6165
|
executionContext
|
|
5035
6166
|
});
|
|
5036
6167
|
const committed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6168
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5037
6169
|
const current = materialize([...latest]);
|
|
5038
6170
|
if (current.runStatus !== "running") return false;
|
|
5039
6171
|
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
@@ -5106,6 +6238,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5106
6238
|
executionContext
|
|
5107
6239
|
});
|
|
5108
6240
|
withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6241
|
+
if (!decisionStillCurrent(latest, node.id)) return;
|
|
5109
6242
|
const current = materialize([...latest]);
|
|
5110
6243
|
if (current.runStatus !== "running") return;
|
|
5111
6244
|
const status = current.nodes.get(node.id)?.status ?? "pending";
|
|
@@ -5204,6 +6337,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5204
6337
|
return blocked;
|
|
5205
6338
|
}
|
|
5206
6339
|
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6340
|
+
if (!decisionStillCurrent(latest, node.id)) return false;
|
|
5207
6341
|
const current = materialize([...latest]);
|
|
5208
6342
|
if (current.runStatus !== "running") return false;
|
|
5209
6343
|
const state = current.nodes.get(node.id);
|
|
@@ -5321,9 +6455,9 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5321
6455
|
continue;
|
|
5322
6456
|
}
|
|
5323
6457
|
const attemptDir = safeAttemptDirFromId(effect.attemptId);
|
|
5324
|
-
const existingManifest = (0,
|
|
5325
|
-
if ((0,
|
|
5326
|
-
const validated = await deps.validateManifest(existingManifest, (0,
|
|
6458
|
+
const existingManifest = (0, import_node_path13.join)(attemptDir, "manifest.json");
|
|
6459
|
+
if ((0, import_node_fs15.existsSync)(existingManifest)) {
|
|
6460
|
+
const validated = await deps.validateManifest(existingManifest, (0, import_node_path13.join)(attemptDir, "work"));
|
|
5327
6461
|
if (validated.ok && validated.manifest?.status === "ok") {
|
|
5328
6462
|
try {
|
|
5329
6463
|
readAndVerifyV3HostSuccessResult({
|
|
@@ -5600,47 +6734,423 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5600
6734
|
function computeHostCanonicalHash(value) {
|
|
5601
6735
|
return computeInputHash(value);
|
|
5602
6736
|
}
|
|
5603
|
-
function
|
|
6737
|
+
function writeWorkflowDataFile(node, attemptDir) {
|
|
6738
|
+
if (!executionContext) return void 0;
|
|
6739
|
+
const bindings = savedWorkflowBindingsForNode(node);
|
|
6740
|
+
if (bindings.params.length === 0 && bindings.context.length === 0) return void 0;
|
|
6741
|
+
const params = /* @__PURE__ */ Object.create(null);
|
|
6742
|
+
const context = /* @__PURE__ */ Object.create(null);
|
|
6743
|
+
for (const name of bindings.params) {
|
|
6744
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.params, name)) {
|
|
6745
|
+
throw new Error(`v3 runtime: resolved params missing referenced key ${name}`);
|
|
6746
|
+
}
|
|
6747
|
+
params[name] = executionContext.params[name];
|
|
6748
|
+
}
|
|
6749
|
+
for (const name of bindings.context) {
|
|
6750
|
+
if (!Object.prototype.hasOwnProperty.call(executionContext.context, name)) {
|
|
6751
|
+
throw new Error(`v3 runtime: resolved context missing referenced key ${name}`);
|
|
6752
|
+
}
|
|
6753
|
+
context[name] = executionContext.context[name];
|
|
6754
|
+
}
|
|
6755
|
+
const workflowDataPath = (0, import_node_path13.join)(attemptDir, "workflow-inputs.json");
|
|
6756
|
+
(0, import_node_fs15.writeFileSync)(
|
|
6757
|
+
workflowDataPath,
|
|
6758
|
+
`${JSON.stringify({ params, context }, null, 2)}
|
|
6759
|
+
`,
|
|
6760
|
+
{ mode: 384 }
|
|
6761
|
+
);
|
|
6762
|
+
return workflowDataPath;
|
|
6763
|
+
}
|
|
6764
|
+
function reportExternalError(error, attemptId, operation) {
|
|
6765
|
+
try {
|
|
6766
|
+
deps.onExternalError?.(error, { attemptId, operation });
|
|
6767
|
+
} catch {
|
|
6768
|
+
}
|
|
6769
|
+
}
|
|
6770
|
+
async function stopExternal(attempt, reason) {
|
|
6771
|
+
const stopped = await stopExternalAttempt(externalExecutor, dag.runId, attempt, reason, externalCallTimeoutMs);
|
|
6772
|
+
if (!stopped.ok) reportExternalError(stopped.error, attempt.attemptId, "stop");
|
|
6773
|
+
}
|
|
6774
|
+
function externalNodeFor(attempt) {
|
|
6775
|
+
return attempt.loop ? instanceNodeFor(attempt.loop) : nodesById.get(attempt.nodeId);
|
|
6776
|
+
}
|
|
6777
|
+
function startExternal(node, botSnap, events, loopRef, omitted, instanceId) {
|
|
6778
|
+
if (!isGoalNode(node)) throw new Error(`v3 runtime: node "${node.id}" is not a goal node`);
|
|
5604
6779
|
const dispatchKey = instanceId ?? node.id;
|
|
5605
6780
|
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
5606
6781
|
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
5607
|
-
const attemptDir = (0,
|
|
5608
|
-
|
|
5609
|
-
|
|
5610
|
-
|
|
5611
|
-
|
|
5612
|
-
|
|
5613
|
-
|
|
5614
|
-
|
|
5615
|
-
|
|
5616
|
-
|
|
5617
|
-
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
6782
|
+
const attemptDir = (0, import_node_path13.join)(runDir, dispatchKey, "attempts", attemptNNN);
|
|
6783
|
+
(0, import_node_fs15.mkdirSync)(attemptDir, { recursive: true });
|
|
6784
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
6785
|
+
(0, import_node_fs15.writeFileSync)(
|
|
6786
|
+
(0, import_node_path13.join)(attemptDir, "inputs.json"),
|
|
6787
|
+
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
6788
|
+
);
|
|
6789
|
+
const claimed = withJournalMutationSync(journalPath, ({ events: latest, append }) => {
|
|
6790
|
+
if (!decisionStillCurrent(latest, node.id, loopRef?.loopId)) return false;
|
|
6791
|
+
const current = materialize([...latest]);
|
|
6792
|
+
if (current.runStatus !== "running") return false;
|
|
6793
|
+
if ((current.nodes.get(node.id)?.status ?? "pending") !== "pending") return false;
|
|
6794
|
+
const identity = { nodeId: node.id, ...instanceId ? { instanceId } : {}, attemptId };
|
|
6795
|
+
append({
|
|
6796
|
+
type: "nodeDispatched",
|
|
6797
|
+
...identity,
|
|
6798
|
+
loop: loopRef,
|
|
6799
|
+
nodeExecutor: "bot",
|
|
6800
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
6801
|
+
}, { durable: true });
|
|
6802
|
+
append({
|
|
6803
|
+
type: "nodeHandoffPrepared",
|
|
6804
|
+
...identity,
|
|
6805
|
+
targetLarkAppId: current.assignments.get(node.id) ?? botSnap.larkAppId,
|
|
6806
|
+
placement: "topic",
|
|
6807
|
+
revision: dagRevision,
|
|
6808
|
+
idempotency: { nodeCard: (0, import_node_crypto9.randomUUID)(), handoff: (0, import_node_crypto9.randomUUID)() },
|
|
6809
|
+
preparedAt: clock()
|
|
6810
|
+
}, { durable: true });
|
|
6811
|
+
return true;
|
|
6812
|
+
});
|
|
6813
|
+
if (!claimed) return false;
|
|
6814
|
+
const attempt = foldV3ExternalAttempts(readJournal(journalPath)).get(attemptId);
|
|
6815
|
+
if (attempt && !attempt.closed) startHandoffSend(attempt, false);
|
|
6816
|
+
return true;
|
|
6817
|
+
}
|
|
6818
|
+
function appendMissingHandoffIntent(attempt) {
|
|
6819
|
+
const node = externalNodeFor(attempt);
|
|
6820
|
+
const frozen = node ? snapshotForSelector(botSnapshots, node.bot) : void 0;
|
|
6821
|
+
if (!frozen) return false;
|
|
6822
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6823
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
6824
|
+
if (!current.ok || current.attempt.phase !== "dispatched") return false;
|
|
6825
|
+
const dispatched = events.find((event) => event.type === "nodeDispatched" && event.attemptId === attempt.attemptId);
|
|
6826
|
+
append({
|
|
6827
|
+
type: "nodeHandoffPrepared",
|
|
6828
|
+
nodeId: attempt.nodeId,
|
|
6829
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6830
|
+
attemptId: attempt.attemptId,
|
|
6831
|
+
targetLarkAppId: current.snap.assignments.get(attempt.nodeId) ?? frozen.larkAppId,
|
|
6832
|
+
placement: "topic",
|
|
6833
|
+
revision: dispatched?.revision ?? 1,
|
|
6834
|
+
idempotency: { nodeCard: (0, import_node_crypto9.randomUUID)(), handoff: (0, import_node_crypto9.randomUUID)() },
|
|
6835
|
+
preparedAt: clock()
|
|
6836
|
+
}, { durable: true });
|
|
6837
|
+
return true;
|
|
6838
|
+
});
|
|
6839
|
+
}
|
|
6840
|
+
function buildExternalDispatchRequest(attempt, redispatch, signal) {
|
|
6841
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
6842
|
+
const node = externalNodeFor(attempt);
|
|
6843
|
+
if (!ref || !attempt.idempotency || !node || !isGoalNode(node)) {
|
|
6844
|
+
throw new Error(`v3 runtime: external attempt ${attempt.attemptId} has no prepared goal handoff`);
|
|
6845
|
+
}
|
|
6846
|
+
const attemptDir = safeAttemptDirFromId(attempt.attemptId);
|
|
6847
|
+
const inputsPath = (0, import_node_path13.join)(attemptDir, "inputs.json");
|
|
6848
|
+
const inputs = JSON.parse((0, import_node_fs15.readFileSync)(inputsPath, "utf-8"));
|
|
6849
|
+
const events = readJournal(journalPath);
|
|
6850
|
+
const key = attempt.instanceId ?? attempt.nodeId;
|
|
6851
|
+
const retry = events.find(
|
|
6852
|
+
(event) => event.type === "nodeRetryRequested" && (event.instanceId ?? event.nodeId) === key && event.nextAttemptId === attempt.attemptId
|
|
6853
|
+
);
|
|
6854
|
+
const previousBlock = retry ? [...events].reverse().find(
|
|
6855
|
+
(event) => event.type === "nodeBlocked" && event.attemptId === retry.previousAttemptId
|
|
6856
|
+
) : void 0;
|
|
6857
|
+
const reassigned = [...events].reverse().find(
|
|
6858
|
+
(event) => event.type === "nodeReassigned" && event.nodeId === attempt.nodeId && event.to === ref.targetLarkAppId
|
|
6859
|
+
);
|
|
6860
|
+
const revised = [...events].reverse().find(
|
|
6861
|
+
(event) => event.type === "dagRevised" && event.revision <= ref.revision && (event.effects?.supersede?.some((item) => item.nodeId === attempt.nodeId) === true || event.effects?.assign?.some((item) => item.nodeId === attempt.nodeId) === true)
|
|
6862
|
+
);
|
|
6863
|
+
const revisedAssign = revised?.effects?.assign?.find((item) => item.nodeId === attempt.nodeId && item.to === ref.targetLarkAppId);
|
|
6864
|
+
return {
|
|
6865
|
+
...ref,
|
|
6866
|
+
idempotency: attempt.idempotency,
|
|
6867
|
+
redispatch,
|
|
6868
|
+
node,
|
|
6869
|
+
attemptNumber: attemptNumber(attempt.attemptId) ?? 1,
|
|
6870
|
+
...attempt.loop ? { loop: attempt.loop } : {},
|
|
6871
|
+
inputs,
|
|
6872
|
+
inputsPath,
|
|
6873
|
+
attemptDir,
|
|
6874
|
+
runDir,
|
|
6875
|
+
timeoutSec: node.timeoutSec ?? DEFAULT_NODE_TIMEOUT_SEC,
|
|
6876
|
+
...opts.chatBinding ? { chatBinding: opts.chatBinding } : {},
|
|
6877
|
+
...retry ? {
|
|
6878
|
+
retryOf: {
|
|
6879
|
+
attemptId: retry.previousAttemptId,
|
|
6880
|
+
...retry.previousErrorCode ? { errorCode: retry.previousErrorCode } : {},
|
|
6881
|
+
...previousBlock?.message ? { message: previousBlock.message } : {}
|
|
6882
|
+
}
|
|
6883
|
+
} : {},
|
|
6884
|
+
...reassigned ? { reassignedFrom: reassigned.from } : revisedAssign?.from ? { reassignedFrom: revisedAssign.from } : {},
|
|
6885
|
+
...revised ? { revisedBy: { revision: revised.revision, summary: revised.summary } } : {},
|
|
6886
|
+
signal
|
|
6887
|
+
};
|
|
6888
|
+
}
|
|
6889
|
+
function startHandoffSend(attempt, redispatch) {
|
|
6890
|
+
const executor = externalExecutor;
|
|
6891
|
+
const key = `${attempt.instanceId ?? attempt.nodeId}::handoff`;
|
|
6892
|
+
if (!executor || inFlight.has(key) || handoffControllers.has(attempt.attemptId)) return;
|
|
6893
|
+
const controller = new AbortController();
|
|
6894
|
+
handoffControllers.set(attempt.attemptId, controller);
|
|
6895
|
+
const p = Promise.resolve().then(() => withExternalCallTimeout(
|
|
6896
|
+
Promise.resolve().then(() => executor.dispatch(buildExternalDispatchRequest(attempt, redispatch, controller.signal))),
|
|
6897
|
+
externalCallTimeoutMs,
|
|
6898
|
+
"dispatch"
|
|
6899
|
+
).catch((err) => {
|
|
6900
|
+
if (!controller.signal.aborted) controller.abort(err);
|
|
6901
|
+
throw err;
|
|
6902
|
+
})).then(async (outcome) => {
|
|
6903
|
+
handoffRetryAt.delete(attempt.attemptId);
|
|
6904
|
+
if (outcome.kind === "rejected") {
|
|
6905
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6906
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
6907
|
+
if (!current.ok || current.attempt.phase !== "prepared") return;
|
|
6908
|
+
append({
|
|
6909
|
+
type: "nodeBlocked",
|
|
6910
|
+
nodeId: attempt.nodeId,
|
|
6911
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6912
|
+
attemptId: attempt.attemptId,
|
|
6913
|
+
errorClass: "workerError",
|
|
6914
|
+
errorCode: outcome.code,
|
|
6915
|
+
message: outcome.message ?? `handoff was not sent (${outcome.code})`
|
|
6916
|
+
}, { durable: true });
|
|
6917
|
+
});
|
|
6918
|
+
return;
|
|
6919
|
+
}
|
|
6920
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
6921
|
+
const sentAt = clock();
|
|
6922
|
+
const recorded = withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6923
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
6924
|
+
if (!latest || latest.closed) return false;
|
|
6925
|
+
if (latest.handoffMessageId !== void 0) return true;
|
|
6926
|
+
append({
|
|
6927
|
+
type: "nodeHandoffSent",
|
|
6928
|
+
nodeId: attempt.nodeId,
|
|
6929
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6930
|
+
attemptId: attempt.attemptId,
|
|
6931
|
+
handoffMessageId: outcome.handoffMessageId,
|
|
6932
|
+
...outcome.nodeCardMessageId ? { nodeCardMessageId: outcome.nodeCardMessageId } : {},
|
|
6933
|
+
...outcome.threadId ? { threadId: outcome.threadId } : {},
|
|
6934
|
+
sentAt,
|
|
6935
|
+
acceptDeadlineAt: sentAt + externalAcceptTimeoutMs
|
|
6936
|
+
}, { durable: true });
|
|
6937
|
+
return true;
|
|
6938
|
+
});
|
|
6939
|
+
if (!recorded) {
|
|
6940
|
+
await stopExternal({ ...attempt, handoffMessageId: outcome.handoffMessageId }, "obsolete");
|
|
6941
|
+
}
|
|
6942
|
+
}).catch((err) => {
|
|
6943
|
+
reportExternalError(err, attempt.attemptId, "dispatch");
|
|
6944
|
+
handoffRetryAt.set(attempt.attemptId, clock() + HANDOFF_RESEND_BACKOFF_MS);
|
|
6945
|
+
}).finally(() => {
|
|
6946
|
+
if (handoffControllers.get(attempt.attemptId) === controller) {
|
|
6947
|
+
handoffControllers.delete(attempt.attemptId);
|
|
6948
|
+
}
|
|
6949
|
+
if (inFlight.get(key) === p) inFlight.delete(key);
|
|
6950
|
+
});
|
|
6951
|
+
inFlight.set(key, p);
|
|
6952
|
+
}
|
|
6953
|
+
async function blockExternalAttempt(attempt, stopReason, verdict) {
|
|
6954
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
6955
|
+
await stopExternal(attempt, stopReason);
|
|
6956
|
+
return withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
6957
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) return true;
|
|
6958
|
+
const current = classifyExternalAttempt(events, attempt.attemptId);
|
|
6959
|
+
if (!current.ok) return false;
|
|
6960
|
+
append({
|
|
6961
|
+
type: "nodeBlocked",
|
|
6962
|
+
nodeId: attempt.nodeId,
|
|
6963
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
6964
|
+
attemptId: attempt.attemptId,
|
|
6965
|
+
...verdict
|
|
6966
|
+
}, { durable: true });
|
|
6967
|
+
return true;
|
|
6968
|
+
});
|
|
6969
|
+
}
|
|
6970
|
+
async function probeExternalAcceptance(attempt) {
|
|
6971
|
+
const ref = externalAttemptRef(dag.runId, attempt);
|
|
6972
|
+
const executor = externalExecutor;
|
|
6973
|
+
if (!executor || !ref || !attempt.handoffMessageId) return { status: "unknown" };
|
|
6974
|
+
const handoffMessageId = attempt.handoffMessageId;
|
|
6975
|
+
try {
|
|
6976
|
+
const probe = await withExternalCallTimeout(
|
|
6977
|
+
Promise.resolve().then(() => executor.probeAcceptance({ ...ref, handoffMessageId })),
|
|
6978
|
+
externalCallTimeoutMs,
|
|
6979
|
+
"probe"
|
|
6980
|
+
);
|
|
6981
|
+
return probe.status === "accepted" ? {
|
|
6982
|
+
status: "accepted",
|
|
6983
|
+
sessionId: probe.sessionId,
|
|
6984
|
+
...probe.acceptedAt !== void 0 ? { acceptedAt: probe.acceptedAt } : {}
|
|
6985
|
+
} : { status: "notAccepted", ...typeof probe.detail === "string" && probe.detail ? { detail: probe.detail.slice(0, 2e3) } : {} };
|
|
6986
|
+
} catch (err) {
|
|
6987
|
+
reportExternalError(err, attempt.attemptId, "probe");
|
|
6988
|
+
return { status: "unknown" };
|
|
6989
|
+
}
|
|
6990
|
+
}
|
|
6991
|
+
async function reconcileExternalAttempts(events) {
|
|
6992
|
+
if (!externalMode) return false;
|
|
6993
|
+
let progressed = false;
|
|
6994
|
+
for (const attempt of openV3ExternalAttempts(events)) {
|
|
6995
|
+
if (handoffControllers.has(attempt.attemptId)) continue;
|
|
6996
|
+
if (!classifyExternalAttempt(events, attempt.attemptId).ok) continue;
|
|
6997
|
+
const now = clock();
|
|
6998
|
+
if (attempt.phase === "dispatched") {
|
|
6999
|
+
progressed = appendMissingHandoffIntent(attempt) || progressed;
|
|
7000
|
+
} else if (attempt.phase === "prepared") {
|
|
7001
|
+
if (now >= attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) {
|
|
7002
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7003
|
+
errorClass: "workerError",
|
|
7004
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7005
|
+
message: "handoff delivery could not be confirmed within the idempotency window; check the chat before retrying"
|
|
7006
|
+
});
|
|
7007
|
+
progressed = true;
|
|
7008
|
+
} else if (now >= (handoffRetryAt.get(attempt.attemptId) ?? 0)) {
|
|
7009
|
+
startHandoffSend(attempt, true);
|
|
5621
7010
|
}
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
|
|
7011
|
+
} else if (attempt.phase === "sent") {
|
|
7012
|
+
const due = now >= attempt.acceptDeadlineAt;
|
|
7013
|
+
const key = probeKey(attempt.attemptId);
|
|
7014
|
+
const failure = probeFailures.get(key);
|
|
7015
|
+
const probeDue = failure ? now >= failure.retryAt : !acceptanceObserved.has(attempt.attemptId) || due;
|
|
7016
|
+
if (probeDue) {
|
|
7017
|
+
const probe = await probeExternalAcceptance(attempt);
|
|
7018
|
+
progressed = true;
|
|
7019
|
+
if (probe.status === "unknown") {
|
|
7020
|
+
const firstFailedAt = failure?.firstFailedAt ?? now;
|
|
7021
|
+
if (due && now - firstFailedAt >= PROBE_UNCERTAIN_AFTER_MS) {
|
|
7022
|
+
probeFailures.delete(key);
|
|
7023
|
+
await blockExternalAttempt(attempt, "uncertain", {
|
|
7024
|
+
errorClass: "workerError",
|
|
7025
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_UNCERTAIN,
|
|
7026
|
+
message: "whether the target bot accepted the handoff could not be confirmed; check the chat before retrying"
|
|
7027
|
+
});
|
|
7028
|
+
} else {
|
|
7029
|
+
probeFailures.set(key, { firstFailedAt, retryAt: now + PROBE_RETRY_BACKOFF_MS });
|
|
7030
|
+
}
|
|
7031
|
+
} else {
|
|
7032
|
+
probeFailures.delete(key);
|
|
7033
|
+
acceptanceObserved.add(attempt.attemptId);
|
|
7034
|
+
if (probe.status === "accepted") {
|
|
7035
|
+
const { sessionId, acceptedAt } = probe;
|
|
7036
|
+
withJournalMutationSync(journalPath, ({ events: latest, append }) => appendAcceptedLocked(latest, append, dag, {
|
|
7037
|
+
attemptId: attempt.attemptId,
|
|
7038
|
+
sessionId,
|
|
7039
|
+
...acceptedAt !== void 0 ? { acceptedAt } : {}
|
|
7040
|
+
}, clock()));
|
|
7041
|
+
} else if (due) {
|
|
7042
|
+
await blockExternalAttempt(attempt, "notAccepted", {
|
|
7043
|
+
errorClass: "workerError",
|
|
7044
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.HANDOFF_NOT_ACCEPTED,
|
|
7045
|
+
message: probe.detail ? `the target bot did not accept the handoff before the acceptance deadline: ${probe.detail}` : "the target bot did not accept the handoff before the acceptance deadline"
|
|
7046
|
+
});
|
|
7047
|
+
}
|
|
5625
7048
|
}
|
|
5626
|
-
context[name] = executionContext.context[name];
|
|
5627
7049
|
}
|
|
5628
|
-
|
|
5629
|
-
(
|
|
5630
|
-
|
|
5631
|
-
|
|
5632
|
-
|
|
5633
|
-
|
|
5634
|
-
|
|
7050
|
+
} else if (now >= attempt.deadlineAt) {
|
|
7051
|
+
await blockExternalAttempt(attempt, "timeout", {
|
|
7052
|
+
errorClass: "timeout",
|
|
7053
|
+
errorCode: V3_EXTERNAL_ERROR_CODES.NODE_TIMEOUT,
|
|
7054
|
+
message: "the node did not submit a valid result before its timeout"
|
|
7055
|
+
});
|
|
7056
|
+
progressed = true;
|
|
7057
|
+
}
|
|
7058
|
+
}
|
|
7059
|
+
return progressed;
|
|
7060
|
+
}
|
|
7061
|
+
function settleExternalLeftovers(events) {
|
|
7062
|
+
let progressed = false;
|
|
7063
|
+
for (const attempt of foldV3ExternalAttempts(events).values()) {
|
|
7064
|
+
if (hasExternalInstallIntent(runDir, attempt.attemptId)) {
|
|
7065
|
+
if (settleExternalInstallIntent(runDir, dag, attempt.attemptId).kind !== "none") progressed = true;
|
|
7066
|
+
} else if (attempt.closed) {
|
|
7067
|
+
(0, import_node_fs15.rmSync)((0, import_node_path13.join)(safeAttemptDirFromId(attempt.attemptId), "submissions"), { recursive: true, force: true });
|
|
5635
7068
|
}
|
|
5636
7069
|
}
|
|
5637
|
-
|
|
7070
|
+
return progressed;
|
|
7071
|
+
}
|
|
7072
|
+
function obsoleteExternalAttempts(events, snap) {
|
|
7073
|
+
return openV3ExternalAttempts(events).filter((attempt) => {
|
|
7074
|
+
const state = attempt.instanceId ? snap.instances.get(attempt.instanceId) : snap.nodes.get(attempt.nodeId);
|
|
7075
|
+
return state?.status === "cancelled" || state?.status === "superseded";
|
|
7076
|
+
});
|
|
7077
|
+
}
|
|
7078
|
+
async function drainExternalAttempts(attempts, stopReason, drainReason) {
|
|
7079
|
+
for (const attempt of attempts) {
|
|
7080
|
+
await stopExternal(attempt, stopReason);
|
|
7081
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7082
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7083
|
+
if (!latest || latest.closed) return;
|
|
7084
|
+
append({
|
|
7085
|
+
type: "nodeAttemptDrained",
|
|
7086
|
+
nodeId: attempt.nodeId,
|
|
7087
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7088
|
+
attemptId: attempt.attemptId,
|
|
7089
|
+
reason: drainReason
|
|
7090
|
+
}, { durable: true });
|
|
7091
|
+
});
|
|
7092
|
+
}
|
|
7093
|
+
}
|
|
7094
|
+
async function closeExternalAttemptsForCancellation(cancelRequestId) {
|
|
7095
|
+
for (const attempt of openV3ExternalAttempts(readJournal(journalPath))) {
|
|
7096
|
+
await stopExternal(attempt, "runCancelled");
|
|
7097
|
+
withJournalMutationSync(journalPath, ({ events, append }) => {
|
|
7098
|
+
const latest = foldV3ExternalAttempts(events).get(attempt.attemptId);
|
|
7099
|
+
if (!latest || latest.closed) return;
|
|
7100
|
+
const snap = materialize([...events]);
|
|
7101
|
+
if (snap.runStatus !== "cancelling" || snap.cancelRequestId !== cancelRequestId) return;
|
|
7102
|
+
append({
|
|
7103
|
+
type: "nodeAttemptDrained",
|
|
7104
|
+
nodeId: attempt.nodeId,
|
|
7105
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7106
|
+
attemptId: attempt.attemptId,
|
|
7107
|
+
reason: "runCancellation"
|
|
7108
|
+
}, { durable: true });
|
|
7109
|
+
});
|
|
7110
|
+
}
|
|
7111
|
+
}
|
|
7112
|
+
function sentWakeAt(attempt, now) {
|
|
7113
|
+
const failure = probeFailures.get(probeKey(attempt.attemptId));
|
|
7114
|
+
if (!failure) return attempt.acceptDeadlineAt;
|
|
7115
|
+
return attempt.acceptDeadlineAt > now ? Math.min(attempt.acceptDeadlineAt, failure.retryAt) : failure.retryAt;
|
|
7116
|
+
}
|
|
7117
|
+
function awaitingExternalOutcome(open, pendingWaits) {
|
|
7118
|
+
const now = clock();
|
|
7119
|
+
const externalAttempts = open.map((attempt) => {
|
|
7120
|
+
const wakeAt = attempt.phase === "dispatched" ? now : attempt.phase === "prepared" ? Math.min(handoffRetryAt.get(attempt.attemptId) ?? now, attempt.preparedAt + V3_HANDOFF_IDEMPOTENCY_WINDOW_MS) : attempt.phase === "sent" ? sentWakeAt(attempt, now) : attempt.deadlineAt;
|
|
7121
|
+
return {
|
|
7122
|
+
nodeId: attempt.nodeId,
|
|
7123
|
+
...attempt.instanceId ? { instanceId: attempt.instanceId } : {},
|
|
7124
|
+
attemptId: attempt.attemptId,
|
|
7125
|
+
phase: attempt.phase,
|
|
7126
|
+
...attempt.targetLarkAppId ? { targetLarkAppId: attempt.targetLarkAppId } : {},
|
|
7127
|
+
...wakeAt !== void 0 ? { wakeAt } : {}
|
|
7128
|
+
};
|
|
7129
|
+
});
|
|
7130
|
+
const wakes = externalAttempts.flatMap((attempt) => attempt.wakeAt !== void 0 ? [attempt.wakeAt] : []);
|
|
7131
|
+
return {
|
|
7132
|
+
reason: "awaitingExternal",
|
|
7133
|
+
pendingWaits,
|
|
7134
|
+
externalAttempts,
|
|
7135
|
+
...wakes.length > 0 ? { wakeAt: Math.min(...wakes) } : {},
|
|
7136
|
+
runDir
|
|
7137
|
+
};
|
|
7138
|
+
}
|
|
7139
|
+
function startWork(node, botSnap, botKey, events, loopRef, omitted, instanceId) {
|
|
7140
|
+
const dispatchKey = instanceId ?? node.id;
|
|
7141
|
+
const attemptId = nextAttemptIdFor(events, dispatchKey);
|
|
7142
|
+
const attemptNNN = attemptId.slice(attemptId.lastIndexOf("/") + 1);
|
|
7143
|
+
const attemptDir = (0, import_node_path13.join)(runDir, dispatchKey, "attempts", attemptNNN);
|
|
7144
|
+
const outputDir = (0, import_node_path13.join)(attemptDir, "work");
|
|
7145
|
+
(0, import_node_fs15.mkdirSync)(outputDir, { recursive: true });
|
|
7146
|
+
const workflowDataPath = writeWorkflowDataFile(node, attemptDir);
|
|
7147
|
+
const goalPath = (0, import_node_path13.join)(attemptDir, "goal.txt");
|
|
5638
7148
|
const loopCtx = loopRef ? {
|
|
5639
7149
|
loopId: loopRef.loopId,
|
|
5640
7150
|
iteration: loopRef.iteration,
|
|
5641
7151
|
maxIterations: nodesById.get(loopRef.loopId).maxIterations
|
|
5642
7152
|
} : void 0;
|
|
5643
|
-
(0,
|
|
7153
|
+
(0, import_node_fs15.writeFileSync)(
|
|
5644
7154
|
goalPath,
|
|
5645
7155
|
renderGoalFile(
|
|
5646
7156
|
node.goal ?? "",
|
|
@@ -5652,12 +7162,12 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5652
7162
|
)
|
|
5653
7163
|
);
|
|
5654
7164
|
const effSnap = mergeNodeCapability(botSnap, node.override);
|
|
5655
|
-
const inputsPath = (0,
|
|
5656
|
-
(0,
|
|
7165
|
+
const inputsPath = (0, import_node_path13.join)(attemptDir, "inputs.json");
|
|
7166
|
+
(0, import_node_fs15.writeFileSync)(
|
|
5657
7167
|
inputsPath,
|
|
5658
7168
|
JSON.stringify(buildInputs(node, events, attemptId, loopRef, omitted, workflowDataPath), null, 2)
|
|
5659
7169
|
);
|
|
5660
|
-
const manifestPath = (0,
|
|
7170
|
+
const manifestPath = (0, import_node_path13.join)(attemptDir, "manifest.json");
|
|
5661
7171
|
const env = {
|
|
5662
7172
|
[GOAL_ENV.GOAL_PATH]: goalPath,
|
|
5663
7173
|
[GOAL_ENV.INPUTS_PATH]: inputsPath,
|
|
@@ -5671,7 +7181,8 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5671
7181
|
nodeId: node.id,
|
|
5672
7182
|
...instanceId ? { instanceId } : {},
|
|
5673
7183
|
attemptId,
|
|
5674
|
-
loop: loopRef
|
|
7184
|
+
loop: loopRef,
|
|
7185
|
+
...dagRevision > 1 ? { revision: dagRevision } : {}
|
|
5675
7186
|
})) return false;
|
|
5676
7187
|
const leaseBinding = {
|
|
5677
7188
|
attemptDir,
|
|
@@ -5877,7 +7388,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5877
7388
|
});
|
|
5878
7389
|
return;
|
|
5879
7390
|
}
|
|
5880
|
-
const res = validateResult((0,
|
|
7391
|
+
const res = validateResult((0, import_node_path13.join)(outputDir, entry.path), node.resultSchema);
|
|
5881
7392
|
if (!res.ok) {
|
|
5882
7393
|
appendWorkerOutcome({
|
|
5883
7394
|
type: "nodeBlocked",
|
|
@@ -5937,7 +7448,7 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5937
7448
|
}
|
|
5938
7449
|
const kind = classifyTerminal(errorClass, { selfReportedFail, retryable });
|
|
5939
7450
|
if (kind === "blocked") {
|
|
5940
|
-
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk((0,
|
|
7451
|
+
const ask = errorCode === ASK_HUMAN_ERROR_CODE ? readGoalAsk((0, import_node_path13.join)(attemptDir, GOAL_ASK_FILE)) : void 0;
|
|
5941
7452
|
appendWorkerOutcome({
|
|
5942
7453
|
type: "nodeBlocked",
|
|
5943
7454
|
nodeId: node.id,
|
|
@@ -5994,58 +7505,19 @@ async function runWorkflow(dag, deps, opts) {
|
|
|
5994
7505
|
inFlight.set(dispatchKey, p);
|
|
5995
7506
|
return true;
|
|
5996
7507
|
}
|
|
5997
|
-
function affectedNodesFrom(root) {
|
|
5998
|
-
const reachable = /* @__PURE__ */ new Set([root]);
|
|
5999
|
-
let changed = true;
|
|
6000
|
-
while (changed) {
|
|
6001
|
-
changed = false;
|
|
6002
|
-
for (const n of dag.nodes) {
|
|
6003
|
-
if (reachable.has(n.id)) continue;
|
|
6004
|
-
if (n.depends.some((d) => reachable.has(d.from))) {
|
|
6005
|
-
reachable.add(n.id);
|
|
6006
|
-
changed = true;
|
|
6007
|
-
}
|
|
6008
|
-
}
|
|
6009
|
-
}
|
|
6010
|
-
return [...reachable];
|
|
6011
|
-
}
|
|
6012
7508
|
function appendRevisitEvents(nodeId, instanceId, attemptId, request, sourceManifestPath) {
|
|
6013
|
-
const
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
let reasonPathRel;
|
|
6018
|
-
if (request.reason) {
|
|
6019
|
-
const dir = (0, import_node_path9.join)(runDir, "revisits");
|
|
6020
|
-
(0, import_node_fs11.mkdirSync)(dir, { recursive: true });
|
|
6021
|
-
const abs = (0, import_node_path9.join)(dir, `${instanceId.replace(/[#/]/g, "-")}-reason.md`);
|
|
6022
|
-
(0, import_node_fs11.writeFileSync)(abs, `# Revisit reason (from ${nodeId} / ${instanceId})
|
|
6023
|
-
|
|
6024
|
-
${request.reason}
|
|
6025
|
-
`);
|
|
6026
|
-
reasonPathRel = (0, import_node_path9.relative)(runDir, abs);
|
|
6027
|
-
}
|
|
6028
|
-
appendEvent(journalPath, {
|
|
6029
|
-
type: "nodeRevisitRequested",
|
|
7509
|
+
const plan = planV3Revisit({
|
|
7510
|
+
runDir,
|
|
7511
|
+
dag,
|
|
7512
|
+
events: readJournal(journalPath),
|
|
6030
7513
|
nodeId,
|
|
6031
7514
|
instanceId,
|
|
6032
7515
|
attemptId,
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
...reasonPathRel ? { reasonPath: reasonPathRel } : {},
|
|
6036
|
-
sourceManifestPath: (0, import_node_path9.relative)(runDir, sourceManifestPath),
|
|
6037
|
-
...targetSucc ? { targetPreviousManifestPath: (0, import_node_path9.relative)(runDir, targetSucc.manifestPath) } : {}
|
|
7516
|
+
request,
|
|
7517
|
+
sourceManifestPath
|
|
6038
7518
|
});
|
|
6039
|
-
for (const
|
|
6040
|
-
|
|
6041
|
-
if (!eff) continue;
|
|
6042
|
-
appendEvent(journalPath, {
|
|
6043
|
-
type: "nodeInstanceSuperseded",
|
|
6044
|
-
nodeId: affectedNodeId,
|
|
6045
|
-
instanceId: eff,
|
|
6046
|
-
byNodeId: request.toNodeId,
|
|
6047
|
-
reason: "refresh"
|
|
6048
|
-
});
|
|
7519
|
+
for (const event of plan.events) appendEvent(journalPath, event);
|
|
7520
|
+
for (const eff of plan.supersededInstanceIds) {
|
|
6049
7521
|
const controller = nodeControllers.get(eff);
|
|
6050
7522
|
if (controller && !controller.signal.aborted) {
|
|
6051
7523
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6167,11 +7639,11 @@ ${request.reason}
|
|
|
6167
7639
|
}
|
|
6168
7640
|
function applyLoopControl(a) {
|
|
6169
7641
|
if (a.kind === "startLoop") {
|
|
6170
|
-
|
|
7642
|
+
appendControlEvent({ type: "loopStarted", loopId: a.loopId });
|
|
6171
7643
|
return;
|
|
6172
7644
|
}
|
|
6173
7645
|
if (a.kind === "startLoopIteration") {
|
|
6174
|
-
|
|
7646
|
+
appendControlEvent({ type: "loopIterationStarted", loopId: a.loopId, iteration: a.iteration });
|
|
6175
7647
|
return;
|
|
6176
7648
|
}
|
|
6177
7649
|
const loopNode = nodesById.get(a.loopId);
|
|
@@ -6180,7 +7652,7 @@ ${request.reason}
|
|
|
6180
7652
|
const outInstId = loopInstanceId(a.loopId, a.iteration, loopNode.output.from);
|
|
6181
7653
|
const succ2 = [...events2].reverse().find((e) => e.type === "nodeSucceeded" && e.nodeId === outInstId);
|
|
6182
7654
|
if (!succ2) {
|
|
6183
|
-
|
|
7655
|
+
appendControlEvent({
|
|
6184
7656
|
type: "nodeFailed",
|
|
6185
7657
|
nodeId: a.loopId,
|
|
6186
7658
|
attemptId: `${a.loopId}/iterations/${String(a.iteration).padStart(3, "0")}`,
|
|
@@ -6189,7 +7661,7 @@ ${request.reason}
|
|
|
6189
7661
|
});
|
|
6190
7662
|
return;
|
|
6191
7663
|
}
|
|
6192
|
-
|
|
7664
|
+
appendControlEvent({
|
|
6193
7665
|
type: "nodeSucceeded",
|
|
6194
7666
|
nodeId: a.loopId,
|
|
6195
7667
|
attemptId: succ2.attemptId,
|
|
@@ -6205,11 +7677,11 @@ ${request.reason}
|
|
|
6205
7677
|
let observed = `${loopNode.exit.when.path}=<unreadable>`;
|
|
6206
7678
|
if (succ) {
|
|
6207
7679
|
try {
|
|
6208
|
-
const manifest = JSON.parse((0,
|
|
7680
|
+
const manifest = JSON.parse((0, import_node_fs15.readFileSync)(succ.manifestPath, "utf-8"));
|
|
6209
7681
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6210
7682
|
if (entry) {
|
|
6211
7683
|
const result = JSON.parse(
|
|
6212
|
-
(0,
|
|
7684
|
+
(0, import_node_fs15.readFileSync)((0, import_node_path13.join)((0, import_node_path13.dirname)(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6213
7685
|
);
|
|
6214
7686
|
observed = `${loopNode.exit.when.path}=${JSON.stringify(result[key])}`;
|
|
6215
7687
|
matched = matchLoopExitWhen(loopNode.exit.when, result[key]);
|
|
@@ -6223,7 +7695,7 @@ ${request.reason}
|
|
|
6223
7695
|
const effectiveMax = loopNode.maxIterations + granted;
|
|
6224
7696
|
const anomalous = observed.endsWith("<unreadable>");
|
|
6225
7697
|
const decision = matched ? "exit" : !anomalous && a.iteration < effectiveMax ? "continue" : "exhausted";
|
|
6226
|
-
|
|
7698
|
+
appendControlEvent({
|
|
6227
7699
|
type: "loopIterationDecision",
|
|
6228
7700
|
loopId: a.loopId,
|
|
6229
7701
|
iteration: a.iteration,
|
|
@@ -6240,7 +7712,7 @@ ${request.reason}
|
|
|
6240
7712
|
const target = nodesById.get(a.to);
|
|
6241
7713
|
const dep = target?.depends.find((d) => d.from === a.from);
|
|
6242
7714
|
if (!target || !dep?.when) {
|
|
6243
|
-
|
|
7715
|
+
appendControlEvent({
|
|
6244
7716
|
type: "edgeResolved",
|
|
6245
7717
|
from: a.from,
|
|
6246
7718
|
to: a.to,
|
|
@@ -6258,11 +7730,11 @@ ${request.reason}
|
|
|
6258
7730
|
let detail = `${dep.when.path}=<unreadable>`;
|
|
6259
7731
|
if (succ) {
|
|
6260
7732
|
try {
|
|
6261
|
-
const manifest = JSON.parse((0,
|
|
7733
|
+
const manifest = JSON.parse((0, import_node_fs15.readFileSync)(succ.manifestPath, "utf-8"));
|
|
6262
7734
|
const entry = manifest.files.find((f) => f.path === "result.json");
|
|
6263
7735
|
if (entry) {
|
|
6264
7736
|
const result = JSON.parse(
|
|
6265
|
-
(0,
|
|
7737
|
+
(0, import_node_fs15.readFileSync)((0, import_node_path13.join)((0, import_node_path13.dirname)(succ.manifestPath), "work", entry.path), "utf-8")
|
|
6266
7738
|
);
|
|
6267
7739
|
detail = `${dep.when.path}=${JSON.stringify(result[key])}`;
|
|
6268
7740
|
active = matchLoopExitWhen(dep.when, result[key]);
|
|
@@ -6270,7 +7742,7 @@ ${request.reason}
|
|
|
6270
7742
|
} catch {
|
|
6271
7743
|
}
|
|
6272
7744
|
}
|
|
6273
|
-
|
|
7745
|
+
appendControlEvent({
|
|
6274
7746
|
type: "edgeResolved",
|
|
6275
7747
|
from: a.from,
|
|
6276
7748
|
to: a.to,
|
|
@@ -6287,7 +7759,7 @@ ${request.reason}
|
|
|
6287
7759
|
if (status !== "pending" && status !== "gateWaiting" && status !== "running") return false;
|
|
6288
7760
|
const instanceId = snap.nodes.get(a.nodeId)?.effectiveInstanceId;
|
|
6289
7761
|
const attemptId = latestAttemptIdFor(events, instanceId ?? a.nodeId);
|
|
6290
|
-
|
|
7762
|
+
const appended = appendControlEvent({
|
|
6291
7763
|
type: "nodeCancelled",
|
|
6292
7764
|
nodeId: a.nodeId,
|
|
6293
7765
|
...instanceId ? { instanceId } : {},
|
|
@@ -6296,6 +7768,7 @@ ${request.reason}
|
|
|
6296
7768
|
byNodeId: a.byNodeId,
|
|
6297
7769
|
detail: a.detail
|
|
6298
7770
|
});
|
|
7771
|
+
if (!appended) return true;
|
|
6299
7772
|
const controller = nodeControllers.get(instanceId ?? a.nodeId);
|
|
6300
7773
|
if (controller && !controller.signal.aborted) {
|
|
6301
7774
|
controller.abort({ kind: "attemptDrain", drainReason: "obsoleteAttempt" });
|
|
@@ -6365,8 +7838,8 @@ ${request.reason}
|
|
|
6365
7838
|
const key = snap.nodes.get(nodeId)?.effectiveInstanceId ?? nodeId;
|
|
6366
7839
|
const succ = latestSuccess(key);
|
|
6367
7840
|
if (!succ) return;
|
|
6368
|
-
const upstreamOutputDir = (0,
|
|
6369
|
-
const manifest = JSON.parse((0,
|
|
7841
|
+
const upstreamOutputDir = (0, import_node_path13.join)((0, import_node_path13.dirname)(succ.manifestPath), "work");
|
|
7842
|
+
const manifest = JSON.parse((0, import_node_fs15.readFileSync)(succ.manifestPath, "utf-8"));
|
|
6370
7843
|
for (const f of manifest.files) {
|
|
6371
7844
|
if (filter && !filter(f)) continue;
|
|
6372
7845
|
inputs.push({
|
|
@@ -6374,29 +7847,29 @@ ${request.reason}
|
|
|
6374
7847
|
...output ? { output } : {},
|
|
6375
7848
|
...succ.instanceId ? { instanceId: succ.instanceId } : {},
|
|
6376
7849
|
name: f.name,
|
|
6377
|
-
path: (0,
|
|
7850
|
+
path: (0, import_node_path13.join)(upstreamOutputDir, f.path),
|
|
6378
7851
|
kind: f.kind,
|
|
6379
7852
|
preview: f.preview
|
|
6380
7853
|
});
|
|
6381
7854
|
}
|
|
6382
7855
|
};
|
|
6383
|
-
const resolveRunPath = (p) => (0,
|
|
7856
|
+
const resolveRunPath = (p) => (0, import_node_path13.isAbsolute)(p) ? p : (0, import_node_path13.join)(runDir, p);
|
|
6384
7857
|
const pushManifestByPath = (label, manifestPath, namePrefix) => {
|
|
6385
7858
|
if (!manifestPath) return;
|
|
6386
7859
|
const abs = resolveRunPath(manifestPath);
|
|
6387
|
-
if (!(0,
|
|
7860
|
+
if (!(0, import_node_fs15.existsSync)(abs)) return;
|
|
6388
7861
|
let manifest;
|
|
6389
7862
|
try {
|
|
6390
|
-
manifest = JSON.parse((0,
|
|
7863
|
+
manifest = JSON.parse((0, import_node_fs15.readFileSync)(abs, "utf-8"));
|
|
6391
7864
|
} catch {
|
|
6392
7865
|
return;
|
|
6393
7866
|
}
|
|
6394
|
-
const outDir = (0,
|
|
7867
|
+
const outDir = (0, import_node_path13.join)((0, import_node_path13.dirname)(abs), "work");
|
|
6395
7868
|
for (const f of manifest.files) {
|
|
6396
7869
|
inputs.push({
|
|
6397
7870
|
from: label,
|
|
6398
7871
|
name: `${namePrefix}:${f.name}`,
|
|
6399
|
-
path: (0,
|
|
7872
|
+
path: (0, import_node_path13.join)(outDir, f.path),
|
|
6400
7873
|
kind: f.kind,
|
|
6401
7874
|
preview: f.preview
|
|
6402
7875
|
});
|
|
@@ -6466,7 +7939,7 @@ ${request.reason}
|
|
|
6466
7939
|
}
|
|
6467
7940
|
|
|
6468
7941
|
// src/workflows/v3/portable-runtime.ts
|
|
6469
|
-
var
|
|
7942
|
+
var import_node_path14 = require("node:path");
|
|
6470
7943
|
function defaultValidateProfile(profile, selector) {
|
|
6471
7944
|
if (!profile.profileId || !profile.executorId || !profile.workingDirectory) {
|
|
6472
7945
|
throw new Error(
|
|
@@ -6515,7 +7988,7 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6515
7988
|
validateProfile(executionProfile, selector);
|
|
6516
7989
|
}
|
|
6517
7990
|
const frozenDefinition = assertOrCreatePortableRunSnapshot(
|
|
6518
|
-
(0,
|
|
7991
|
+
(0, import_node_path14.join)(options.baseDir, normalizedDag.runId),
|
|
6519
7992
|
normalizedDag,
|
|
6520
7993
|
profiles
|
|
6521
7994
|
);
|
|
@@ -6593,6 +8066,9 @@ async function runPortableWorkflow(dag, deps, options) {
|
|
|
6593
8066
|
])
|
|
6594
8067
|
)
|
|
6595
8068
|
});
|
|
8069
|
+
if (outcome.reason === "awaitingExternal") {
|
|
8070
|
+
throw new Error("portable workflow runtime does not drive bot-executed goal nodes");
|
|
8071
|
+
}
|
|
6596
8072
|
if (outcome.reason !== "terminal") return outcome;
|
|
6597
8073
|
return {
|
|
6598
8074
|
...outcome,
|