@dudousxd/nestjs-durable-core 0.85.5 → 0.85.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1176 -635
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +160 -21
- package/dist/index.d.ts +160 -21
- package/dist/index.js +1176 -635
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -646,6 +646,386 @@ var BoundedRunner = class {
|
|
|
646
646
|
};
|
|
647
647
|
var LOCAL_RUN_CONCURRENCY = 256;
|
|
648
648
|
|
|
649
|
+
// src/fan-join.ts
|
|
650
|
+
function allChildId(runId, firstSeq, index, wave) {
|
|
651
|
+
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
652
|
+
}
|
|
653
|
+
__name(allChildId, "allChildId");
|
|
654
|
+
function allGroup(firstSeq, size, mode) {
|
|
655
|
+
const group = `all:${firstSeq}:n${size}`;
|
|
656
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
657
|
+
}
|
|
658
|
+
__name(allGroup, "allGroup");
|
|
659
|
+
function stepFanGroup(firstSeq, size, mode) {
|
|
660
|
+
const group = `steps:${firstSeq}:${size}`;
|
|
661
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
662
|
+
}
|
|
663
|
+
__name(stepFanGroup, "stepFanGroup");
|
|
664
|
+
function parseStepFanGroup(group) {
|
|
665
|
+
const fan = parseFanGroup(group);
|
|
666
|
+
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
667
|
+
return {
|
|
668
|
+
firstSeq: fan.firstSeq,
|
|
669
|
+
size: fan.size
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
673
|
+
function parseFanGroup(group) {
|
|
674
|
+
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
675
|
+
if (!m) return void 0;
|
|
676
|
+
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
677
|
+
return {
|
|
678
|
+
kind: m[1],
|
|
679
|
+
firstSeq: Number(m[2]),
|
|
680
|
+
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
681
|
+
failFast: m[4] !== void 0
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
__name(parseFanGroup, "parseFanGroup");
|
|
685
|
+
function allWait(runId, firstSeq, wave) {
|
|
686
|
+
const base = allBase(runId, firstSeq, wave);
|
|
687
|
+
return {
|
|
688
|
+
waiterPrefix: `child:${base}`,
|
|
689
|
+
checkpointPrefix: `signal:child:${base}`
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
__name(allWait, "allWait");
|
|
693
|
+
function fanJoin(waits) {
|
|
694
|
+
return {
|
|
695
|
+
waits
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
__name(fanJoin, "fanJoin");
|
|
699
|
+
var FOLDED_PREFIX = "fan:";
|
|
700
|
+
function allFoldedName(runId, firstSeq, wave) {
|
|
701
|
+
return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
|
|
702
|
+
}
|
|
703
|
+
__name(allFoldedName, "allFoldedName");
|
|
704
|
+
function foldedChildIds(checkpoint) {
|
|
705
|
+
const base = checkpoint.name.slice(FOLDED_PREFIX.length);
|
|
706
|
+
const { from, values } = checkpoint.output;
|
|
707
|
+
return values.map((_, j) => `${base}${from + j}`);
|
|
708
|
+
}
|
|
709
|
+
__name(foldedChildIds, "foldedChildIds");
|
|
710
|
+
function allBase(runId, firstSeq, wave) {
|
|
711
|
+
return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
|
|
712
|
+
}
|
|
713
|
+
__name(allBase, "allBase");
|
|
714
|
+
function allWaves(size, concurrency) {
|
|
715
|
+
if (concurrency === void 0 || concurrency >= size) return void 0;
|
|
716
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
717
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
|
|
718
|
+
}
|
|
719
|
+
if (concurrency === 1) return {
|
|
720
|
+
waveSize: 1,
|
|
721
|
+
maxWaves: 1
|
|
722
|
+
};
|
|
723
|
+
return {
|
|
724
|
+
waveSize: Math.floor(concurrency / 2),
|
|
725
|
+
maxWaves: 2
|
|
726
|
+
};
|
|
727
|
+
}
|
|
728
|
+
__name(allWaves, "allWaves");
|
|
729
|
+
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
|
|
730
|
+
function parentRunIdOf(runId) {
|
|
731
|
+
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
732
|
+
}
|
|
733
|
+
__name(parentRunIdOf, "parentRunIdOf");
|
|
734
|
+
var GROUP = /^all:(\d+)(?::n\d+)?(:failFast)?$/;
|
|
735
|
+
var REST = /^(?:w(\d+)\.)?\d+$/;
|
|
736
|
+
function fanOfWaiter(waiter) {
|
|
737
|
+
const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
|
|
738
|
+
if (!group) return void 0;
|
|
739
|
+
const base = `child:${waiter.runId}.all.${group[1]}.`;
|
|
740
|
+
if (!waiter.token.startsWith(base)) return void 0;
|
|
741
|
+
const rest = REST.exec(waiter.token.slice(base.length));
|
|
742
|
+
if (!rest) return void 0;
|
|
743
|
+
const waiterPrefix = rest[1] === void 0 ? base : `${base}w${rest[1]}.`;
|
|
744
|
+
return {
|
|
745
|
+
waiterPrefix,
|
|
746
|
+
failFast: group[2] !== void 0
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
__name(fanOfWaiter, "fanOfWaiter");
|
|
750
|
+
|
|
751
|
+
// src/interfaces.ts
|
|
752
|
+
function checkpointNameKind(name) {
|
|
753
|
+
if (name.startsWith("signal:child:")) return "child";
|
|
754
|
+
if (name.startsWith("fan:")) return "folded";
|
|
755
|
+
if (name.startsWith("spawn:")) return "spawn";
|
|
756
|
+
return "step";
|
|
757
|
+
}
|
|
758
|
+
__name(checkpointNameKind, "checkpointNameKind");
|
|
759
|
+
function poolNamespace(namespace) {
|
|
760
|
+
return namespace === "" ? "default" : namespace;
|
|
761
|
+
}
|
|
762
|
+
__name(poolNamespace, "poolNamespace");
|
|
763
|
+
var TERMINAL_RUN_STATUSES = [
|
|
764
|
+
"completed",
|
|
765
|
+
"failed",
|
|
766
|
+
"cancelled",
|
|
767
|
+
"dead"
|
|
768
|
+
];
|
|
769
|
+
function isNonEmptyRunScope(scope) {
|
|
770
|
+
if (!scope) return false;
|
|
771
|
+
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
772
|
+
}
|
|
773
|
+
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
774
|
+
|
|
775
|
+
// src/events.ts
|
|
776
|
+
var enc = /* @__PURE__ */ __name((s) => Buffer.from(s, "utf8").toString("base64"), "enc");
|
|
777
|
+
var dec = /* @__PURE__ */ __name((s) => Buffer.from(s, "base64").toString("utf8"), "dec");
|
|
778
|
+
var EVENT_BUFFER_SCAN_LIMIT = 50;
|
|
779
|
+
var EVENT_TOKEN_PREFIX = "event:";
|
|
780
|
+
function eventPrefix(name) {
|
|
781
|
+
return `${EVENT_TOKEN_PREFIX}${enc(name)}:`;
|
|
782
|
+
}
|
|
783
|
+
__name(eventPrefix, "eventPrefix");
|
|
784
|
+
function isEventToken(token) {
|
|
785
|
+
return token.startsWith(EVENT_TOKEN_PREFIX);
|
|
786
|
+
}
|
|
787
|
+
__name(isEventToken, "isEventToken");
|
|
788
|
+
function eventToken(name, match, runId, seq) {
|
|
789
|
+
return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
|
|
790
|
+
}
|
|
791
|
+
__name(eventToken, "eventToken");
|
|
792
|
+
function eventNameOf(token) {
|
|
793
|
+
try {
|
|
794
|
+
return dec(token.split(":")[1] ?? "");
|
|
795
|
+
} catch {
|
|
796
|
+
return token;
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
__name(eventNameOf, "eventNameOf");
|
|
800
|
+
function eventMatchOf(token) {
|
|
801
|
+
try {
|
|
802
|
+
return JSON.parse(dec(token.split(":")[2] ?? ""));
|
|
803
|
+
} catch {
|
|
804
|
+
return {};
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
__name(eventMatchOf, "eventMatchOf");
|
|
808
|
+
function eventMatches(payload, match) {
|
|
809
|
+
const keys = Object.keys(match);
|
|
810
|
+
if (keys.length === 0) return true;
|
|
811
|
+
if (!payload || typeof payload !== "object") return false;
|
|
812
|
+
const p = payload;
|
|
813
|
+
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
814
|
+
}
|
|
815
|
+
__name(eventMatches, "eventMatches");
|
|
816
|
+
|
|
817
|
+
// src/run-waiting.ts
|
|
818
|
+
var WEBHOOK_PREFIX = "wh:";
|
|
819
|
+
var CHILD_PREFIX = "child:";
|
|
820
|
+
var BREAKPOINT_PREFIX = "bp:";
|
|
821
|
+
var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
|
|
822
|
+
function parseWebhookToken(token) {
|
|
823
|
+
if (!token.startsWith(WEBHOOK_PREFIX)) return null;
|
|
824
|
+
const rest = token.slice(WEBHOOK_PREFIX.length);
|
|
825
|
+
const at = rest.lastIndexOf(":");
|
|
826
|
+
if (at <= 0) return null;
|
|
827
|
+
const seqText = rest.slice(at + 1);
|
|
828
|
+
if (!/^\d+$/.test(seqText)) return null;
|
|
829
|
+
return {
|
|
830
|
+
runId: rest.slice(0, at),
|
|
831
|
+
seq: Number(seqText)
|
|
832
|
+
};
|
|
833
|
+
}
|
|
834
|
+
__name(parseWebhookToken, "parseWebhookToken");
|
|
835
|
+
function classifyWaiterToken(token) {
|
|
836
|
+
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
837
|
+
on: "webhook",
|
|
838
|
+
name: token
|
|
839
|
+
};
|
|
840
|
+
if (token.startsWith(BREAKPOINT_PREFIX)) return {
|
|
841
|
+
on: "breakpoint",
|
|
842
|
+
name: "breakpoint"
|
|
843
|
+
};
|
|
844
|
+
if (token.startsWith(CHILD_PREFIX)) return {
|
|
845
|
+
on: "child",
|
|
846
|
+
name: token.slice(CHILD_PREFIX.length)
|
|
847
|
+
};
|
|
848
|
+
if (token.startsWith(EVENT_PREFIX)) return {
|
|
849
|
+
on: "signal",
|
|
850
|
+
name: eventNameOf(token)
|
|
851
|
+
};
|
|
852
|
+
return {
|
|
853
|
+
on: "signal",
|
|
854
|
+
name: token
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
__name(classifyWaiterToken, "classifyWaiterToken");
|
|
858
|
+
function resolveRunWaiting(run, waiterByRun) {
|
|
859
|
+
if (run.status !== "suspended") return void 0;
|
|
860
|
+
const waiter = waiterByRun.get(run.id);
|
|
861
|
+
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
862
|
+
}
|
|
863
|
+
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
864
|
+
var RUN_ID_BATCH = 500;
|
|
865
|
+
async function listSignalWaitersOfRuns(store, runIds) {
|
|
866
|
+
const unique = [
|
|
867
|
+
...new Set(runIds)
|
|
868
|
+
];
|
|
869
|
+
if (unique.length === 0) return [];
|
|
870
|
+
if (!store.listSignalWaitersByRun) {
|
|
871
|
+
const ids = new Set(unique);
|
|
872
|
+
return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
|
|
873
|
+
}
|
|
874
|
+
const out = [];
|
|
875
|
+
for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
|
|
876
|
+
out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
|
|
877
|
+
}
|
|
878
|
+
return out;
|
|
879
|
+
}
|
|
880
|
+
__name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
|
|
881
|
+
function indexWaitersByRun(waiters) {
|
|
882
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
883
|
+
for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
|
|
884
|
+
return byRun;
|
|
885
|
+
}
|
|
886
|
+
__name(indexWaitersByRun, "indexWaitersByRun");
|
|
887
|
+
var GET_RUN_FALLBACK_PARALLEL = 50;
|
|
888
|
+
async function getRunsByIds(store, runIds) {
|
|
889
|
+
const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
|
|
890
|
+
if (unread.length > 0) throw error2;
|
|
891
|
+
return runs;
|
|
892
|
+
}
|
|
893
|
+
__name(getRunsByIds, "getRunsByIds");
|
|
894
|
+
async function readRunsByIds(store, runIds) {
|
|
895
|
+
const unique = [
|
|
896
|
+
...new Set(runIds)
|
|
897
|
+
];
|
|
898
|
+
const runs = /* @__PURE__ */ new Map();
|
|
899
|
+
const unread = [];
|
|
900
|
+
let error2;
|
|
901
|
+
const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
|
|
902
|
+
for (let at = 0; at < unique.length; at += batch) {
|
|
903
|
+
const ids = unique.slice(at, at + batch);
|
|
904
|
+
if (store.getRuns) {
|
|
905
|
+
try {
|
|
906
|
+
for (const run of await store.getRuns(ids)) runs.set(run.id, run);
|
|
907
|
+
} catch (err) {
|
|
908
|
+
error2 ??= err;
|
|
909
|
+
unread.push(...ids);
|
|
910
|
+
}
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
|
|
914
|
+
reads.forEach((read, k) => {
|
|
915
|
+
if (read.status === "fulfilled") {
|
|
916
|
+
if (read.value) runs.set(read.value.id, read.value);
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
error2 ??= read.reason;
|
|
920
|
+
unread.push(ids[k]);
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
return {
|
|
924
|
+
runs,
|
|
925
|
+
unread,
|
|
926
|
+
...unread.length > 0 ? {
|
|
927
|
+
error: error2
|
|
928
|
+
} : {}
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
__name(readRunsByIds, "readRunsByIds");
|
|
932
|
+
|
|
933
|
+
// src/buffered-child-signals.ts
|
|
934
|
+
var CHILD_TOKEN_PREFIX = "child:";
|
|
935
|
+
var CHILD_SIGNAL_NAME_PREFIX = "signal:child:";
|
|
936
|
+
var RETRYABLE = /* @__PURE__ */ new Set([
|
|
937
|
+
"failed",
|
|
938
|
+
"dead"
|
|
939
|
+
]);
|
|
940
|
+
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
941
|
+
function canonicalJson(value) {
|
|
942
|
+
const plain = JSON.stringify(value);
|
|
943
|
+
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
944
|
+
}
|
|
945
|
+
__name(canonicalJson, "canonicalJson");
|
|
946
|
+
function sortedJson(value) {
|
|
947
|
+
if (Array.isArray(value)) return `[${value.map(sortedJson).join(",")}]`;
|
|
948
|
+
if (value !== null && typeof value === "object") {
|
|
949
|
+
const entries = Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
950
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${sortedJson(v)}`).join(",")}}`;
|
|
951
|
+
}
|
|
952
|
+
return JSON.stringify(value);
|
|
953
|
+
}
|
|
954
|
+
__name(sortedJson, "sortedJson");
|
|
955
|
+
function settledNames(childId) {
|
|
956
|
+
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
957
|
+
const fan = FAN_CHILD.exec(childId);
|
|
958
|
+
if (!fan) return {
|
|
959
|
+
signal
|
|
960
|
+
};
|
|
961
|
+
return {
|
|
962
|
+
signal,
|
|
963
|
+
fold: {
|
|
964
|
+
name: `${FOLDED_PREFIX}${fan[1]}`,
|
|
965
|
+
index: Number(fan[2])
|
|
966
|
+
}
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
__name(settledNames, "settledNames");
|
|
970
|
+
function parentHoldsChild(checkpoints, childId, completion) {
|
|
971
|
+
const { signal, fold } = settledNames(childId);
|
|
972
|
+
const expected = canonicalJson(completion);
|
|
973
|
+
return checkpoints.some((cp) => {
|
|
974
|
+
if (cp.status !== "completed") return false;
|
|
975
|
+
if (cp.name === signal) return canonicalJson(cp.output) === expected;
|
|
976
|
+
if (fold && cp.name === fold.name) {
|
|
977
|
+
const { from, values } = cp.output ?? {};
|
|
978
|
+
return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
|
|
979
|
+
}
|
|
980
|
+
return false;
|
|
981
|
+
});
|
|
982
|
+
}
|
|
983
|
+
__name(parentHoldsChild, "parentHoldsChild");
|
|
984
|
+
async function parentCheckpoints(store, parentId, childId) {
|
|
985
|
+
const { signal, fold } = settledNames(childId);
|
|
986
|
+
const prefixes = fold ? [
|
|
987
|
+
signal,
|
|
988
|
+
fold.name
|
|
989
|
+
] : [
|
|
990
|
+
signal
|
|
991
|
+
];
|
|
992
|
+
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
993
|
+
}
|
|
994
|
+
__name(parentCheckpoints, "parentCheckpoints");
|
|
995
|
+
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
996
|
+
const parentId = parentRunIdOf(childId);
|
|
997
|
+
if (!parentId) return false;
|
|
998
|
+
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
999
|
+
if (!run) return false;
|
|
1000
|
+
if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
|
|
1001
|
+
return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
|
|
1002
|
+
}
|
|
1003
|
+
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1004
|
+
async function unneededChildSignals(store, rows) {
|
|
1005
|
+
const parentIds = rows.flatMap((row) => {
|
|
1006
|
+
const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
|
|
1007
|
+
return parentId ? [
|
|
1008
|
+
parentId
|
|
1009
|
+
] : [];
|
|
1010
|
+
});
|
|
1011
|
+
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1012
|
+
const unreadable = new Set(unread);
|
|
1013
|
+
const doomed = [];
|
|
1014
|
+
for (const row of rows) {
|
|
1015
|
+
if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
|
|
1016
|
+
const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
|
|
1017
|
+
const parentId = parentRunIdOf(childId);
|
|
1018
|
+
if (parentId && unreadable.has(parentId)) continue;
|
|
1019
|
+
const parent = parentId ? runs.get(parentId) ?? null : null;
|
|
1020
|
+
try {
|
|
1021
|
+
if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
|
|
1022
|
+
} catch {
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
return doomed;
|
|
1026
|
+
}
|
|
1027
|
+
__name(unneededChildSignals, "unneededChildSignals");
|
|
1028
|
+
|
|
649
1029
|
// src/execution-hooks.ts
|
|
650
1030
|
var wrappers = [];
|
|
651
1031
|
function useDurableExecution(wrapper) {
|
|
@@ -975,6 +1355,32 @@ function completionError(c) {
|
|
|
975
1355
|
};
|
|
976
1356
|
}
|
|
977
1357
|
__name(completionError, "completionError");
|
|
1358
|
+
function childCompletion(child) {
|
|
1359
|
+
if (child.status === "completed") return {
|
|
1360
|
+
ok: true,
|
|
1361
|
+
value: child.output
|
|
1362
|
+
};
|
|
1363
|
+
if (child.status === "failed") return failedCompletion(child.error ?? {
|
|
1364
|
+
message: "failed"
|
|
1365
|
+
});
|
|
1366
|
+
if (child.status === "dead") return failedCompletion(child.error ?? {
|
|
1367
|
+
message: "dead"
|
|
1368
|
+
});
|
|
1369
|
+
if (child.status === "cancelled") return cancelledCompletion(child.error);
|
|
1370
|
+
return void 0;
|
|
1371
|
+
}
|
|
1372
|
+
__name(childCompletion, "childCompletion");
|
|
1373
|
+
var CHILD_CANCELLED = "cancelled";
|
|
1374
|
+
function cancelledCompletion(error2) {
|
|
1375
|
+
const base = error2 ?? {
|
|
1376
|
+
message: "cancelled"
|
|
1377
|
+
};
|
|
1378
|
+
return failedCompletion({
|
|
1379
|
+
...base,
|
|
1380
|
+
code: base.code ?? CHILD_CANCELLED
|
|
1381
|
+
});
|
|
1382
|
+
}
|
|
1383
|
+
__name(cancelledCompletion, "cancelledCompletion");
|
|
978
1384
|
|
|
979
1385
|
// src/concurrency.ts
|
|
980
1386
|
var DEFAULT_CONCURRENCY_STATUSES = [
|
|
@@ -1180,210 +1586,66 @@ var EventAccumulators = class {
|
|
|
1180
1586
|
const [workflow, input] = config.mode === "debounce" ? [
|
|
1181
1587
|
"__evt_debounce",
|
|
1182
1588
|
{
|
|
1183
|
-
target,
|
|
1184
|
-
windowMs: config.windowMs
|
|
1185
|
-
}
|
|
1186
|
-
] : [
|
|
1187
|
-
"__evt_batch",
|
|
1188
|
-
{
|
|
1189
|
-
target,
|
|
1190
|
-
maxSize: config.maxSize,
|
|
1191
|
-
windowMs: config.windowMs
|
|
1192
|
-
}
|
|
1193
|
-
];
|
|
1194
|
-
return this.host.signalWithStart(workflow, input, accToken(target), {
|
|
1195
|
-
token: accToken(target),
|
|
1196
|
-
payload
|
|
1197
|
-
});
|
|
1198
|
-
}
|
|
1199
|
-
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1200
|
-
async debounce(ctx, input) {
|
|
1201
|
-
const { target, windowMs } = input;
|
|
1202
|
-
const token = accToken(target);
|
|
1203
|
-
let last = await ctx.waitForSignal(token);
|
|
1204
|
-
for (; ; ) {
|
|
1205
|
-
try {
|
|
1206
|
-
last = await ctx.waitForSignal(token, {
|
|
1207
|
-
timeoutMs: windowMs
|
|
1208
|
-
});
|
|
1209
|
-
} catch (e) {
|
|
1210
|
-
if (e instanceof SignalTimeoutError) break;
|
|
1211
|
-
throw e;
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
await ctx.startChild(target, last);
|
|
1215
|
-
return ctx.continueAsNew(input);
|
|
1216
|
-
}
|
|
1217
|
-
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
1218
|
-
async batch(ctx, input) {
|
|
1219
|
-
const { target, maxSize, windowMs } = input;
|
|
1220
|
-
const token = accToken(target);
|
|
1221
|
-
const events = [
|
|
1222
|
-
await ctx.waitForSignal(token)
|
|
1223
|
-
];
|
|
1224
|
-
const deadline = await ctx.now() + windowMs;
|
|
1225
|
-
while (events.length < maxSize) {
|
|
1226
|
-
const remaining = deadline - await ctx.now();
|
|
1227
|
-
if (remaining <= 0) break;
|
|
1228
|
-
try {
|
|
1229
|
-
events.push(await ctx.waitForSignal(token, {
|
|
1230
|
-
timeoutMs: remaining
|
|
1231
|
-
}));
|
|
1232
|
-
} catch (e) {
|
|
1233
|
-
if (e instanceof SignalTimeoutError) break;
|
|
1234
|
-
throw e;
|
|
1235
|
-
}
|
|
1236
|
-
}
|
|
1237
|
-
await ctx.startChild(target, {
|
|
1238
|
-
events
|
|
1239
|
-
});
|
|
1240
|
-
return ctx.continueAsNew(input);
|
|
1241
|
-
}
|
|
1242
|
-
};
|
|
1243
|
-
|
|
1244
|
-
// src/events.ts
|
|
1245
|
-
var enc = /* @__PURE__ */ __name((s) => Buffer.from(s, "utf8").toString("base64"), "enc");
|
|
1246
|
-
var dec = /* @__PURE__ */ __name((s) => Buffer.from(s, "base64").toString("utf8"), "dec");
|
|
1247
|
-
var EVENT_BUFFER_SCAN_LIMIT = 50;
|
|
1248
|
-
var EVENT_TOKEN_PREFIX = "event:";
|
|
1249
|
-
function eventPrefix(name) {
|
|
1250
|
-
return `${EVENT_TOKEN_PREFIX}${enc(name)}:`;
|
|
1251
|
-
}
|
|
1252
|
-
__name(eventPrefix, "eventPrefix");
|
|
1253
|
-
function isEventToken(token) {
|
|
1254
|
-
return token.startsWith(EVENT_TOKEN_PREFIX);
|
|
1255
|
-
}
|
|
1256
|
-
__name(isEventToken, "isEventToken");
|
|
1257
|
-
function eventToken(name, match, runId, seq) {
|
|
1258
|
-
return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
|
|
1259
|
-
}
|
|
1260
|
-
__name(eventToken, "eventToken");
|
|
1261
|
-
function eventNameOf(token) {
|
|
1262
|
-
try {
|
|
1263
|
-
return dec(token.split(":")[1] ?? "");
|
|
1264
|
-
} catch {
|
|
1265
|
-
return token;
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
__name(eventNameOf, "eventNameOf");
|
|
1269
|
-
function eventMatchOf(token) {
|
|
1270
|
-
try {
|
|
1271
|
-
return JSON.parse(dec(token.split(":")[2] ?? ""));
|
|
1272
|
-
} catch {
|
|
1273
|
-
return {};
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
__name(eventMatchOf, "eventMatchOf");
|
|
1277
|
-
function eventMatches(payload, match) {
|
|
1278
|
-
const keys = Object.keys(match);
|
|
1279
|
-
if (keys.length === 0) return true;
|
|
1280
|
-
if (!payload || typeof payload !== "object") return false;
|
|
1281
|
-
const p = payload;
|
|
1282
|
-
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
1283
|
-
}
|
|
1284
|
-
__name(eventMatches, "eventMatches");
|
|
1285
|
-
|
|
1286
|
-
// src/fan-join.ts
|
|
1287
|
-
function allChildId(runId, firstSeq, index, wave) {
|
|
1288
|
-
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
1289
|
-
}
|
|
1290
|
-
__name(allChildId, "allChildId");
|
|
1291
|
-
function allGroup(firstSeq, size, mode) {
|
|
1292
|
-
const group = `all:${firstSeq}:n${size}`;
|
|
1293
|
-
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1294
|
-
}
|
|
1295
|
-
__name(allGroup, "allGroup");
|
|
1296
|
-
function stepFanGroup(firstSeq, size, mode) {
|
|
1297
|
-
const group = `steps:${firstSeq}:${size}`;
|
|
1298
|
-
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1299
|
-
}
|
|
1300
|
-
__name(stepFanGroup, "stepFanGroup");
|
|
1301
|
-
function parseStepFanGroup(group) {
|
|
1302
|
-
const fan = parseFanGroup(group);
|
|
1303
|
-
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
1304
|
-
return {
|
|
1305
|
-
firstSeq: fan.firstSeq,
|
|
1306
|
-
size: fan.size
|
|
1307
|
-
};
|
|
1308
|
-
}
|
|
1309
|
-
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1310
|
-
function parseFanGroup(group) {
|
|
1311
|
-
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
1312
|
-
if (!m) return void 0;
|
|
1313
|
-
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
1314
|
-
return {
|
|
1315
|
-
kind: m[1],
|
|
1316
|
-
firstSeq: Number(m[2]),
|
|
1317
|
-
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
1318
|
-
failFast: m[4] !== void 0
|
|
1319
|
-
};
|
|
1320
|
-
}
|
|
1321
|
-
__name(parseFanGroup, "parseFanGroup");
|
|
1322
|
-
function allWait(runId, firstSeq, wave) {
|
|
1323
|
-
const base = allBase(runId, firstSeq, wave);
|
|
1324
|
-
return {
|
|
1325
|
-
waiterPrefix: `child:${base}`,
|
|
1326
|
-
checkpointPrefix: `signal:child:${base}`
|
|
1327
|
-
};
|
|
1328
|
-
}
|
|
1329
|
-
__name(allWait, "allWait");
|
|
1330
|
-
function fanJoin(waits) {
|
|
1331
|
-
return {
|
|
1332
|
-
waits
|
|
1333
|
-
};
|
|
1334
|
-
}
|
|
1335
|
-
__name(fanJoin, "fanJoin");
|
|
1336
|
-
var FOLDED_PREFIX = "fan:";
|
|
1337
|
-
function allFoldedName(runId, firstSeq, wave) {
|
|
1338
|
-
return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
|
|
1339
|
-
}
|
|
1340
|
-
__name(allFoldedName, "allFoldedName");
|
|
1341
|
-
function foldedChildIds(checkpoint) {
|
|
1342
|
-
const base = checkpoint.name.slice(FOLDED_PREFIX.length);
|
|
1343
|
-
const { from, values } = checkpoint.output;
|
|
1344
|
-
return values.map((_, j) => `${base}${from + j}`);
|
|
1345
|
-
}
|
|
1346
|
-
__name(foldedChildIds, "foldedChildIds");
|
|
1347
|
-
function allBase(runId, firstSeq, wave) {
|
|
1348
|
-
return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
|
|
1349
|
-
}
|
|
1350
|
-
__name(allBase, "allBase");
|
|
1351
|
-
function allWaves(size, concurrency) {
|
|
1352
|
-
if (concurrency === void 0 || concurrency >= size) return void 0;
|
|
1353
|
-
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
1354
|
-
throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
|
|
1589
|
+
target,
|
|
1590
|
+
windowMs: config.windowMs
|
|
1591
|
+
}
|
|
1592
|
+
] : [
|
|
1593
|
+
"__evt_batch",
|
|
1594
|
+
{
|
|
1595
|
+
target,
|
|
1596
|
+
maxSize: config.maxSize,
|
|
1597
|
+
windowMs: config.windowMs
|
|
1598
|
+
}
|
|
1599
|
+
];
|
|
1600
|
+
return this.host.signalWithStart(workflow, input, accToken(target), {
|
|
1601
|
+
token: accToken(target),
|
|
1602
|
+
payload
|
|
1603
|
+
});
|
|
1355
1604
|
}
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
}
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1605
|
+
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1606
|
+
async debounce(ctx, input) {
|
|
1607
|
+
const { target, windowMs } = input;
|
|
1608
|
+
const token = accToken(target);
|
|
1609
|
+
let last = await ctx.waitForSignal(token);
|
|
1610
|
+
for (; ; ) {
|
|
1611
|
+
try {
|
|
1612
|
+
last = await ctx.waitForSignal(token, {
|
|
1613
|
+
timeoutMs: windowMs
|
|
1614
|
+
});
|
|
1615
|
+
} catch (e) {
|
|
1616
|
+
if (e instanceof SignalTimeoutError) break;
|
|
1617
|
+
throw e;
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
await ctx.startChild(target, last);
|
|
1621
|
+
return ctx.continueAsNew(input);
|
|
1622
|
+
}
|
|
1623
|
+
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
1624
|
+
async batch(ctx, input) {
|
|
1625
|
+
const { target, maxSize, windowMs } = input;
|
|
1626
|
+
const token = accToken(target);
|
|
1627
|
+
const events = [
|
|
1628
|
+
await ctx.waitForSignal(token)
|
|
1629
|
+
];
|
|
1630
|
+
const deadline = await ctx.now() + windowMs;
|
|
1631
|
+
while (events.length < maxSize) {
|
|
1632
|
+
const remaining = deadline - await ctx.now();
|
|
1633
|
+
if (remaining <= 0) break;
|
|
1634
|
+
try {
|
|
1635
|
+
events.push(await ctx.waitForSignal(token, {
|
|
1636
|
+
timeoutMs: remaining
|
|
1637
|
+
}));
|
|
1638
|
+
} catch (e) {
|
|
1639
|
+
if (e instanceof SignalTimeoutError) break;
|
|
1640
|
+
throw e;
|
|
1641
|
+
}
|
|
1642
|
+
}
|
|
1643
|
+
await ctx.startChild(target, {
|
|
1644
|
+
events
|
|
1645
|
+
});
|
|
1646
|
+
return ctx.continueAsNew(input);
|
|
1647
|
+
}
|
|
1648
|
+
};
|
|
1387
1649
|
|
|
1388
1650
|
// src/handshake/descriptor.ts
|
|
1389
1651
|
var CURRENT_PROTOCOL_VERSION = 1;
|
|
@@ -2072,30 +2334,6 @@ function planDispatch(requires, liveDescriptors, cp, token) {
|
|
|
2072
2334
|
}
|
|
2073
2335
|
__name(planDispatch, "planDispatch");
|
|
2074
2336
|
|
|
2075
|
-
// src/interfaces.ts
|
|
2076
|
-
function checkpointNameKind(name) {
|
|
2077
|
-
if (name.startsWith("signal:child:")) return "child";
|
|
2078
|
-
if (name.startsWith("fan:")) return "folded";
|
|
2079
|
-
if (name.startsWith("spawn:")) return "spawn";
|
|
2080
|
-
return "step";
|
|
2081
|
-
}
|
|
2082
|
-
__name(checkpointNameKind, "checkpointNameKind");
|
|
2083
|
-
function poolNamespace(namespace) {
|
|
2084
|
-
return namespace === "" ? "default" : namespace;
|
|
2085
|
-
}
|
|
2086
|
-
__name(poolNamespace, "poolNamespace");
|
|
2087
|
-
var TERMINAL_RUN_STATUSES = [
|
|
2088
|
-
"completed",
|
|
2089
|
-
"failed",
|
|
2090
|
-
"cancelled",
|
|
2091
|
-
"dead"
|
|
2092
|
-
];
|
|
2093
|
-
function isNonEmptyRunScope(scope) {
|
|
2094
|
-
if (!scope) return false;
|
|
2095
|
-
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
2096
|
-
}
|
|
2097
|
-
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
2098
|
-
|
|
2099
2337
|
// src/on-error.ts
|
|
2100
2338
|
var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
|
|
2101
2339
|
"non_determinism",
|
|
@@ -2270,122 +2508,6 @@ var RunDispatchBatcher = class {
|
|
|
2270
2508
|
}
|
|
2271
2509
|
};
|
|
2272
2510
|
|
|
2273
|
-
// src/run-waiting.ts
|
|
2274
|
-
var WEBHOOK_PREFIX = "wh:";
|
|
2275
|
-
var CHILD_PREFIX = "child:";
|
|
2276
|
-
var BREAKPOINT_PREFIX = "bp:";
|
|
2277
|
-
var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
|
|
2278
|
-
function parseWebhookToken(token) {
|
|
2279
|
-
if (!token.startsWith(WEBHOOK_PREFIX)) return null;
|
|
2280
|
-
const rest = token.slice(WEBHOOK_PREFIX.length);
|
|
2281
|
-
const at = rest.lastIndexOf(":");
|
|
2282
|
-
if (at <= 0) return null;
|
|
2283
|
-
const seqText = rest.slice(at + 1);
|
|
2284
|
-
if (!/^\d+$/.test(seqText)) return null;
|
|
2285
|
-
return {
|
|
2286
|
-
runId: rest.slice(0, at),
|
|
2287
|
-
seq: Number(seqText)
|
|
2288
|
-
};
|
|
2289
|
-
}
|
|
2290
|
-
__name(parseWebhookToken, "parseWebhookToken");
|
|
2291
|
-
function classifyWaiterToken(token) {
|
|
2292
|
-
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
2293
|
-
on: "webhook",
|
|
2294
|
-
name: token
|
|
2295
|
-
};
|
|
2296
|
-
if (token.startsWith(BREAKPOINT_PREFIX)) return {
|
|
2297
|
-
on: "breakpoint",
|
|
2298
|
-
name: "breakpoint"
|
|
2299
|
-
};
|
|
2300
|
-
if (token.startsWith(CHILD_PREFIX)) return {
|
|
2301
|
-
on: "child",
|
|
2302
|
-
name: token.slice(CHILD_PREFIX.length)
|
|
2303
|
-
};
|
|
2304
|
-
if (token.startsWith(EVENT_PREFIX)) return {
|
|
2305
|
-
on: "signal",
|
|
2306
|
-
name: eventNameOf(token)
|
|
2307
|
-
};
|
|
2308
|
-
return {
|
|
2309
|
-
on: "signal",
|
|
2310
|
-
name: token
|
|
2311
|
-
};
|
|
2312
|
-
}
|
|
2313
|
-
__name(classifyWaiterToken, "classifyWaiterToken");
|
|
2314
|
-
function resolveRunWaiting(run, waiterByRun) {
|
|
2315
|
-
if (run.status !== "suspended") return void 0;
|
|
2316
|
-
const waiter = waiterByRun.get(run.id);
|
|
2317
|
-
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
2318
|
-
}
|
|
2319
|
-
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
2320
|
-
var RUN_ID_BATCH = 500;
|
|
2321
|
-
async function listSignalWaitersOfRuns(store, runIds) {
|
|
2322
|
-
const unique = [
|
|
2323
|
-
...new Set(runIds)
|
|
2324
|
-
];
|
|
2325
|
-
if (unique.length === 0) return [];
|
|
2326
|
-
if (!store.listSignalWaitersByRun) {
|
|
2327
|
-
const ids = new Set(unique);
|
|
2328
|
-
return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
|
|
2329
|
-
}
|
|
2330
|
-
const out = [];
|
|
2331
|
-
for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
|
|
2332
|
-
out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
|
|
2333
|
-
}
|
|
2334
|
-
return out;
|
|
2335
|
-
}
|
|
2336
|
-
__name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
|
|
2337
|
-
function indexWaitersByRun(waiters) {
|
|
2338
|
-
const byRun = /* @__PURE__ */ new Map();
|
|
2339
|
-
for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
|
|
2340
|
-
return byRun;
|
|
2341
|
-
}
|
|
2342
|
-
__name(indexWaitersByRun, "indexWaitersByRun");
|
|
2343
|
-
var GET_RUN_FALLBACK_PARALLEL = 50;
|
|
2344
|
-
async function getRunsByIds(store, runIds) {
|
|
2345
|
-
const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
|
|
2346
|
-
if (unread.length > 0) throw error2;
|
|
2347
|
-
return runs;
|
|
2348
|
-
}
|
|
2349
|
-
__name(getRunsByIds, "getRunsByIds");
|
|
2350
|
-
async function readRunsByIds(store, runIds) {
|
|
2351
|
-
const unique = [
|
|
2352
|
-
...new Set(runIds)
|
|
2353
|
-
];
|
|
2354
|
-
const runs = /* @__PURE__ */ new Map();
|
|
2355
|
-
const unread = [];
|
|
2356
|
-
let error2;
|
|
2357
|
-
const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
|
|
2358
|
-
for (let at = 0; at < unique.length; at += batch) {
|
|
2359
|
-
const ids = unique.slice(at, at + batch);
|
|
2360
|
-
if (store.getRuns) {
|
|
2361
|
-
try {
|
|
2362
|
-
for (const run of await store.getRuns(ids)) runs.set(run.id, run);
|
|
2363
|
-
} catch (err) {
|
|
2364
|
-
error2 ??= err;
|
|
2365
|
-
unread.push(...ids);
|
|
2366
|
-
}
|
|
2367
|
-
continue;
|
|
2368
|
-
}
|
|
2369
|
-
const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
|
|
2370
|
-
reads.forEach((read, k) => {
|
|
2371
|
-
if (read.status === "fulfilled") {
|
|
2372
|
-
if (read.value) runs.set(read.value.id, read.value);
|
|
2373
|
-
return;
|
|
2374
|
-
}
|
|
2375
|
-
error2 ??= read.reason;
|
|
2376
|
-
unread.push(ids[k]);
|
|
2377
|
-
});
|
|
2378
|
-
}
|
|
2379
|
-
return {
|
|
2380
|
-
runs,
|
|
2381
|
-
unread,
|
|
2382
|
-
...unread.length > 0 ? {
|
|
2383
|
-
error: error2
|
|
2384
|
-
} : {}
|
|
2385
|
-
};
|
|
2386
|
-
}
|
|
2387
|
-
__name(readRunsByIds, "readRunsByIds");
|
|
2388
|
-
|
|
2389
2511
|
// src/scheduler.ts
|
|
2390
2512
|
import { createRequire } from "node:module";
|
|
2391
2513
|
|
|
@@ -3225,6 +3347,7 @@ function constantTimeEquals(a, b) {
|
|
|
3225
3347
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3226
3348
|
|
|
3227
3349
|
// src/transport-pool.ts
|
|
3350
|
+
var DISPATCH_CHUNK = 200;
|
|
3228
3351
|
var TransportPool = class {
|
|
3229
3352
|
static {
|
|
3230
3353
|
__name(this, "TransportPool");
|
|
@@ -3284,6 +3407,28 @@ var TransportPool = class {
|
|
|
3284
3407
|
}
|
|
3285
3408
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3286
3409
|
}
|
|
3410
|
+
/**
|
|
3411
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3412
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3413
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3414
|
+
*/
|
|
3415
|
+
async dispatchMany(tasks, preferId) {
|
|
3416
|
+
if (tasks.length === 0) return;
|
|
3417
|
+
const first = this.ordered(preferId)[0];
|
|
3418
|
+
if (first?.transport.dispatchMany) {
|
|
3419
|
+
try {
|
|
3420
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3421
|
+
...task,
|
|
3422
|
+
transport: first.id
|
|
3423
|
+
})));
|
|
3424
|
+
return;
|
|
3425
|
+
} catch {
|
|
3426
|
+
}
|
|
3427
|
+
}
|
|
3428
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3429
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3287
3432
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3288
3433
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3289
3434
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -3842,7 +3987,23 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3842
3987
|
status: "running"
|
|
3843
3988
|
}));
|
|
3844
3989
|
}
|
|
3845
|
-
|
|
3990
|
+
const childRun = await store.getRun(id);
|
|
3991
|
+
if (!childRun) {
|
|
3992
|
+
host.startChild(workflowName(workflow2), input, id, priority, version);
|
|
3993
|
+
} else {
|
|
3994
|
+
const done = (await settleFinishedChildren([
|
|
3995
|
+
{
|
|
3996
|
+
id,
|
|
3997
|
+
seq: current
|
|
3998
|
+
}
|
|
3999
|
+
], /* @__PURE__ */ new Map([
|
|
4000
|
+
[
|
|
4001
|
+
id,
|
|
4002
|
+
childRun
|
|
4003
|
+
]
|
|
4004
|
+
]))).get(id);
|
|
4005
|
+
if (done) return unwrapCompletion(done, `child "${id}"`);
|
|
4006
|
+
}
|
|
3846
4007
|
throw new WorkflowSuspended(void 0, fanJoin([
|
|
3847
4008
|
{
|
|
3848
4009
|
waiterPrefix: `child:${id}`,
|
|
@@ -3863,164 +4024,174 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
3863
4024
|
const group = allGroup(firstSeq, inputs.length, mode);
|
|
3864
4025
|
const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
|
|
3865
4026
|
const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
const
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
const
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
throw new GatherError([
|
|
3893
|
-
{
|
|
3894
|
-
index: i,
|
|
3895
|
-
id: id(i),
|
|
3896
|
-
error: c.error ?? "unknown"
|
|
4027
|
+
for (; ; ) {
|
|
4028
|
+
const { checkpoints: existing, folded } = await readFan(positions, [
|
|
4029
|
+
allWait(runId, firstSeq).checkpointPrefix,
|
|
4030
|
+
allFoldedName(runId, firstSeq)
|
|
4031
|
+
]);
|
|
4032
|
+
const waveCount = waves ? Math.ceil(inputs.length / waves.waveSize) : 1;
|
|
4033
|
+
const waveItems = /* @__PURE__ */ __name((k) => {
|
|
4034
|
+
if (!waves) return inputs.map((_, i) => i);
|
|
4035
|
+
const from = k * waves.waveSize;
|
|
4036
|
+
return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
|
|
4037
|
+
}, "waveItems");
|
|
4038
|
+
if (mode === "failFast") {
|
|
4039
|
+
const failure = /* @__PURE__ */ __name((i2) => {
|
|
4040
|
+
const cp = existing[i2];
|
|
4041
|
+
const c2 = cp?.status === "completed" ? cp.output : null;
|
|
4042
|
+
return c2 && c2.ok === false ? c2 : null;
|
|
4043
|
+
}, "failure");
|
|
4044
|
+
const first = /* @__PURE__ */ __name((cancelled) => inputs.findIndex((_, i2) => {
|
|
4045
|
+
const c2 = failure(i2);
|
|
4046
|
+
return c2 !== null && c2.code === "cancelled" === cancelled;
|
|
4047
|
+
}), "first");
|
|
4048
|
+
const i = first(false) !== -1 ? first(false) : first(true);
|
|
4049
|
+
const c = i === -1 ? null : failure(i);
|
|
4050
|
+
if (c) {
|
|
4051
|
+
for (let j = 0; j < inputs.length; j += 1) {
|
|
4052
|
+
if (j !== i && existing[j]?.status !== "completed") host.cancelChild?.(id(j));
|
|
3897
4053
|
}
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
const last = items[items.length - 1];
|
|
3906
|
-
if (folded[first]) continue;
|
|
3907
|
-
const values = [];
|
|
3908
|
-
for (const i of items) {
|
|
3909
|
-
const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
|
|
3910
|
-
if (!c || typeof c !== "object" || c.ok !== true) break;
|
|
3911
|
-
values.push(c.value);
|
|
3912
|
-
}
|
|
3913
|
-
if (values.length < items.length) continue;
|
|
3914
|
-
const into = instantCheckpoint({
|
|
3915
|
-
runId,
|
|
3916
|
-
seq: positions[first],
|
|
3917
|
-
name: allFoldedName(runId, firstSeq, waves ? k : void 0),
|
|
3918
|
-
kind: "signal",
|
|
3919
|
-
output: {
|
|
3920
|
-
from: first,
|
|
3921
|
-
values
|
|
3922
|
-
},
|
|
3923
|
-
parallelGroup: group
|
|
3924
|
-
});
|
|
3925
|
-
await store.compactCheckpoints(runId, positions[first], positions[last], into);
|
|
3926
|
-
for (const i of items) replay?.delete(positions[i]);
|
|
3927
|
-
replay?.set(into.seq, into);
|
|
3928
|
-
}
|
|
3929
|
-
}
|
|
3930
|
-
const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
|
|
3931
|
-
const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
|
|
3932
|
-
const active = [];
|
|
3933
|
-
let unsettled = 0;
|
|
3934
|
-
for (let k = 0; k < waveCount; k += 1) {
|
|
3935
|
-
if (settled(k)) continue;
|
|
3936
|
-
if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
|
|
3937
|
-
active.push(k);
|
|
3938
|
-
unsettled += 1;
|
|
3939
|
-
}
|
|
3940
|
-
}
|
|
3941
|
-
if (active.length > 0) {
|
|
3942
|
-
const toRegister = [];
|
|
3943
|
-
const toEnsure = [];
|
|
3944
|
-
for (const k of active) {
|
|
3945
|
-
for (const i of waveItems(k)) {
|
|
3946
|
-
const cp = existing[i];
|
|
3947
|
-
if (cp?.status === "completed") continue;
|
|
3948
|
-
(cp ? toEnsure : toRegister).push(i);
|
|
4054
|
+
throw new GatherError([
|
|
4055
|
+
{
|
|
4056
|
+
index: i,
|
|
4057
|
+
id: id(i),
|
|
4058
|
+
error: c.error ?? "unknown"
|
|
4059
|
+
}
|
|
4060
|
+
], inputs.length);
|
|
3949
4061
|
}
|
|
3950
4062
|
}
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
|
|
3961
|
-
|
|
3962
|
-
|
|
3963
|
-
|
|
3964
|
-
|
|
3965
|
-
checkpoint: instantCheckpoint({
|
|
4063
|
+
if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
|
|
4064
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
4065
|
+
const items = waveItems(k);
|
|
4066
|
+
const first = items[0];
|
|
4067
|
+
const last = items[items.length - 1];
|
|
4068
|
+
if (folded[first]) continue;
|
|
4069
|
+
const values = [];
|
|
4070
|
+
for (const i of items) {
|
|
4071
|
+
const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
|
|
4072
|
+
if (!c || typeof c !== "object" || c.ok !== true) break;
|
|
4073
|
+
values.push(c.value);
|
|
4074
|
+
}
|
|
4075
|
+
if (values.length < items.length) continue;
|
|
4076
|
+
const into = instantCheckpoint({
|
|
3966
4077
|
runId,
|
|
3967
|
-
seq: positions[
|
|
3968
|
-
name:
|
|
4078
|
+
seq: positions[first],
|
|
4079
|
+
name: allFoldedName(runId, firstSeq, waves ? k : void 0),
|
|
3969
4080
|
kind: "signal",
|
|
3970
|
-
|
|
4081
|
+
output: {
|
|
4082
|
+
from: first,
|
|
4083
|
+
values
|
|
4084
|
+
},
|
|
3971
4085
|
parallelGroup: group
|
|
3972
|
-
})
|
|
3973
|
-
|
|
3974
|
-
|
|
3975
|
-
|
|
3976
|
-
}
|
|
3977
|
-
if (toEnsure.length > 0) {
|
|
3978
|
-
const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
|
|
3979
|
-
const skip = new Set(unread);
|
|
3980
|
-
for (const i of toEnsure) {
|
|
3981
|
-
if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
|
|
4086
|
+
});
|
|
4087
|
+
await store.compactCheckpoints(runId, positions[first], positions[last], into);
|
|
4088
|
+
for (const i of items) replay?.delete(positions[i]);
|
|
4089
|
+
replay?.set(into.seq, into);
|
|
3982
4090
|
}
|
|
3983
4091
|
}
|
|
3984
|
-
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
|
|
3994
|
-
|
|
3995
|
-
|
|
3996
|
-
|
|
3997
|
-
|
|
3998
|
-
const
|
|
4092
|
+
const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
|
|
4093
|
+
const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
|
|
4094
|
+
const active = [];
|
|
4095
|
+
let unsettled = 0;
|
|
4096
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
4097
|
+
if (settled(k)) continue;
|
|
4098
|
+
if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
|
|
4099
|
+
active.push(k);
|
|
4100
|
+
unsettled += 1;
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
let known;
|
|
4104
|
+
if (active.length > 0) {
|
|
4105
|
+
const toRegister = [];
|
|
4106
|
+
const toEnsure = [];
|
|
4107
|
+
for (const k of active) {
|
|
4108
|
+
for (const i of waveItems(k)) {
|
|
4109
|
+
const cp = existing[i];
|
|
4110
|
+
if (cp?.status === "completed") continue;
|
|
4111
|
+
(cp ? toEnsure : toRegister).push(i);
|
|
4112
|
+
}
|
|
4113
|
+
}
|
|
4114
|
+
for (let at = 0; at < toRegister.length; at += FAN_CHUNK) {
|
|
4115
|
+
const chunk = toRegister.slice(at, at + FAN_CHUNK);
|
|
4116
|
+
const entries = chunk.map((i) => ({
|
|
4117
|
+
// Carry the fan `group` onto the waiter too: the child's terminal `signal:child:`
|
|
4118
|
+
// checkpoint is (re)written by engine.signal when the child notifies the parent,
|
|
4119
|
+
// OVERWRITING the running placeholder at the same seq — so without this the resolved
|
|
4120
|
+
// checkpoint would lose the group and the dashboard would render the resolved fan as a
|
|
4121
|
+
// sequential chain.
|
|
4122
|
+
waiter: {
|
|
4123
|
+
token: `child:${id(i)}`,
|
|
4124
|
+
runId,
|
|
4125
|
+
seq: positions[i],
|
|
4126
|
+
parallelGroup: group
|
|
4127
|
+
},
|
|
4128
|
+
checkpoint: instantCheckpoint({
|
|
4129
|
+
runId,
|
|
4130
|
+
seq: positions[i],
|
|
4131
|
+
name: `signal:child:${id(i)}`,
|
|
4132
|
+
kind: "signal",
|
|
4133
|
+
status: "running",
|
|
4134
|
+
parallelGroup: group
|
|
4135
|
+
})
|
|
4136
|
+
}));
|
|
4137
|
+
await registerChildWaits(entries);
|
|
4138
|
+
for (const i of chunk) host.startChild(name, inputs[i], id(i));
|
|
4139
|
+
}
|
|
4140
|
+
if (toEnsure.length > 0) {
|
|
4141
|
+
const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
|
|
4142
|
+
const skip = new Set(unread);
|
|
4143
|
+
for (const i of toEnsure) {
|
|
4144
|
+
if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
|
|
4145
|
+
}
|
|
4146
|
+
known = started;
|
|
4147
|
+
}
|
|
4148
|
+
const now2 = await settleFinishedChildren(toEnsure.map((i) => ({
|
|
3999
4149
|
id: id(i),
|
|
4000
|
-
|
|
4001
|
-
|
|
4002
|
-
};
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4014
|
-
|
|
4015
|
-
|
|
4016
|
-
|
|
4017
|
-
|
|
4018
|
-
|
|
4150
|
+
seq: positions[i],
|
|
4151
|
+
parallelGroup: group
|
|
4152
|
+
})), known);
|
|
4153
|
+
if (now2.size > 0) continue;
|
|
4154
|
+
}
|
|
4155
|
+
if (active.length > 0) {
|
|
4156
|
+
throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
|
|
4157
|
+
}
|
|
4158
|
+
const outputs = [];
|
|
4159
|
+
const results = [];
|
|
4160
|
+
const failures = [];
|
|
4161
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
4162
|
+
const c = existing[i]?.output;
|
|
4163
|
+
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
4164
|
+
const { message, ...rest } = completionError({
|
|
4165
|
+
...c,
|
|
4166
|
+
error: c.error ?? "unknown"
|
|
4167
|
+
});
|
|
4168
|
+
const failure = {
|
|
4169
|
+
id: id(i),
|
|
4170
|
+
error: message,
|
|
4171
|
+
...rest
|
|
4172
|
+
};
|
|
4173
|
+
failures.push({
|
|
4174
|
+
index: i,
|
|
4175
|
+
...failure
|
|
4176
|
+
});
|
|
4177
|
+
results.push({
|
|
4178
|
+
status: "rejected",
|
|
4179
|
+
...failure
|
|
4180
|
+
});
|
|
4181
|
+
outputs.push(void 0);
|
|
4182
|
+
} else {
|
|
4183
|
+
const value = c?.value;
|
|
4184
|
+
results.push({
|
|
4185
|
+
status: "fulfilled",
|
|
4186
|
+
value
|
|
4187
|
+
});
|
|
4188
|
+
outputs.push(value);
|
|
4189
|
+
}
|
|
4019
4190
|
}
|
|
4191
|
+
if (settle) return results;
|
|
4192
|
+
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
4193
|
+
return outputs;
|
|
4020
4194
|
}
|
|
4021
|
-
if (settle) return results;
|
|
4022
|
-
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
4023
|
-
return outputs;
|
|
4024
4195
|
}, "fan");
|
|
4025
4196
|
const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
|
|
4026
4197
|
const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
|
|
@@ -4106,6 +4277,39 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4106
4277
|
}
|
|
4107
4278
|
for (const e of entries) replay?.set(e.checkpoint.seq, e.checkpoint);
|
|
4108
4279
|
}, "registerChildWaits");
|
|
4280
|
+
const settleFinishedChildren = /* @__PURE__ */ __name(async (awaited, known) => {
|
|
4281
|
+
const settled = /* @__PURE__ */ new Map();
|
|
4282
|
+
if (awaited.length === 0) return settled;
|
|
4283
|
+
const runs = new Map(known ?? []);
|
|
4284
|
+
const toRead = awaited.filter((a) => !runs.has(a.id)).map((a) => a.id);
|
|
4285
|
+
if (toRead.length > 0) {
|
|
4286
|
+
for (const [id, run] of (await readRunsByIds(store, toRead)).runs) runs.set(id, run);
|
|
4287
|
+
}
|
|
4288
|
+
for (const a of awaited) {
|
|
4289
|
+
const child2 = runs.get(a.id);
|
|
4290
|
+
const completion = child2 ? childCompletion(child2) : void 0;
|
|
4291
|
+
if (!completion) continue;
|
|
4292
|
+
const token = `child:${a.id}`;
|
|
4293
|
+
const taken = await store.takeSignalWaiter(token);
|
|
4294
|
+
if (!taken) continue;
|
|
4295
|
+
if (taken.runId !== runId || taken.seq !== a.seq) {
|
|
4296
|
+
await store.putSignalWaiter(taken);
|
|
4297
|
+
continue;
|
|
4298
|
+
}
|
|
4299
|
+
await writeCheckpoint(instantCheckpoint({
|
|
4300
|
+
runId,
|
|
4301
|
+
seq: a.seq,
|
|
4302
|
+
name: `signal:child:${a.id}`,
|
|
4303
|
+
kind: "signal",
|
|
4304
|
+
output: completion,
|
|
4305
|
+
...a.parallelGroup !== void 0 ? {
|
|
4306
|
+
parallelGroup: a.parallelGroup
|
|
4307
|
+
} : {}
|
|
4308
|
+
}));
|
|
4309
|
+
settled.set(a.id, completion);
|
|
4310
|
+
}
|
|
4311
|
+
return settled;
|
|
4312
|
+
}, "settleFinishedChildren");
|
|
4109
4313
|
const startChild = /* @__PURE__ */ __name(async (workflow2, input, options) => {
|
|
4110
4314
|
const { childId, priority, version } = normalizeChildOptions(options);
|
|
4111
4315
|
const current = pos.next();
|
|
@@ -4444,6 +4648,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4444
4648
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4445
4649
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4446
4650
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4651
|
+
function wakeSeen(request, settledFans, served) {
|
|
4652
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
4653
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
4654
|
+
}
|
|
4655
|
+
__name(wakeSeen, "wakeSeen");
|
|
4656
|
+
function isFanWakeRequest(payload) {
|
|
4657
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
4658
|
+
}
|
|
4659
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4447
4660
|
function deferred() {
|
|
4448
4661
|
let resolve;
|
|
4449
4662
|
let reject;
|
|
@@ -4462,35 +4675,51 @@ __name(deferred, "deferred");
|
|
|
4462
4675
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4463
4676
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4464
4677
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
4678
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4465
4679
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4680
|
+
var BUFFERED_SWEEP_PAGE = 500;
|
|
4681
|
+
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
4682
|
+
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4466
4683
|
var DELIVER_CHUNK = 50;
|
|
4467
|
-
function childCompletion(child) {
|
|
4468
|
-
if (child.status === "completed") return {
|
|
4469
|
-
ok: true,
|
|
4470
|
-
value: child.output
|
|
4471
|
-
};
|
|
4472
|
-
if (child.status === "failed") return failedCompletion(child.error ?? {
|
|
4473
|
-
message: "failed"
|
|
4474
|
-
});
|
|
4475
|
-
if (child.status === "dead") return failedCompletion(child.error ?? {
|
|
4476
|
-
message: "dead"
|
|
4477
|
-
});
|
|
4478
|
-
if (child.status === "cancelled") return cancelledCompletion(child.error);
|
|
4479
|
-
return void 0;
|
|
4480
|
-
}
|
|
4481
|
-
__name(childCompletion, "childCompletion");
|
|
4482
|
-
var CHILD_CANCELLED = "cancelled";
|
|
4483
|
-
function cancelledCompletion(error2) {
|
|
4484
|
-
const base = error2 ?? {
|
|
4485
|
-
message: "cancelled"
|
|
4486
|
-
};
|
|
4487
|
-
return failedCompletion({
|
|
4488
|
-
...base,
|
|
4489
|
-
code: base.code ?? CHILD_CANCELLED
|
|
4490
|
-
});
|
|
4491
|
-
}
|
|
4492
|
-
__name(cancelledCompletion, "cancelledCompletion");
|
|
4493
4684
|
var RESULT_BATCH_MAX = 256;
|
|
4685
|
+
function isResultError(outcome) {
|
|
4686
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
4687
|
+
}
|
|
4688
|
+
__name(isResultError, "isResultError");
|
|
4689
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
4690
|
+
static {
|
|
4691
|
+
__name(this, "FanWakeCollector");
|
|
4692
|
+
}
|
|
4693
|
+
carrier;
|
|
4694
|
+
generic = false;
|
|
4695
|
+
fan;
|
|
4696
|
+
/** The highest seq the batch settled, per fan. */
|
|
4697
|
+
highest = /* @__PURE__ */ new Map();
|
|
4698
|
+
/** A checkpoint the batch settled. */
|
|
4699
|
+
note(cp) {
|
|
4700
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4701
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
4702
|
+
}
|
|
4703
|
+
/** A settled checkpoint that resumes the run. */
|
|
4704
|
+
add(index, cp, status) {
|
|
4705
|
+
this.carrier = index;
|
|
4706
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4707
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
4708
|
+
this.generic = true;
|
|
4709
|
+
return;
|
|
4710
|
+
}
|
|
4711
|
+
this.fan = fan.firstSeq;
|
|
4712
|
+
}
|
|
4713
|
+
applyTo(outcomes) {
|
|
4714
|
+
if (this.carrier === void 0) return outcomes;
|
|
4715
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
4716
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
4717
|
+
first: this.fan,
|
|
4718
|
+
seq
|
|
4719
|
+
};
|
|
4720
|
+
return outcomes;
|
|
4721
|
+
}
|
|
4722
|
+
};
|
|
4494
4723
|
function shuffle(items) {
|
|
4495
4724
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4496
4725
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -4679,6 +4908,23 @@ var WorkflowEngine = class {
|
|
|
4679
4908
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4680
4909
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
4681
4910
|
/**
|
|
4911
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
4912
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
4913
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
4914
|
+
*/
|
|
4915
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
4916
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
4917
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
4918
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
4919
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
4920
|
+
/**
|
|
4921
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
4922
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
4923
|
+
*/
|
|
4924
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
4925
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
4926
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
4927
|
+
/**
|
|
4682
4928
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4683
4929
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
4684
4930
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -4705,6 +4951,10 @@ var WorkflowEngine = class {
|
|
|
4705
4951
|
/** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
|
|
4706
4952
|
notifiedStalled = /* @__PURE__ */ new Map();
|
|
4707
4953
|
lastStalledSweepAt = Number.NEGATIVE_INFINITY;
|
|
4954
|
+
/** {@link sweepBufferedSignals}: the id the pass in progress resumes after, `undefined` between passes. */
|
|
4955
|
+
bufferedSweepCursor;
|
|
4956
|
+
bufferedSweepActive = false;
|
|
4957
|
+
lastBufferedSweepEndedAt = Number.NEGATIVE_INFINITY;
|
|
4708
4958
|
stalledAfterMs;
|
|
4709
4959
|
draining = false;
|
|
4710
4960
|
/** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
|
|
@@ -4798,8 +5048,9 @@ var WorkflowEngine = class {
|
|
|
4798
5048
|
}
|
|
4799
5049
|
return;
|
|
4800
5050
|
}
|
|
4801
|
-
|
|
4802
|
-
|
|
5051
|
+
const wake = await this.completeRemoteResult(result);
|
|
5052
|
+
if (!wake) return;
|
|
5053
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
4803
5054
|
if (err instanceof NamespaceMismatch) return;
|
|
4804
5055
|
throw err;
|
|
4805
5056
|
}));
|
|
@@ -5016,7 +5267,10 @@ var WorkflowEngine = class {
|
|
|
5016
5267
|
const resumed = await this.resumeLeased([
|
|
5017
5268
|
run
|
|
5018
5269
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5019
|
-
if (resumed.length === 0)
|
|
5270
|
+
if (resumed.length === 0) {
|
|
5271
|
+
this.unconditionalRechecks.add(runId);
|
|
5272
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5273
|
+
}
|
|
5020
5274
|
return;
|
|
5021
5275
|
}
|
|
5022
5276
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5310,6 +5564,50 @@ var WorkflowEngine = class {
|
|
|
5310
5564
|
return () => this.deadListeners.delete(listener);
|
|
5311
5565
|
}
|
|
5312
5566
|
/**
|
|
5567
|
+
* Delete buffered child completions no parent will ever consume (see
|
|
5568
|
+
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
5569
|
+
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
5570
|
+
* completion that found no waiter, and nothing else ever removes them.
|
|
5571
|
+
*
|
|
5572
|
+
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5573
|
+
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5574
|
+
* few timer ticks without a long query or a big delete. A pass that reaches the end of the table
|
|
5575
|
+
* rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
|
|
5576
|
+
* poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
|
|
5577
|
+
* namespace, and a scoped read would report another tenant's parent as gone), and on a store
|
|
5578
|
+
* without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5579
|
+
*/
|
|
5580
|
+
async sweepBufferedSignals(now = this.clock()) {
|
|
5581
|
+
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5582
|
+
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5583
|
+
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
5584
|
+
if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
|
|
5585
|
+
return 0;
|
|
5586
|
+
}
|
|
5587
|
+
this.bufferedSweepActive = true;
|
|
5588
|
+
let deleted = 0;
|
|
5589
|
+
try {
|
|
5590
|
+
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5591
|
+
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5592
|
+
const doomed = await unneededChildSignals(this.store, rows);
|
|
5593
|
+
if (doomed.length > 0) {
|
|
5594
|
+
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5595
|
+
deleted += doomed.length;
|
|
5596
|
+
}
|
|
5597
|
+
const last = rows.at(-1);
|
|
5598
|
+
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5599
|
+
this.bufferedSweepCursor = void 0;
|
|
5600
|
+
this.lastBufferedSweepEndedAt = now;
|
|
5601
|
+
break;
|
|
5602
|
+
}
|
|
5603
|
+
this.bufferedSweepCursor = last.id;
|
|
5604
|
+
}
|
|
5605
|
+
} finally {
|
|
5606
|
+
this.bufferedSweepActive = false;
|
|
5607
|
+
}
|
|
5608
|
+
return deleted;
|
|
5609
|
+
}
|
|
5610
|
+
/**
|
|
5313
5611
|
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
5314
5612
|
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
5315
5613
|
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
@@ -5539,15 +5837,27 @@ var WorkflowEngine = class {
|
|
|
5539
5837
|
* terminal too.
|
|
5540
5838
|
*/
|
|
5541
5839
|
resume(runId) {
|
|
5840
|
+
this.noteWakeReason(runId, void 0);
|
|
5542
5841
|
return this.resumeRun(runId, true);
|
|
5543
5842
|
}
|
|
5843
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
5844
|
+
noteWakeReason(runId, fan) {
|
|
5845
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
5846
|
+
generic: false,
|
|
5847
|
+
fans: /* @__PURE__ */ new Map()
|
|
5848
|
+
};
|
|
5849
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
5850
|
+
else reason.generic = true;
|
|
5851
|
+
this.wakeReasons.set(runId, reason);
|
|
5852
|
+
}
|
|
5544
5853
|
/**
|
|
5545
5854
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5546
5855
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5547
5856
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5548
5857
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5549
5858
|
*/
|
|
5550
|
-
resumeImplicit(runId) {
|
|
5859
|
+
resumeImplicit(runId, fan) {
|
|
5860
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5551
5861
|
const turn = this.wakeTurns.get(runId);
|
|
5552
5862
|
if (turn) {
|
|
5553
5863
|
turn.again ??= deferred();
|
|
@@ -5572,7 +5882,7 @@ var WorkflowEngine = class {
|
|
|
5572
5882
|
});
|
|
5573
5883
|
return;
|
|
5574
5884
|
}
|
|
5575
|
-
const next = this.resumeImplicit(runId);
|
|
5885
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5576
5886
|
next.then(again.resolve, again.reject);
|
|
5577
5887
|
this.inBackground(next);
|
|
5578
5888
|
}, "finish");
|
|
@@ -7016,7 +7326,9 @@ var WorkflowEngine = class {
|
|
|
7016
7326
|
await this.deliverSignal(waiter, completion);
|
|
7017
7327
|
return;
|
|
7018
7328
|
}
|
|
7019
|
-
if (cancelled
|
|
7329
|
+
if (cancelled) return;
|
|
7330
|
+
const childId = token.slice(CHILD_TOKEN_PREFIX.length);
|
|
7331
|
+
if (!await childCompletionNeedsBuffer(this.store, childId, completion)) return;
|
|
7020
7332
|
await this.signal(token, completion);
|
|
7021
7333
|
return;
|
|
7022
7334
|
} catch (err) {
|
|
@@ -7030,24 +7342,6 @@ var WorkflowEngine = class {
|
|
|
7030
7342
|
}
|
|
7031
7343
|
}
|
|
7032
7344
|
/**
|
|
7033
|
-
* Whether the parent of the child behind `token` (`child:<id>`) already holds `completion` as that
|
|
7034
|
-
* child's settled `signal:child:<id>` checkpoint: its own turn settled the child from the child's
|
|
7035
|
-
* row, so this notification is a duplicate. A different completion (a fix-and-replay run's success
|
|
7036
|
-
* for a child the parent saw fail) is not, and is buffered for the parent's retry. Only for a child
|
|
7037
|
-
* whose id the engine minted (its parent is derivable from it).
|
|
7038
|
-
*/
|
|
7039
|
-
async parentHoldsCompletion(token, completion) {
|
|
7040
|
-
const childId = token.slice("child:".length);
|
|
7041
|
-
const parentId = parentRunIdOf(childId);
|
|
7042
|
-
if (!parentId) return false;
|
|
7043
|
-
const name = `${CHILD_SIGNAL_PREFIX}${childId}`;
|
|
7044
|
-
const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentId, [
|
|
7045
|
-
name
|
|
7046
|
-
]) : await this.store.listCheckpoints(parentId);
|
|
7047
|
-
const expected = JSON.stringify(completion);
|
|
7048
|
-
return checkpoints.some((cp) => cp.name === name && cp.status === "completed" && JSON.stringify(cp.output) === expected);
|
|
7049
|
-
}
|
|
7050
|
-
/**
|
|
7051
7345
|
* Deferred child start shared by the in-process ctx host and the remote `startChild` command.
|
|
7052
7346
|
* Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
|
|
7053
7347
|
* that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
|
|
@@ -7062,15 +7356,22 @@ var WorkflowEngine = class {
|
|
|
7062
7356
|
startChildDeferred(workflow, input, childId, opts) {
|
|
7063
7357
|
this.localWork.submit(`start:${childId}`, async () => {
|
|
7064
7358
|
if (this.draining) return;
|
|
7065
|
-
await this.start(workflow, input, childId, opts).catch((err) => {
|
|
7359
|
+
const started = await this.start(workflow, input, childId, opts).catch((err) => {
|
|
7066
7360
|
const message = err instanceof Error ? err.message : String(err);
|
|
7067
7361
|
this.notifyParent(childId, {
|
|
7068
7362
|
ok: false,
|
|
7069
7363
|
error: `child workflow "${workflow}" failed to start: ${message}`
|
|
7070
7364
|
});
|
|
7365
|
+
return void 0;
|
|
7071
7366
|
});
|
|
7367
|
+
if (started) this.redeliverFinishedChild(childId, started);
|
|
7072
7368
|
});
|
|
7073
7369
|
}
|
|
7370
|
+
/** {@link startChildDeferred}: re-deliver the completion of a child `start` found finished. */
|
|
7371
|
+
redeliverFinishedChild(childId, run) {
|
|
7372
|
+
const completion = childCompletion(run);
|
|
7373
|
+
if (completion) this.inBackground(this.deliverToParent(`child:${childId}`, completion));
|
|
7374
|
+
}
|
|
7074
7375
|
/**
|
|
7075
7376
|
* Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
|
|
7076
7377
|
* `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
|
|
@@ -7646,9 +7947,26 @@ var WorkflowEngine = class {
|
|
|
7646
7947
|
async execute(run, fn) {
|
|
7647
7948
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
7648
7949
|
const lockNow = this.clock();
|
|
7950
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
7951
|
+
generic: true,
|
|
7952
|
+
fans: /* @__PURE__ */ new Map()
|
|
7953
|
+
};
|
|
7954
|
+
this.wakeReasons.delete(run.id);
|
|
7649
7955
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
7650
7956
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
7651
|
-
if (
|
|
7957
|
+
if (reason.generic) {
|
|
7958
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
7959
|
+
this.genericWakeRequested.add(run.id);
|
|
7960
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
7961
|
+
}
|
|
7962
|
+
} else {
|
|
7963
|
+
const fans = {
|
|
7964
|
+
fans: [
|
|
7965
|
+
...reason.fans
|
|
7966
|
+
]
|
|
7967
|
+
};
|
|
7968
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
7969
|
+
}
|
|
7652
7970
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
7653
7971
|
return {
|
|
7654
7972
|
runId: run.id,
|
|
@@ -7656,56 +7974,142 @@ var WorkflowEngine = class {
|
|
|
7656
7974
|
};
|
|
7657
7975
|
}
|
|
7658
7976
|
}
|
|
7977
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
7978
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
7659
7979
|
const held = this.holdExecutionLease(run.id);
|
|
7660
7980
|
try {
|
|
7981
|
+
const served = /* @__PURE__ */ new Set();
|
|
7982
|
+
this.turnServedWakes.set(run.id, served);
|
|
7983
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
7661
7984
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
7662
7985
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
7663
7986
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
7664
7987
|
}
|
|
7665
|
-
if (result.status === "suspended")
|
|
7666
|
-
|
|
7667
|
-
|
|
7988
|
+
if (result.status === "suspended") {
|
|
7989
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
7990
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
7991
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
7668
7992
|
}
|
|
7669
7993
|
return result;
|
|
7670
7994
|
} finally {
|
|
7671
7995
|
held.stop();
|
|
7996
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
7997
|
+
this.turnServedWakes.delete(run.id);
|
|
7672
7998
|
}
|
|
7673
7999
|
}
|
|
7674
8000
|
/**
|
|
7675
8001
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7676
8002
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
7677
|
-
* (a signal, a child's completion). The request is a buffered
|
|
7678
|
-
* on
|
|
8003
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8004
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8005
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
7679
8006
|
*
|
|
7680
8007
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
7681
8008
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
7682
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
7683
|
-
*
|
|
8009
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8010
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8011
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8012
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
7684
8013
|
*/
|
|
7685
|
-
async requestWakeAfterTurn(runId) {
|
|
7686
|
-
|
|
8014
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8015
|
+
const id = globalThis.crypto.randomUUID();
|
|
8016
|
+
await this.store.bufferEvent({
|
|
8017
|
+
name: wakeRequestToken(runId),
|
|
8018
|
+
payload: request,
|
|
8019
|
+
id,
|
|
8020
|
+
publishedAt: this.clock()
|
|
8021
|
+
});
|
|
7687
8022
|
const run = await this.store.getRun(runId);
|
|
7688
|
-
if (run
|
|
7689
|
-
|
|
7690
|
-
|
|
8023
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8024
|
+
await this.store.removeBufferedEvent(id);
|
|
8025
|
+
return;
|
|
8026
|
+
}
|
|
8027
|
+
if (run.status === "running") return;
|
|
8028
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8029
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
7691
8030
|
return;
|
|
7692
8031
|
}
|
|
7693
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
7694
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
7695
8032
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
7696
8033
|
}
|
|
7697
|
-
/**
|
|
7698
|
-
|
|
7699
|
-
|
|
8034
|
+
/**
|
|
8035
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8036
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8037
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8038
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8039
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8040
|
+
*/
|
|
8041
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8042
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8043
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8044
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8045
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8046
|
+
if (needed.length === 0) return;
|
|
8047
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
7700
8048
|
await this.wake(runId).catch((err) => {
|
|
7701
8049
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
7702
8050
|
});
|
|
7703
8051
|
}
|
|
7704
|
-
/**
|
|
7705
|
-
|
|
7706
|
-
|
|
7707
|
-
|
|
7708
|
-
|
|
8052
|
+
/**
|
|
8053
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8054
|
+
* signals, which are taken as they are read.
|
|
8055
|
+
*/
|
|
8056
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8057
|
+
const token = wakeRequestToken(runId);
|
|
8058
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8059
|
+
id: request.id,
|
|
8060
|
+
payload: request.payload
|
|
8061
|
+
}));
|
|
8062
|
+
if (!legacy) return kept;
|
|
8063
|
+
for (; ; ) {
|
|
8064
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8065
|
+
if (!signal) return kept;
|
|
8066
|
+
kept.push({
|
|
8067
|
+
payload: signal.payload
|
|
8068
|
+
});
|
|
8069
|
+
}
|
|
8070
|
+
}
|
|
8071
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8072
|
+
async dropWakeRequests(requests) {
|
|
8073
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8074
|
+
this.store.removeBufferedEvent(request.id)
|
|
8075
|
+
]));
|
|
8076
|
+
}
|
|
8077
|
+
/**
|
|
8078
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8079
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8080
|
+
* until then stay for whoever runs the run next.
|
|
8081
|
+
*/
|
|
8082
|
+
async noteServedWakes(runId) {
|
|
8083
|
+
const served = this.turnServedWakes.get(runId);
|
|
8084
|
+
if (!served) return;
|
|
8085
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8086
|
+
served.add(request.id);
|
|
8087
|
+
}
|
|
8088
|
+
}
|
|
8089
|
+
/**
|
|
8090
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8091
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8092
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8093
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8094
|
+
*/
|
|
8095
|
+
async keepDueForKeptWakes(runId) {
|
|
8096
|
+
const served = this.turnServedWakes.get(runId);
|
|
8097
|
+
if (!served) return;
|
|
8098
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8099
|
+
const kept = await this.listWakeRequests(runId);
|
|
8100
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8101
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8102
|
+
}
|
|
8103
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8104
|
+
async makeDueBy(runId, at) {
|
|
8105
|
+
const run = await this.store.getRun(runId);
|
|
8106
|
+
if (run?.status !== "suspended") return;
|
|
8107
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8108
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8109
|
+
"suspended"
|
|
8110
|
+
], {
|
|
8111
|
+
wakeAt: at
|
|
8112
|
+
});
|
|
7709
8113
|
}
|
|
7710
8114
|
/**
|
|
7711
8115
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -7724,6 +8128,7 @@ var WorkflowEngine = class {
|
|
|
7724
8128
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
7725
8129
|
const timer = setTimeout(() => {
|
|
7726
8130
|
this.leaseRetries.delete(runId);
|
|
8131
|
+
this.genericWakeRequested.delete(runId);
|
|
7727
8132
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
7728
8133
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
7729
8134
|
timer.unref?.();
|
|
@@ -7732,12 +8137,15 @@ var WorkflowEngine = class {
|
|
|
7732
8137
|
async retryDroppedWake(runId, attempt) {
|
|
7733
8138
|
if (this.draining) return;
|
|
7734
8139
|
const run = await this.store.getRun(runId);
|
|
7735
|
-
if (run?.status !== "suspended") return;
|
|
7736
8140
|
const nowMs = this.clock();
|
|
7737
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
7738
|
-
|
|
8141
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8142
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
7739
8143
|
return;
|
|
7740
8144
|
}
|
|
8145
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8146
|
+
if (run?.status !== "suspended") return;
|
|
8147
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8148
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
7741
8149
|
if (this.queuesRunsOverTransport) {
|
|
7742
8150
|
await this.wake(runId);
|
|
7743
8151
|
return;
|
|
@@ -8076,6 +8484,7 @@ ${run.workflowVersion}`;
|
|
|
8076
8484
|
updatedAt
|
|
8077
8485
|
});
|
|
8078
8486
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8487
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8079
8488
|
this.emit({
|
|
8080
8489
|
type: "run.suspended",
|
|
8081
8490
|
runId: run.id,
|
|
@@ -8675,10 +9084,13 @@ ${run.workflowVersion}`;
|
|
|
8675
9084
|
// dashboard renders the fan as one parallel group. Undefined for a lone (non-fan) child.
|
|
8676
9085
|
parallelGroup: cmd.parallelGroup
|
|
8677
9086
|
});
|
|
8678
|
-
|
|
9087
|
+
const existingChild = await this.store.getRun(childId);
|
|
9088
|
+
if (!existingChild) {
|
|
8679
9089
|
this.startChildDeferred(cmd.workflow, cmd.input, childId, {
|
|
8680
9090
|
namespace: run.namespace
|
|
8681
9091
|
});
|
|
9092
|
+
} else {
|
|
9093
|
+
this.redeliverFinishedChild(childId, existingChild);
|
|
8682
9094
|
}
|
|
8683
9095
|
} else {
|
|
8684
9096
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
@@ -8766,6 +9178,7 @@ ${run.workflowVersion}`;
|
|
|
8766
9178
|
namespace: run.namespace
|
|
8767
9179
|
});
|
|
8768
9180
|
}
|
|
9181
|
+
await this.noteServedWakes(run.id);
|
|
8769
9182
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
8770
9183
|
return this.singletonWaitResult(run);
|
|
8771
9184
|
}
|
|
@@ -9439,6 +9852,14 @@ ${run.workflowVersion}`;
|
|
|
9439
9852
|
throw new WorkflowSuspended();
|
|
9440
9853
|
}
|
|
9441
9854
|
/**
|
|
9855
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
9856
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
9857
|
+
*/
|
|
9858
|
+
stepIsWaiting(cp) {
|
|
9859
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
9860
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
9861
|
+
}
|
|
9862
|
+
/**
|
|
9442
9863
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9443
9864
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9444
9865
|
*
|
|
@@ -9468,6 +9889,7 @@ ${run.workflowVersion}`;
|
|
|
9468
9889
|
const failures = [];
|
|
9469
9890
|
const unsent = [];
|
|
9470
9891
|
let inFlight = 0;
|
|
9892
|
+
let highest = -1;
|
|
9471
9893
|
let wakeAt;
|
|
9472
9894
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9473
9895
|
const seq = positions[i];
|
|
@@ -9477,10 +9899,19 @@ ${run.workflowVersion}`;
|
|
|
9477
9899
|
continue;
|
|
9478
9900
|
}
|
|
9479
9901
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
9902
|
+
if (seq > highest) highest = seq;
|
|
9480
9903
|
if (cp.status === "completed") {
|
|
9481
9904
|
outputs[i] = cp.output;
|
|
9482
9905
|
continue;
|
|
9483
9906
|
}
|
|
9907
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
9908
|
+
inFlight += 1;
|
|
9909
|
+
if (cp.wakeAt != null) {
|
|
9910
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
9911
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
9912
|
+
}
|
|
9913
|
+
continue;
|
|
9914
|
+
}
|
|
9484
9915
|
try {
|
|
9485
9916
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9486
9917
|
...routing,
|
|
@@ -9508,6 +9939,7 @@ ${run.workflowVersion}`;
|
|
|
9508
9939
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9509
9940
|
}
|
|
9510
9941
|
}
|
|
9942
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9511
9943
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9512
9944
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9513
9945
|
if (inFlight + batch.length > 0) {
|
|
@@ -9538,53 +9970,62 @@ ${run.workflowVersion}`;
|
|
|
9538
9970
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9539
9971
|
return outputs;
|
|
9540
9972
|
}
|
|
9541
|
-
/**
|
|
9973
|
+
/**
|
|
9974
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
9975
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
9976
|
+
* batch, where the store and transport have them.
|
|
9977
|
+
*/
|
|
9542
9978
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9543
9979
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9544
9980
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9545
9981
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9546
|
-
|
|
9547
|
-
|
|
9548
|
-
|
|
9549
|
-
|
|
9550
|
-
|
|
9551
|
-
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
|
|
9556
|
-
|
|
9557
|
-
|
|
9558
|
-
|
|
9559
|
-
|
|
9560
|
-
|
|
9561
|
-
|
|
9562
|
-
|
|
9563
|
-
|
|
9564
|
-
|
|
9982
|
+
const checkpoints = batch.map((i) => {
|
|
9983
|
+
const seq = positions[i];
|
|
9984
|
+
return {
|
|
9985
|
+
runId,
|
|
9986
|
+
seq,
|
|
9987
|
+
name: step.name,
|
|
9988
|
+
kind: "remote",
|
|
9989
|
+
stepId: stepId(runId, seq),
|
|
9990
|
+
status: "pending",
|
|
9991
|
+
input: inputs[i],
|
|
9992
|
+
attempts: 1,
|
|
9993
|
+
workerGroup,
|
|
9994
|
+
parallelGroup: group,
|
|
9995
|
+
enqueuedAt,
|
|
9996
|
+
startedAt: enqueuedAt,
|
|
9997
|
+
finishedAt: enqueuedAt
|
|
9998
|
+
};
|
|
9999
|
+
});
|
|
10000
|
+
if (this.store.saveCheckpoints) {
|
|
10001
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10002
|
+
} else {
|
|
10003
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10004
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10005
|
+
}
|
|
9565
10006
|
}
|
|
9566
|
-
|
|
9567
|
-
|
|
9568
|
-
|
|
9569
|
-
|
|
9570
|
-
|
|
9571
|
-
|
|
9572
|
-
|
|
9573
|
-
|
|
9574
|
-
|
|
9575
|
-
|
|
9576
|
-
|
|
9577
|
-
|
|
9578
|
-
|
|
9579
|
-
|
|
9580
|
-
|
|
9581
|
-
|
|
9582
|
-
|
|
9583
|
-
|
|
9584
|
-
|
|
9585
|
-
|
|
9586
|
-
|
|
9587
|
-
})
|
|
10007
|
+
const traceparent = this.traceparent?.();
|
|
10008
|
+
const context = this.context?.();
|
|
10009
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10010
|
+
runId,
|
|
10011
|
+
seq: cp.seq,
|
|
10012
|
+
name: step.name,
|
|
10013
|
+
stepId: cp.stepId,
|
|
10014
|
+
group: workerGroup,
|
|
10015
|
+
input: inputs[batch[k]],
|
|
10016
|
+
traceparent,
|
|
10017
|
+
context,
|
|
10018
|
+
priority: opts.admission?.priority,
|
|
10019
|
+
attempt: 1
|
|
10020
|
+
})), opts.transport);
|
|
10021
|
+
for (const cp of checkpoints) {
|
|
10022
|
+
this.emit({
|
|
10023
|
+
type: "step.started",
|
|
10024
|
+
runId,
|
|
10025
|
+
seq: cp.seq,
|
|
10026
|
+
name: step.name,
|
|
10027
|
+
kind: "remote"
|
|
10028
|
+
});
|
|
9588
10029
|
}
|
|
9589
10030
|
}
|
|
9590
10031
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -9625,6 +10066,27 @@ ${run.workflowVersion}`;
|
|
|
9625
10066
|
error: s.reason
|
|
9626
10067
|
});
|
|
9627
10068
|
}
|
|
10069
|
+
/**
|
|
10070
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10071
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10072
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10073
|
+
*/
|
|
10074
|
+
async readCheckpoints(runId, seqs) {
|
|
10075
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10076
|
+
try {
|
|
10077
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10078
|
+
cp.seq,
|
|
10079
|
+
cp
|
|
10080
|
+
]));
|
|
10081
|
+
return seqs.map((seq) => ({
|
|
10082
|
+
status: "fulfilled",
|
|
10083
|
+
value: bySeq.get(seq) ?? null
|
|
10084
|
+
}));
|
|
10085
|
+
} catch {
|
|
10086
|
+
}
|
|
10087
|
+
}
|
|
10088
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10089
|
+
}
|
|
9628
10090
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
9629
10091
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
9630
10092
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -9666,9 +10128,10 @@ ${run.workflowVersion}`;
|
|
|
9666
10128
|
*
|
|
9667
10129
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
9668
10130
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
9669
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
9670
|
-
*
|
|
9671
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10131
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10132
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10133
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10134
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
9672
10135
|
*/
|
|
9673
10136
|
completeRemoteResult(result) {
|
|
9674
10137
|
return new Promise((resolve, reject) => {
|
|
@@ -9696,7 +10159,7 @@ ${run.workflowVersion}`;
|
|
|
9696
10159
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
9697
10160
|
let outcomes;
|
|
9698
10161
|
try {
|
|
9699
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10162
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
9700
10163
|
} catch (err) {
|
|
9701
10164
|
outcomes = batch.map(() => ({
|
|
9702
10165
|
error: err
|
|
@@ -9704,8 +10167,8 @@ ${run.workflowVersion}`;
|
|
|
9704
10167
|
}
|
|
9705
10168
|
batch.forEach((q, i) => {
|
|
9706
10169
|
const outcome = outcomes[i] ?? false;
|
|
9707
|
-
if (
|
|
9708
|
-
else q.
|
|
10170
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10171
|
+
else q.resolve(outcome);
|
|
9709
10172
|
});
|
|
9710
10173
|
}
|
|
9711
10174
|
this.resultQueues.delete(runId);
|
|
@@ -9714,8 +10177,13 @@ ${run.workflowVersion}`;
|
|
|
9714
10177
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
9715
10178
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
9716
10179
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10180
|
+
*
|
|
10181
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10182
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10183
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10184
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
9717
10185
|
*/
|
|
9718
|
-
async completeRemoteResults(runId, results) {
|
|
10186
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
9719
10187
|
const outcomes = results.map(() => false);
|
|
9720
10188
|
const seen = /* @__PURE__ */ new Set();
|
|
9721
10189
|
const live = results.flatMap((r, i) => {
|
|
@@ -9725,7 +10193,7 @@ ${run.workflowVersion}`;
|
|
|
9725
10193
|
i
|
|
9726
10194
|
];
|
|
9727
10195
|
});
|
|
9728
|
-
const reads = await
|
|
10196
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
9729
10197
|
const pending = [];
|
|
9730
10198
|
const resettled = [];
|
|
9731
10199
|
reads.forEach((read, k) => {
|
|
@@ -9746,7 +10214,7 @@ ${run.workflowVersion}`;
|
|
|
9746
10214
|
});
|
|
9747
10215
|
if (pending.length === 0) {
|
|
9748
10216
|
if (resettled.length === 0) return outcomes;
|
|
9749
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10217
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
9750
10218
|
}
|
|
9751
10219
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
9752
10220
|
const run = await this.store.getRun(runId);
|
|
@@ -9773,7 +10241,9 @@ ${run.workflowVersion}`;
|
|
|
9773
10241
|
cp,
|
|
9774
10242
|
result: results[index]
|
|
9775
10243
|
});
|
|
9776
|
-
else if (claim !== false) outcomes[index] =
|
|
10244
|
+
else if (claim !== false) outcomes[index] = {
|
|
10245
|
+
error: claim.error
|
|
10246
|
+
};
|
|
9777
10247
|
});
|
|
9778
10248
|
for (const { cp, result } of settled) {
|
|
9779
10249
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -9791,19 +10261,21 @@ ${run.workflowVersion}`;
|
|
|
9791
10261
|
});
|
|
9792
10262
|
}
|
|
9793
10263
|
const barriers = /* @__PURE__ */ new Map();
|
|
9794
|
-
|
|
10264
|
+
const wake = new FanWakeCollector();
|
|
10265
|
+
for (const { cp } of settled) wake.note(cp);
|
|
9795
10266
|
for (const { index, cp, result } of settled) {
|
|
9796
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10267
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10268
|
+
wake.add(index, cp, result.status);
|
|
10269
|
+
}
|
|
9797
10270
|
}
|
|
9798
|
-
|
|
9799
|
-
return outcomes;
|
|
10271
|
+
return wake.applyTo(outcomes);
|
|
9800
10272
|
}
|
|
9801
10273
|
/**
|
|
9802
10274
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
9803
10275
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
9804
10276
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
9805
10277
|
*/
|
|
9806
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10278
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
9807
10279
|
const run = await this.store.getRun(runId);
|
|
9808
10280
|
if (run?.status !== "suspended") return outcomes;
|
|
9809
10281
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -9814,9 +10286,11 @@ ${run.workflowVersion}`;
|
|
|
9814
10286
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
9815
10287
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
9816
10288
|
};
|
|
9817
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
9818
|
-
|
|
9819
|
-
|
|
10289
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10290
|
+
const wake = new FanWakeCollector();
|
|
10291
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10292
|
+
wake.add(index, cp, asResult.status);
|
|
10293
|
+
return wake.applyTo(outcomes);
|
|
9820
10294
|
}
|
|
9821
10295
|
}
|
|
9822
10296
|
return outcomes;
|
|
@@ -9827,15 +10301,17 @@ ${run.workflowVersion}`;
|
|
|
9827
10301
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
9828
10302
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
9829
10303
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
9830
|
-
* re-suspends, a lost one strands the run.
|
|
10304
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10305
|
+
* that result (see {@link completeRemoteResults}).
|
|
9831
10306
|
*/
|
|
9832
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10307
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
9833
10308
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
9834
10309
|
if (!fan) return false;
|
|
9835
10310
|
if (result.status === "failed") return false;
|
|
9836
10311
|
let closed = barriers.get(fan.firstSeq);
|
|
9837
10312
|
if (!closed) {
|
|
9838
|
-
|
|
10313
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10314
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
9839
10315
|
barriers.set(fan.firstSeq, closed);
|
|
9840
10316
|
}
|
|
9841
10317
|
if (await closed) return true;
|
|
@@ -10669,6 +11145,10 @@ var CodecStateStore = class {
|
|
|
10669
11145
|
saveCheckpointIfPending;
|
|
10670
11146
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
10671
11147
|
saveCheckpointsIfPending;
|
|
11148
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11149
|
+
saveCheckpoints;
|
|
11150
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11151
|
+
getCheckpoints;
|
|
10672
11152
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
10673
11153
|
countCheckpoints;
|
|
10674
11154
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -10689,6 +11169,10 @@ var CodecStateStore = class {
|
|
|
10689
11169
|
compactCheckpoints;
|
|
10690
11170
|
/** Decodes each run like {@link getRun}. Bound only when the inner store has it. */
|
|
10691
11171
|
getRuns;
|
|
11172
|
+
/** Decodes each payload like {@link takeBufferedSignal}. Bound only when the inner store has it. */
|
|
11173
|
+
listBufferedSignals;
|
|
11174
|
+
/** Forwarded verbatim — deletes by row id. */
|
|
11175
|
+
deleteBufferedSignals;
|
|
10692
11176
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
10693
11177
|
listSignalWaitersByRun;
|
|
10694
11178
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
@@ -10715,6 +11199,10 @@ var CodecStateStore = class {
|
|
|
10715
11199
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
10716
11200
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
10717
11201
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11202
|
+
const saveMany = inner.saveCheckpoints;
|
|
11203
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11204
|
+
const getMany = inner.getCheckpoints;
|
|
11205
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
10718
11206
|
const countCps = inner.countCheckpoints;
|
|
10719
11207
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
10720
11208
|
const stats = inner.checkpointStats;
|
|
@@ -10734,6 +11222,13 @@ var CodecStateStore = class {
|
|
|
10734
11222
|
if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
|
|
10735
11223
|
const getRuns = inner.getRuns;
|
|
10736
11224
|
if (getRuns) this.getRuns = async (runIds) => (await getRuns.call(inner, runIds)).map((r) => this.decRun(r));
|
|
11225
|
+
const listBuffered = inner.listBufferedSignals;
|
|
11226
|
+
if (listBuffered) this.listBufferedSignals = async (tokenPrefix, afterId, limit) => (await listBuffered.call(inner, tokenPrefix, afterId, limit)).map((row) => ({
|
|
11227
|
+
...row,
|
|
11228
|
+
payload: this.dec(row.payload)
|
|
11229
|
+
}));
|
|
11230
|
+
const deleteBuffered = inner.deleteBufferedSignals;
|
|
11231
|
+
if (deleteBuffered) this.deleteBufferedSignals = (ids) => deleteBuffered.call(inner, ids);
|
|
10737
11232
|
const byRun = inner.listSignalWaitersByRun;
|
|
10738
11233
|
if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
|
|
10739
11234
|
const hasWaiter = inner.hasSignalWaiter;
|
|
@@ -11344,11 +11839,30 @@ var InMemoryStateStore = class {
|
|
|
11344
11839
|
...cp
|
|
11345
11840
|
} : null;
|
|
11346
11841
|
}
|
|
11842
|
+
async getCheckpoints(runId, seqs) {
|
|
11843
|
+
return [
|
|
11844
|
+
...new Set(seqs)
|
|
11845
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
11846
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
11847
|
+
return cp ? [
|
|
11848
|
+
{
|
|
11849
|
+
...cp
|
|
11850
|
+
}
|
|
11851
|
+
] : [];
|
|
11852
|
+
});
|
|
11853
|
+
}
|
|
11347
11854
|
async saveCheckpoint(checkpoint) {
|
|
11348
11855
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11349
11856
|
...checkpoint
|
|
11350
11857
|
});
|
|
11351
11858
|
}
|
|
11859
|
+
async saveCheckpoints(checkpoints) {
|
|
11860
|
+
for (const checkpoint of checkpoints) {
|
|
11861
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11862
|
+
...checkpoint
|
|
11863
|
+
});
|
|
11864
|
+
}
|
|
11865
|
+
}
|
|
11352
11866
|
async saveCheckpointIfPending(checkpoint) {
|
|
11353
11867
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11354
11868
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|
|
@@ -11479,20 +11993,47 @@ var InMemoryStateStore = class {
|
|
|
11479
11993
|
});
|
|
11480
11994
|
}
|
|
11481
11995
|
bufferedSignals = /* @__PURE__ */ new Map();
|
|
11996
|
+
bufferedSignalSeq = 0;
|
|
11482
11997
|
async bufferSignal(token, payload) {
|
|
11483
11998
|
const queue = this.bufferedSignals.get(token) ?? [];
|
|
11484
|
-
queue.push(
|
|
11999
|
+
queue.push({
|
|
12000
|
+
id: ++this.bufferedSignalSeq,
|
|
12001
|
+
payload
|
|
12002
|
+
});
|
|
11485
12003
|
this.bufferedSignals.set(token, queue);
|
|
11486
12004
|
}
|
|
11487
12005
|
async takeBufferedSignal(token) {
|
|
11488
12006
|
const queue = this.bufferedSignals.get(token);
|
|
11489
12007
|
if (!queue || queue.length === 0) return null;
|
|
11490
|
-
const
|
|
12008
|
+
const head = queue.shift();
|
|
11491
12009
|
if (queue.length === 0) this.bufferedSignals.delete(token);
|
|
11492
12010
|
return {
|
|
11493
|
-
payload
|
|
12011
|
+
payload: head.payload
|
|
11494
12012
|
};
|
|
11495
12013
|
}
|
|
12014
|
+
async listBufferedSignals(tokenPrefix, afterId, limit) {
|
|
12015
|
+
const after = afterId === void 0 ? 0 : Number(afterId);
|
|
12016
|
+
const rows = [];
|
|
12017
|
+
for (const [token, queue] of this.bufferedSignals) {
|
|
12018
|
+
if (!token.startsWith(tokenPrefix)) continue;
|
|
12019
|
+
for (const { id, payload } of queue) {
|
|
12020
|
+
if (id > after) rows.push({
|
|
12021
|
+
id: String(id),
|
|
12022
|
+
token,
|
|
12023
|
+
payload
|
|
12024
|
+
});
|
|
12025
|
+
}
|
|
12026
|
+
}
|
|
12027
|
+
return rows.sort((a, b) => Number(a.id) - Number(b.id)).slice(0, limit);
|
|
12028
|
+
}
|
|
12029
|
+
async deleteBufferedSignals(ids) {
|
|
12030
|
+
const doomed = new Set(ids.map(Number));
|
|
12031
|
+
for (const [token, queue] of this.bufferedSignals) {
|
|
12032
|
+
const kept = queue.filter((row) => !doomed.has(row.id));
|
|
12033
|
+
if (kept.length === 0) this.bufferedSignals.delete(token);
|
|
12034
|
+
else this.bufferedSignals.set(token, kept);
|
|
12035
|
+
}
|
|
12036
|
+
}
|
|
11496
12037
|
/** `id -> row`, so {@link removeBufferedEvent} can delete an exact row regardless of iteration
|
|
11497
12038
|
* order — mirrors the SQL stores' PK-by-id delete. */
|
|
11498
12039
|
bufferedEvents = /* @__PURE__ */ new Map();
|