@dudousxd/nestjs-durable-core 0.85.5 → 0.85.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1176 -635
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +160 -21
- package/dist/index.d.ts +160 -21
- package/dist/index.js +1176 -635
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -860,6 +860,386 @@ var BoundedRunner = class {
|
|
|
860
860
|
};
|
|
861
861
|
var LOCAL_RUN_CONCURRENCY = 256;
|
|
862
862
|
|
|
863
|
+
// src/fan-join.ts
|
|
864
|
+
function allChildId(runId, firstSeq, index, wave) {
|
|
865
|
+
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
866
|
+
}
|
|
867
|
+
__name(allChildId, "allChildId");
|
|
868
|
+
function allGroup(firstSeq, size, mode) {
|
|
869
|
+
const group = `all:${firstSeq}:n${size}`;
|
|
870
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
871
|
+
}
|
|
872
|
+
__name(allGroup, "allGroup");
|
|
873
|
+
function stepFanGroup(firstSeq, size, mode) {
|
|
874
|
+
const group = `steps:${firstSeq}:${size}`;
|
|
875
|
+
return mode === "failFast" ? `${group}:failFast` : group;
|
|
876
|
+
}
|
|
877
|
+
__name(stepFanGroup, "stepFanGroup");
|
|
878
|
+
function parseStepFanGroup(group) {
|
|
879
|
+
const fan = parseFanGroup(group);
|
|
880
|
+
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
881
|
+
return {
|
|
882
|
+
firstSeq: fan.firstSeq,
|
|
883
|
+
size: fan.size
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
887
|
+
function parseFanGroup(group) {
|
|
888
|
+
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
889
|
+
if (!m) return void 0;
|
|
890
|
+
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
891
|
+
return {
|
|
892
|
+
kind: m[1],
|
|
893
|
+
firstSeq: Number(m[2]),
|
|
894
|
+
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
895
|
+
failFast: m[4] !== void 0
|
|
896
|
+
};
|
|
897
|
+
}
|
|
898
|
+
__name(parseFanGroup, "parseFanGroup");
|
|
899
|
+
function allWait(runId, firstSeq, wave) {
|
|
900
|
+
const base = allBase(runId, firstSeq, wave);
|
|
901
|
+
return {
|
|
902
|
+
waiterPrefix: `child:${base}`,
|
|
903
|
+
checkpointPrefix: `signal:child:${base}`
|
|
904
|
+
};
|
|
905
|
+
}
|
|
906
|
+
__name(allWait, "allWait");
|
|
907
|
+
function fanJoin(waits) {
|
|
908
|
+
return {
|
|
909
|
+
waits
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
__name(fanJoin, "fanJoin");
|
|
913
|
+
var FOLDED_PREFIX = "fan:";
|
|
914
|
+
function allFoldedName(runId, firstSeq, wave) {
|
|
915
|
+
return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
|
|
916
|
+
}
|
|
917
|
+
__name(allFoldedName, "allFoldedName");
|
|
918
|
+
function foldedChildIds(checkpoint) {
|
|
919
|
+
const base = checkpoint.name.slice(FOLDED_PREFIX.length);
|
|
920
|
+
const { from, values } = checkpoint.output;
|
|
921
|
+
return values.map((_, j) => `${base}${from + j}`);
|
|
922
|
+
}
|
|
923
|
+
__name(foldedChildIds, "foldedChildIds");
|
|
924
|
+
function allBase(runId, firstSeq, wave) {
|
|
925
|
+
return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
|
|
926
|
+
}
|
|
927
|
+
__name(allBase, "allBase");
|
|
928
|
+
function allWaves(size, concurrency) {
|
|
929
|
+
if (concurrency === void 0 || concurrency >= size) return void 0;
|
|
930
|
+
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
931
|
+
throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
|
|
932
|
+
}
|
|
933
|
+
if (concurrency === 1) return {
|
|
934
|
+
waveSize: 1,
|
|
935
|
+
maxWaves: 1
|
|
936
|
+
};
|
|
937
|
+
return {
|
|
938
|
+
waveSize: Math.floor(concurrency / 2),
|
|
939
|
+
maxWaves: 2
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
__name(allWaves, "allWaves");
|
|
943
|
+
var ENGINE_CHILD_ID = /^(.+)\.(?:all\.\d+\.(?:w\d+\.)?\d+|child\.\d+|onError\.\d+)$/;
|
|
944
|
+
function parentRunIdOf(runId) {
|
|
945
|
+
return ENGINE_CHILD_ID.exec(runId)?.[1];
|
|
946
|
+
}
|
|
947
|
+
__name(parentRunIdOf, "parentRunIdOf");
|
|
948
|
+
var GROUP = /^all:(\d+)(?::n\d+)?(:failFast)?$/;
|
|
949
|
+
var REST = /^(?:w(\d+)\.)?\d+$/;
|
|
950
|
+
function fanOfWaiter(waiter) {
|
|
951
|
+
const group = waiter.parallelGroup ? GROUP.exec(waiter.parallelGroup) : null;
|
|
952
|
+
if (!group) return void 0;
|
|
953
|
+
const base = `child:${waiter.runId}.all.${group[1]}.`;
|
|
954
|
+
if (!waiter.token.startsWith(base)) return void 0;
|
|
955
|
+
const rest = REST.exec(waiter.token.slice(base.length));
|
|
956
|
+
if (!rest) return void 0;
|
|
957
|
+
const waiterPrefix = rest[1] === void 0 ? base : `${base}w${rest[1]}.`;
|
|
958
|
+
return {
|
|
959
|
+
waiterPrefix,
|
|
960
|
+
failFast: group[2] !== void 0
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
__name(fanOfWaiter, "fanOfWaiter");
|
|
964
|
+
|
|
965
|
+
// src/interfaces.ts
|
|
966
|
+
function checkpointNameKind(name) {
|
|
967
|
+
if (name.startsWith("signal:child:")) return "child";
|
|
968
|
+
if (name.startsWith("fan:")) return "folded";
|
|
969
|
+
if (name.startsWith("spawn:")) return "spawn";
|
|
970
|
+
return "step";
|
|
971
|
+
}
|
|
972
|
+
__name(checkpointNameKind, "checkpointNameKind");
|
|
973
|
+
function poolNamespace(namespace) {
|
|
974
|
+
return namespace === "" ? "default" : namespace;
|
|
975
|
+
}
|
|
976
|
+
__name(poolNamespace, "poolNamespace");
|
|
977
|
+
var TERMINAL_RUN_STATUSES = [
|
|
978
|
+
"completed",
|
|
979
|
+
"failed",
|
|
980
|
+
"cancelled",
|
|
981
|
+
"dead"
|
|
982
|
+
];
|
|
983
|
+
function isNonEmptyRunScope(scope) {
|
|
984
|
+
if (!scope) return false;
|
|
985
|
+
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
986
|
+
}
|
|
987
|
+
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
988
|
+
|
|
989
|
+
// src/events.ts
|
|
990
|
+
var enc = /* @__PURE__ */ __name((s) => Buffer.from(s, "utf8").toString("base64"), "enc");
|
|
991
|
+
var dec = /* @__PURE__ */ __name((s) => Buffer.from(s, "base64").toString("utf8"), "dec");
|
|
992
|
+
var EVENT_BUFFER_SCAN_LIMIT = 50;
|
|
993
|
+
var EVENT_TOKEN_PREFIX = "event:";
|
|
994
|
+
function eventPrefix(name) {
|
|
995
|
+
return `${EVENT_TOKEN_PREFIX}${enc(name)}:`;
|
|
996
|
+
}
|
|
997
|
+
__name(eventPrefix, "eventPrefix");
|
|
998
|
+
function isEventToken(token) {
|
|
999
|
+
return token.startsWith(EVENT_TOKEN_PREFIX);
|
|
1000
|
+
}
|
|
1001
|
+
__name(isEventToken, "isEventToken");
|
|
1002
|
+
function eventToken(name, match, runId, seq) {
|
|
1003
|
+
return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
|
|
1004
|
+
}
|
|
1005
|
+
__name(eventToken, "eventToken");
|
|
1006
|
+
function eventNameOf(token) {
|
|
1007
|
+
try {
|
|
1008
|
+
return dec(token.split(":")[1] ?? "");
|
|
1009
|
+
} catch {
|
|
1010
|
+
return token;
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
__name(eventNameOf, "eventNameOf");
|
|
1014
|
+
function eventMatchOf(token) {
|
|
1015
|
+
try {
|
|
1016
|
+
return JSON.parse(dec(token.split(":")[2] ?? ""));
|
|
1017
|
+
} catch {
|
|
1018
|
+
return {};
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
__name(eventMatchOf, "eventMatchOf");
|
|
1022
|
+
function eventMatches(payload, match) {
|
|
1023
|
+
const keys = Object.keys(match);
|
|
1024
|
+
if (keys.length === 0) return true;
|
|
1025
|
+
if (!payload || typeof payload !== "object") return false;
|
|
1026
|
+
const p = payload;
|
|
1027
|
+
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
1028
|
+
}
|
|
1029
|
+
__name(eventMatches, "eventMatches");
|
|
1030
|
+
|
|
1031
|
+
// src/run-waiting.ts
|
|
1032
|
+
var WEBHOOK_PREFIX = "wh:";
|
|
1033
|
+
var CHILD_PREFIX = "child:";
|
|
1034
|
+
var BREAKPOINT_PREFIX = "bp:";
|
|
1035
|
+
var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
|
|
1036
|
+
function parseWebhookToken(token) {
|
|
1037
|
+
if (!token.startsWith(WEBHOOK_PREFIX)) return null;
|
|
1038
|
+
const rest = token.slice(WEBHOOK_PREFIX.length);
|
|
1039
|
+
const at = rest.lastIndexOf(":");
|
|
1040
|
+
if (at <= 0) return null;
|
|
1041
|
+
const seqText = rest.slice(at + 1);
|
|
1042
|
+
if (!/^\d+$/.test(seqText)) return null;
|
|
1043
|
+
return {
|
|
1044
|
+
runId: rest.slice(0, at),
|
|
1045
|
+
seq: Number(seqText)
|
|
1046
|
+
};
|
|
1047
|
+
}
|
|
1048
|
+
__name(parseWebhookToken, "parseWebhookToken");
|
|
1049
|
+
function classifyWaiterToken(token) {
|
|
1050
|
+
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
1051
|
+
on: "webhook",
|
|
1052
|
+
name: token
|
|
1053
|
+
};
|
|
1054
|
+
if (token.startsWith(BREAKPOINT_PREFIX)) return {
|
|
1055
|
+
on: "breakpoint",
|
|
1056
|
+
name: "breakpoint"
|
|
1057
|
+
};
|
|
1058
|
+
if (token.startsWith(CHILD_PREFIX)) return {
|
|
1059
|
+
on: "child",
|
|
1060
|
+
name: token.slice(CHILD_PREFIX.length)
|
|
1061
|
+
};
|
|
1062
|
+
if (token.startsWith(EVENT_PREFIX)) return {
|
|
1063
|
+
on: "signal",
|
|
1064
|
+
name: eventNameOf(token)
|
|
1065
|
+
};
|
|
1066
|
+
return {
|
|
1067
|
+
on: "signal",
|
|
1068
|
+
name: token
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
__name(classifyWaiterToken, "classifyWaiterToken");
|
|
1072
|
+
function resolveRunWaiting(run, waiterByRun) {
|
|
1073
|
+
if (run.status !== "suspended") return void 0;
|
|
1074
|
+
const waiter = waiterByRun.get(run.id);
|
|
1075
|
+
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
1076
|
+
}
|
|
1077
|
+
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
1078
|
+
var RUN_ID_BATCH = 500;
|
|
1079
|
+
async function listSignalWaitersOfRuns(store, runIds) {
|
|
1080
|
+
const unique = [
|
|
1081
|
+
...new Set(runIds)
|
|
1082
|
+
];
|
|
1083
|
+
if (unique.length === 0) return [];
|
|
1084
|
+
if (!store.listSignalWaitersByRun) {
|
|
1085
|
+
const ids = new Set(unique);
|
|
1086
|
+
return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
|
|
1087
|
+
}
|
|
1088
|
+
const out = [];
|
|
1089
|
+
for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
|
|
1090
|
+
out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
|
|
1091
|
+
}
|
|
1092
|
+
return out;
|
|
1093
|
+
}
|
|
1094
|
+
__name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
|
|
1095
|
+
function indexWaitersByRun(waiters) {
|
|
1096
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
1097
|
+
for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
|
|
1098
|
+
return byRun;
|
|
1099
|
+
}
|
|
1100
|
+
__name(indexWaitersByRun, "indexWaitersByRun");
|
|
1101
|
+
var GET_RUN_FALLBACK_PARALLEL = 50;
|
|
1102
|
+
async function getRunsByIds(store, runIds) {
|
|
1103
|
+
const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
|
|
1104
|
+
if (unread.length > 0) throw error2;
|
|
1105
|
+
return runs;
|
|
1106
|
+
}
|
|
1107
|
+
__name(getRunsByIds, "getRunsByIds");
|
|
1108
|
+
async function readRunsByIds(store, runIds) {
|
|
1109
|
+
const unique = [
|
|
1110
|
+
...new Set(runIds)
|
|
1111
|
+
];
|
|
1112
|
+
const runs = /* @__PURE__ */ new Map();
|
|
1113
|
+
const unread = [];
|
|
1114
|
+
let error2;
|
|
1115
|
+
const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
|
|
1116
|
+
for (let at = 0; at < unique.length; at += batch) {
|
|
1117
|
+
const ids = unique.slice(at, at + batch);
|
|
1118
|
+
if (store.getRuns) {
|
|
1119
|
+
try {
|
|
1120
|
+
for (const run of await store.getRuns(ids)) runs.set(run.id, run);
|
|
1121
|
+
} catch (err) {
|
|
1122
|
+
error2 ??= err;
|
|
1123
|
+
unread.push(...ids);
|
|
1124
|
+
}
|
|
1125
|
+
continue;
|
|
1126
|
+
}
|
|
1127
|
+
const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
|
|
1128
|
+
reads.forEach((read, k) => {
|
|
1129
|
+
if (read.status === "fulfilled") {
|
|
1130
|
+
if (read.value) runs.set(read.value.id, read.value);
|
|
1131
|
+
return;
|
|
1132
|
+
}
|
|
1133
|
+
error2 ??= read.reason;
|
|
1134
|
+
unread.push(ids[k]);
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
return {
|
|
1138
|
+
runs,
|
|
1139
|
+
unread,
|
|
1140
|
+
...unread.length > 0 ? {
|
|
1141
|
+
error: error2
|
|
1142
|
+
} : {}
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
__name(readRunsByIds, "readRunsByIds");
|
|
1146
|
+
|
|
1147
|
+
// src/buffered-child-signals.ts
|
|
1148
|
+
var CHILD_TOKEN_PREFIX = "child:";
|
|
1149
|
+
var CHILD_SIGNAL_NAME_PREFIX = "signal:child:";
|
|
1150
|
+
var RETRYABLE = /* @__PURE__ */ new Set([
|
|
1151
|
+
"failed",
|
|
1152
|
+
"dead"
|
|
1153
|
+
]);
|
|
1154
|
+
var FAN_CHILD = /^(.+\.all\.\d+\.(?:w\d+\.)?)(\d+)$/;
|
|
1155
|
+
function canonicalJson(value) {
|
|
1156
|
+
const plain = JSON.stringify(value);
|
|
1157
|
+
return plain === void 0 ? "null" : sortedJson(JSON.parse(plain));
|
|
1158
|
+
}
|
|
1159
|
+
__name(canonicalJson, "canonicalJson");
|
|
1160
|
+
function sortedJson(value) {
|
|
1161
|
+
if (Array.isArray(value)) return `[${value.map(sortedJson).join(",")}]`;
|
|
1162
|
+
if (value !== null && typeof value === "object") {
|
|
1163
|
+
const entries = Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0);
|
|
1164
|
+
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${sortedJson(v)}`).join(",")}}`;
|
|
1165
|
+
}
|
|
1166
|
+
return JSON.stringify(value);
|
|
1167
|
+
}
|
|
1168
|
+
__name(sortedJson, "sortedJson");
|
|
1169
|
+
function settledNames(childId) {
|
|
1170
|
+
const signal = `${CHILD_SIGNAL_NAME_PREFIX}${childId}`;
|
|
1171
|
+
const fan = FAN_CHILD.exec(childId);
|
|
1172
|
+
if (!fan) return {
|
|
1173
|
+
signal
|
|
1174
|
+
};
|
|
1175
|
+
return {
|
|
1176
|
+
signal,
|
|
1177
|
+
fold: {
|
|
1178
|
+
name: `${FOLDED_PREFIX}${fan[1]}`,
|
|
1179
|
+
index: Number(fan[2])
|
|
1180
|
+
}
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
__name(settledNames, "settledNames");
|
|
1184
|
+
function parentHoldsChild(checkpoints, childId, completion) {
|
|
1185
|
+
const { signal, fold } = settledNames(childId);
|
|
1186
|
+
const expected = canonicalJson(completion);
|
|
1187
|
+
return checkpoints.some((cp) => {
|
|
1188
|
+
if (cp.status !== "completed") return false;
|
|
1189
|
+
if (cp.name === signal) return canonicalJson(cp.output) === expected;
|
|
1190
|
+
if (fold && cp.name === fold.name) {
|
|
1191
|
+
const { from, values } = cp.output ?? {};
|
|
1192
|
+
return typeof from === "number" && Array.isArray(values) && fold.index >= from && fold.index < from + values.length;
|
|
1193
|
+
}
|
|
1194
|
+
return false;
|
|
1195
|
+
});
|
|
1196
|
+
}
|
|
1197
|
+
__name(parentHoldsChild, "parentHoldsChild");
|
|
1198
|
+
async function parentCheckpoints(store, parentId, childId) {
|
|
1199
|
+
const { signal, fold } = settledNames(childId);
|
|
1200
|
+
const prefixes = fold ? [
|
|
1201
|
+
signal,
|
|
1202
|
+
fold.name
|
|
1203
|
+
] : [
|
|
1204
|
+
signal
|
|
1205
|
+
];
|
|
1206
|
+
return store.listCheckpointsByNamePrefix ? store.listCheckpointsByNamePrefix(parentId, prefixes) : store.listCheckpoints(parentId);
|
|
1207
|
+
}
|
|
1208
|
+
__name(parentCheckpoints, "parentCheckpoints");
|
|
1209
|
+
async function childCompletionNeedsBuffer(store, childId, completion, parent) {
|
|
1210
|
+
const parentId = parentRunIdOf(childId);
|
|
1211
|
+
if (!parentId) return false;
|
|
1212
|
+
const run = parent === void 0 ? await store.getRun(parentId) : parent;
|
|
1213
|
+
if (!run) return false;
|
|
1214
|
+
if (TERMINAL_RUN_STATUSES.includes(run.status) && !RETRYABLE.has(run.status)) return false;
|
|
1215
|
+
return !parentHoldsChild(await parentCheckpoints(store, parentId, childId), childId, completion);
|
|
1216
|
+
}
|
|
1217
|
+
__name(childCompletionNeedsBuffer, "childCompletionNeedsBuffer");
|
|
1218
|
+
async function unneededChildSignals(store, rows) {
|
|
1219
|
+
const parentIds = rows.flatMap((row) => {
|
|
1220
|
+
const parentId = parentRunIdOf(row.token.slice(CHILD_TOKEN_PREFIX.length));
|
|
1221
|
+
return parentId ? [
|
|
1222
|
+
parentId
|
|
1223
|
+
] : [];
|
|
1224
|
+
});
|
|
1225
|
+
const { runs, unread } = await readRunsByIds(store, parentIds);
|
|
1226
|
+
const unreadable = new Set(unread);
|
|
1227
|
+
const doomed = [];
|
|
1228
|
+
for (const row of rows) {
|
|
1229
|
+
if (!row.token.startsWith(CHILD_TOKEN_PREFIX)) continue;
|
|
1230
|
+
const childId = row.token.slice(CHILD_TOKEN_PREFIX.length);
|
|
1231
|
+
const parentId = parentRunIdOf(childId);
|
|
1232
|
+
if (parentId && unreadable.has(parentId)) continue;
|
|
1233
|
+
const parent = parentId ? runs.get(parentId) ?? null : null;
|
|
1234
|
+
try {
|
|
1235
|
+
if (!await childCompletionNeedsBuffer(store, childId, row.payload, parent)) doomed.push(row);
|
|
1236
|
+
} catch {
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
return doomed;
|
|
1240
|
+
}
|
|
1241
|
+
__name(unneededChildSignals, "unneededChildSignals");
|
|
1242
|
+
|
|
863
1243
|
// src/execution-hooks.ts
|
|
864
1244
|
var wrappers = [];
|
|
865
1245
|
function useDurableExecution(wrapper) {
|
|
@@ -1189,6 +1569,32 @@ function completionError(c) {
|
|
|
1189
1569
|
};
|
|
1190
1570
|
}
|
|
1191
1571
|
__name(completionError, "completionError");
|
|
1572
|
+
function childCompletion(child) {
|
|
1573
|
+
if (child.status === "completed") return {
|
|
1574
|
+
ok: true,
|
|
1575
|
+
value: child.output
|
|
1576
|
+
};
|
|
1577
|
+
if (child.status === "failed") return failedCompletion(child.error ?? {
|
|
1578
|
+
message: "failed"
|
|
1579
|
+
});
|
|
1580
|
+
if (child.status === "dead") return failedCompletion(child.error ?? {
|
|
1581
|
+
message: "dead"
|
|
1582
|
+
});
|
|
1583
|
+
if (child.status === "cancelled") return cancelledCompletion(child.error);
|
|
1584
|
+
return void 0;
|
|
1585
|
+
}
|
|
1586
|
+
__name(childCompletion, "childCompletion");
|
|
1587
|
+
var CHILD_CANCELLED = "cancelled";
|
|
1588
|
+
function cancelledCompletion(error2) {
|
|
1589
|
+
const base = error2 ?? {
|
|
1590
|
+
message: "cancelled"
|
|
1591
|
+
};
|
|
1592
|
+
return failedCompletion({
|
|
1593
|
+
...base,
|
|
1594
|
+
code: base.code ?? CHILD_CANCELLED
|
|
1595
|
+
});
|
|
1596
|
+
}
|
|
1597
|
+
__name(cancelledCompletion, "cancelledCompletion");
|
|
1192
1598
|
|
|
1193
1599
|
// src/concurrency.ts
|
|
1194
1600
|
var DEFAULT_CONCURRENCY_STATUSES = [
|
|
@@ -1394,210 +1800,66 @@ var EventAccumulators = class {
|
|
|
1394
1800
|
const [workflow, input] = config.mode === "debounce" ? [
|
|
1395
1801
|
"__evt_debounce",
|
|
1396
1802
|
{
|
|
1397
|
-
target,
|
|
1398
|
-
windowMs: config.windowMs
|
|
1399
|
-
}
|
|
1400
|
-
] : [
|
|
1401
|
-
"__evt_batch",
|
|
1402
|
-
{
|
|
1403
|
-
target,
|
|
1404
|
-
maxSize: config.maxSize,
|
|
1405
|
-
windowMs: config.windowMs
|
|
1406
|
-
}
|
|
1407
|
-
];
|
|
1408
|
-
return this.host.signalWithStart(workflow, input, accToken(target), {
|
|
1409
|
-
token: accToken(target),
|
|
1410
|
-
payload
|
|
1411
|
-
});
|
|
1412
|
-
}
|
|
1413
|
-
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1414
|
-
async debounce(ctx, input) {
|
|
1415
|
-
const { target, windowMs } = input;
|
|
1416
|
-
const token = accToken(target);
|
|
1417
|
-
let last = await ctx.waitForSignal(token);
|
|
1418
|
-
for (; ; ) {
|
|
1419
|
-
try {
|
|
1420
|
-
last = await ctx.waitForSignal(token, {
|
|
1421
|
-
timeoutMs: windowMs
|
|
1422
|
-
});
|
|
1423
|
-
} catch (e) {
|
|
1424
|
-
if (e instanceof SignalTimeoutError) break;
|
|
1425
|
-
throw e;
|
|
1426
|
-
}
|
|
1427
|
-
}
|
|
1428
|
-
await ctx.startChild(target, last);
|
|
1429
|
-
return ctx.continueAsNew(input);
|
|
1430
|
-
}
|
|
1431
|
-
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
1432
|
-
async batch(ctx, input) {
|
|
1433
|
-
const { target, maxSize, windowMs } = input;
|
|
1434
|
-
const token = accToken(target);
|
|
1435
|
-
const events = [
|
|
1436
|
-
await ctx.waitForSignal(token)
|
|
1437
|
-
];
|
|
1438
|
-
const deadline = await ctx.now() + windowMs;
|
|
1439
|
-
while (events.length < maxSize) {
|
|
1440
|
-
const remaining = deadline - await ctx.now();
|
|
1441
|
-
if (remaining <= 0) break;
|
|
1442
|
-
try {
|
|
1443
|
-
events.push(await ctx.waitForSignal(token, {
|
|
1444
|
-
timeoutMs: remaining
|
|
1445
|
-
}));
|
|
1446
|
-
} catch (e) {
|
|
1447
|
-
if (e instanceof SignalTimeoutError) break;
|
|
1448
|
-
throw e;
|
|
1449
|
-
}
|
|
1450
|
-
}
|
|
1451
|
-
await ctx.startChild(target, {
|
|
1452
|
-
events
|
|
1453
|
-
});
|
|
1454
|
-
return ctx.continueAsNew(input);
|
|
1455
|
-
}
|
|
1456
|
-
};
|
|
1457
|
-
|
|
1458
|
-
// src/events.ts
|
|
1459
|
-
var enc = /* @__PURE__ */ __name((s) => Buffer.from(s, "utf8").toString("base64"), "enc");
|
|
1460
|
-
var dec = /* @__PURE__ */ __name((s) => Buffer.from(s, "base64").toString("utf8"), "dec");
|
|
1461
|
-
var EVENT_BUFFER_SCAN_LIMIT = 50;
|
|
1462
|
-
var EVENT_TOKEN_PREFIX = "event:";
|
|
1463
|
-
function eventPrefix(name) {
|
|
1464
|
-
return `${EVENT_TOKEN_PREFIX}${enc(name)}:`;
|
|
1465
|
-
}
|
|
1466
|
-
__name(eventPrefix, "eventPrefix");
|
|
1467
|
-
function isEventToken(token) {
|
|
1468
|
-
return token.startsWith(EVENT_TOKEN_PREFIX);
|
|
1469
|
-
}
|
|
1470
|
-
__name(isEventToken, "isEventToken");
|
|
1471
|
-
function eventToken(name, match, runId, seq) {
|
|
1472
|
-
return `${eventPrefix(name)}${enc(JSON.stringify(match ?? {}))}:${runId}#${seq}`;
|
|
1473
|
-
}
|
|
1474
|
-
__name(eventToken, "eventToken");
|
|
1475
|
-
function eventNameOf(token) {
|
|
1476
|
-
try {
|
|
1477
|
-
return dec(token.split(":")[1] ?? "");
|
|
1478
|
-
} catch {
|
|
1479
|
-
return token;
|
|
1480
|
-
}
|
|
1481
|
-
}
|
|
1482
|
-
__name(eventNameOf, "eventNameOf");
|
|
1483
|
-
function eventMatchOf(token) {
|
|
1484
|
-
try {
|
|
1485
|
-
return JSON.parse(dec(token.split(":")[2] ?? ""));
|
|
1486
|
-
} catch {
|
|
1487
|
-
return {};
|
|
1488
|
-
}
|
|
1489
|
-
}
|
|
1490
|
-
__name(eventMatchOf, "eventMatchOf");
|
|
1491
|
-
function eventMatches(payload, match) {
|
|
1492
|
-
const keys = Object.keys(match);
|
|
1493
|
-
if (keys.length === 0) return true;
|
|
1494
|
-
if (!payload || typeof payload !== "object") return false;
|
|
1495
|
-
const p = payload;
|
|
1496
|
-
return keys.every((k) => JSON.stringify(p[k]) === JSON.stringify(match[k]));
|
|
1497
|
-
}
|
|
1498
|
-
__name(eventMatches, "eventMatches");
|
|
1499
|
-
|
|
1500
|
-
// src/fan-join.ts
|
|
1501
|
-
function allChildId(runId, firstSeq, index, wave) {
|
|
1502
|
-
return `${allBase(runId, firstSeq, wave)}${index}`;
|
|
1503
|
-
}
|
|
1504
|
-
__name(allChildId, "allChildId");
|
|
1505
|
-
function allGroup(firstSeq, size, mode) {
|
|
1506
|
-
const group = `all:${firstSeq}:n${size}`;
|
|
1507
|
-
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1508
|
-
}
|
|
1509
|
-
__name(allGroup, "allGroup");
|
|
1510
|
-
function stepFanGroup(firstSeq, size, mode) {
|
|
1511
|
-
const group = `steps:${firstSeq}:${size}`;
|
|
1512
|
-
return mode === "failFast" ? `${group}:failFast` : group;
|
|
1513
|
-
}
|
|
1514
|
-
__name(stepFanGroup, "stepFanGroup");
|
|
1515
|
-
function parseStepFanGroup(group) {
|
|
1516
|
-
const fan = parseFanGroup(group);
|
|
1517
|
-
if (fan?.kind !== "steps" || fan.size === void 0) return void 0;
|
|
1518
|
-
return {
|
|
1519
|
-
firstSeq: fan.firstSeq,
|
|
1520
|
-
size: fan.size
|
|
1521
|
-
};
|
|
1522
|
-
}
|
|
1523
|
-
__name(parseStepFanGroup, "parseStepFanGroup");
|
|
1524
|
-
function parseFanGroup(group) {
|
|
1525
|
-
const m = group ? /^(all|steps):(\d+)(?::n?(\d+))?(:failFast)?$/.exec(group) : null;
|
|
1526
|
-
if (!m) return void 0;
|
|
1527
|
-
if (m[1] === "steps" && m[3] === void 0) return void 0;
|
|
1528
|
-
return {
|
|
1529
|
-
kind: m[1],
|
|
1530
|
-
firstSeq: Number(m[2]),
|
|
1531
|
-
size: m[3] === void 0 ? void 0 : Number(m[3]),
|
|
1532
|
-
failFast: m[4] !== void 0
|
|
1533
|
-
};
|
|
1534
|
-
}
|
|
1535
|
-
__name(parseFanGroup, "parseFanGroup");
|
|
1536
|
-
function allWait(runId, firstSeq, wave) {
|
|
1537
|
-
const base = allBase(runId, firstSeq, wave);
|
|
1538
|
-
return {
|
|
1539
|
-
waiterPrefix: `child:${base}`,
|
|
1540
|
-
checkpointPrefix: `signal:child:${base}`
|
|
1541
|
-
};
|
|
1542
|
-
}
|
|
1543
|
-
__name(allWait, "allWait");
|
|
1544
|
-
function fanJoin(waits) {
|
|
1545
|
-
return {
|
|
1546
|
-
waits
|
|
1547
|
-
};
|
|
1548
|
-
}
|
|
1549
|
-
__name(fanJoin, "fanJoin");
|
|
1550
|
-
var FOLDED_PREFIX = "fan:";
|
|
1551
|
-
function allFoldedName(runId, firstSeq, wave) {
|
|
1552
|
-
return `${FOLDED_PREFIX}${allBase(runId, firstSeq, wave)}`;
|
|
1553
|
-
}
|
|
1554
|
-
__name(allFoldedName, "allFoldedName");
|
|
1555
|
-
function foldedChildIds(checkpoint) {
|
|
1556
|
-
const base = checkpoint.name.slice(FOLDED_PREFIX.length);
|
|
1557
|
-
const { from, values } = checkpoint.output;
|
|
1558
|
-
return values.map((_, j) => `${base}${from + j}`);
|
|
1559
|
-
}
|
|
1560
|
-
__name(foldedChildIds, "foldedChildIds");
|
|
1561
|
-
function allBase(runId, firstSeq, wave) {
|
|
1562
|
-
return wave === void 0 ? `${runId}.all.${firstSeq}.` : `${runId}.all.${firstSeq}.w${wave}.`;
|
|
1563
|
-
}
|
|
1564
|
-
__name(allBase, "allBase");
|
|
1565
|
-
function allWaves(size, concurrency) {
|
|
1566
|
-
if (concurrency === void 0 || concurrency >= size) return void 0;
|
|
1567
|
-
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
|
1568
|
-
throw new Error(`ctx.all: concurrency must be a positive integer, got ${concurrency}`);
|
|
1803
|
+
target,
|
|
1804
|
+
windowMs: config.windowMs
|
|
1805
|
+
}
|
|
1806
|
+
] : [
|
|
1807
|
+
"__evt_batch",
|
|
1808
|
+
{
|
|
1809
|
+
target,
|
|
1810
|
+
maxSize: config.maxSize,
|
|
1811
|
+
windowMs: config.windowMs
|
|
1812
|
+
}
|
|
1813
|
+
];
|
|
1814
|
+
return this.host.signalWithStart(workflow, input, accToken(target), {
|
|
1815
|
+
token: accToken(target),
|
|
1816
|
+
payload
|
|
1817
|
+
});
|
|
1569
1818
|
}
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
}
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1819
|
+
/** Fire the target with the LAST payload once events have been quiet for `windowMs`. */
|
|
1820
|
+
async debounce(ctx, input) {
|
|
1821
|
+
const { target, windowMs } = input;
|
|
1822
|
+
const token = accToken(target);
|
|
1823
|
+
let last = await ctx.waitForSignal(token);
|
|
1824
|
+
for (; ; ) {
|
|
1825
|
+
try {
|
|
1826
|
+
last = await ctx.waitForSignal(token, {
|
|
1827
|
+
timeoutMs: windowMs
|
|
1828
|
+
});
|
|
1829
|
+
} catch (e) {
|
|
1830
|
+
if (e instanceof SignalTimeoutError) break;
|
|
1831
|
+
throw e;
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
await ctx.startChild(target, last);
|
|
1835
|
+
return ctx.continueAsNew(input);
|
|
1836
|
+
}
|
|
1837
|
+
/** Fire the target with all payloads (`{ events }`) once `maxSize` or `windowMs` from the first. */
|
|
1838
|
+
async batch(ctx, input) {
|
|
1839
|
+
const { target, maxSize, windowMs } = input;
|
|
1840
|
+
const token = accToken(target);
|
|
1841
|
+
const events = [
|
|
1842
|
+
await ctx.waitForSignal(token)
|
|
1843
|
+
];
|
|
1844
|
+
const deadline = await ctx.now() + windowMs;
|
|
1845
|
+
while (events.length < maxSize) {
|
|
1846
|
+
const remaining = deadline - await ctx.now();
|
|
1847
|
+
if (remaining <= 0) break;
|
|
1848
|
+
try {
|
|
1849
|
+
events.push(await ctx.waitForSignal(token, {
|
|
1850
|
+
timeoutMs: remaining
|
|
1851
|
+
}));
|
|
1852
|
+
} catch (e) {
|
|
1853
|
+
if (e instanceof SignalTimeoutError) break;
|
|
1854
|
+
throw e;
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
await ctx.startChild(target, {
|
|
1858
|
+
events
|
|
1859
|
+
});
|
|
1860
|
+
return ctx.continueAsNew(input);
|
|
1861
|
+
}
|
|
1862
|
+
};
|
|
1601
1863
|
|
|
1602
1864
|
// src/handshake/descriptor.ts
|
|
1603
1865
|
var CURRENT_PROTOCOL_VERSION = 1;
|
|
@@ -2286,30 +2548,6 @@ function planDispatch(requires, liveDescriptors, cp, token) {
|
|
|
2286
2548
|
}
|
|
2287
2549
|
__name(planDispatch, "planDispatch");
|
|
2288
2550
|
|
|
2289
|
-
// src/interfaces.ts
|
|
2290
|
-
function checkpointNameKind(name) {
|
|
2291
|
-
if (name.startsWith("signal:child:")) return "child";
|
|
2292
|
-
if (name.startsWith("fan:")) return "folded";
|
|
2293
|
-
if (name.startsWith("spawn:")) return "spawn";
|
|
2294
|
-
return "step";
|
|
2295
|
-
}
|
|
2296
|
-
__name(checkpointNameKind, "checkpointNameKind");
|
|
2297
|
-
function poolNamespace(namespace) {
|
|
2298
|
-
return namespace === "" ? "default" : namespace;
|
|
2299
|
-
}
|
|
2300
|
-
__name(poolNamespace, "poolNamespace");
|
|
2301
|
-
var TERMINAL_RUN_STATUSES = [
|
|
2302
|
-
"completed",
|
|
2303
|
-
"failed",
|
|
2304
|
-
"cancelled",
|
|
2305
|
-
"dead"
|
|
2306
|
-
];
|
|
2307
|
-
function isNonEmptyRunScope(scope) {
|
|
2308
|
-
if (!scope) return false;
|
|
2309
|
-
return scope.workflow !== void 0 || scope.workflows !== void 0 || scope.tag !== void 0 || scope.tags !== void 0 || scope.namespace !== void 0 || scope.namespaces !== void 0 || scope.attributes !== void 0 && scope.attributes.length > 0;
|
|
2310
|
-
}
|
|
2311
|
-
__name(isNonEmptyRunScope, "isNonEmptyRunScope");
|
|
2312
|
-
|
|
2313
2551
|
// src/on-error.ts
|
|
2314
2552
|
var UNROUTABLE_CODES = /* @__PURE__ */ new Set([
|
|
2315
2553
|
"non_determinism",
|
|
@@ -2484,122 +2722,6 @@ var RunDispatchBatcher = class {
|
|
|
2484
2722
|
}
|
|
2485
2723
|
};
|
|
2486
2724
|
|
|
2487
|
-
// src/run-waiting.ts
|
|
2488
|
-
var WEBHOOK_PREFIX = "wh:";
|
|
2489
|
-
var CHILD_PREFIX = "child:";
|
|
2490
|
-
var BREAKPOINT_PREFIX = "bp:";
|
|
2491
|
-
var EVENT_PREFIX = EVENT_TOKEN_PREFIX;
|
|
2492
|
-
function parseWebhookToken(token) {
|
|
2493
|
-
if (!token.startsWith(WEBHOOK_PREFIX)) return null;
|
|
2494
|
-
const rest = token.slice(WEBHOOK_PREFIX.length);
|
|
2495
|
-
const at = rest.lastIndexOf(":");
|
|
2496
|
-
if (at <= 0) return null;
|
|
2497
|
-
const seqText = rest.slice(at + 1);
|
|
2498
|
-
if (!/^\d+$/.test(seqText)) return null;
|
|
2499
|
-
return {
|
|
2500
|
-
runId: rest.slice(0, at),
|
|
2501
|
-
seq: Number(seqText)
|
|
2502
|
-
};
|
|
2503
|
-
}
|
|
2504
|
-
__name(parseWebhookToken, "parseWebhookToken");
|
|
2505
|
-
function classifyWaiterToken(token) {
|
|
2506
|
-
if (token.startsWith(WEBHOOK_PREFIX)) return {
|
|
2507
|
-
on: "webhook",
|
|
2508
|
-
name: token
|
|
2509
|
-
};
|
|
2510
|
-
if (token.startsWith(BREAKPOINT_PREFIX)) return {
|
|
2511
|
-
on: "breakpoint",
|
|
2512
|
-
name: "breakpoint"
|
|
2513
|
-
};
|
|
2514
|
-
if (token.startsWith(CHILD_PREFIX)) return {
|
|
2515
|
-
on: "child",
|
|
2516
|
-
name: token.slice(CHILD_PREFIX.length)
|
|
2517
|
-
};
|
|
2518
|
-
if (token.startsWith(EVENT_PREFIX)) return {
|
|
2519
|
-
on: "signal",
|
|
2520
|
-
name: eventNameOf(token)
|
|
2521
|
-
};
|
|
2522
|
-
return {
|
|
2523
|
-
on: "signal",
|
|
2524
|
-
name: token
|
|
2525
|
-
};
|
|
2526
|
-
}
|
|
2527
|
-
__name(classifyWaiterToken, "classifyWaiterToken");
|
|
2528
|
-
function resolveRunWaiting(run, waiterByRun) {
|
|
2529
|
-
if (run.status !== "suspended") return void 0;
|
|
2530
|
-
const waiter = waiterByRun.get(run.id);
|
|
2531
|
-
return waiter ? classifyWaiterToken(waiter.token) : void 0;
|
|
2532
|
-
}
|
|
2533
|
-
__name(resolveRunWaiting, "resolveRunWaiting");
|
|
2534
|
-
var RUN_ID_BATCH = 500;
|
|
2535
|
-
async function listSignalWaitersOfRuns(store, runIds) {
|
|
2536
|
-
const unique = [
|
|
2537
|
-
...new Set(runIds)
|
|
2538
|
-
];
|
|
2539
|
-
if (unique.length === 0) return [];
|
|
2540
|
-
if (!store.listSignalWaitersByRun) {
|
|
2541
|
-
const ids = new Set(unique);
|
|
2542
|
-
return (await store.listSignalWaiters("")).filter((w) => ids.has(w.runId));
|
|
2543
|
-
}
|
|
2544
|
-
const out = [];
|
|
2545
|
-
for (let at = 0; at < unique.length; at += RUN_ID_BATCH) {
|
|
2546
|
-
out.push(...await store.listSignalWaitersByRun(unique.slice(at, at + RUN_ID_BATCH)));
|
|
2547
|
-
}
|
|
2548
|
-
return out;
|
|
2549
|
-
}
|
|
2550
|
-
__name(listSignalWaitersOfRuns, "listSignalWaitersOfRuns");
|
|
2551
|
-
function indexWaitersByRun(waiters) {
|
|
2552
|
-
const byRun = /* @__PURE__ */ new Map();
|
|
2553
|
-
for (const w of waiters) if (!byRun.has(w.runId)) byRun.set(w.runId, w);
|
|
2554
|
-
return byRun;
|
|
2555
|
-
}
|
|
2556
|
-
__name(indexWaitersByRun, "indexWaitersByRun");
|
|
2557
|
-
var GET_RUN_FALLBACK_PARALLEL = 50;
|
|
2558
|
-
async function getRunsByIds(store, runIds) {
|
|
2559
|
-
const { runs, unread, error: error2 } = await readRunsByIds(store, runIds);
|
|
2560
|
-
if (unread.length > 0) throw error2;
|
|
2561
|
-
return runs;
|
|
2562
|
-
}
|
|
2563
|
-
__name(getRunsByIds, "getRunsByIds");
|
|
2564
|
-
async function readRunsByIds(store, runIds) {
|
|
2565
|
-
const unique = [
|
|
2566
|
-
...new Set(runIds)
|
|
2567
|
-
];
|
|
2568
|
-
const runs = /* @__PURE__ */ new Map();
|
|
2569
|
-
const unread = [];
|
|
2570
|
-
let error2;
|
|
2571
|
-
const batch = store.getRuns ? RUN_ID_BATCH : GET_RUN_FALLBACK_PARALLEL;
|
|
2572
|
-
for (let at = 0; at < unique.length; at += batch) {
|
|
2573
|
-
const ids = unique.slice(at, at + batch);
|
|
2574
|
-
if (store.getRuns) {
|
|
2575
|
-
try {
|
|
2576
|
-
for (const run of await store.getRuns(ids)) runs.set(run.id, run);
|
|
2577
|
-
} catch (err) {
|
|
2578
|
-
error2 ??= err;
|
|
2579
|
-
unread.push(...ids);
|
|
2580
|
-
}
|
|
2581
|
-
continue;
|
|
2582
|
-
}
|
|
2583
|
-
const reads = await Promise.allSettled(ids.map((id) => store.getRun(id)));
|
|
2584
|
-
reads.forEach((read, k) => {
|
|
2585
|
-
if (read.status === "fulfilled") {
|
|
2586
|
-
if (read.value) runs.set(read.value.id, read.value);
|
|
2587
|
-
return;
|
|
2588
|
-
}
|
|
2589
|
-
error2 ??= read.reason;
|
|
2590
|
-
unread.push(ids[k]);
|
|
2591
|
-
});
|
|
2592
|
-
}
|
|
2593
|
-
return {
|
|
2594
|
-
runs,
|
|
2595
|
-
unread,
|
|
2596
|
-
...unread.length > 0 ? {
|
|
2597
|
-
error: error2
|
|
2598
|
-
} : {}
|
|
2599
|
-
};
|
|
2600
|
-
}
|
|
2601
|
-
__name(readRunsByIds, "readRunsByIds");
|
|
2602
|
-
|
|
2603
2725
|
// src/scheduler.ts
|
|
2604
2726
|
var import_node_module = require("module");
|
|
2605
2727
|
|
|
@@ -3439,6 +3561,7 @@ function constantTimeEquals(a, b) {
|
|
|
3439
3561
|
__name(constantTimeEquals, "constantTimeEquals");
|
|
3440
3562
|
|
|
3441
3563
|
// src/transport-pool.ts
|
|
3564
|
+
var DISPATCH_CHUNK = 200;
|
|
3442
3565
|
var TransportPool = class {
|
|
3443
3566
|
static {
|
|
3444
3567
|
__name(this, "TransportPool");
|
|
@@ -3498,6 +3621,28 @@ var TransportPool = class {
|
|
|
3498
3621
|
}
|
|
3499
3622
|
throw lastErr ?? new Error("no transport accepted the dispatch");
|
|
3500
3623
|
}
|
|
3624
|
+
/**
|
|
3625
|
+
* {@link dispatch} for many tasks: one {@link Transport.dispatchMany} call on the preferred (or
|
|
3626
|
+
* first) transport when it has one, else (or when that call rejects) each task through
|
|
3627
|
+
* {@link dispatch}, a bounded number at a time.
|
|
3628
|
+
*/
|
|
3629
|
+
async dispatchMany(tasks, preferId) {
|
|
3630
|
+
if (tasks.length === 0) return;
|
|
3631
|
+
const first = this.ordered(preferId)[0];
|
|
3632
|
+
if (first?.transport.dispatchMany) {
|
|
3633
|
+
try {
|
|
3634
|
+
await first.transport.dispatchMany(tasks.map((task) => ({
|
|
3635
|
+
...task,
|
|
3636
|
+
transport: first.id
|
|
3637
|
+
})));
|
|
3638
|
+
return;
|
|
3639
|
+
} catch {
|
|
3640
|
+
}
|
|
3641
|
+
}
|
|
3642
|
+
for (let at = 0; at < tasks.length; at += DISPATCH_CHUNK) {
|
|
3643
|
+
await Promise.all(tasks.slice(at, at + DISPATCH_CHUNK).map((task) => this.dispatch(task, preferId)));
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3501
3646
|
/** Worker-health for `group`, merged across every transport that can report it (a group's queue
|
|
3502
3647
|
* and its workers may live on any pinned transport). Undefined when no transport implements
|
|
3503
3648
|
* `groupHealth` (e.g. a pure in-process transport — nothing to introspect). */
|
|
@@ -4056,7 +4201,23 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4056
4201
|
status: "running"
|
|
4057
4202
|
}));
|
|
4058
4203
|
}
|
|
4059
|
-
|
|
4204
|
+
const childRun = await store.getRun(id);
|
|
4205
|
+
if (!childRun) {
|
|
4206
|
+
host.startChild(workflowName(workflow2), input, id, priority, version);
|
|
4207
|
+
} else {
|
|
4208
|
+
const done = (await settleFinishedChildren([
|
|
4209
|
+
{
|
|
4210
|
+
id,
|
|
4211
|
+
seq: current
|
|
4212
|
+
}
|
|
4213
|
+
], /* @__PURE__ */ new Map([
|
|
4214
|
+
[
|
|
4215
|
+
id,
|
|
4216
|
+
childRun
|
|
4217
|
+
]
|
|
4218
|
+
]))).get(id);
|
|
4219
|
+
if (done) return unwrapCompletion(done, `child "${id}"`);
|
|
4220
|
+
}
|
|
4060
4221
|
throw new WorkflowSuspended(void 0, fanJoin([
|
|
4061
4222
|
{
|
|
4062
4223
|
waiterPrefix: `child:${id}`,
|
|
@@ -4077,164 +4238,174 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4077
4238
|
const group = allGroup(firstSeq, inputs.length, mode);
|
|
4078
4239
|
const waveOf = /* @__PURE__ */ __name((i) => waves ? Math.floor(i / waves.waveSize) : void 0, "waveOf");
|
|
4079
4240
|
const id = /* @__PURE__ */ __name((i) => allChildId(runId, firstSeq, i, waveOf(i)), "id");
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
const
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
const
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
throw new GatherError([
|
|
4107
|
-
{
|
|
4108
|
-
index: i,
|
|
4109
|
-
id: id(i),
|
|
4110
|
-
error: c.error ?? "unknown"
|
|
4241
|
+
for (; ; ) {
|
|
4242
|
+
const { checkpoints: existing, folded } = await readFan(positions, [
|
|
4243
|
+
allWait(runId, firstSeq).checkpointPrefix,
|
|
4244
|
+
allFoldedName(runId, firstSeq)
|
|
4245
|
+
]);
|
|
4246
|
+
const waveCount = waves ? Math.ceil(inputs.length / waves.waveSize) : 1;
|
|
4247
|
+
const waveItems = /* @__PURE__ */ __name((k) => {
|
|
4248
|
+
if (!waves) return inputs.map((_, i) => i);
|
|
4249
|
+
const from = k * waves.waveSize;
|
|
4250
|
+
return inputs.map((_, i) => i).slice(from, from + waves.waveSize);
|
|
4251
|
+
}, "waveItems");
|
|
4252
|
+
if (mode === "failFast") {
|
|
4253
|
+
const failure = /* @__PURE__ */ __name((i2) => {
|
|
4254
|
+
const cp = existing[i2];
|
|
4255
|
+
const c2 = cp?.status === "completed" ? cp.output : null;
|
|
4256
|
+
return c2 && c2.ok === false ? c2 : null;
|
|
4257
|
+
}, "failure");
|
|
4258
|
+
const first = /* @__PURE__ */ __name((cancelled) => inputs.findIndex((_, i2) => {
|
|
4259
|
+
const c2 = failure(i2);
|
|
4260
|
+
return c2 !== null && c2.code === "cancelled" === cancelled;
|
|
4261
|
+
}), "first");
|
|
4262
|
+
const i = first(false) !== -1 ? first(false) : first(true);
|
|
4263
|
+
const c = i === -1 ? null : failure(i);
|
|
4264
|
+
if (c) {
|
|
4265
|
+
for (let j = 0; j < inputs.length; j += 1) {
|
|
4266
|
+
if (j !== i && existing[j]?.status !== "completed") host.cancelChild?.(id(j));
|
|
4111
4267
|
}
|
|
4112
|
-
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
const last = items[items.length - 1];
|
|
4120
|
-
if (folded[first]) continue;
|
|
4121
|
-
const values = [];
|
|
4122
|
-
for (const i of items) {
|
|
4123
|
-
const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
|
|
4124
|
-
if (!c || typeof c !== "object" || c.ok !== true) break;
|
|
4125
|
-
values.push(c.value);
|
|
4126
|
-
}
|
|
4127
|
-
if (values.length < items.length) continue;
|
|
4128
|
-
const into = instantCheckpoint({
|
|
4129
|
-
runId,
|
|
4130
|
-
seq: positions[first],
|
|
4131
|
-
name: allFoldedName(runId, firstSeq, waves ? k : void 0),
|
|
4132
|
-
kind: "signal",
|
|
4133
|
-
output: {
|
|
4134
|
-
from: first,
|
|
4135
|
-
values
|
|
4136
|
-
},
|
|
4137
|
-
parallelGroup: group
|
|
4138
|
-
});
|
|
4139
|
-
await store.compactCheckpoints(runId, positions[first], positions[last], into);
|
|
4140
|
-
for (const i of items) replay?.delete(positions[i]);
|
|
4141
|
-
replay?.set(into.seq, into);
|
|
4142
|
-
}
|
|
4143
|
-
}
|
|
4144
|
-
const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
|
|
4145
|
-
const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
|
|
4146
|
-
const active = [];
|
|
4147
|
-
let unsettled = 0;
|
|
4148
|
-
for (let k = 0; k < waveCount; k += 1) {
|
|
4149
|
-
if (settled(k)) continue;
|
|
4150
|
-
if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
|
|
4151
|
-
active.push(k);
|
|
4152
|
-
unsettled += 1;
|
|
4153
|
-
}
|
|
4154
|
-
}
|
|
4155
|
-
if (active.length > 0) {
|
|
4156
|
-
const toRegister = [];
|
|
4157
|
-
const toEnsure = [];
|
|
4158
|
-
for (const k of active) {
|
|
4159
|
-
for (const i of waveItems(k)) {
|
|
4160
|
-
const cp = existing[i];
|
|
4161
|
-
if (cp?.status === "completed") continue;
|
|
4162
|
-
(cp ? toEnsure : toRegister).push(i);
|
|
4268
|
+
throw new GatherError([
|
|
4269
|
+
{
|
|
4270
|
+
index: i,
|
|
4271
|
+
id: id(i),
|
|
4272
|
+
error: c.error ?? "unknown"
|
|
4273
|
+
}
|
|
4274
|
+
], inputs.length);
|
|
4163
4275
|
}
|
|
4164
4276
|
}
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
checkpoint: instantCheckpoint({
|
|
4277
|
+
if (store.compactCheckpoints && inputs.length >= FOLD_MIN_FAN) {
|
|
4278
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
4279
|
+
const items = waveItems(k);
|
|
4280
|
+
const first = items[0];
|
|
4281
|
+
const last = items[items.length - 1];
|
|
4282
|
+
if (folded[first]) continue;
|
|
4283
|
+
const values = [];
|
|
4284
|
+
for (const i of items) {
|
|
4285
|
+
const c = existing[i]?.status === "completed" ? existing[i]?.output : void 0;
|
|
4286
|
+
if (!c || typeof c !== "object" || c.ok !== true) break;
|
|
4287
|
+
values.push(c.value);
|
|
4288
|
+
}
|
|
4289
|
+
if (values.length < items.length) continue;
|
|
4290
|
+
const into = instantCheckpoint({
|
|
4180
4291
|
runId,
|
|
4181
|
-
seq: positions[
|
|
4182
|
-
name:
|
|
4292
|
+
seq: positions[first],
|
|
4293
|
+
name: allFoldedName(runId, firstSeq, waves ? k : void 0),
|
|
4183
4294
|
kind: "signal",
|
|
4184
|
-
|
|
4295
|
+
output: {
|
|
4296
|
+
from: first,
|
|
4297
|
+
values
|
|
4298
|
+
},
|
|
4185
4299
|
parallelGroup: group
|
|
4186
|
-
})
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
}
|
|
4191
|
-
if (toEnsure.length > 0) {
|
|
4192
|
-
const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
|
|
4193
|
-
const skip = new Set(unread);
|
|
4194
|
-
for (const i of toEnsure) {
|
|
4195
|
-
if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
|
|
4300
|
+
});
|
|
4301
|
+
await store.compactCheckpoints(runId, positions[first], positions[last], into);
|
|
4302
|
+
for (const i of items) replay?.delete(positions[i]);
|
|
4303
|
+
replay?.set(into.seq, into);
|
|
4196
4304
|
}
|
|
4197
4305
|
}
|
|
4198
|
-
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4204
|
-
|
|
4205
|
-
|
|
4206
|
-
|
|
4207
|
-
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
|
|
4211
|
-
|
|
4212
|
-
const
|
|
4306
|
+
const sent = /* @__PURE__ */ __name((k) => waveItems(k).some((i) => existing[i]), "sent");
|
|
4307
|
+
const settled = /* @__PURE__ */ __name((k) => waveItems(k).every((i) => existing[i]?.status === "completed"), "settled");
|
|
4308
|
+
const active = [];
|
|
4309
|
+
let unsettled = 0;
|
|
4310
|
+
for (let k = 0; k < waveCount; k += 1) {
|
|
4311
|
+
if (settled(k)) continue;
|
|
4312
|
+
if (sent(k) || unsettled < (waves?.maxWaves ?? 1)) {
|
|
4313
|
+
active.push(k);
|
|
4314
|
+
unsettled += 1;
|
|
4315
|
+
}
|
|
4316
|
+
}
|
|
4317
|
+
let known;
|
|
4318
|
+
if (active.length > 0) {
|
|
4319
|
+
const toRegister = [];
|
|
4320
|
+
const toEnsure = [];
|
|
4321
|
+
for (const k of active) {
|
|
4322
|
+
for (const i of waveItems(k)) {
|
|
4323
|
+
const cp = existing[i];
|
|
4324
|
+
if (cp?.status === "completed") continue;
|
|
4325
|
+
(cp ? toEnsure : toRegister).push(i);
|
|
4326
|
+
}
|
|
4327
|
+
}
|
|
4328
|
+
for (let at = 0; at < toRegister.length; at += FAN_CHUNK) {
|
|
4329
|
+
const chunk = toRegister.slice(at, at + FAN_CHUNK);
|
|
4330
|
+
const entries = chunk.map((i) => ({
|
|
4331
|
+
// Carry the fan `group` onto the waiter too: the child's terminal `signal:child:`
|
|
4332
|
+
// checkpoint is (re)written by engine.signal when the child notifies the parent,
|
|
4333
|
+
// OVERWRITING the running placeholder at the same seq — so without this the resolved
|
|
4334
|
+
// checkpoint would lose the group and the dashboard would render the resolved fan as a
|
|
4335
|
+
// sequential chain.
|
|
4336
|
+
waiter: {
|
|
4337
|
+
token: `child:${id(i)}`,
|
|
4338
|
+
runId,
|
|
4339
|
+
seq: positions[i],
|
|
4340
|
+
parallelGroup: group
|
|
4341
|
+
},
|
|
4342
|
+
checkpoint: instantCheckpoint({
|
|
4343
|
+
runId,
|
|
4344
|
+
seq: positions[i],
|
|
4345
|
+
name: `signal:child:${id(i)}`,
|
|
4346
|
+
kind: "signal",
|
|
4347
|
+
status: "running",
|
|
4348
|
+
parallelGroup: group
|
|
4349
|
+
})
|
|
4350
|
+
}));
|
|
4351
|
+
await registerChildWaits(entries);
|
|
4352
|
+
for (const i of chunk) host.startChild(name, inputs[i], id(i));
|
|
4353
|
+
}
|
|
4354
|
+
if (toEnsure.length > 0) {
|
|
4355
|
+
const { runs: started, unread } = await readRunsByIds(store, toEnsure.map((i) => id(i)));
|
|
4356
|
+
const skip = new Set(unread);
|
|
4357
|
+
for (const i of toEnsure) {
|
|
4358
|
+
if (!started.has(id(i)) && !skip.has(id(i))) host.startChild(name, inputs[i], id(i));
|
|
4359
|
+
}
|
|
4360
|
+
known = started;
|
|
4361
|
+
}
|
|
4362
|
+
const now2 = await settleFinishedChildren(toEnsure.map((i) => ({
|
|
4213
4363
|
id: id(i),
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
};
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
|
|
4221
|
-
|
|
4222
|
-
|
|
4223
|
-
|
|
4224
|
-
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
|
|
4228
|
-
|
|
4229
|
-
|
|
4230
|
-
|
|
4231
|
-
|
|
4232
|
-
|
|
4364
|
+
seq: positions[i],
|
|
4365
|
+
parallelGroup: group
|
|
4366
|
+
})), known);
|
|
4367
|
+
if (now2.size > 0) continue;
|
|
4368
|
+
}
|
|
4369
|
+
if (active.length > 0) {
|
|
4370
|
+
throw new WorkflowSuspended(void 0, fanJoin(active.map((k) => allWait(runId, firstSeq, waves ? k : void 0))));
|
|
4371
|
+
}
|
|
4372
|
+
const outputs = [];
|
|
4373
|
+
const results = [];
|
|
4374
|
+
const failures = [];
|
|
4375
|
+
for (let i = 0; i < inputs.length; i += 1) {
|
|
4376
|
+
const c = existing[i]?.output;
|
|
4377
|
+
if (c && typeof c === "object" && "ok" in c && c.ok === false) {
|
|
4378
|
+
const { message, ...rest } = completionError({
|
|
4379
|
+
...c,
|
|
4380
|
+
error: c.error ?? "unknown"
|
|
4381
|
+
});
|
|
4382
|
+
const failure = {
|
|
4383
|
+
id: id(i),
|
|
4384
|
+
error: message,
|
|
4385
|
+
...rest
|
|
4386
|
+
};
|
|
4387
|
+
failures.push({
|
|
4388
|
+
index: i,
|
|
4389
|
+
...failure
|
|
4390
|
+
});
|
|
4391
|
+
results.push({
|
|
4392
|
+
status: "rejected",
|
|
4393
|
+
...failure
|
|
4394
|
+
});
|
|
4395
|
+
outputs.push(void 0);
|
|
4396
|
+
} else {
|
|
4397
|
+
const value = c?.value;
|
|
4398
|
+
results.push({
|
|
4399
|
+
status: "fulfilled",
|
|
4400
|
+
value
|
|
4401
|
+
});
|
|
4402
|
+
outputs.push(value);
|
|
4403
|
+
}
|
|
4233
4404
|
}
|
|
4405
|
+
if (settle) return results;
|
|
4406
|
+
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
4407
|
+
return outputs;
|
|
4234
4408
|
}
|
|
4235
|
-
if (settle) return results;
|
|
4236
|
-
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
4237
|
-
return outputs;
|
|
4238
4409
|
}, "fan");
|
|
4239
4410
|
const all = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, false), "all");
|
|
4240
4411
|
const allSettled = /* @__PURE__ */ __name((workflow2, inputs, opts) => fan(workflow2, inputs, opts, true), "allSettled");
|
|
@@ -4320,6 +4491,39 @@ function createWorkflowCtx(host, runId, compensations, workflow = "", searchAttr
|
|
|
4320
4491
|
}
|
|
4321
4492
|
for (const e of entries) replay?.set(e.checkpoint.seq, e.checkpoint);
|
|
4322
4493
|
}, "registerChildWaits");
|
|
4494
|
+
const settleFinishedChildren = /* @__PURE__ */ __name(async (awaited, known) => {
|
|
4495
|
+
const settled = /* @__PURE__ */ new Map();
|
|
4496
|
+
if (awaited.length === 0) return settled;
|
|
4497
|
+
const runs = new Map(known ?? []);
|
|
4498
|
+
const toRead = awaited.filter((a) => !runs.has(a.id)).map((a) => a.id);
|
|
4499
|
+
if (toRead.length > 0) {
|
|
4500
|
+
for (const [id, run] of (await readRunsByIds(store, toRead)).runs) runs.set(id, run);
|
|
4501
|
+
}
|
|
4502
|
+
for (const a of awaited) {
|
|
4503
|
+
const child2 = runs.get(a.id);
|
|
4504
|
+
const completion = child2 ? childCompletion(child2) : void 0;
|
|
4505
|
+
if (!completion) continue;
|
|
4506
|
+
const token = `child:${a.id}`;
|
|
4507
|
+
const taken = await store.takeSignalWaiter(token);
|
|
4508
|
+
if (!taken) continue;
|
|
4509
|
+
if (taken.runId !== runId || taken.seq !== a.seq) {
|
|
4510
|
+
await store.putSignalWaiter(taken);
|
|
4511
|
+
continue;
|
|
4512
|
+
}
|
|
4513
|
+
await writeCheckpoint(instantCheckpoint({
|
|
4514
|
+
runId,
|
|
4515
|
+
seq: a.seq,
|
|
4516
|
+
name: `signal:child:${a.id}`,
|
|
4517
|
+
kind: "signal",
|
|
4518
|
+
output: completion,
|
|
4519
|
+
...a.parallelGroup !== void 0 ? {
|
|
4520
|
+
parallelGroup: a.parallelGroup
|
|
4521
|
+
} : {}
|
|
4522
|
+
}));
|
|
4523
|
+
settled.set(a.id, completion);
|
|
4524
|
+
}
|
|
4525
|
+
return settled;
|
|
4526
|
+
}, "settleFinishedChildren");
|
|
4323
4527
|
const startChild = /* @__PURE__ */ __name(async (workflow2, input, options) => {
|
|
4324
4528
|
const { childId, priority, version } = normalizeChildOptions(options);
|
|
4325
4529
|
const current = pos.next();
|
|
@@ -4658,6 +4862,15 @@ var PARENT_NOTIFY_BACKOFF_MS = 250;
|
|
|
4658
4862
|
var LEASE_RECHECK_MAX_MS = 2e3;
|
|
4659
4863
|
var LEASE_RECHECK_SLACK_MS = 100;
|
|
4660
4864
|
var LEASE_RECHECK_ATTEMPTS = 30;
|
|
4865
|
+
function wakeSeen(request, settledFans, served) {
|
|
4866
|
+
if (request.id !== void 0 && served.has(request.id)) return true;
|
|
4867
|
+
return isFanWakeRequest(request.payload) && request.payload.fans.every(([first, seq]) => (settledFans.get(first) ?? -1) >= seq);
|
|
4868
|
+
}
|
|
4869
|
+
__name(wakeSeen, "wakeSeen");
|
|
4870
|
+
function isFanWakeRequest(payload) {
|
|
4871
|
+
return typeof payload === "object" && payload !== null && Array.isArray(payload.fans);
|
|
4872
|
+
}
|
|
4873
|
+
__name(isFanWakeRequest, "isFanWakeRequest");
|
|
4661
4874
|
function deferred() {
|
|
4662
4875
|
let resolve;
|
|
4663
4876
|
let reject;
|
|
@@ -4676,35 +4889,51 @@ __name(deferred, "deferred");
|
|
|
4676
4889
|
var pendingStateKey = /* @__PURE__ */ __name((at) => String(new Date(at).getTime()), "pendingStateKey");
|
|
4677
4890
|
var freshPendingKey = /* @__PURE__ */ __name(() => `${Date.now()}-${globalThis.crypto.randomUUID().slice(0, 8)}`, "freshPendingKey");
|
|
4678
4891
|
var wakeRequestToken = /* @__PURE__ */ __name((runId) => `durable:wake-after-turn:${runId}`, "wakeRequestToken");
|
|
4892
|
+
var WAKE_REQUEST_SCAN = 100;
|
|
4679
4893
|
var CHILD_SIGNAL_PREFIX = "signal:child:";
|
|
4894
|
+
var BUFFERED_SWEEP_PAGE = 500;
|
|
4895
|
+
var BUFFERED_SWEEP_PAGES_PER_CALL = 4;
|
|
4896
|
+
var BUFFERED_SWEEP_REST_MS = 10 * 6e4;
|
|
4680
4897
|
var DELIVER_CHUNK = 50;
|
|
4681
|
-
function childCompletion(child) {
|
|
4682
|
-
if (child.status === "completed") return {
|
|
4683
|
-
ok: true,
|
|
4684
|
-
value: child.output
|
|
4685
|
-
};
|
|
4686
|
-
if (child.status === "failed") return failedCompletion(child.error ?? {
|
|
4687
|
-
message: "failed"
|
|
4688
|
-
});
|
|
4689
|
-
if (child.status === "dead") return failedCompletion(child.error ?? {
|
|
4690
|
-
message: "dead"
|
|
4691
|
-
});
|
|
4692
|
-
if (child.status === "cancelled") return cancelledCompletion(child.error);
|
|
4693
|
-
return void 0;
|
|
4694
|
-
}
|
|
4695
|
-
__name(childCompletion, "childCompletion");
|
|
4696
|
-
var CHILD_CANCELLED = "cancelled";
|
|
4697
|
-
function cancelledCompletion(error2) {
|
|
4698
|
-
const base = error2 ?? {
|
|
4699
|
-
message: "cancelled"
|
|
4700
|
-
};
|
|
4701
|
-
return failedCompletion({
|
|
4702
|
-
...base,
|
|
4703
|
-
code: base.code ?? CHILD_CANCELLED
|
|
4704
|
-
});
|
|
4705
|
-
}
|
|
4706
|
-
__name(cancelledCompletion, "cancelledCompletion");
|
|
4707
4898
|
var RESULT_BATCH_MAX = 256;
|
|
4899
|
+
function isResultError(outcome) {
|
|
4900
|
+
return typeof outcome === "object" && "error" in outcome;
|
|
4901
|
+
}
|
|
4902
|
+
__name(isResultError, "isResultError");
|
|
4903
|
+
var FanWakeCollector = class FanWakeCollector2 {
|
|
4904
|
+
static {
|
|
4905
|
+
__name(this, "FanWakeCollector");
|
|
4906
|
+
}
|
|
4907
|
+
carrier;
|
|
4908
|
+
generic = false;
|
|
4909
|
+
fan;
|
|
4910
|
+
/** The highest seq the batch settled, per fan. */
|
|
4911
|
+
highest = /* @__PURE__ */ new Map();
|
|
4912
|
+
/** A checkpoint the batch settled. */
|
|
4913
|
+
note(cp) {
|
|
4914
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4915
|
+
if (fan) this.highest.set(fan.firstSeq, Math.max(this.highest.get(fan.firstSeq) ?? cp.seq, cp.seq));
|
|
4916
|
+
}
|
|
4917
|
+
/** A settled checkpoint that resumes the run. */
|
|
4918
|
+
add(index, cp, status) {
|
|
4919
|
+
this.carrier = index;
|
|
4920
|
+
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
4921
|
+
if (!fan || status === "failed" || this.fan !== void 0 && this.fan !== fan.firstSeq) {
|
|
4922
|
+
this.generic = true;
|
|
4923
|
+
return;
|
|
4924
|
+
}
|
|
4925
|
+
this.fan = fan.firstSeq;
|
|
4926
|
+
}
|
|
4927
|
+
applyTo(outcomes) {
|
|
4928
|
+
if (this.carrier === void 0) return outcomes;
|
|
4929
|
+
const seq = this.fan === void 0 ? void 0 : this.highest.get(this.fan);
|
|
4930
|
+
outcomes[this.carrier] = this.generic || this.fan === void 0 || seq === void 0 ? true : {
|
|
4931
|
+
first: this.fan,
|
|
4932
|
+
seq
|
|
4933
|
+
};
|
|
4934
|
+
return outcomes;
|
|
4935
|
+
}
|
|
4936
|
+
};
|
|
4708
4937
|
function shuffle(items) {
|
|
4709
4938
|
for (let i = items.length - 1; i > 0; i -= 1) {
|
|
4710
4939
|
const j = Math.floor(Math.random() * (i + 1));
|
|
@@ -4893,6 +5122,23 @@ var WorkflowEngine = class {
|
|
|
4893
5122
|
/** Re-checks scheduled for runs whose wake found another instance's lease (see {@link execute}). */
|
|
4894
5123
|
leaseRetries = /* @__PURE__ */ new Map();
|
|
4895
5124
|
/**
|
|
5125
|
+
* Runs whose scheduled re-check resumes them whatever happened meanwhile. Without an entry the
|
|
5126
|
+
* re-check only stood for fan wakes kept as requests, and resumes the run only if a request is
|
|
5127
|
+
* still there: the turn that held the lease takes them once it ends (see {@link retryDroppedWake}).
|
|
5128
|
+
*/
|
|
5129
|
+
unconditionalRechecks = /* @__PURE__ */ new Set();
|
|
5130
|
+
/** Runs a generic wake request was written for in the current re-check cycle (see {@link execute}). */
|
|
5131
|
+
genericWakeRequested = /* @__PURE__ */ new Set();
|
|
5132
|
+
/** Why each run is being woken on this instance, until the turn that answers it starts. */
|
|
5133
|
+
wakeReasons = /* @__PURE__ */ new Map();
|
|
5134
|
+
/**
|
|
5135
|
+
* The step fans each executing turn on this instance read with nothing in flight: first seq -> the
|
|
5136
|
+
* highest seq read. Written by {@link callRemoteAll}, read once the turn ends.
|
|
5137
|
+
*/
|
|
5138
|
+
turnSettledFans = /* @__PURE__ */ new Map();
|
|
5139
|
+
/** The wake requests each executing turn on this instance serves (see {@link noteServedWakes}). */
|
|
5140
|
+
turnServedWakes = /* @__PURE__ */ new Map();
|
|
5141
|
+
/**
|
|
4896
5142
|
* Every run turn and child start this instance takes on itself, under one budget of
|
|
4897
5143
|
* {@link LOCAL_RUN_CONCURRENCY}: the runs it dispatches to itself, a fan's deferred child starts,
|
|
4898
5144
|
* enqueued pickups and the runs it takes off the transport's run queue.
|
|
@@ -4919,6 +5165,10 @@ var WorkflowEngine = class {
|
|
|
4919
5165
|
/** Runs already paged about, by id → the `updatedAt` of the paged episode (see {@link sweepStalled}). */
|
|
4920
5166
|
notifiedStalled = /* @__PURE__ */ new Map();
|
|
4921
5167
|
lastStalledSweepAt = Number.NEGATIVE_INFINITY;
|
|
5168
|
+
/** {@link sweepBufferedSignals}: the id the pass in progress resumes after, `undefined` between passes. */
|
|
5169
|
+
bufferedSweepCursor;
|
|
5170
|
+
bufferedSweepActive = false;
|
|
5171
|
+
lastBufferedSweepEndedAt = Number.NEGATIVE_INFINITY;
|
|
4922
5172
|
stalledAfterMs;
|
|
4923
5173
|
draining = false;
|
|
4924
5174
|
/** Wait already spent on this shutdown before `drain()` (see `beginDrain`). */
|
|
@@ -5012,8 +5262,9 @@ var WorkflowEngine = class {
|
|
|
5012
5262
|
}
|
|
5013
5263
|
return;
|
|
5014
5264
|
}
|
|
5015
|
-
|
|
5016
|
-
|
|
5265
|
+
const wake = await this.completeRemoteResult(result);
|
|
5266
|
+
if (!wake) return;
|
|
5267
|
+
await this.serializedPerRun(result.runId, () => this.resumeImplicit(result.runId, wake === true ? void 0 : wake).then(() => void 0, (err) => {
|
|
5017
5268
|
if (err instanceof NamespaceMismatch) return;
|
|
5018
5269
|
throw err;
|
|
5019
5270
|
}));
|
|
@@ -5230,7 +5481,10 @@ var WorkflowEngine = class {
|
|
|
5230
5481
|
const resumed = await this.resumeLeased([
|
|
5231
5482
|
run
|
|
5232
5483
|
], nowMs, (locked) => this.reconcileSignalWaiter(locked, waiters.get(locked.id)));
|
|
5233
|
-
if (resumed.length === 0)
|
|
5484
|
+
if (resumed.length === 0) {
|
|
5485
|
+
this.unconditionalRechecks.add(runId);
|
|
5486
|
+
this.recheckAfterLease(runId, run.lockedUntil);
|
|
5487
|
+
}
|
|
5234
5488
|
return;
|
|
5235
5489
|
}
|
|
5236
5490
|
await this.resumeImplicit(runId).catch((err) => {
|
|
@@ -5524,6 +5778,50 @@ var WorkflowEngine = class {
|
|
|
5524
5778
|
return () => this.deadListeners.delete(listener);
|
|
5525
5779
|
}
|
|
5526
5780
|
/**
|
|
5781
|
+
* Delete buffered child completions no parent will ever consume (see
|
|
5782
|
+
* `childCompletionNeedsBuffer`): those of root runs, of children whose parent already settled them
|
|
5783
|
+
* or is gone, or terminal for good. Such rows were written by earlier versions, which buffered every
|
|
5784
|
+
* completion that found no waiter, and nothing else ever removes them.
|
|
5785
|
+
*
|
|
5786
|
+
* Walks `durable_buffered_signals` by primary key, {@link BUFFERED_SWEEP_PAGE} rows at a time and
|
|
5787
|
+
* at most {@link BUFFERED_SWEEP_PAGES_PER_CALL} pages per call, so a large backlog drains over a
|
|
5788
|
+
* few timer ticks without a long query or a big delete. A pass that reaches the end of the table
|
|
5789
|
+
* rests {@link BUFFERED_SWEEP_REST_MS} before the next one. Call it every tick (the NestJS timer
|
|
5790
|
+
* poller does). Skipped while draining, on a namespaced engine (a buffered signal carries no
|
|
5791
|
+
* namespace, and a scoped read would report another tenant's parent as gone), and on a store
|
|
5792
|
+
* without `listBufferedSignals`/`deleteBufferedSignals`. Returns how many rows it deleted.
|
|
5793
|
+
*/
|
|
5794
|
+
async sweepBufferedSignals(now = this.clock()) {
|
|
5795
|
+
const { listBufferedSignals, deleteBufferedSignals } = this.store;
|
|
5796
|
+
if (!listBufferedSignals || !deleteBufferedSignals) return 0;
|
|
5797
|
+
if (this.draining || this.namespace !== void 0 || this.bufferedSweepActive) return 0;
|
|
5798
|
+
if (this.bufferedSweepCursor === void 0 && now - this.lastBufferedSweepEndedAt < BUFFERED_SWEEP_REST_MS) {
|
|
5799
|
+
return 0;
|
|
5800
|
+
}
|
|
5801
|
+
this.bufferedSweepActive = true;
|
|
5802
|
+
let deleted = 0;
|
|
5803
|
+
try {
|
|
5804
|
+
for (let page = 0; page < BUFFERED_SWEEP_PAGES_PER_CALL; page++) {
|
|
5805
|
+
const rows = await listBufferedSignals.call(this.store, CHILD_TOKEN_PREFIX, this.bufferedSweepCursor, BUFFERED_SWEEP_PAGE);
|
|
5806
|
+
const doomed = await unneededChildSignals(this.store, rows);
|
|
5807
|
+
if (doomed.length > 0) {
|
|
5808
|
+
await deleteBufferedSignals.call(this.store, doomed.map((row) => row.id));
|
|
5809
|
+
deleted += doomed.length;
|
|
5810
|
+
}
|
|
5811
|
+
const last = rows.at(-1);
|
|
5812
|
+
if (rows.length < BUFFERED_SWEEP_PAGE || !last) {
|
|
5813
|
+
this.bufferedSweepCursor = void 0;
|
|
5814
|
+
this.lastBufferedSweepEndedAt = now;
|
|
5815
|
+
break;
|
|
5816
|
+
}
|
|
5817
|
+
this.bufferedSweepCursor = last.id;
|
|
5818
|
+
}
|
|
5819
|
+
} finally {
|
|
5820
|
+
this.bufferedSweepActive = false;
|
|
5821
|
+
}
|
|
5822
|
+
return deleted;
|
|
5823
|
+
}
|
|
5824
|
+
/**
|
|
5527
5825
|
* Register a pager hook for STRANDED runs: an in-flight run untouched past `stalledAfterMs` that is
|
|
5528
5826
|
* either suspended with NO wake timer (wake-forever: nothing will ever re-drive it) or parked on an
|
|
5529
5827
|
* old pending remote step with no live lease (dispatched and, most likely, lost). The timer poller
|
|
@@ -5753,15 +6051,27 @@ var WorkflowEngine = class {
|
|
|
5753
6051
|
* terminal too.
|
|
5754
6052
|
*/
|
|
5755
6053
|
resume(runId) {
|
|
6054
|
+
this.noteWakeReason(runId, void 0);
|
|
5756
6055
|
return this.resumeRun(runId, true);
|
|
5757
6056
|
}
|
|
6057
|
+
/** Record why `runId` is being woken: a step fan's barrier opening, or (`undefined`) anything else. */
|
|
6058
|
+
noteWakeReason(runId, fan) {
|
|
6059
|
+
const reason = this.wakeReasons.get(runId) ?? {
|
|
6060
|
+
generic: false,
|
|
6061
|
+
fans: /* @__PURE__ */ new Map()
|
|
6062
|
+
};
|
|
6063
|
+
if (fan) reason.fans.set(fan.first, Math.max(reason.fans.get(fan.first) ?? fan.seq, fan.seq));
|
|
6064
|
+
else reason.generic = true;
|
|
6065
|
+
this.wakeReasons.set(runId, reason);
|
|
6066
|
+
}
|
|
5758
6067
|
/**
|
|
5759
6068
|
* {@link resume} for every IMPLICIT wake-up — a delivered signal/child completion, a late step
|
|
5760
6069
|
* result, a due timer, a redelivered dispatch, a queue-slot wake. A `failed` run is terminal to
|
|
5761
6070
|
* these: only an explicit retry ({@link requeue} / {@link resume}) may re-drive it. Otherwise a
|
|
5762
6071
|
* stale waiter or a late result left behind by a failed run silently re-executes its body.
|
|
5763
6072
|
*/
|
|
5764
|
-
resumeImplicit(runId) {
|
|
6073
|
+
resumeImplicit(runId, fan) {
|
|
6074
|
+
if (fan !== "coalesced") this.noteWakeReason(runId, fan);
|
|
5765
6075
|
const turn = this.wakeTurns.get(runId);
|
|
5766
6076
|
if (turn) {
|
|
5767
6077
|
turn.again ??= deferred();
|
|
@@ -5786,7 +6096,7 @@ var WorkflowEngine = class {
|
|
|
5786
6096
|
});
|
|
5787
6097
|
return;
|
|
5788
6098
|
}
|
|
5789
|
-
const next = this.resumeImplicit(runId);
|
|
6099
|
+
const next = this.resumeImplicit(runId, "coalesced");
|
|
5790
6100
|
next.then(again.resolve, again.reject);
|
|
5791
6101
|
this.inBackground(next);
|
|
5792
6102
|
}, "finish");
|
|
@@ -7230,7 +7540,9 @@ var WorkflowEngine = class {
|
|
|
7230
7540
|
await this.deliverSignal(waiter, completion);
|
|
7231
7541
|
return;
|
|
7232
7542
|
}
|
|
7233
|
-
if (cancelled
|
|
7543
|
+
if (cancelled) return;
|
|
7544
|
+
const childId = token.slice(CHILD_TOKEN_PREFIX.length);
|
|
7545
|
+
if (!await childCompletionNeedsBuffer(this.store, childId, completion)) return;
|
|
7234
7546
|
await this.signal(token, completion);
|
|
7235
7547
|
return;
|
|
7236
7548
|
} catch (err) {
|
|
@@ -7244,24 +7556,6 @@ var WorkflowEngine = class {
|
|
|
7244
7556
|
}
|
|
7245
7557
|
}
|
|
7246
7558
|
/**
|
|
7247
|
-
* Whether the parent of the child behind `token` (`child:<id>`) already holds `completion` as that
|
|
7248
|
-
* child's settled `signal:child:<id>` checkpoint: its own turn settled the child from the child's
|
|
7249
|
-
* row, so this notification is a duplicate. A different completion (a fix-and-replay run's success
|
|
7250
|
-
* for a child the parent saw fail) is not, and is buffered for the parent's retry. Only for a child
|
|
7251
|
-
* whose id the engine minted (its parent is derivable from it).
|
|
7252
|
-
*/
|
|
7253
|
-
async parentHoldsCompletion(token, completion) {
|
|
7254
|
-
const childId = token.slice("child:".length);
|
|
7255
|
-
const parentId = parentRunIdOf(childId);
|
|
7256
|
-
if (!parentId) return false;
|
|
7257
|
-
const name = `${CHILD_SIGNAL_PREFIX}${childId}`;
|
|
7258
|
-
const checkpoints = this.store.listCheckpointsByNamePrefix ? await this.store.listCheckpointsByNamePrefix(parentId, [
|
|
7259
|
-
name
|
|
7260
|
-
]) : await this.store.listCheckpoints(parentId);
|
|
7261
|
-
const expected = JSON.stringify(completion);
|
|
7262
|
-
return checkpoints.some((cp) => cp.name === name && cp.status === "completed" && JSON.stringify(cp.output) === expected);
|
|
7263
|
-
}
|
|
7264
|
-
/**
|
|
7265
7559
|
* Deferred child start shared by the in-process ctx host and the remote `startChild` command.
|
|
7266
7560
|
* Deferred (microtask) so a fast child can't reentrantly resume a still-suspending parent. A start
|
|
7267
7561
|
* that THROWS (unregistered/unroutable workflow, input validation, singleton back-pressure, store
|
|
@@ -7276,15 +7570,22 @@ var WorkflowEngine = class {
|
|
|
7276
7570
|
startChildDeferred(workflow, input, childId, opts) {
|
|
7277
7571
|
this.localWork.submit(`start:${childId}`, async () => {
|
|
7278
7572
|
if (this.draining) return;
|
|
7279
|
-
await this.start(workflow, input, childId, opts).catch((err) => {
|
|
7573
|
+
const started = await this.start(workflow, input, childId, opts).catch((err) => {
|
|
7280
7574
|
const message = err instanceof Error ? err.message : String(err);
|
|
7281
7575
|
this.notifyParent(childId, {
|
|
7282
7576
|
ok: false,
|
|
7283
7577
|
error: `child workflow "${workflow}" failed to start: ${message}`
|
|
7284
7578
|
});
|
|
7579
|
+
return void 0;
|
|
7285
7580
|
});
|
|
7581
|
+
if (started) this.redeliverFinishedChild(childId, started);
|
|
7286
7582
|
});
|
|
7287
7583
|
}
|
|
7584
|
+
/** {@link startChildDeferred}: re-deliver the completion of a child `start` found finished. */
|
|
7585
|
+
redeliverFinishedChild(childId, run) {
|
|
7586
|
+
const completion = childCompletion(run);
|
|
7587
|
+
if (completion) this.inBackground(this.deliverToParent(`child:${childId}`, completion));
|
|
7588
|
+
}
|
|
7288
7589
|
/**
|
|
7289
7590
|
* Cancel a run (e.g. from the dashboard). Returns null if the run does not exist. Pass
|
|
7290
7591
|
* `{ compensate: true }` to undo the saga first: the suspended run is resumed so its completed
|
|
@@ -7860,9 +8161,26 @@ var WorkflowEngine = class {
|
|
|
7860
8161
|
async execute(run, fn) {
|
|
7861
8162
|
const registered = this.workflows.get(versionKey(run.workflow, run.workflowVersion)) ?? await this.findInheritedRegistration(run) ?? await this.resolveRemoteByConvention(run);
|
|
7862
8163
|
const lockNow = this.clock();
|
|
8164
|
+
const reason = this.wakeReasons.get(run.id) ?? {
|
|
8165
|
+
generic: true,
|
|
8166
|
+
fans: /* @__PURE__ */ new Map()
|
|
8167
|
+
};
|
|
8168
|
+
this.wakeReasons.delete(run.id);
|
|
7863
8169
|
if (run.lockedBy !== this.instanceId || (run.lockedUntil ?? 0) <= lockNow) {
|
|
7864
8170
|
if (!await this.store.tryLockRun(run.id, this.instanceId, lockNow + this.leaseMs, lockNow)) {
|
|
7865
|
-
if (
|
|
8171
|
+
if (reason.generic) {
|
|
8172
|
+
if (!this.genericWakeRequested.has(run.id)) {
|
|
8173
|
+
this.genericWakeRequested.add(run.id);
|
|
8174
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, null));
|
|
8175
|
+
}
|
|
8176
|
+
} else {
|
|
8177
|
+
const fans = {
|
|
8178
|
+
fans: [
|
|
8179
|
+
...reason.fans
|
|
8180
|
+
]
|
|
8181
|
+
};
|
|
8182
|
+
this.inBackground(this.requestWakeAfterTurn(run.id, fans));
|
|
8183
|
+
}
|
|
7866
8184
|
this.recheckAfterLease(run.id, run.lockedUntil);
|
|
7867
8185
|
return {
|
|
7868
8186
|
runId: run.id,
|
|
@@ -7870,56 +8188,142 @@ var WorkflowEngine = class {
|
|
|
7870
8188
|
};
|
|
7871
8189
|
}
|
|
7872
8190
|
}
|
|
8191
|
+
const settledFans = /* @__PURE__ */ new Map();
|
|
8192
|
+
this.turnSettledFans.set(run.id, settledFans);
|
|
7873
8193
|
const held = this.holdExecutionLease(run.id);
|
|
7874
8194
|
try {
|
|
8195
|
+
const served = /* @__PURE__ */ new Set();
|
|
8196
|
+
this.turnServedWakes.set(run.id, served);
|
|
8197
|
+
if (registered?.remote) await this.noteServedWakes(run.id);
|
|
7875
8198
|
const result = registered?.remote ? await this.runRemoteExecution(run, registered) : await this.runExecution(run, fn, registered);
|
|
7876
8199
|
if (registered?.singleton && (result.status === "completed" || result.status === "failed" || result.status === "cancelled" || result.status === "dead")) {
|
|
7877
8200
|
void this.singletons.wakeNext(run).catch(() => void 0);
|
|
7878
8201
|
}
|
|
7879
|
-
if (result.status === "suspended")
|
|
7880
|
-
|
|
7881
|
-
|
|
8202
|
+
if (result.status === "suspended") {
|
|
8203
|
+
this.inBackground(this.honorWakeRequests(run.id, settledFans, served));
|
|
8204
|
+
} else if (TERMINAL_RUN_STATUSES.includes(result.status)) {
|
|
8205
|
+
this.inBackground(this.listWakeRequests(run.id, true).then((kept) => this.dropWakeRequests(kept)));
|
|
7882
8206
|
}
|
|
7883
8207
|
return result;
|
|
7884
8208
|
} finally {
|
|
7885
8209
|
held.stop();
|
|
8210
|
+
if (this.turnSettledFans.get(run.id) === settledFans) this.turnSettledFans.delete(run.id);
|
|
8211
|
+
this.turnServedWakes.delete(run.id);
|
|
7886
8212
|
}
|
|
7887
8213
|
}
|
|
7888
8214
|
/**
|
|
7889
8215
|
* Keep a wake that found `runId`'s lease held by a turn executing elsewhere, durably, so that turn
|
|
7890
8216
|
* runs again once it ends instead of suspending on a snapshot that predates what the wake brought
|
|
7891
|
-
* (a signal, a child's completion). The request is a buffered
|
|
7892
|
-
* on
|
|
8217
|
+
* (a signal, a child's completion). The request is a buffered event under a name no workflow waits
|
|
8218
|
+
* on. It stays until a turn that started after it has settled: the holder looks at it after it
|
|
8219
|
+
* releases the lease ({@link honorWakeRequests}), and the turn that serves it drops it.
|
|
7893
8220
|
*
|
|
7894
8221
|
* Written first and looked at second, against a holder that releases first and looks second:
|
|
7895
8222
|
* whichever look runs last sees the other side's write. So when the run is no longer executing by
|
|
7896
|
-
* the time this re-reads it, the holder may have looked already, and this side
|
|
7897
|
-
*
|
|
8223
|
+
* the time this re-reads it, the holder may have looked already, and this side wakes the run
|
|
8224
|
+
* itself once the lease is free ({@link retryDroppedWake}). While the holder still runs, the run is
|
|
8225
|
+
* made due when the holder's lease runs out, so a holder that goes down before it settles leaves
|
|
8226
|
+
* the run to any instance's timer poll; a holder that settles sets the run's timer itself.
|
|
7898
8227
|
*/
|
|
7899
|
-
async requestWakeAfterTurn(runId) {
|
|
7900
|
-
|
|
8228
|
+
async requestWakeAfterTurn(runId, request) {
|
|
8229
|
+
const id = globalThis.crypto.randomUUID();
|
|
8230
|
+
await this.store.bufferEvent({
|
|
8231
|
+
name: wakeRequestToken(runId),
|
|
8232
|
+
payload: request,
|
|
8233
|
+
id,
|
|
8234
|
+
publishedAt: this.clock()
|
|
8235
|
+
});
|
|
7901
8236
|
const run = await this.store.getRun(runId);
|
|
7902
|
-
if (run
|
|
7903
|
-
|
|
7904
|
-
|
|
8237
|
+
if (!run || TERMINAL_RUN_STATUSES.includes(run.status)) {
|
|
8238
|
+
await this.store.removeBufferedEvent(id);
|
|
8239
|
+
return;
|
|
8240
|
+
}
|
|
8241
|
+
if (run.status === "running") return;
|
|
8242
|
+
if (run.lockedBy !== void 0 && run.lockedBy !== this.instanceId) {
|
|
8243
|
+
await this.makeDueBy(runId, (run.lockedUntil ?? this.clock()) + LEASE_RECHECK_SLACK_MS);
|
|
7905
8244
|
return;
|
|
7906
8245
|
}
|
|
7907
|
-
if (!await this.takeWakeRequests(runId)) return;
|
|
7908
|
-
if (TERMINAL_RUN_STATUSES.includes(run.status)) return;
|
|
7909
8246
|
this.recheckAfterLease(runId, run.lockedUntil);
|
|
7910
8247
|
}
|
|
7911
|
-
/**
|
|
7912
|
-
|
|
7913
|
-
|
|
8248
|
+
/**
|
|
8249
|
+
* After a turn that suspended released its lease: drop the wakes it served, and run again if a
|
|
8250
|
+
* wake was kept for it meanwhile, unless every such wake was a fan wake this turn already saw: it
|
|
8251
|
+
* read that fan with nothing in flight, at least as far as the batch that opened its barrier.
|
|
8252
|
+
* The run is made due before it is woken, so if this instance goes down before that turn runs,
|
|
8253
|
+
* the timer poll of another instance runs it once this instance's hold on it would have lapsed.
|
|
8254
|
+
*/
|
|
8255
|
+
async honorWakeRequests(runId, settledFans, served) {
|
|
8256
|
+
const kept = await this.listWakeRequests(runId, true);
|
|
8257
|
+
const seen = /* @__PURE__ */ __name((request) => wakeSeen(request, settledFans, served), "seen");
|
|
8258
|
+
const needed = kept.filter((request) => !seen(request));
|
|
8259
|
+
await this.dropWakeRequests(kept.filter(seen));
|
|
8260
|
+
if (needed.length === 0) return;
|
|
8261
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
7914
8262
|
await this.wake(runId).catch((err) => {
|
|
7915
8263
|
if (!(err instanceof NamespaceMismatch)) throw err;
|
|
7916
8264
|
});
|
|
7917
8265
|
}
|
|
7918
|
-
/**
|
|
7919
|
-
|
|
7920
|
-
|
|
7921
|
-
|
|
7922
|
-
|
|
8266
|
+
/**
|
|
8267
|
+
* The wake requests kept for `runId`. With `legacy`, also those an older engine kept as buffered
|
|
8268
|
+
* signals, which are taken as they are read.
|
|
8269
|
+
*/
|
|
8270
|
+
async listWakeRequests(runId, legacy = false) {
|
|
8271
|
+
const token = wakeRequestToken(runId);
|
|
8272
|
+
const kept = (await this.store.listBufferedEvents(token, WAKE_REQUEST_SCAN)).map((request) => ({
|
|
8273
|
+
id: request.id,
|
|
8274
|
+
payload: request.payload
|
|
8275
|
+
}));
|
|
8276
|
+
if (!legacy) return kept;
|
|
8277
|
+
for (; ; ) {
|
|
8278
|
+
const signal = await this.store.takeBufferedSignal(token);
|
|
8279
|
+
if (!signal) return kept;
|
|
8280
|
+
kept.push({
|
|
8281
|
+
payload: signal.payload
|
|
8282
|
+
});
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
8285
|
+
/** Delete `requests` (the ones kept as buffered events). */
|
|
8286
|
+
async dropWakeRequests(requests) {
|
|
8287
|
+
await Promise.all(requests.flatMap((request) => request.id === void 0 ? [] : [
|
|
8288
|
+
this.store.removeBufferedEvent(request.id)
|
|
8289
|
+
]));
|
|
8290
|
+
}
|
|
8291
|
+
/**
|
|
8292
|
+
* Record, for the turn executing `runId` here, the wakes kept for the run so far. Called before the
|
|
8293
|
+
* turn reads the run's state, so the turn serves them: they are dropped once it has settled, and
|
|
8294
|
+
* until then stay for whoever runs the run next.
|
|
8295
|
+
*/
|
|
8296
|
+
async noteServedWakes(runId) {
|
|
8297
|
+
const served = this.turnServedWakes.get(runId);
|
|
8298
|
+
if (!served) return;
|
|
8299
|
+
for (const request of await this.store.listBufferedEvents(wakeRequestToken(runId), WAKE_REQUEST_SCAN)) {
|
|
8300
|
+
served.add(request.id);
|
|
8301
|
+
}
|
|
8302
|
+
}
|
|
8303
|
+
/**
|
|
8304
|
+
* Right after a turn of `runId` wrote its suspension, while it still holds the lease: if a wake
|
|
8305
|
+
* was kept for the run during the turn that the turn did not see, keep the run due when the lease
|
|
8306
|
+
* runs out. The suspension overwrote the timer the instance keeping the wake set, and this instance
|
|
8307
|
+
* may go down before it looks at the wake once it has released the lease.
|
|
8308
|
+
*/
|
|
8309
|
+
async keepDueForKeptWakes(runId) {
|
|
8310
|
+
const served = this.turnServedWakes.get(runId);
|
|
8311
|
+
if (!served) return;
|
|
8312
|
+
const settledFans = this.turnSettledFans.get(runId) ?? /* @__PURE__ */ new Map();
|
|
8313
|
+
const kept = await this.listWakeRequests(runId);
|
|
8314
|
+
if (kept.every((request) => wakeSeen(request, settledFans, served))) return;
|
|
8315
|
+
await this.makeDueBy(runId, this.clock() + this.leaseMs);
|
|
8316
|
+
}
|
|
8317
|
+
/** Bring a suspended run's timer forward to `at` (never later than the one it has). */
|
|
8318
|
+
async makeDueBy(runId, at) {
|
|
8319
|
+
const run = await this.store.getRun(runId);
|
|
8320
|
+
if (run?.status !== "suspended") return;
|
|
8321
|
+
if (run.wakeAt !== void 0 && run.wakeAt <= at) return;
|
|
8322
|
+
await (this.store.updateRunIf ?? this.updateRunIfFallback).call(this.store, runId, [
|
|
8323
|
+
"suspended"
|
|
8324
|
+
], {
|
|
8325
|
+
wakeAt: at
|
|
8326
|
+
});
|
|
7923
8327
|
}
|
|
7924
8328
|
/**
|
|
7925
8329
|
* A wake that found the run leased by another instance is dropped, and normally that is fine: the
|
|
@@ -7938,6 +8342,7 @@ var WorkflowEngine = class {
|
|
|
7938
8342
|
const remaining = Math.max(0, (lockedUntil ?? this.clock()) - this.clock());
|
|
7939
8343
|
const timer = setTimeout(() => {
|
|
7940
8344
|
this.leaseRetries.delete(runId);
|
|
8345
|
+
this.genericWakeRequested.delete(runId);
|
|
7941
8346
|
this.inBackground(this.retryDroppedWake(runId, attempt));
|
|
7942
8347
|
}, Math.min(remaining, LEASE_RECHECK_MAX_MS) + LEASE_RECHECK_SLACK_MS);
|
|
7943
8348
|
timer.unref?.();
|
|
@@ -7946,12 +8351,15 @@ var WorkflowEngine = class {
|
|
|
7946
8351
|
async retryDroppedWake(runId, attempt) {
|
|
7947
8352
|
if (this.draining) return;
|
|
7948
8353
|
const run = await this.store.getRun(runId);
|
|
7949
|
-
if (run?.status !== "suspended") return;
|
|
7950
8354
|
const nowMs = this.clock();
|
|
7951
|
-
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) {
|
|
7952
|
-
|
|
8355
|
+
if (run?.status === "suspended" && run.lockedUntil !== void 0 && run.lockedUntil > nowMs && attempt < LEASE_RECHECK_ATTEMPTS) {
|
|
8356
|
+
this.recheckAfterLease(runId, run.lockedUntil, attempt + 1);
|
|
7953
8357
|
return;
|
|
7954
8358
|
}
|
|
8359
|
+
const unconditional = this.unconditionalRechecks.delete(runId);
|
|
8360
|
+
if (run?.status !== "suspended") return;
|
|
8361
|
+
if (run.lockedUntil !== void 0 && run.lockedUntil > nowMs) return;
|
|
8362
|
+
if (!unconditional && (await this.listWakeRequests(runId)).length === 0) return;
|
|
7955
8363
|
if (this.queuesRunsOverTransport) {
|
|
7956
8364
|
await this.wake(runId);
|
|
7957
8365
|
return;
|
|
@@ -8290,6 +8698,7 @@ ${run.workflowVersion}`;
|
|
|
8290
8698
|
updatedAt
|
|
8291
8699
|
});
|
|
8292
8700
|
if (!applied) return this.echoCurrentStatus(run.id);
|
|
8701
|
+
await this.keepDueForKeptWakes(run.id);
|
|
8293
8702
|
this.emit({
|
|
8294
8703
|
type: "run.suspended",
|
|
8295
8704
|
runId: run.id,
|
|
@@ -8889,10 +9298,13 @@ ${run.workflowVersion}`;
|
|
|
8889
9298
|
// dashboard renders the fan as one parallel group. Undefined for a lone (non-fan) child.
|
|
8890
9299
|
parallelGroup: cmd.parallelGroup
|
|
8891
9300
|
});
|
|
8892
|
-
|
|
9301
|
+
const existingChild = await this.store.getRun(childId);
|
|
9302
|
+
if (!existingChild) {
|
|
8893
9303
|
this.startChildDeferred(cmd.workflow, cmd.input, childId, {
|
|
8894
9304
|
namespace: run.namespace
|
|
8895
9305
|
});
|
|
9306
|
+
} else {
|
|
9307
|
+
this.redeliverFinishedChild(childId, existingChild);
|
|
8896
9308
|
}
|
|
8897
9309
|
} else {
|
|
8898
9310
|
throw new Error(`remote workflow command '${cmd.kind}' is not supported yet`);
|
|
@@ -8980,6 +9392,7 @@ ${run.workflowVersion}`;
|
|
|
8980
9392
|
namespace: run.namespace
|
|
8981
9393
|
});
|
|
8982
9394
|
}
|
|
9395
|
+
await this.noteServedWakes(run.id);
|
|
8983
9396
|
if (run.status !== "cancelling" && registered?.singleton && !await this.singletons.admit(run, registered.singleton)) {
|
|
8984
9397
|
return this.singletonWaitResult(run);
|
|
8985
9398
|
}
|
|
@@ -9653,6 +10066,14 @@ ${run.workflowVersion}`;
|
|
|
9653
10066
|
throw new WorkflowSuspended();
|
|
9654
10067
|
}
|
|
9655
10068
|
/**
|
|
10069
|
+
* Whether a `pending` step a turn just read only waits for its result: {@link callRemote} would
|
|
10070
|
+
* suspend on it without writing (no redispatch deadline to stamp, the one stamped not yet due).
|
|
10071
|
+
*/
|
|
10072
|
+
stepIsWaiting(cp) {
|
|
10073
|
+
if (this.remoteRedispatchMs == null) return true;
|
|
10074
|
+
return cp.wakeAt != null && this.clock() < cp.wakeAt;
|
|
10075
|
+
}
|
|
10076
|
+
/**
|
|
9656
10077
|
* `ctx.all` over a `@Step`: one durable step per input, dispatched together, joined once. The ctx
|
|
9657
10078
|
* reserves the whole position block up front, so replay re-derives the same seqs.
|
|
9658
10079
|
*
|
|
@@ -9682,6 +10103,7 @@ ${run.workflowVersion}`;
|
|
|
9682
10103
|
const failures = [];
|
|
9683
10104
|
const unsent = [];
|
|
9684
10105
|
let inFlight = 0;
|
|
10106
|
+
let highest = -1;
|
|
9685
10107
|
let wakeAt;
|
|
9686
10108
|
for (let i = 0; i < inputs.length; i += 1) {
|
|
9687
10109
|
const seq = positions[i];
|
|
@@ -9691,10 +10113,19 @@ ${run.workflowVersion}`;
|
|
|
9691
10113
|
continue;
|
|
9692
10114
|
}
|
|
9693
10115
|
if (cp.name !== step.name) throw new NonDeterminismError(runId, seq, step.name, cp.name);
|
|
10116
|
+
if (seq > highest) highest = seq;
|
|
9694
10117
|
if (cp.status === "completed") {
|
|
9695
10118
|
outputs[i] = cp.output;
|
|
9696
10119
|
continue;
|
|
9697
10120
|
}
|
|
10121
|
+
if (cp.status === "pending" && this.stepIsWaiting(cp)) {
|
|
10122
|
+
inFlight += 1;
|
|
10123
|
+
if (cp.wakeAt != null) {
|
|
10124
|
+
const at = this.stepLeaseWakeAt(cp.wakeAt);
|
|
10125
|
+
wakeAt = Math.min(wakeAt ?? at, at);
|
|
10126
|
+
}
|
|
10127
|
+
continue;
|
|
10128
|
+
}
|
|
9698
10129
|
try {
|
|
9699
10130
|
outputs[i] = await this.callRemote(runId, seq, step, inputs[i], void 0, opts.transport, void 0, opts.admission, {
|
|
9700
10131
|
...routing,
|
|
@@ -9722,6 +10153,7 @@ ${run.workflowVersion}`;
|
|
|
9722
10153
|
this.awaitedRemoteSteps.get(runId)?.delete(seq);
|
|
9723
10154
|
}
|
|
9724
10155
|
}
|
|
10156
|
+
if (inFlight === 0 && highest >= 0) this.turnSettledFans.get(runId)?.set(first, highest);
|
|
9725
10157
|
const batch = inFlight > 0 ? [] : opts.concurrency === void 0 ? unsent : unsent.slice(0, opts.concurrency);
|
|
9726
10158
|
if (batch.length > 0) await this.sendStepFan(runId, positions, step, inputs, batch, group, opts, routing);
|
|
9727
10159
|
if (inFlight + batch.length > 0) {
|
|
@@ -9752,53 +10184,62 @@ ${run.workflowVersion}`;
|
|
|
9752
10184
|
if (failures.length > 0) throw new GatherError(failures, inputs.length);
|
|
9753
10185
|
return outputs;
|
|
9754
10186
|
}
|
|
9755
|
-
/**
|
|
10187
|
+
/**
|
|
10188
|
+
* Write the `pending` checkpoints of a step fan's new steps, then dispatch them: one
|
|
10189
|
+
* {@link StateStore.saveCheckpoints} call and one {@link TransportPool.dispatchMany} for the
|
|
10190
|
+
* batch, where the store and transport have them.
|
|
10191
|
+
*/
|
|
9756
10192
|
async sendStepFan(runId, positions, step, inputs, batch, group, opts, routing) {
|
|
9757
10193
|
const workerGroup = stepGroup(step, routing?.namespace);
|
|
9758
10194
|
await this.ensureRoutable(workerGroup, step.requires);
|
|
9759
10195
|
const enqueuedAt = /* @__PURE__ */ new Date();
|
|
9760
|
-
|
|
9761
|
-
|
|
9762
|
-
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
9778
|
-
|
|
10196
|
+
const checkpoints = batch.map((i) => {
|
|
10197
|
+
const seq = positions[i];
|
|
10198
|
+
return {
|
|
10199
|
+
runId,
|
|
10200
|
+
seq,
|
|
10201
|
+
name: step.name,
|
|
10202
|
+
kind: "remote",
|
|
10203
|
+
stepId: stepId(runId, seq),
|
|
10204
|
+
status: "pending",
|
|
10205
|
+
input: inputs[i],
|
|
10206
|
+
attempts: 1,
|
|
10207
|
+
workerGroup,
|
|
10208
|
+
parallelGroup: group,
|
|
10209
|
+
enqueuedAt,
|
|
10210
|
+
startedAt: enqueuedAt,
|
|
10211
|
+
finishedAt: enqueuedAt
|
|
10212
|
+
};
|
|
10213
|
+
});
|
|
10214
|
+
if (this.store.saveCheckpoints) {
|
|
10215
|
+
await this.store.saveCheckpoints(checkpoints);
|
|
10216
|
+
} else {
|
|
10217
|
+
for (let at = 0; at < checkpoints.length; at += STEP_FAN_CHUNK) {
|
|
10218
|
+
await Promise.all(checkpoints.slice(at, at + STEP_FAN_CHUNK).map((cp) => this.store.saveCheckpoint(cp)));
|
|
10219
|
+
}
|
|
9779
10220
|
}
|
|
9780
|
-
|
|
9781
|
-
|
|
9782
|
-
|
|
9783
|
-
|
|
9784
|
-
|
|
9785
|
-
|
|
9786
|
-
|
|
9787
|
-
|
|
9788
|
-
|
|
9789
|
-
|
|
9790
|
-
|
|
9791
|
-
|
|
9792
|
-
|
|
9793
|
-
|
|
9794
|
-
|
|
9795
|
-
|
|
9796
|
-
|
|
9797
|
-
|
|
9798
|
-
|
|
9799
|
-
|
|
9800
|
-
|
|
9801
|
-
})
|
|
10221
|
+
const traceparent = this.traceparent?.();
|
|
10222
|
+
const context = this.context?.();
|
|
10223
|
+
await this.pool.dispatchMany(checkpoints.map((cp, k) => ({
|
|
10224
|
+
runId,
|
|
10225
|
+
seq: cp.seq,
|
|
10226
|
+
name: step.name,
|
|
10227
|
+
stepId: cp.stepId,
|
|
10228
|
+
group: workerGroup,
|
|
10229
|
+
input: inputs[batch[k]],
|
|
10230
|
+
traceparent,
|
|
10231
|
+
context,
|
|
10232
|
+
priority: opts.admission?.priority,
|
|
10233
|
+
attempt: 1
|
|
10234
|
+
})), opts.transport);
|
|
10235
|
+
for (const cp of checkpoints) {
|
|
10236
|
+
this.emit({
|
|
10237
|
+
type: "step.started",
|
|
10238
|
+
runId,
|
|
10239
|
+
seq: cp.seq,
|
|
10240
|
+
name: step.name,
|
|
10241
|
+
kind: "remote"
|
|
10242
|
+
});
|
|
9802
10243
|
}
|
|
9803
10244
|
}
|
|
9804
10245
|
/** Remember a step settled here, forgetting entries older than a lease so the map stays small. */
|
|
@@ -9839,6 +10280,27 @@ ${run.workflowVersion}`;
|
|
|
9839
10280
|
error: s.reason
|
|
9840
10281
|
});
|
|
9841
10282
|
}
|
|
10283
|
+
/**
|
|
10284
|
+
* The checkpoints of one run at distinct `seqs`, each as its own outcome: one
|
|
10285
|
+
* {@link StateStore.getCheckpoints} call when the store has it, else one read each. The batch
|
|
10286
|
+
* call rejecting falls back to one read each, so a failing read fails only its own entry.
|
|
10287
|
+
*/
|
|
10288
|
+
async readCheckpoints(runId, seqs) {
|
|
10289
|
+
if (seqs.length > 1 && this.store.getCheckpoints) {
|
|
10290
|
+
try {
|
|
10291
|
+
const bySeq = new Map((await this.store.getCheckpoints(runId, seqs)).map((cp) => [
|
|
10292
|
+
cp.seq,
|
|
10293
|
+
cp
|
|
10294
|
+
]));
|
|
10295
|
+
return seqs.map((seq) => ({
|
|
10296
|
+
status: "fulfilled",
|
|
10297
|
+
value: bySeq.get(seq) ?? null
|
|
10298
|
+
}));
|
|
10299
|
+
} catch {
|
|
10300
|
+
}
|
|
10301
|
+
}
|
|
10302
|
+
return Promise.allSettled(seqs.map((seq) => this.store.getCheckpoint(runId, seq)));
|
|
10303
|
+
}
|
|
9842
10304
|
/** Enqueue a remote step task to its worker group via the transport pool. Shared by the initial
|
|
9843
10305
|
* dispatch, the opt-in self-heal re-dispatch (see `remoteRedispatchMs`), and the explicit
|
|
9844
10306
|
* {@link redispatchPending}. Carries the current trace/context so a re-dispatch is traceable. */
|
|
@@ -9880,9 +10342,10 @@ ${run.workflowVersion}`;
|
|
|
9880
10342
|
*
|
|
9881
10343
|
* Results of one run are settled in batches: the first starts a batch, and the ones that arrive
|
|
9882
10344
|
* while it is being written form the next. Under load a wide fan's results are written by one
|
|
9883
|
-
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked
|
|
9884
|
-
*
|
|
9885
|
-
* transport acknowledges nothing that is not durable yet.
|
|
10345
|
+
* {@link StateStore.saveCheckpointsIfPending} call per batch, and the fan's barrier is asked only
|
|
10346
|
+
* by a batch with no more of the fan's results queued behind it. A result resolves only once its
|
|
10347
|
+
* own write has committed, so the transport acknowledges nothing that is not durable yet. How
|
|
10348
|
+
* large a batch gets is bounded by how many results the transport hands over at once.
|
|
9886
10349
|
*/
|
|
9887
10350
|
completeRemoteResult(result) {
|
|
9888
10351
|
return new Promise((resolve, reject) => {
|
|
@@ -9910,7 +10373,7 @@ ${run.workflowVersion}`;
|
|
|
9910
10373
|
const batch = queue.splice(0, RESULT_BATCH_MAX);
|
|
9911
10374
|
let outcomes;
|
|
9912
10375
|
try {
|
|
9913
|
-
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result));
|
|
10376
|
+
outcomes = await this.completeRemoteResults(runId, batch.map((q) => q.result), (from, to) => queue.some(({ result }) => result.seq >= from && result.seq <= to));
|
|
9914
10377
|
} catch (err) {
|
|
9915
10378
|
outcomes = batch.map(() => ({
|
|
9916
10379
|
error: err
|
|
@@ -9918,8 +10381,8 @@ ${run.workflowVersion}`;
|
|
|
9918
10381
|
}
|
|
9919
10382
|
batch.forEach((q, i) => {
|
|
9920
10383
|
const outcome = outcomes[i] ?? false;
|
|
9921
|
-
if (
|
|
9922
|
-
else q.
|
|
10384
|
+
if (isResultError(outcome)) q.reject(outcome.error);
|
|
10385
|
+
else q.resolve(outcome);
|
|
9923
10386
|
});
|
|
9924
10387
|
}
|
|
9925
10388
|
this.resultQueues.delete(runId);
|
|
@@ -9928,8 +10391,13 @@ ${run.workflowVersion}`;
|
|
|
9928
10391
|
* Settle a batch of one run's step results. At most one of them resolves `true` (resume the run):
|
|
9929
10392
|
* that resume replays every write of the batch, so a second would only replay it again. A result
|
|
9930
10393
|
* whose read or write failed resolves its error, and the transport redelivers just that one.
|
|
10394
|
+
*
|
|
10395
|
+
* `queuedIn(from, to)` says whether a result for a seq in that range is already waiting for this
|
|
10396
|
+
* run's next batch on this instance. A fan with one waiting is not asked its barrier: that result
|
|
10397
|
+
* asks it once its own batch is written (as a settle, or as a redelivery of a settled step), so
|
|
10398
|
+
* under a backlog the barrier is read once per drained queue rather than once per batch.
|
|
9931
10399
|
*/
|
|
9932
|
-
async completeRemoteResults(runId, results) {
|
|
10400
|
+
async completeRemoteResults(runId, results, queuedIn = () => false) {
|
|
9933
10401
|
const outcomes = results.map(() => false);
|
|
9934
10402
|
const seen = /* @__PURE__ */ new Set();
|
|
9935
10403
|
const live = results.flatMap((r, i) => {
|
|
@@ -9939,7 +10407,7 @@ ${run.workflowVersion}`;
|
|
|
9939
10407
|
i
|
|
9940
10408
|
];
|
|
9941
10409
|
});
|
|
9942
|
-
const reads = await
|
|
10410
|
+
const reads = await this.readCheckpoints(runId, live.map((i) => results[i].seq));
|
|
9943
10411
|
const pending = [];
|
|
9944
10412
|
const resettled = [];
|
|
9945
10413
|
reads.forEach((read, k) => {
|
|
@@ -9960,7 +10428,7 @@ ${run.workflowVersion}`;
|
|
|
9960
10428
|
});
|
|
9961
10429
|
if (pending.length === 0) {
|
|
9962
10430
|
if (resettled.length === 0) return outcomes;
|
|
9963
|
-
return this.resumeForRedelivered(runId, resettled, outcomes);
|
|
10431
|
+
return this.resumeForRedelivered(runId, resettled, outcomes, queuedIn);
|
|
9964
10432
|
}
|
|
9965
10433
|
for (const { cp } of pending) void this.releaseQueueSlot(cp.stepId).catch(() => void 0);
|
|
9966
10434
|
const run = await this.store.getRun(runId);
|
|
@@ -9987,7 +10455,9 @@ ${run.workflowVersion}`;
|
|
|
9987
10455
|
cp,
|
|
9988
10456
|
result: results[index]
|
|
9989
10457
|
});
|
|
9990
|
-
else if (claim !== false) outcomes[index] =
|
|
10458
|
+
else if (claim !== false) outcomes[index] = {
|
|
10459
|
+
error: claim.error
|
|
10460
|
+
};
|
|
9991
10461
|
});
|
|
9992
10462
|
for (const { cp, result } of settled) {
|
|
9993
10463
|
this.noteSettledHere(`${runId}:${result.seq}`);
|
|
@@ -10005,19 +10475,21 @@ ${run.workflowVersion}`;
|
|
|
10005
10475
|
});
|
|
10006
10476
|
}
|
|
10007
10477
|
const barriers = /* @__PURE__ */ new Map();
|
|
10008
|
-
|
|
10478
|
+
const wake = new FanWakeCollector();
|
|
10479
|
+
for (const { cp } of settled) wake.note(cp);
|
|
10009
10480
|
for (const { index, cp, result } of settled) {
|
|
10010
|
-
if (!await this.stepFanStillRunning(cp, result, barriers))
|
|
10481
|
+
if (!await this.stepFanStillRunning(cp, result, barriers, queuedIn)) {
|
|
10482
|
+
wake.add(index, cp, result.status);
|
|
10483
|
+
}
|
|
10011
10484
|
}
|
|
10012
|
-
|
|
10013
|
-
return outcomes;
|
|
10485
|
+
return wake.applyTo(outcomes);
|
|
10014
10486
|
}
|
|
10015
10487
|
/**
|
|
10016
10488
|
* The outcomes for a batch of redelivered results (their steps already settled): one of them
|
|
10017
10489
|
* resumes the run if it is `suspended` and, for a step of a fan, the fan's barrier is open. A
|
|
10018
10490
|
* resume of a run that had in fact moved on replays to its current wait and re-parks.
|
|
10019
10491
|
*/
|
|
10020
|
-
async resumeForRedelivered(runId, resettled, outcomes) {
|
|
10492
|
+
async resumeForRedelivered(runId, resettled, outcomes, queuedIn) {
|
|
10021
10493
|
const run = await this.store.getRun(runId);
|
|
10022
10494
|
if (run?.status !== "suspended") return outcomes;
|
|
10023
10495
|
const barriers = /* @__PURE__ */ new Map();
|
|
@@ -10028,9 +10500,11 @@ ${run.workflowVersion}`;
|
|
|
10028
10500
|
stepId: cp.stepId ?? stepId(runId, cp.seq),
|
|
10029
10501
|
status: cp.status === "failed" ? "failed" : "completed"
|
|
10030
10502
|
};
|
|
10031
|
-
if (!await this.stepFanStillRunning(cp, asResult, barriers)) {
|
|
10032
|
-
|
|
10033
|
-
|
|
10503
|
+
if (!await this.stepFanStillRunning(cp, asResult, barriers, queuedIn)) {
|
|
10504
|
+
const wake = new FanWakeCollector();
|
|
10505
|
+
for (const settled of resettled) wake.note(settled.cp);
|
|
10506
|
+
wake.add(index, cp, asResult.status);
|
|
10507
|
+
return wake.applyTo(outcomes);
|
|
10034
10508
|
}
|
|
10035
10509
|
}
|
|
10036
10510
|
return outcomes;
|
|
@@ -10041,15 +10515,17 @@ ${run.workflowVersion}`;
|
|
|
10041
10515
|
* retry needs the parent to schedule it). Without it every result would replay the whole parent,
|
|
10042
10516
|
* which makes a fan of N steps cost O(N²) checkpoint reads. `barriers` holds the answer per fan
|
|
10043
10517
|
* for the rest of a batch. A barrier that cannot be read counts as open: a spare resume replays and
|
|
10044
|
-
* re-suspends, a lost one strands the run.
|
|
10518
|
+
* re-suspends, a lost one strands the run. A fan with a result waiting in `queuedIn` is left to
|
|
10519
|
+
* that result (see {@link completeRemoteResults}).
|
|
10045
10520
|
*/
|
|
10046
|
-
async stepFanStillRunning(cp, result, barriers) {
|
|
10521
|
+
async stepFanStillRunning(cp, result, barriers, queuedIn) {
|
|
10047
10522
|
const fan = parseStepFanGroup(cp.parallelGroup);
|
|
10048
10523
|
if (!fan) return false;
|
|
10049
10524
|
if (result.status === "failed") return false;
|
|
10050
10525
|
let closed = barriers.get(fan.firstSeq);
|
|
10051
10526
|
if (!closed) {
|
|
10052
|
-
|
|
10527
|
+
const lastSeq = fan.firstSeq + fan.size - 1;
|
|
10528
|
+
closed = queuedIn(fan.firstSeq, lastSeq) ? Promise.resolve(true) : this.stepFanPending(result.runId, fan.firstSeq, lastSeq).then((pending) => pending > 0, () => false);
|
|
10053
10529
|
barriers.set(fan.firstSeq, closed);
|
|
10054
10530
|
}
|
|
10055
10531
|
if (await closed) return true;
|
|
@@ -10883,6 +11359,10 @@ var CodecStateStore = class {
|
|
|
10883
11359
|
saveCheckpointIfPending;
|
|
10884
11360
|
/** Encodes each checkpoint like {@link saveCheckpoint}. Bound only when the inner store has it. */
|
|
10885
11361
|
saveCheckpointsIfPending;
|
|
11362
|
+
/** Encodes each checkpoint written. Bound only when the inner store has it. */
|
|
11363
|
+
saveCheckpoints;
|
|
11364
|
+
/** Decodes each checkpoint read. Bound only when the inner store has it. */
|
|
11365
|
+
getCheckpoints;
|
|
10886
11366
|
/** Forwarded verbatim — a count reads no payload. Bound only when the inner store has it. */
|
|
10887
11367
|
countCheckpoints;
|
|
10888
11368
|
/** Forwarded verbatim — retention selects by status/updatedAt/scope, never by payload. Bound only
|
|
@@ -10903,6 +11383,10 @@ var CodecStateStore = class {
|
|
|
10903
11383
|
compactCheckpoints;
|
|
10904
11384
|
/** Decodes each run like {@link getRun}. Bound only when the inner store has it. */
|
|
10905
11385
|
getRuns;
|
|
11386
|
+
/** Decodes each payload like {@link takeBufferedSignal}. Bound only when the inner store has it. */
|
|
11387
|
+
listBufferedSignals;
|
|
11388
|
+
/** Forwarded verbatim — deletes by row id. */
|
|
11389
|
+
deleteBufferedSignals;
|
|
10906
11390
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
10907
11391
|
listSignalWaitersByRun;
|
|
10908
11392
|
/** Forwarded verbatim — a waiter carries no payload. */
|
|
@@ -10929,6 +11413,10 @@ var CodecStateStore = class {
|
|
|
10929
11413
|
if (saveIfPending) this.saveCheckpointIfPending = (cp) => saveIfPending.call(inner, this.encCp(cp));
|
|
10930
11414
|
const saveManyIfPending = inner.saveCheckpointsIfPending;
|
|
10931
11415
|
if (saveManyIfPending) this.saveCheckpointsIfPending = (cps) => saveManyIfPending.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11416
|
+
const saveMany = inner.saveCheckpoints;
|
|
11417
|
+
if (saveMany) this.saveCheckpoints = (cps) => saveMany.call(inner, cps.map((cp) => this.encCp(cp)));
|
|
11418
|
+
const getMany = inner.getCheckpoints;
|
|
11419
|
+
if (getMany) this.getCheckpoints = async (runId, seqs) => (await getMany.call(inner, runId, seqs)).map((cp) => this.decCp(cp));
|
|
10932
11420
|
const countCps = inner.countCheckpoints;
|
|
10933
11421
|
if (countCps) this.countCheckpoints = (runId, from, to, statuses) => countCps.call(inner, runId, from, to, statuses);
|
|
10934
11422
|
const stats = inner.checkpointStats;
|
|
@@ -10948,6 +11436,13 @@ var CodecStateStore = class {
|
|
|
10948
11436
|
if (compact) this.compactCheckpoints = (runId, fromSeq, toSeq, into) => compact.call(inner, runId, fromSeq, toSeq, this.encCp(into));
|
|
10949
11437
|
const getRuns = inner.getRuns;
|
|
10950
11438
|
if (getRuns) this.getRuns = async (runIds) => (await getRuns.call(inner, runIds)).map((r) => this.decRun(r));
|
|
11439
|
+
const listBuffered = inner.listBufferedSignals;
|
|
11440
|
+
if (listBuffered) this.listBufferedSignals = async (tokenPrefix, afterId, limit) => (await listBuffered.call(inner, tokenPrefix, afterId, limit)).map((row) => ({
|
|
11441
|
+
...row,
|
|
11442
|
+
payload: this.dec(row.payload)
|
|
11443
|
+
}));
|
|
11444
|
+
const deleteBuffered = inner.deleteBufferedSignals;
|
|
11445
|
+
if (deleteBuffered) this.deleteBufferedSignals = (ids) => deleteBuffered.call(inner, ids);
|
|
10951
11446
|
const byRun = inner.listSignalWaitersByRun;
|
|
10952
11447
|
if (byRun) this.listSignalWaitersByRun = (runIds) => byRun.call(inner, runIds);
|
|
10953
11448
|
const hasWaiter = inner.hasSignalWaiter;
|
|
@@ -11558,11 +12053,30 @@ var InMemoryStateStore = class {
|
|
|
11558
12053
|
...cp
|
|
11559
12054
|
} : null;
|
|
11560
12055
|
}
|
|
12056
|
+
async getCheckpoints(runId, seqs) {
|
|
12057
|
+
return [
|
|
12058
|
+
...new Set(seqs)
|
|
12059
|
+
].sort((a, b) => a - b).flatMap((seq) => {
|
|
12060
|
+
const cp = this.checkpoints.get(this.key(runId, seq));
|
|
12061
|
+
return cp ? [
|
|
12062
|
+
{
|
|
12063
|
+
...cp
|
|
12064
|
+
}
|
|
12065
|
+
] : [];
|
|
12066
|
+
});
|
|
12067
|
+
}
|
|
11561
12068
|
async saveCheckpoint(checkpoint) {
|
|
11562
12069
|
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
11563
12070
|
...checkpoint
|
|
11564
12071
|
});
|
|
11565
12072
|
}
|
|
12073
|
+
async saveCheckpoints(checkpoints) {
|
|
12074
|
+
for (const checkpoint of checkpoints) {
|
|
12075
|
+
this.checkpoints.set(this.key(checkpoint.runId, checkpoint.seq), {
|
|
12076
|
+
...checkpoint
|
|
12077
|
+
});
|
|
12078
|
+
}
|
|
12079
|
+
}
|
|
11566
12080
|
async saveCheckpointIfPending(checkpoint) {
|
|
11567
12081
|
const key = this.key(checkpoint.runId, checkpoint.seq);
|
|
11568
12082
|
if (this.checkpoints.get(key)?.status !== "pending") return false;
|
|
@@ -11693,20 +12207,47 @@ var InMemoryStateStore = class {
|
|
|
11693
12207
|
});
|
|
11694
12208
|
}
|
|
11695
12209
|
bufferedSignals = /* @__PURE__ */ new Map();
|
|
12210
|
+
bufferedSignalSeq = 0;
|
|
11696
12211
|
async bufferSignal(token, payload) {
|
|
11697
12212
|
const queue = this.bufferedSignals.get(token) ?? [];
|
|
11698
|
-
queue.push(
|
|
12213
|
+
queue.push({
|
|
12214
|
+
id: ++this.bufferedSignalSeq,
|
|
12215
|
+
payload
|
|
12216
|
+
});
|
|
11699
12217
|
this.bufferedSignals.set(token, queue);
|
|
11700
12218
|
}
|
|
11701
12219
|
async takeBufferedSignal(token) {
|
|
11702
12220
|
const queue = this.bufferedSignals.get(token);
|
|
11703
12221
|
if (!queue || queue.length === 0) return null;
|
|
11704
|
-
const
|
|
12222
|
+
const head = queue.shift();
|
|
11705
12223
|
if (queue.length === 0) this.bufferedSignals.delete(token);
|
|
11706
12224
|
return {
|
|
11707
|
-
payload
|
|
12225
|
+
payload: head.payload
|
|
11708
12226
|
};
|
|
11709
12227
|
}
|
|
12228
|
+
async listBufferedSignals(tokenPrefix, afterId, limit) {
|
|
12229
|
+
const after = afterId === void 0 ? 0 : Number(afterId);
|
|
12230
|
+
const rows = [];
|
|
12231
|
+
for (const [token, queue] of this.bufferedSignals) {
|
|
12232
|
+
if (!token.startsWith(tokenPrefix)) continue;
|
|
12233
|
+
for (const { id, payload } of queue) {
|
|
12234
|
+
if (id > after) rows.push({
|
|
12235
|
+
id: String(id),
|
|
12236
|
+
token,
|
|
12237
|
+
payload
|
|
12238
|
+
});
|
|
12239
|
+
}
|
|
12240
|
+
}
|
|
12241
|
+
return rows.sort((a, b) => Number(a.id) - Number(b.id)).slice(0, limit);
|
|
12242
|
+
}
|
|
12243
|
+
async deleteBufferedSignals(ids) {
|
|
12244
|
+
const doomed = new Set(ids.map(Number));
|
|
12245
|
+
for (const [token, queue] of this.bufferedSignals) {
|
|
12246
|
+
const kept = queue.filter((row) => !doomed.has(row.id));
|
|
12247
|
+
if (kept.length === 0) this.bufferedSignals.delete(token);
|
|
12248
|
+
else this.bufferedSignals.set(token, kept);
|
|
12249
|
+
}
|
|
12250
|
+
}
|
|
11710
12251
|
/** `id -> row`, so {@link removeBufferedEvent} can delete an exact row regardless of iteration
|
|
11711
12252
|
* order — mirrors the SQL stores' PK-by-id delete. */
|
|
11712
12253
|
bufferedEvents = /* @__PURE__ */ new Map();
|