@fabricorg/platform-host 8.2.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +134 -4
- package/dist/index.cjs +1193 -22
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +348 -11
- package/dist/index.d.ts +348 -11
- package/dist/index.js +1185 -24
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AdapterCircuitBreakerRegistry, AdapterRegistry, FABRIC_GOVERNANCE_CONTRACT_VERSION, aggregatePolicyOutcomes, assertActionFinalization, assertAdapterCircuitBreakerConfig, assertAdapterOutcomeConsistency, assertAdapterStepDeadlines, assertGovernanceRuntimeEvidence, assertMutationGovernanceContext, canonicalJson, createFabricId, evaluatePolicyDefinitions, evaluateStateMachineTransition, executeWithAdapterRetry, isAdapterCircuitOpenError, isReconciliationRequiredError, isRetryableExecutionError } from "@fabricorg/platform";
|
|
1
|
+
import { AdapterCircuitBreakerRegistry, AdapterRegistry, FABRIC_GOVERNANCE_CONTRACT_VERSION, ReconciliationRequiredError, aggregatePolicyOutcomes, assertActionFinalization, assertAdapterCircuitBreakerConfig, assertAdapterOutcomeConsistency, assertAdapterStepDeadlines, assertGovernanceRuntimeEvidence, assertMutationGovernanceContext, canonicalJson, createFabricId, evaluatePolicyDefinitions, evaluateStateMachineTransition, executeWithAdapterRetry, isAdapterCircuitOpenError, isReconciliationRequiredError, isRetryableExecutionError } from "@fabricorg/platform";
|
|
2
2
|
import { performance } from "perf_hooks";
|
|
3
3
|
import { assertAssemblyRuntimeCompatible } from "@fabricorg/assembly";
|
|
4
4
|
import { createHash } from "crypto";
|
|
@@ -35,6 +35,18 @@ var ActionAdmissionError = class extends Error {
|
|
|
35
35
|
this.name = "ActionAdmissionError";
|
|
36
36
|
}
|
|
37
37
|
};
|
|
38
|
+
/** A definitive authorization-boundary refusal before an invocation is created. */
|
|
39
|
+
var SubmissionRefusalError = class extends Error {
|
|
40
|
+
actionId;
|
|
41
|
+
refusal;
|
|
42
|
+
code = "ACTION_SUBMISSION_REFUSED";
|
|
43
|
+
constructor(actionId, refusal, message) {
|
|
44
|
+
super(message);
|
|
45
|
+
this.actionId = actionId;
|
|
46
|
+
this.refusal = refusal;
|
|
47
|
+
this.name = "SubmissionRefusalError";
|
|
48
|
+
}
|
|
49
|
+
};
|
|
38
50
|
/**
|
|
39
51
|
* Why a completion was not applied, typed so a webhook handler can answer
|
|
40
52
|
* the external system correctly: `retryable` means the invocation is still
|
|
@@ -128,7 +140,11 @@ function omitUndefinedOptionalFields(value, keys) {
|
|
|
128
140
|
return copy;
|
|
129
141
|
}
|
|
130
142
|
function validateFinalization(value, action, privateFields, contracts) {
|
|
131
|
-
const normalized = omitUndefinedOptionalFields(value, [
|
|
143
|
+
const normalized = omitUndefinedOptionalFields(value, [
|
|
144
|
+
"events",
|
|
145
|
+
"resultPatch",
|
|
146
|
+
"stagedInvocations"
|
|
147
|
+
]);
|
|
132
148
|
if (normalized !== value && normalized && typeof normalized === "object" && !Array.isArray(normalized)) {
|
|
133
149
|
const record = normalized;
|
|
134
150
|
if (Array.isArray(record.events)) {
|
|
@@ -138,7 +154,7 @@ function validateFinalization(value, action, privateFields, contracts) {
|
|
|
138
154
|
}
|
|
139
155
|
}
|
|
140
156
|
const result = finalizationJson(normalized);
|
|
141
|
-
if (!result || typeof result !== "object" || Array.isArray(result) || Object.keys(result).some((key) => key !== "events" && key !== "resultPatch")) throw new Error("Finalization must return only events and
|
|
157
|
+
if (!result || typeof result !== "object" || Array.isArray(result) || Object.keys(result).some((key) => key !== "events" && key !== "resultPatch" && key !== "stagedInvocations")) throw new Error("Finalization must return only events, resultPatch and stagedInvocations.");
|
|
142
158
|
if (result.resultPatch !== void 0) {
|
|
143
159
|
if (!result.resultPatch || typeof result.resultPatch !== "object" || Array.isArray(result.resultPatch)) throw new Error("Finalization resultPatch must be an object.");
|
|
144
160
|
if (Object.keys(result.resultPatch).some((key) => privateFields.includes(key))) throw new Error("Finalization resultPatch contains a private result key.");
|
|
@@ -1148,6 +1164,120 @@ function createOutboxRequeue(deps) {
|
|
|
1148
1164
|
};
|
|
1149
1165
|
}
|
|
1150
1166
|
//#endregion
|
|
1167
|
+
//#region src/staged-invocations.ts
|
|
1168
|
+
const MAX_STAGED_INVOCATIONS = 32;
|
|
1169
|
+
const MAX_STAGED_INVOCATION_KEY_LENGTH = 128;
|
|
1170
|
+
const MAX_STAGED_INVOCATION_BYTES = 65536;
|
|
1171
|
+
const MAX_STAGED_INVOCATION_BATCH_BYTES = 262144;
|
|
1172
|
+
const STAGED_INVOCATION_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/;
|
|
1173
|
+
/** Validate and normalize one immutable batch before its parent transaction commits. */
|
|
1174
|
+
function prepareStagedInvocationBatch(input) {
|
|
1175
|
+
if (!Array.isArray(input.requests)) throw new Error("stagedInvocations must be an array.");
|
|
1176
|
+
if (input.requests.length > 32) throw new Error(`stagedInvocations exceeds the 32 request limit.`);
|
|
1177
|
+
const normalized = input.requests.map((candidate, index) => {
|
|
1178
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate) || Object.keys(candidate).some((key) => ![
|
|
1179
|
+
"key",
|
|
1180
|
+
"actionId",
|
|
1181
|
+
"parameters"
|
|
1182
|
+
].includes(key))) throw new Error(`stagedInvocations[${index}] must contain only key, actionId and parameters.`);
|
|
1183
|
+
const request = {
|
|
1184
|
+
key: candidate.key,
|
|
1185
|
+
actionId: candidate.actionId,
|
|
1186
|
+
parameters: candidate.parameters
|
|
1187
|
+
};
|
|
1188
|
+
if (typeof request.key !== "string" || request.key.length === 0 || request.key.length > 128 || !STAGED_INVOCATION_KEY_PATTERN.test(request.key)) throw new Error(`stagedInvocations[${index}] key must match ${STAGED_INVOCATION_KEY_PATTERN} and be at most 128 characters.`);
|
|
1189
|
+
if (!(input.parentAction.stagesInvocations ?? []).includes(request.actionId)) throw new Error(`Action "${input.parentAction.actionId}" may not stage "${String(request.actionId)}".`);
|
|
1190
|
+
const child = input.resolveAction(request.actionId);
|
|
1191
|
+
if (!child) throw new Error(`Staged action "${String(request.actionId)}" is not registered.`);
|
|
1192
|
+
const parameters = finalizationJson(request.parameters);
|
|
1193
|
+
if (!parameters || typeof parameters !== "object" || Array.isArray(parameters)) throw new Error(`stagedInvocations[${index}] parameters must be a JSON object.`);
|
|
1194
|
+
if (!child.schema.safeParse(parameters).success) throw new Error(`stagedInvocations[${index}] parameters do not match action "${child.actionId}".`);
|
|
1195
|
+
let redactedParameters = parameters;
|
|
1196
|
+
if (input.redactParameters && !input.redactedKeys?.includes(request.key)) try {
|
|
1197
|
+
redactedParameters = input.redactParameters(child.actionId, structuredClone(parameters));
|
|
1198
|
+
} catch {
|
|
1199
|
+
throw new Error("Staged invocation parameter redaction failed.");
|
|
1200
|
+
}
|
|
1201
|
+
const durableParameters = finalizationJson(redactedParameters);
|
|
1202
|
+
if (!durableParameters || typeof durableParameters !== "object" || Array.isArray(durableParameters) || !child.schema.safeParse(durableParameters).success) throw new Error(`stagedInvocations[${index}] redacted parameters do not match action "${child.actionId}".`);
|
|
1203
|
+
if (byteLength(canonicalJson({
|
|
1204
|
+
key: request.key,
|
|
1205
|
+
actionId: child.actionId,
|
|
1206
|
+
parameters: durableParameters
|
|
1207
|
+
})) > 65536) throw new Error(`stagedInvocations[${index}] exceeds ${MAX_STAGED_INVOCATION_BYTES} bytes.`);
|
|
1208
|
+
return {
|
|
1209
|
+
key: request.key,
|
|
1210
|
+
actionId: child.actionId,
|
|
1211
|
+
expectedActionVersion: child.version,
|
|
1212
|
+
parameters: durableParameters,
|
|
1213
|
+
parameterDigest: digestParameters(durableParameters)
|
|
1214
|
+
};
|
|
1215
|
+
});
|
|
1216
|
+
for (const [index, request] of normalized.entries()) if (normalized.findIndex((candidate) => candidate.key === request.key) !== index) throw new Error(`stagedInvocations contains duplicate key "${request.key}".`);
|
|
1217
|
+
normalized.sort((left, right) => compareText(left.key, right.key) || compareText(left.actionId, right.actionId));
|
|
1218
|
+
const batchIdentity = normalized.map(({ key, actionId, expectedActionVersion, parameters, parameterDigest }) => ({
|
|
1219
|
+
key,
|
|
1220
|
+
actionId,
|
|
1221
|
+
expectedActionVersion,
|
|
1222
|
+
parameters,
|
|
1223
|
+
parameterDigest
|
|
1224
|
+
}));
|
|
1225
|
+
if (byteLength(canonicalJson(batchIdentity)) > 262144) throw new Error(`stagedInvocations batch exceeds ${MAX_STAGED_INVOCATION_BATCH_BYTES} bytes.`);
|
|
1226
|
+
const batchDigest = digestParameters({
|
|
1227
|
+
parentInvocationId: input.parent.id,
|
|
1228
|
+
requests: batchIdentity
|
|
1229
|
+
});
|
|
1230
|
+
return normalized.map((request) => ({
|
|
1231
|
+
id: lifecycleId("stg", input.parent.id, request.key),
|
|
1232
|
+
tenantId: input.parent.tenantId,
|
|
1233
|
+
spaceId: input.parent.spaceId,
|
|
1234
|
+
parentInvocationId: input.parent.id,
|
|
1235
|
+
parentActionId: input.parent.actionId,
|
|
1236
|
+
...request,
|
|
1237
|
+
batchDigest,
|
|
1238
|
+
status: "pending",
|
|
1239
|
+
attemptCount: 0,
|
|
1240
|
+
availableAt: new Date(input.now),
|
|
1241
|
+
leaseToken: 0,
|
|
1242
|
+
createdAt: new Date(input.now),
|
|
1243
|
+
updatedAt: new Date(input.now)
|
|
1244
|
+
}));
|
|
1245
|
+
}
|
|
1246
|
+
function stagedInvocationRequestIdentity(record) {
|
|
1247
|
+
return canonicalJson({
|
|
1248
|
+
id: record.id,
|
|
1249
|
+
tenantId: record.tenantId,
|
|
1250
|
+
spaceId: record.spaceId,
|
|
1251
|
+
parentInvocationId: record.parentInvocationId,
|
|
1252
|
+
parentActionId: record.parentActionId,
|
|
1253
|
+
key: record.key,
|
|
1254
|
+
actionId: record.actionId,
|
|
1255
|
+
expectedActionVersion: record.expectedActionVersion,
|
|
1256
|
+
parameters: record.parameters,
|
|
1257
|
+
parameterDigest: record.parameterDigest,
|
|
1258
|
+
batchDigest: record.batchDigest
|
|
1259
|
+
});
|
|
1260
|
+
}
|
|
1261
|
+
function cloneStagedInvocationRecord(record) {
|
|
1262
|
+
return {
|
|
1263
|
+
...structuredClone(record),
|
|
1264
|
+
availableAt: new Date(record.availableAt),
|
|
1265
|
+
...record.leaseExpiresAt ? { leaseExpiresAt: new Date(record.leaseExpiresAt) } : {},
|
|
1266
|
+
createdAt: new Date(record.createdAt),
|
|
1267
|
+
updatedAt: new Date(record.updatedAt)
|
|
1268
|
+
};
|
|
1269
|
+
}
|
|
1270
|
+
function asStagedInvocationStore(store) {
|
|
1271
|
+
const candidate = store;
|
|
1272
|
+
return candidate.transactionalStagedInvocations === true && typeof candidate.getActionInvocationByStagedOrigin === "function" && typeof candidate.createActionInvocationForStagedLease === "function" && typeof candidate.claimStagedInvocations === "function" && typeof candidate.markStagedInvocationDispatched === "function" && typeof candidate.markStagedInvocationRejected === "function" && typeof candidate.markStagedInvocationFailed === "function" && typeof candidate.renewStagedInvocationLease === "function" && typeof candidate.rotateStagedInvocationLease === "function" && typeof candidate.cancelStagedInvocationsForFailedParents === "function" && typeof candidate.listStagedInvocations === "function" ? candidate : void 0;
|
|
1273
|
+
}
|
|
1274
|
+
function byteLength(value) {
|
|
1275
|
+
return new TextEncoder().encode(value).byteLength;
|
|
1276
|
+
}
|
|
1277
|
+
function compareText(left, right) {
|
|
1278
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
1279
|
+
}
|
|
1280
|
+
//#endregion
|
|
1151
1281
|
//#region src/host.ts
|
|
1152
1282
|
const DEFAULT_EXTRACT_EVENTS = (data) => {
|
|
1153
1283
|
const value = data._events;
|
|
@@ -1172,6 +1302,25 @@ const ADMISSION_FAILED = Object.freeze({ kind: "failed" });
|
|
|
1172
1302
|
const ADMISSION_TIMED_OUT = Object.freeze({ kind: "timed_out" });
|
|
1173
1303
|
/** What an admission refusal's code may look like; anything else is `invalid_decision`. */
|
|
1174
1304
|
const ADMISSION_REFUSAL_CODE = /^[a-z][A-Za-z0-9_.:-]{0,127}$/;
|
|
1305
|
+
var StagedDispatchRejectionError = class extends Error {
|
|
1306
|
+
name = "StagedDispatchRejectionError";
|
|
1307
|
+
};
|
|
1308
|
+
var StagedDispatchLeaseLostError = class extends Error {
|
|
1309
|
+
name = "StagedDispatchLeaseLostError";
|
|
1310
|
+
};
|
|
1311
|
+
var StagedDispatcherHandoffError = class extends Error {
|
|
1312
|
+
name = "StagedDispatcherHandoffError";
|
|
1313
|
+
};
|
|
1314
|
+
var StagedSubmissionTimeoutError = class extends Error {
|
|
1315
|
+
name = "StagedSubmissionTimeoutError";
|
|
1316
|
+
};
|
|
1317
|
+
/** Fixed stage error texts: a stage records no caught error text and no parameters. */
|
|
1318
|
+
const STAGED_ERROR = Object.freeze({
|
|
1319
|
+
dispatch: "Staged invocation dispatch failed.",
|
|
1320
|
+
refused: "Staged invocation submission was definitively refused.",
|
|
1321
|
+
handoff: "Staged child dispatcher handoff was not confirmed.",
|
|
1322
|
+
lookup: "Staged child lookup failed."
|
|
1323
|
+
});
|
|
1175
1324
|
function createGovernedActionHost(options) {
|
|
1176
1325
|
const retryableExecutionMaxAttempts = options.retryableExecutionMaxAttempts ?? 5;
|
|
1177
1326
|
if (!Number.isInteger(retryableExecutionMaxAttempts) || retryableExecutionMaxAttempts <= 0) throw new Error("retryableExecutionMaxAttempts must be a positive integer.");
|
|
@@ -1201,6 +1350,9 @@ function createGovernedActionHost(options) {
|
|
|
1201
1350
|
const configuredActionResolver = options.resolveAction ?? options.registry?.resolveAction;
|
|
1202
1351
|
if (!configuredActionResolver) throw new Error("Platform Host requires an explicit module registry or resolveAction function.");
|
|
1203
1352
|
const actionResolver = configuredActionResolver;
|
|
1353
|
+
const stagedStore = asStagedInvocationStore(options.store);
|
|
1354
|
+
const declaresStaging = options.registry?.orderedModules?.some((module) => (module.actions ?? []).some((action) => (action.stagesInvocations?.length ?? 0) > 0)) ?? false;
|
|
1355
|
+
if (declaresStaging && (!stagedStore || !asAtomicMutationStore(options.store))) throw new Error("Staged descendant invocations require a store with transactionalStagedInvocations and transactionWithEvents capabilities.");
|
|
1204
1356
|
const stateMachineResolver = options.registry?.resolveStateMachine ?? (() => void 0);
|
|
1205
1357
|
const eventResultFields = options.eventResultFields ?? ["_events"];
|
|
1206
1358
|
const admissionTimeoutMs = options.admissionTimeoutMs ?? DEFAULT_ADMISSION_TIMEOUT_MS;
|
|
@@ -1209,8 +1361,12 @@ function createGovernedActionHost(options) {
|
|
|
1209
1361
|
return error instanceof HostError ? errorMessage(error) : redactErrorText(options.redactErrorText, context, errorMessage(error));
|
|
1210
1362
|
}
|
|
1211
1363
|
async function submitAction(input) {
|
|
1364
|
+
return submitActionInternal(input);
|
|
1365
|
+
}
|
|
1366
|
+
async function submitActionInternal(input, internal = {}) {
|
|
1212
1367
|
const read = snapshotSubmission(input);
|
|
1213
|
-
|
|
1368
|
+
const submittedExecutionReason = read.executionReason;
|
|
1369
|
+
if (submittedExecutionReason && !["initial", "offline_replay"].includes(submittedExecutionReason) && !(internal.staged && submittedExecutionReason === "staged_dispatch")) throw new Error(`Unsupported submitted execution reason: ${submittedExecutionReason}`);
|
|
1214
1370
|
const action = actionResolver(read.actionId);
|
|
1215
1371
|
if (!action) throw new Error(`Unknown action: ${read.actionId}`);
|
|
1216
1372
|
const declaresAdmission = action.admitSubmission !== void 0;
|
|
@@ -1218,15 +1374,15 @@ function createGovernedActionHost(options) {
|
|
|
1218
1374
|
if (action.execution?.connectivity === "online-required" && submission.executionReason === "offline_replay") throw new Error(`Action ${submission.actionId} is online-required and cannot be submitted as an offline replay.`);
|
|
1219
1375
|
if (action.execution?.sensitivity === "restricted" && Object.keys(submission.provenance?.auditAttributes ?? {}).length > 0) throw new Error(`Action ${submission.actionId} declares restricted sensitivity; provenance audit attributes must not be durably recorded.`);
|
|
1220
1376
|
const authorizationInput = toAuthorizationInput(action, submission);
|
|
1221
|
-
if (!await options.authorization.checkEntitlement(authorizationInput)) throw new
|
|
1222
|
-
if (!await options.authorization.authorize(authorizationInput)) throw new
|
|
1377
|
+
if (!await options.authorization.checkEntitlement(authorizationInput)) throw new SubmissionRefusalError(action.actionId, "entitlement", `Module "${action.namespace}" is not enabled for tenant ${submission.tenantId}`);
|
|
1378
|
+
if (!await options.authorization.authorize(authorizationInput)) throw new SubmissionRefusalError(action.actionId, "authorization", `Actor ${submission.actorId} is not authorized for action ${submission.actionId}`);
|
|
1223
1379
|
if (declaresAdmission && (!submission.parametersArePrivate || !isRecord(submission.parameters))) refuseAdmission(action, submission, "invalid_parameters");
|
|
1224
1380
|
const actionInvocationId = createFabricId("act");
|
|
1225
1381
|
if (submission.correlationId && submission.provenance?.correlationId && submission.correlationId !== submission.provenance.correlationId) throw new Error("Invocation provenance correlationId does not match submission correlationId.");
|
|
1226
1382
|
if (submission.causationId && submission.provenance?.causationId && submission.causationId !== submission.provenance.causationId) throw new Error("Invocation provenance causationId does not match submission causationId.");
|
|
1227
1383
|
const correlationId = submission.correlationId ?? submission.provenance?.correlationId ?? createFabricId("corr");
|
|
1228
1384
|
const prepare = () => {
|
|
1229
|
-
const durableParameters = options.redactActionParameters ? options.redactActionParameters(submission.actionId, submission.parameters) : submission.parameters;
|
|
1385
|
+
const durableParameters = !internal.parametersAlreadyRedacted && options.redactActionParameters ? options.redactActionParameters(submission.actionId, submission.parameters) : submission.parameters;
|
|
1230
1386
|
const parameterDigest = digestParameters(submission.parameters);
|
|
1231
1387
|
if (submission.authorizationBinding) validateAuthorizationBinding(submission, parameterDigest, action.execution?.authorityMoment, now());
|
|
1232
1388
|
return {
|
|
@@ -1247,7 +1403,7 @@ function createGovernedActionHost(options) {
|
|
|
1247
1403
|
assertGovernanceRuntimeEvidence(runtimeEvidence);
|
|
1248
1404
|
const admission = declaresAdmission ? await admitSubmission(action, submission) : void 0;
|
|
1249
1405
|
const { durableParameters, parameterDigest } = early ?? prepare();
|
|
1250
|
-
const
|
|
1406
|
+
const invocationInput = {
|
|
1251
1407
|
id: actionInvocationId,
|
|
1252
1408
|
tenantId: submission.tenantId,
|
|
1253
1409
|
spaceId: submission.spaceId,
|
|
@@ -1273,7 +1429,24 @@ function createGovernedActionHost(options) {
|
|
|
1273
1429
|
...submission.executionReason ? { executionReason: submission.executionReason } : {},
|
|
1274
1430
|
...submission.authorizationBindingId ? { authorizationBindingId: submission.authorizationBindingId } : {},
|
|
1275
1431
|
...admission ? { admittedAt: admission.admittedAt } : {}
|
|
1276
|
-
}
|
|
1432
|
+
};
|
|
1433
|
+
const staged = internal.staged;
|
|
1434
|
+
const durableInvocation = staged ? await (async () => {
|
|
1435
|
+
if (!stagedStore) throw new Error("Staged invocation store capability disappeared.");
|
|
1436
|
+
const created = await stagedStore.createActionInvocationForStagedLease({
|
|
1437
|
+
stageId: staged.id,
|
|
1438
|
+
workerId: staged.workerId,
|
|
1439
|
+
leaseToken: staged.leaseToken,
|
|
1440
|
+
now: now(),
|
|
1441
|
+
invocation: {
|
|
1442
|
+
...invocationInput,
|
|
1443
|
+
stagedOrigin: staged.origin
|
|
1444
|
+
}
|
|
1445
|
+
});
|
|
1446
|
+
if (created.kind === "lease_lost") throw new StagedDispatchLeaseLostError("Staged invocation lease lost before child creation.");
|
|
1447
|
+
staged.onChild(created.invocation);
|
|
1448
|
+
return created.invocation;
|
|
1449
|
+
})() : await options.store.createActionInvocation(invocationInput);
|
|
1277
1450
|
emitPlatformHostTelemetry(options.telemetry, {
|
|
1278
1451
|
kind: "event",
|
|
1279
1452
|
name: "invocation.submitted",
|
|
@@ -1307,6 +1480,7 @@ function createGovernedActionHost(options) {
|
|
|
1307
1480
|
}
|
|
1308
1481
|
}
|
|
1309
1482
|
const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
|
|
1483
|
+
if (internal.staged && durableInvocation.id !== actionInvocationId) return replayExistingStagedInvocation(durableInvocation);
|
|
1310
1484
|
if (durableInvocation.id !== actionInvocationId) {
|
|
1311
1485
|
if (options.dispatcher && durableInvocation.status === "pending") {
|
|
1312
1486
|
const dispatched = await options.dispatcher.dispatch({
|
|
@@ -1355,6 +1529,7 @@ function createGovernedActionHost(options) {
|
|
|
1355
1529
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
1356
1530
|
}, submission.actionId);
|
|
1357
1531
|
} catch (error) {
|
|
1532
|
+
if (internal.staged) throw new StagedDispatcherHandoffError("Staged child dispatcher handoff failed.");
|
|
1358
1533
|
throw error;
|
|
1359
1534
|
}
|
|
1360
1535
|
return {
|
|
@@ -1363,6 +1538,48 @@ function createGovernedActionHost(options) {
|
|
|
1363
1538
|
};
|
|
1364
1539
|
}
|
|
1365
1540
|
/**
|
|
1541
|
+
* Hand an existing staged child to the dispatcher again under its stable
|
|
1542
|
+
* workflow id, as the public replay branch above does, when the child is
|
|
1543
|
+
* still pending and a dispatcher is configured. Returns undefined when no
|
|
1544
|
+
* handoff is needed. Raw dispatcher errors are replaced by a fixed one.
|
|
1545
|
+
*/
|
|
1546
|
+
async function handOffExistingStagedChild(existing) {
|
|
1547
|
+
if (!options.dispatcher || existing.status !== "pending") return void 0;
|
|
1548
|
+
try {
|
|
1549
|
+
return await options.dispatcher.dispatch({
|
|
1550
|
+
actionInvocationId: existing.id,
|
|
1551
|
+
actionId: existing.actionId,
|
|
1552
|
+
tenantId: existing.tenantId,
|
|
1553
|
+
spaceId: existing.spaceId,
|
|
1554
|
+
workflowId: `action-invocation-${existing.id}`
|
|
1555
|
+
});
|
|
1556
|
+
} catch {
|
|
1557
|
+
throw new StagedDispatcherHandoffError("Staged child dispatcher handoff failed.");
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
async function replayExistingStagedInvocation(existing) {
|
|
1561
|
+
const workflowId = `action-invocation-${existing.id}`;
|
|
1562
|
+
const dispatched = await handOffExistingStagedChild(existing);
|
|
1563
|
+
if (dispatched) return withConsistency({
|
|
1564
|
+
actionInvocationId: existing.id,
|
|
1565
|
+
status: existing.status,
|
|
1566
|
+
workflowId: dispatched.workflowId,
|
|
1567
|
+
...dispatched.runId ? { runId: dispatched.runId } : {}
|
|
1568
|
+
}, existing.actionId);
|
|
1569
|
+
return withConsistency({
|
|
1570
|
+
actionInvocationId: existing.id,
|
|
1571
|
+
status: existing.status,
|
|
1572
|
+
workflowId,
|
|
1573
|
+
result: existing.result,
|
|
1574
|
+
...existing.error ? { error: existing.error } : {},
|
|
1575
|
+
...existing.hitlRoute ? { hitlRoute: existing.hitlRoute } : {},
|
|
1576
|
+
...existing.hitlRiskTier ? { hitlRiskTier: existing.hitlRiskTier } : {},
|
|
1577
|
+
...existing.authorizationReconciliation ? { reconciliation: existing.authorizationReconciliation } : {},
|
|
1578
|
+
...existing.adapterReconciliation ? { adapterReconciliation: existing.adapterReconciliation } : {},
|
|
1579
|
+
...existing.pendingCompletion ? { pendingCompletion: existing.pendingCompletion } : {}
|
|
1580
|
+
}, existing.actionId);
|
|
1581
|
+
}
|
|
1582
|
+
/**
|
|
1366
1583
|
* Run the action's `admitSubmission` check on a copy of the submission
|
|
1367
1584
|
* snapshot's parameters, with the snapshot's scope and actor, bounded by
|
|
1368
1585
|
* `admissionTimeoutMs`. Returns when it was admitted; anything but
|
|
@@ -1869,6 +2086,13 @@ function createGovernedActionHost(options) {
|
|
|
1869
2086
|
try {
|
|
1870
2087
|
assertAdapterStepDeadlines(action);
|
|
1871
2088
|
assertActionFinalization(action);
|
|
2089
|
+
if ((action.stagesInvocations?.length ?? 0) > 0) {
|
|
2090
|
+
const store = asAtomicMutationStore(options.store);
|
|
2091
|
+
if (!stagedStore || !store) throw new Error("Staged descendant invocations require an atomic staging-capable store.");
|
|
2092
|
+
await store.transactionWithEvents(async (transaction) => {
|
|
2093
|
+
if (typeof transaction.stageInvocations !== "function") throw new Error("Staged descendant invocations require transaction-scoped stageInvocations support.");
|
|
2094
|
+
});
|
|
2095
|
+
}
|
|
1872
2096
|
if (action.finalize) {
|
|
1873
2097
|
const store = asAtomicMutationStore(options.store);
|
|
1874
2098
|
if (!store) throw new Error("Action finalization requires atomic mutation transactions.");
|
|
@@ -2126,6 +2350,7 @@ function createGovernedActionHost(options) {
|
|
|
2126
2350
|
let finalizationInput;
|
|
2127
2351
|
let domainEvents = [];
|
|
2128
2352
|
let pendingTransitionEvent;
|
|
2353
|
+
let handlerStagedInvocations = replayCheckpoint?.stagedInvocations ?? [];
|
|
2129
2354
|
try {
|
|
2130
2355
|
if (replayCheckpoint) {
|
|
2131
2356
|
data = structuredClone(replayCheckpoint.result);
|
|
@@ -2220,6 +2445,24 @@ function createGovernedActionHost(options) {
|
|
|
2220
2445
|
};
|
|
2221
2446
|
if (!handlerResult.success) throw handlerResult.error != null ? new Error(handlerResult.error) : new HostError("Action handler failed");
|
|
2222
2447
|
const handlerData = handlerResult.data ?? {};
|
|
2448
|
+
const rawStagedInvocations = handlerResult.stagedInvocations ?? [];
|
|
2449
|
+
const stagedRecords = prepareStagedInvocationBatch({
|
|
2450
|
+
parent: invocation,
|
|
2451
|
+
parentAction: action,
|
|
2452
|
+
requests: rawStagedInvocations,
|
|
2453
|
+
resolveAction: actionResolver,
|
|
2454
|
+
redactParameters: options.redactActionParameters,
|
|
2455
|
+
now: now()
|
|
2456
|
+
});
|
|
2457
|
+
handlerStagedInvocations = stagedRecords.map((record) => ({
|
|
2458
|
+
key: record.key,
|
|
2459
|
+
actionId: record.actionId,
|
|
2460
|
+
parameters: structuredClone(record.parameters)
|
|
2461
|
+
}));
|
|
2462
|
+
if (stagedRecords.length > 0 && action.eventPhase !== "after_adapters") {
|
|
2463
|
+
if (!transaction?.stageInvocations) throw new Error("Staged descendant invocations require an atomic staging transaction.");
|
|
2464
|
+
await transaction.stageInvocations(stagedRecords);
|
|
2465
|
+
}
|
|
2223
2466
|
if (action.resultSchema && !action.finalize) {
|
|
2224
2467
|
const publicResult = withoutPrivateHostFields(handlerData, eventResultFields);
|
|
2225
2468
|
const parsedResult = action.resultSchema.safeParse(publicResult);
|
|
@@ -2270,7 +2513,8 @@ function createGovernedActionHost(options) {
|
|
|
2270
2513
|
handlerEvents: events,
|
|
2271
2514
|
result: preparedFinalizationInput.handlerData,
|
|
2272
2515
|
...transitionEvent ? { stateTransition: transitionEvent } : {},
|
|
2273
|
-
skipped
|
|
2516
|
+
skipped,
|
|
2517
|
+
...handlerStagedInvocations.length > 0 ? { stagedInvocations: handlerStagedInvocations } : {}
|
|
2274
2518
|
});
|
|
2275
2519
|
if (new TextEncoder().encode(canonicalJson(preparation)).byteLength > 262144) throw new HostError("Finalization checkpoint exceeds 256 KiB.");
|
|
2276
2520
|
if (executionLease) {
|
|
@@ -3069,6 +3313,19 @@ function createGovernedActionHost(options) {
|
|
|
3069
3313
|
if (action.resultSchema && !action.resultSchema.safeParse(result).success) throw new Error("Finalization merged result does not match resultSchema.");
|
|
3070
3314
|
finalPatch.result = result;
|
|
3071
3315
|
}
|
|
3316
|
+
const stagedRequests = [...handlerStagedInvocations, ...finalized?.stagedInvocations ?? []];
|
|
3317
|
+
if (stagedRequests.length > 0) {
|
|
3318
|
+
if (!transaction.stageInvocations) throw new Error("Staged descendant invocations require an atomic staging transaction.");
|
|
3319
|
+
await transaction.stageInvocations(prepareStagedInvocationBatch({
|
|
3320
|
+
parent: invocation,
|
|
3321
|
+
parentAction: action,
|
|
3322
|
+
requests: stagedRequests,
|
|
3323
|
+
resolveAction: actionResolver,
|
|
3324
|
+
redactParameters: options.redactActionParameters,
|
|
3325
|
+
redactedKeys: handlerStagedInvocations.map((request) => request.key),
|
|
3326
|
+
now: now()
|
|
3327
|
+
}));
|
|
3328
|
+
}
|
|
3072
3329
|
for (const [index, event] of domainEvents.entries()) await appendEvent(invocation, event, `domain:${index}`, action.version, transaction);
|
|
3073
3330
|
for (const [index, event] of (finalized?.events ?? []).entries()) await appendEvent(invocation, event, `finalize:${index}`, action.version, transaction);
|
|
3074
3331
|
if (pendingTransitionEvent) await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1, transaction);
|
|
@@ -3103,6 +3360,21 @@ function createGovernedActionHost(options) {
|
|
|
3103
3360
|
await hostOperation(() => persistExecutionInvocation(invocation, finalPatch));
|
|
3104
3361
|
}
|
|
3105
3362
|
} catch (error) {
|
|
3363
|
+
if (isReconciliationRequiredError(error) && error.message.startsWith("Staged invocation batch drift")) {
|
|
3364
|
+
const message = caughtError({
|
|
3365
|
+
actionId: action.actionId,
|
|
3366
|
+
source: "execution"
|
|
3367
|
+
}, error);
|
|
3368
|
+
await hostOperation(() => persistExecutionInvocation(invocation, {
|
|
3369
|
+
status: "reconciliation_required",
|
|
3370
|
+
error: message
|
|
3371
|
+
}));
|
|
3372
|
+
return withConsistency({
|
|
3373
|
+
actionInvocationId,
|
|
3374
|
+
status: "reconciliation_required",
|
|
3375
|
+
error: message
|
|
3376
|
+
}, action.actionId);
|
|
3377
|
+
}
|
|
3106
3378
|
if (action.finalize) {
|
|
3107
3379
|
if (error instanceof RecoverableFinalizationError) throw error;
|
|
3108
3380
|
if (!finalizationCallbackFailure || finalizationCallbackFailure.error !== error) throw new RecoverableFinalizationError(caughtError({
|
|
@@ -3399,8 +3671,282 @@ function createGovernedActionHost(options) {
|
|
|
3399
3671
|
actionResult,
|
|
3400
3672
|
withConsistency
|
|
3401
3673
|
});
|
|
3674
|
+
async function dispatchStagedInvocations(input) {
|
|
3675
|
+
if (!stagedStore) throw new Error("This Platform Host store does not support staged descendant invocations.");
|
|
3676
|
+
const leaseDurationMs = input.leaseDurationMs ?? 3e4;
|
|
3677
|
+
const submissionTimeoutMs = input.submissionTimeoutMs ?? 1e4;
|
|
3678
|
+
const maxAttempts = input.maxAttempts ?? 10;
|
|
3679
|
+
for (const [name, value] of [
|
|
3680
|
+
["leaseDurationMs", leaseDurationMs],
|
|
3681
|
+
["submissionTimeoutMs", submissionTimeoutMs],
|
|
3682
|
+
["maxAttempts", maxAttempts]
|
|
3683
|
+
]) if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer.`);
|
|
3684
|
+
const cancelled = await stagedStore.cancelStagedInvocationsForFailedParents({
|
|
3685
|
+
...input.tenantId ? { tenantId: input.tenantId } : {},
|
|
3686
|
+
...input.spaceId ? { spaceId: input.spaceId } : {},
|
|
3687
|
+
now: now()
|
|
3688
|
+
});
|
|
3689
|
+
const records = await stagedStore.claimStagedInvocations({
|
|
3690
|
+
workerId: input.workerId,
|
|
3691
|
+
leaseDurationMs,
|
|
3692
|
+
limit: input.limit ?? 10,
|
|
3693
|
+
...input.tenantId ? { tenantId: input.tenantId } : {},
|
|
3694
|
+
...input.spaceId ? { spaceId: input.spaceId } : {},
|
|
3695
|
+
now: now()
|
|
3696
|
+
});
|
|
3697
|
+
const result = {
|
|
3698
|
+
claimed: records.length,
|
|
3699
|
+
dispatched: 0,
|
|
3700
|
+
rejected: 0,
|
|
3701
|
+
failed: 0,
|
|
3702
|
+
deadLettered: 0,
|
|
3703
|
+
cancelled
|
|
3704
|
+
};
|
|
3705
|
+
const sweep = {
|
|
3706
|
+
store: stagedStore,
|
|
3707
|
+
input,
|
|
3708
|
+
leaseDurationMs,
|
|
3709
|
+
submissionTimeoutMs,
|
|
3710
|
+
maxAttempts
|
|
3711
|
+
};
|
|
3712
|
+
for (const record of records) {
|
|
3713
|
+
let outcome;
|
|
3714
|
+
try {
|
|
3715
|
+
outcome = await dispatchStagedRow(record, sweep);
|
|
3716
|
+
} catch (error) {
|
|
3717
|
+
reportStagedError(input, error);
|
|
3718
|
+
outcome = "unsettled";
|
|
3719
|
+
}
|
|
3720
|
+
if (outcome === "dispatched") result.dispatched += 1;
|
|
3721
|
+
else if (outcome === "rejected") result.rejected += 1;
|
|
3722
|
+
else {
|
|
3723
|
+
result.failed += 1;
|
|
3724
|
+
if (outcome === "dead_lettered") result.deadLettered += 1;
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
return result;
|
|
3728
|
+
}
|
|
3729
|
+
/**
|
|
3730
|
+
* Settle one claimed row through a single sequence:
|
|
3731
|
+
*
|
|
3732
|
+
* 1. Renew the claim's lease, then look the child up by staged origin. An
|
|
3733
|
+
* earlier attempt's creation still in flight holds an older lease token
|
|
3734
|
+
* and fails the creation fence, so this lookup is authoritative for
|
|
3735
|
+
* earlier attempts. A child found here goes to step 4: no
|
|
3736
|
+
* pre-submission checks, no governance.
|
|
3737
|
+
* 2. No child: pre-submission checks and the shared submission pipeline.
|
|
3738
|
+
* 3. Unless this attempt's own create returned the child, rotate the lease
|
|
3739
|
+
* token. Only this attempt's own timed-out submission could still create
|
|
3740
|
+
* the child, it holds the old token, and the store orders the rotation
|
|
3741
|
+
* after any create the fence already admitted. The lookup that follows is
|
|
3742
|
+
* authoritative; a child found there goes to step 4, otherwise the
|
|
3743
|
+
* outcome is classified as rejected, retry or dead letter.
|
|
3744
|
+
* 4. A known child that is pending is handed to the configured dispatcher
|
|
3745
|
+
* again under its stable workflow id; the stage settles with its id.
|
|
3746
|
+
*
|
|
3747
|
+
* Each settlement is one compare-and-set on the current owner and token. A
|
|
3748
|
+
* lookup that fails settles the row for retry, never terminally.
|
|
3749
|
+
*/
|
|
3750
|
+
async function dispatchStagedRow(record, sweep) {
|
|
3751
|
+
const { store, input, leaseDurationMs, maxAttempts } = sweep;
|
|
3752
|
+
const claimLease = {
|
|
3753
|
+
id: record.id,
|
|
3754
|
+
workerId: input.workerId,
|
|
3755
|
+
leaseToken: record.leaseToken
|
|
3756
|
+
};
|
|
3757
|
+
if (!await store.renewStagedInvocationLease({
|
|
3758
|
+
...claimLease,
|
|
3759
|
+
leaseDurationMs,
|
|
3760
|
+
now: now()
|
|
3761
|
+
})) return "unsettled";
|
|
3762
|
+
const prior = await lookUpStagedChild(record, sweep);
|
|
3763
|
+
if (prior.kind === "unreadable") return settleStagedRetry(record, claimLease, sweep, {
|
|
3764
|
+
error: STAGED_ERROR.lookup,
|
|
3765
|
+
terminal: false
|
|
3766
|
+
});
|
|
3767
|
+
if (prior.kind === "found") return settleExistingStagedChild(record, claimLease, prior.child, sweep);
|
|
3768
|
+
const known = {};
|
|
3769
|
+
let failure;
|
|
3770
|
+
let submitted = false;
|
|
3771
|
+
const stopHeartbeat = startStagedHeartbeat(store, claimLease, input, leaseDurationMs);
|
|
3772
|
+
try {
|
|
3773
|
+
await withSubmissionTimeout(submitStagedChild(record, claimLease, (child) => {
|
|
3774
|
+
known.child = child;
|
|
3775
|
+
}), sweep.submissionTimeoutMs);
|
|
3776
|
+
submitted = true;
|
|
3777
|
+
} catch (error) {
|
|
3778
|
+
failure = error;
|
|
3779
|
+
} finally {
|
|
3780
|
+
await stopHeartbeat();
|
|
3781
|
+
}
|
|
3782
|
+
const createdChild = known.child;
|
|
3783
|
+
if (createdChild) {
|
|
3784
|
+
if (submitted || !options.dispatcher) return settleStagedDispatched(claimLease, createdChild.id, sweep);
|
|
3785
|
+
return settleStagedRetry(record, claimLease, sweep, {
|
|
3786
|
+
error: STAGED_ERROR.handoff,
|
|
3787
|
+
childInvocationId: createdChild.id,
|
|
3788
|
+
terminal: record.attemptCount >= maxAttempts
|
|
3789
|
+
});
|
|
3790
|
+
}
|
|
3791
|
+
const rotatedToken = await store.rotateStagedInvocationLease({
|
|
3792
|
+
...claimLease,
|
|
3793
|
+
leaseDurationMs,
|
|
3794
|
+
now: now()
|
|
3795
|
+
});
|
|
3796
|
+
if (rotatedToken === void 0) return "unsettled";
|
|
3797
|
+
const lease = {
|
|
3798
|
+
...claimLease,
|
|
3799
|
+
leaseToken: rotatedToken
|
|
3800
|
+
};
|
|
3801
|
+
const after = await lookUpStagedChild(record, sweep);
|
|
3802
|
+
if (after.kind === "unreadable") return settleStagedRetry(record, lease, sweep, {
|
|
3803
|
+
error: STAGED_ERROR.lookup,
|
|
3804
|
+
terminal: false
|
|
3805
|
+
});
|
|
3806
|
+
if (after.kind === "found") return settleExistingStagedChild(record, lease, after.child, sweep);
|
|
3807
|
+
if (isDefinitiveStagedRefusal(failure)) return await store.markStagedInvocationRejected({
|
|
3808
|
+
...lease,
|
|
3809
|
+
error: STAGED_ERROR.refused,
|
|
3810
|
+
now: now()
|
|
3811
|
+
}) ? "rejected" : "unsettled";
|
|
3812
|
+
return settleStagedRetry(record, lease, sweep, {
|
|
3813
|
+
error: STAGED_ERROR.dispatch,
|
|
3814
|
+
terminal: record.attemptCount >= maxAttempts
|
|
3815
|
+
});
|
|
3816
|
+
}
|
|
3817
|
+
/** Pre-submission checks, then the shared submission pipeline under the claim's lease. */
|
|
3818
|
+
async function submitStagedChild(record, lease, onChild) {
|
|
3819
|
+
const parent = await options.store.getActionInvocation(record.parentInvocationId, record.tenantId, record.spaceId);
|
|
3820
|
+
if (!parent || parent.status !== "completed") throw new Error("Staged parent is not completed.");
|
|
3821
|
+
const child = actionResolver(record.actionId);
|
|
3822
|
+
if (!child) throw new StagedDispatchRejectionError("Staged action is no longer registered.");
|
|
3823
|
+
if (child.execution?.authorityMoment === "capture") throw new StagedDispatchRejectionError("Staged action now requires capture-time authority.");
|
|
3824
|
+
if (!child.schema.safeParse(structuredClone(record.parameters)).success) throw new StagedDispatchRejectionError("Staged parameters no longer match the registered action.");
|
|
3825
|
+
return submitActionInternal({
|
|
3826
|
+
actionId: record.actionId,
|
|
3827
|
+
parameters: structuredClone(record.parameters),
|
|
3828
|
+
tenantId: parent.tenantId,
|
|
3829
|
+
spaceId: parent.spaceId,
|
|
3830
|
+
actorId: parent.actorId,
|
|
3831
|
+
actorType: parent.actorType,
|
|
3832
|
+
correlationId: parent.correlationId,
|
|
3833
|
+
causationId: parent.id,
|
|
3834
|
+
provenance: {
|
|
3835
|
+
source: "worker",
|
|
3836
|
+
correlationId: parent.correlationId,
|
|
3837
|
+
causationId: parent.id
|
|
3838
|
+
},
|
|
3839
|
+
executionReason: "staged_dispatch"
|
|
3840
|
+
}, {
|
|
3841
|
+
staged: {
|
|
3842
|
+
id: record.id,
|
|
3843
|
+
origin: {
|
|
3844
|
+
parentInvocationId: parent.id,
|
|
3845
|
+
stageKey: record.key
|
|
3846
|
+
},
|
|
3847
|
+
workerId: lease.workerId,
|
|
3848
|
+
leaseToken: lease.leaseToken,
|
|
3849
|
+
onChild
|
|
3850
|
+
},
|
|
3851
|
+
parametersAlreadyRedacted: true
|
|
3852
|
+
});
|
|
3853
|
+
}
|
|
3854
|
+
async function lookUpStagedChild(record, sweep) {
|
|
3855
|
+
try {
|
|
3856
|
+
const child = await sweep.store.getActionInvocationByStagedOrigin(record.tenantId, record.spaceId, {
|
|
3857
|
+
parentInvocationId: record.parentInvocationId,
|
|
3858
|
+
stageKey: record.key
|
|
3859
|
+
});
|
|
3860
|
+
return child ? {
|
|
3861
|
+
kind: "found",
|
|
3862
|
+
child
|
|
3863
|
+
} : { kind: "absent" };
|
|
3864
|
+
} catch (error) {
|
|
3865
|
+
reportStagedError(sweep.input, error);
|
|
3866
|
+
return { kind: "unreadable" };
|
|
3867
|
+
}
|
|
3868
|
+
}
|
|
3869
|
+
async function settleExistingStagedChild(record, lease, child, sweep) {
|
|
3870
|
+
if (options.dispatcher && child.status === "pending") {
|
|
3871
|
+
const stopHeartbeat = startStagedHeartbeat(sweep.store, lease, sweep.input, sweep.leaseDurationMs);
|
|
3872
|
+
let handedOff = false;
|
|
3873
|
+
try {
|
|
3874
|
+
await withSubmissionTimeout(handOffExistingStagedChild(child), sweep.submissionTimeoutMs);
|
|
3875
|
+
handedOff = true;
|
|
3876
|
+
} catch {} finally {
|
|
3877
|
+
await stopHeartbeat();
|
|
3878
|
+
}
|
|
3879
|
+
if (!handedOff) return settleStagedRetry(record, lease, sweep, {
|
|
3880
|
+
error: STAGED_ERROR.handoff,
|
|
3881
|
+
childInvocationId: child.id,
|
|
3882
|
+
terminal: record.attemptCount >= sweep.maxAttempts
|
|
3883
|
+
});
|
|
3884
|
+
}
|
|
3885
|
+
return settleStagedDispatched(lease, child.id, sweep);
|
|
3886
|
+
}
|
|
3887
|
+
async function settleStagedDispatched(lease, childInvocationId, sweep) {
|
|
3888
|
+
return await sweep.store.markStagedInvocationDispatched({
|
|
3889
|
+
...lease,
|
|
3890
|
+
childInvocationId,
|
|
3891
|
+
now: now()
|
|
3892
|
+
}) ? "dispatched" : "unsettled";
|
|
3893
|
+
}
|
|
3894
|
+
async function settleStagedRetry(record, lease, sweep, settlement) {
|
|
3895
|
+
if (!await sweep.store.markStagedInvocationFailed({
|
|
3896
|
+
...lease,
|
|
3897
|
+
error: settlement.error,
|
|
3898
|
+
availableAt: new Date(now().getTime() + (sweep.input.retryDelayMs?.(record.attemptCount) ?? 1e3)),
|
|
3899
|
+
deadLetter: settlement.terminal,
|
|
3900
|
+
...settlement.childInvocationId ? { childInvocationId: settlement.childInvocationId } : {},
|
|
3901
|
+
now: now()
|
|
3902
|
+
})) return "unsettled";
|
|
3903
|
+
return settlement.terminal ? "dead_lettered" : "retry";
|
|
3904
|
+
}
|
|
3905
|
+
function isDefinitiveStagedRefusal(error) {
|
|
3906
|
+
return error instanceof SubmissionRefusalError || error instanceof StagedDispatchRejectionError || error instanceof ActionAdmissionError && (error.refusal === "refused" || error.refusal === "invalid_parameters");
|
|
3907
|
+
}
|
|
3908
|
+
function reportStagedError(input, error) {
|
|
3909
|
+
try {
|
|
3910
|
+
input.onError?.(error);
|
|
3911
|
+
} catch {}
|
|
3912
|
+
}
|
|
3913
|
+
/** Renew one lease token while a slow step runs; the returned stop awaits a renewal in flight. */
|
|
3914
|
+
function startStagedHeartbeat(store, lease, input, leaseDurationMs) {
|
|
3915
|
+
const intervalMs = input.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
3916
|
+
let stopped = false;
|
|
3917
|
+
let timer;
|
|
3918
|
+
let running;
|
|
3919
|
+
const schedule = () => {
|
|
3920
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
3921
|
+
};
|
|
3922
|
+
const tick = () => {
|
|
3923
|
+
running = store.renewStagedInvocationLease({
|
|
3924
|
+
...lease,
|
|
3925
|
+
leaseDurationMs,
|
|
3926
|
+
now: now()
|
|
3927
|
+
}).then(() => void 0).catch(() => void 0).finally(schedule);
|
|
3928
|
+
};
|
|
3929
|
+
schedule();
|
|
3930
|
+
return async () => {
|
|
3931
|
+
stopped = true;
|
|
3932
|
+
if (timer) clearTimeout(timer);
|
|
3933
|
+
await running;
|
|
3934
|
+
};
|
|
3935
|
+
}
|
|
3936
|
+
async function withSubmissionTimeout(submission, timeoutMs) {
|
|
3937
|
+
let timer;
|
|
3938
|
+
try {
|
|
3939
|
+
return await Promise.race([submission, new Promise((_, reject) => {
|
|
3940
|
+
timer = setTimeout(() => reject(new StagedSubmissionTimeoutError("Staged invocation submission timed out.")), timeoutMs);
|
|
3941
|
+
})]);
|
|
3942
|
+
} finally {
|
|
3943
|
+
if (timer) clearTimeout(timer);
|
|
3944
|
+
}
|
|
3945
|
+
}
|
|
3402
3946
|
return {
|
|
3403
3947
|
submitAction,
|
|
3948
|
+
hasStagedInvocationActions: declaresStaging,
|
|
3949
|
+
...stagedStore ? { dispatchStagedInvocations } : {},
|
|
3404
3950
|
executeInvocation,
|
|
3405
3951
|
resumeApprovedInvocation,
|
|
3406
3952
|
recordExecutionAttestation,
|
|
@@ -3730,6 +4276,7 @@ function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
|
|
|
3730
4276
|
var MemoryPlatformHostStore = class {
|
|
3731
4277
|
db;
|
|
3732
4278
|
transactionalOutbox;
|
|
4279
|
+
transactionalStagedInvocations;
|
|
3733
4280
|
adapterCommit;
|
|
3734
4281
|
transactionTail = Promise.resolve();
|
|
3735
4282
|
invocations = [];
|
|
@@ -3740,6 +4287,7 @@ var MemoryPlatformHostStore = class {
|
|
|
3740
4287
|
executionAttestations = [];
|
|
3741
4288
|
externalReconciliations = [];
|
|
3742
4289
|
outbox = [];
|
|
4290
|
+
stagedInvocations = [];
|
|
3743
4291
|
/**
|
|
3744
4292
|
* Requeue evidence is append-only audit history, so unlike the ledger
|
|
3745
4293
|
* arrays above, which tests have always inspected and seeded directly, it
|
|
@@ -3750,6 +4298,7 @@ var MemoryPlatformHostStore = class {
|
|
|
3750
4298
|
constructor(db, transactionProvider) {
|
|
3751
4299
|
this.db = db;
|
|
3752
4300
|
this.transactionalOutbox = transactionProvider !== void 0;
|
|
4301
|
+
this.transactionalStagedInvocations = transactionProvider !== void 0;
|
|
3753
4302
|
if (transactionProvider) this.transactionWithEvents = (run) => this.runTransactionWithEvents(run, transactionProvider);
|
|
3754
4303
|
}
|
|
3755
4304
|
async transaction(run) {
|
|
@@ -3775,6 +4324,7 @@ var MemoryPlatformHostStore = class {
|
|
|
3775
4324
|
const conditionalAdapters = [];
|
|
3776
4325
|
const leasedUpdates = [];
|
|
3777
4326
|
const pendingReconciliations = [];
|
|
4327
|
+
const pendingStagedBatches = [];
|
|
3778
4328
|
const appendPending = async (event, metadata) => {
|
|
3779
4329
|
if (this.events.some((candidate) => candidate.id === event.id) || pendingEvents.some((candidate) => candidate.event.id === event.id)) return;
|
|
3780
4330
|
pendingEvents.push({
|
|
@@ -3831,6 +4381,9 @@ var MemoryPlatformHostStore = class {
|
|
|
3831
4381
|
appendExternalReconciliation: async (record) => {
|
|
3832
4382
|
pendingReconciliations.push(record);
|
|
3833
4383
|
},
|
|
4384
|
+
stageInvocations: async (records) => {
|
|
4385
|
+
pendingStagedBatches.push(records.map(cloneStagedInvocationRecord));
|
|
4386
|
+
},
|
|
3834
4387
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
3835
4388
|
pendingUpdates.push({
|
|
3836
4389
|
id,
|
|
@@ -3884,6 +4437,15 @@ var MemoryPlatformHostStore = class {
|
|
|
3884
4437
|
rememberAppend(this.externalReconciliations, record.id);
|
|
3885
4438
|
await this.appendExternalReconciliation(record);
|
|
3886
4439
|
}
|
|
4440
|
+
for (const records of pendingStagedBatches) {
|
|
4441
|
+
const appended = this.applyStagedBatch(records);
|
|
4442
|
+
if (appended.length > 0) undoLedgerWrites.push(() => {
|
|
4443
|
+
for (const id of appended) {
|
|
4444
|
+
const index = this.stagedInvocations.findIndex((record) => record.id === id);
|
|
4445
|
+
if (index !== -1) this.stagedInvocations.splice(index, 1);
|
|
4446
|
+
}
|
|
4447
|
+
});
|
|
4448
|
+
}
|
|
3887
4449
|
for (const [id, patch, scope] of pendingAdapterUpdates) {
|
|
3888
4450
|
const record = await this.getAdapterInvocation(id, scope);
|
|
3889
4451
|
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
@@ -3934,9 +4496,50 @@ var MemoryPlatformHostStore = class {
|
|
|
3934
4496
|
this.invocations.push(record);
|
|
3935
4497
|
return record;
|
|
3936
4498
|
}
|
|
4499
|
+
async createActionInvocationForStagedLease(input) {
|
|
4500
|
+
const { invocation, now = /* @__PURE__ */ new Date() } = input;
|
|
4501
|
+
const existing = this.invocations.find((record) => record.tenantId === invocation.tenantId && record.spaceId === invocation.spaceId && record.stagedOrigin?.parentInvocationId === invocation.stagedOrigin.parentInvocationId && record.stagedOrigin.stageKey === invocation.stagedOrigin.stageKey);
|
|
4502
|
+
if (existing) return {
|
|
4503
|
+
kind: "existing",
|
|
4504
|
+
invocation: existing
|
|
4505
|
+
};
|
|
4506
|
+
if (!this.stagedInvocations.find((record) => record.id === input.stageId && record.tenantId === invocation.tenantId && record.spaceId === invocation.spaceId && record.parentInvocationId === invocation.stagedOrigin.parentInvocationId && record.key === invocation.stagedOrigin.stageKey && record.actionId === invocation.actionId && record.status === "pending" && record.leaseOwner === input.workerId && record.leaseToken === input.leaseToken && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt > now)) return { kind: "lease_lost" };
|
|
4507
|
+
const createdAt = /* @__PURE__ */ new Date();
|
|
4508
|
+
const record = {
|
|
4509
|
+
...invocation,
|
|
4510
|
+
attemptCount: 0,
|
|
4511
|
+
leaseToken: 0,
|
|
4512
|
+
createdAt,
|
|
4513
|
+
updatedAt: createdAt
|
|
4514
|
+
};
|
|
4515
|
+
this.invocations.push(record);
|
|
4516
|
+
return {
|
|
4517
|
+
kind: "created",
|
|
4518
|
+
invocation: record
|
|
4519
|
+
};
|
|
4520
|
+
}
|
|
4521
|
+
applyStagedBatch(records) {
|
|
4522
|
+
if (records.length === 0) return [];
|
|
4523
|
+
const first = records[0];
|
|
4524
|
+
const existing = this.stagedInvocations.filter((record) => record.tenantId === first.tenantId && record.spaceId === first.spaceId && record.parentInvocationId === first.parentInvocationId);
|
|
4525
|
+
if (existing.length > 0) {
|
|
4526
|
+
if (!(existing.length === records.length && records.every((record) => {
|
|
4527
|
+
const stored = existing.find((candidate) => candidate.key === record.key);
|
|
4528
|
+
return stored !== void 0 && stored.batchDigest === record.batchDigest && stagedInvocationRequestIdentity(stored) === stagedInvocationRequestIdentity(record);
|
|
4529
|
+
}))) throw new ReconciliationRequiredError(`Staged invocation batch drift for parent ${first.parentInvocationId}.`);
|
|
4530
|
+
return [];
|
|
4531
|
+
}
|
|
4532
|
+
const keys = records.map((record) => record.key);
|
|
4533
|
+
if (new Set(keys).size !== keys.length || records.some((record) => record.tenantId !== first.tenantId || record.spaceId !== first.spaceId || record.parentInvocationId !== first.parentInvocationId || record.batchDigest !== first.batchDigest)) throw new Error(`Invalid staged invocation batch for parent ${first.parentInvocationId}.`);
|
|
4534
|
+
for (const record of records) this.stagedInvocations.push(cloneStagedInvocationRecord(record));
|
|
4535
|
+
return records.map((record) => record.id);
|
|
4536
|
+
}
|
|
3937
4537
|
async getActionInvocation(id, tenantId, spaceId) {
|
|
3938
4538
|
return this.invocations.find((record) => record.id === id && record.tenantId === tenantId && record.spaceId === spaceId);
|
|
3939
4539
|
}
|
|
4540
|
+
async getActionInvocationByStagedOrigin(tenantId, spaceId, origin) {
|
|
4541
|
+
return this.invocations.find((record) => record.tenantId === tenantId && record.spaceId === spaceId && record.stagedOrigin?.parentInvocationId === origin.parentInvocationId && record.stagedOrigin.stageKey === origin.stageKey);
|
|
4542
|
+
}
|
|
3940
4543
|
async updateActionInvocation(id, tenantId, spaceId, patch) {
|
|
3941
4544
|
const record = this.invocations.find((row) => row.id === id && row.tenantId === tenantId && row.spaceId === spaceId);
|
|
3942
4545
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
@@ -4266,6 +4869,96 @@ var MemoryPlatformHostStore = class {
|
|
|
4266
4869
|
record.updatedAt = current;
|
|
4267
4870
|
return true;
|
|
4268
4871
|
}
|
|
4872
|
+
async claimStagedInvocations(input) {
|
|
4873
|
+
while (this.adapterCommit) await this.adapterCommit;
|
|
4874
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
4875
|
+
const eligible = this.stagedInvocations.filter((record) => {
|
|
4876
|
+
return this.invocations.find((candidate) => candidate.id === record.parentInvocationId && candidate.tenantId === record.tenantId && candidate.spaceId === record.spaceId)?.status === "completed" && record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId);
|
|
4877
|
+
}).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3)));
|
|
4878
|
+
for (const record of eligible) {
|
|
4879
|
+
record.leaseOwner = input.workerId;
|
|
4880
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
4881
|
+
record.leaseToken += 1;
|
|
4882
|
+
record.attemptCount += 1;
|
|
4883
|
+
record.updatedAt = current;
|
|
4884
|
+
}
|
|
4885
|
+
return eligible.map(cloneStagedInvocationRecord);
|
|
4886
|
+
}
|
|
4887
|
+
async markStagedInvocationDispatched(input) {
|
|
4888
|
+
return this.settleStagedInvocation(input, "dispatched", input.childInvocationId, void 0);
|
|
4889
|
+
}
|
|
4890
|
+
async markStagedInvocationRejected(input) {
|
|
4891
|
+
return this.settleStagedInvocation(input, "rejected", void 0, input.error);
|
|
4892
|
+
}
|
|
4893
|
+
async markStagedInvocationFailed(input) {
|
|
4894
|
+
const record = this.leasedStagedInvocation(input.id, input.workerId, input.leaseToken);
|
|
4895
|
+
if (!record) return false;
|
|
4896
|
+
record.status = input.deadLetter ? "dead_lettered" : "pending";
|
|
4897
|
+
record.lastError = input.error;
|
|
4898
|
+
if (input.childInvocationId) record.childInvocationId = input.childInvocationId;
|
|
4899
|
+
record.availableAt = new Date(input.availableAt);
|
|
4900
|
+
record.updatedAt = input.now ?? /* @__PURE__ */ new Date();
|
|
4901
|
+
delete record.leaseOwner;
|
|
4902
|
+
delete record.leaseExpiresAt;
|
|
4903
|
+
return true;
|
|
4904
|
+
}
|
|
4905
|
+
async renewStagedInvocationLease(input) {
|
|
4906
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
4907
|
+
const record = this.leasedStagedInvocation(input.id, input.workerId, input.leaseToken);
|
|
4908
|
+
if (!record || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
4909
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
4910
|
+
record.updatedAt = current;
|
|
4911
|
+
return true;
|
|
4912
|
+
}
|
|
4913
|
+
/** Await-free, like the fenced create above, so a create runs wholly before or after it. */
|
|
4914
|
+
async rotateStagedInvocationLease(input) {
|
|
4915
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
4916
|
+
const record = this.leasedStagedInvocation(input.id, input.workerId, input.leaseToken);
|
|
4917
|
+
if (!record || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return void 0;
|
|
4918
|
+
record.leaseToken += 1;
|
|
4919
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
4920
|
+
record.updatedAt = current;
|
|
4921
|
+
return record.leaseToken;
|
|
4922
|
+
}
|
|
4923
|
+
async cancelStagedInvocationsForFailedParents(input = {}) {
|
|
4924
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
4925
|
+
let cancelled = 0;
|
|
4926
|
+
for (const record of this.stagedInvocations) {
|
|
4927
|
+
if (record.status !== "pending" || input.tenantId && record.tenantId !== input.tenantId || input.spaceId && record.spaceId !== input.spaceId) continue;
|
|
4928
|
+
const parent = this.invocations.find((candidate) => candidate.id === record.parentInvocationId && candidate.tenantId === record.tenantId && candidate.spaceId === record.spaceId);
|
|
4929
|
+
if (!parent || ![
|
|
4930
|
+
"failed",
|
|
4931
|
+
"blocked_by_policy",
|
|
4932
|
+
"validation_failed"
|
|
4933
|
+
].includes(parent.status)) continue;
|
|
4934
|
+
record.status = "cancelled";
|
|
4935
|
+
record.lastError = `Parent invocation ended with ${parent.status}.`;
|
|
4936
|
+
record.updatedAt = current;
|
|
4937
|
+
delete record.leaseOwner;
|
|
4938
|
+
delete record.leaseExpiresAt;
|
|
4939
|
+
cancelled += 1;
|
|
4940
|
+
}
|
|
4941
|
+
return cancelled;
|
|
4942
|
+
}
|
|
4943
|
+
async listStagedInvocations(input) {
|
|
4944
|
+
return this.stagedInvocations.filter((record) => record.tenantId === input.tenantId && record.spaceId === input.spaceId && (input.parentInvocationId === void 0 || record.parentInvocationId === input.parentInvocationId) && (!input.statuses?.length || input.statuses.includes(record.status))).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).map(cloneStagedInvocationRecord);
|
|
4945
|
+
}
|
|
4946
|
+
leasedStagedInvocation(id, workerId, leaseToken) {
|
|
4947
|
+
return this.stagedInvocations.find((record) => record.id === id && record.status === "pending" && record.leaseOwner === workerId && record.leaseToken === leaseToken);
|
|
4948
|
+
}
|
|
4949
|
+
/** Mirrors PostgreSQL: a missing child id keeps the recorded one; `lastError` is replaced. */
|
|
4950
|
+
settleStagedInvocation(input, status, childInvocationId, lastError) {
|
|
4951
|
+
const record = this.leasedStagedInvocation(input.id, input.workerId, input.leaseToken);
|
|
4952
|
+
if (!record) return false;
|
|
4953
|
+
record.status = status;
|
|
4954
|
+
if (childInvocationId !== void 0) record.childInvocationId = childInvocationId;
|
|
4955
|
+
if (lastError !== void 0) record.lastError = lastError;
|
|
4956
|
+
else delete record.lastError;
|
|
4957
|
+
record.updatedAt = input.now ?? /* @__PURE__ */ new Date();
|
|
4958
|
+
delete record.leaseOwner;
|
|
4959
|
+
delete record.leaseExpiresAt;
|
|
4960
|
+
return true;
|
|
4961
|
+
}
|
|
4269
4962
|
async listEvents(tenantId, spaceId) {
|
|
4270
4963
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
|
|
4271
4964
|
}
|
|
@@ -4359,10 +5052,12 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4359
5052
|
sql;
|
|
4360
5053
|
transactionWithEvents;
|
|
4361
5054
|
transactionalOutbox;
|
|
5055
|
+
transactionalStagedInvocations;
|
|
4362
5056
|
constructor(db, sql, transactionProvider) {
|
|
4363
5057
|
this.db = db;
|
|
4364
5058
|
this.sql = sql;
|
|
4365
5059
|
this.transactionalOutbox = transactionProvider !== void 0;
|
|
5060
|
+
this.transactionalStagedInvocations = transactionProvider !== void 0;
|
|
4366
5061
|
if (transactionProvider) this.transactionWithEvents = async (run) => transactionProvider.run(async ({ db, sql }) => {
|
|
4367
5062
|
const scoped = new PostgresPlatformHostStore(db, sql);
|
|
4368
5063
|
return run({
|
|
@@ -4388,7 +5083,8 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4388
5083
|
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input),
|
|
4389
5084
|
updateAdapterInvocation: (id, patch, scope) => scoped.updateAdapterInvocation(id, patch, scope),
|
|
4390
5085
|
writeAdapterInvocation: (input) => scoped.writeAdapterInvocation(input),
|
|
4391
|
-
appendExternalReconciliation: (record) => scoped.appendExternalReconciliation(record)
|
|
5086
|
+
appendExternalReconciliation: (record) => scoped.appendExternalReconciliation(record),
|
|
5087
|
+
stageInvocations: (records) => scoped.stageInvocations(records)
|
|
4392
5088
|
});
|
|
4393
5089
|
});
|
|
4394
5090
|
}
|
|
@@ -4592,6 +5288,38 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4592
5288
|
version: 10,
|
|
4593
5289
|
name: "finalization_preparation_checkpoint",
|
|
4594
5290
|
sql: `ALTER TABLE fabric_platform.action_invocations ADD COLUMN IF NOT EXISTS finalization_checkpoint jsonb;`
|
|
5291
|
+
},
|
|
5292
|
+
{
|
|
5293
|
+
version: 11,
|
|
5294
|
+
name: "staged_descendant_invocations",
|
|
5295
|
+
sql: `
|
|
5296
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
5297
|
+
ADD COLUMN IF NOT EXISTS staged_parent_invocation_id text,
|
|
5298
|
+
ADD COLUMN IF NOT EXISTS staged_stage_key text;
|
|
5299
|
+
CREATE UNIQUE INDEX IF NOT EXISTS action_invocations_staged_origin_idx
|
|
5300
|
+
ON fabric_platform.action_invocations
|
|
5301
|
+
(tenant_id, space_id, staged_parent_invocation_id, staged_stage_key)
|
|
5302
|
+
WHERE staged_parent_invocation_id IS NOT NULL AND staged_stage_key IS NOT NULL;
|
|
5303
|
+
CREATE TABLE IF NOT EXISTS fabric_platform.staged_invocations (
|
|
5304
|
+
id text PRIMARY KEY,
|
|
5305
|
+
tenant_id text NOT NULL, space_id text NOT NULL,
|
|
5306
|
+
parent_invocation_id text NOT NULL, parent_action_id text NOT NULL,
|
|
5307
|
+
stage_key text NOT NULL, action_id text NOT NULL,
|
|
5308
|
+
expected_action_version integer NOT NULL,
|
|
5309
|
+
parameters jsonb NOT NULL, parameter_digest text NOT NULL, batch_digest text NOT NULL,
|
|
5310
|
+
status text NOT NULL, child_invocation_id text,
|
|
5311
|
+
attempt_count integer NOT NULL DEFAULT 0,
|
|
5312
|
+
available_at timestamptz NOT NULL,
|
|
5313
|
+
lease_owner text, lease_expires_at timestamptz,
|
|
5314
|
+
lease_token bigint NOT NULL DEFAULT 0,
|
|
5315
|
+
last_error text, created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL,
|
|
5316
|
+
UNIQUE (tenant_id, space_id, parent_invocation_id, stage_key)
|
|
5317
|
+
);
|
|
5318
|
+
CREATE INDEX IF NOT EXISTS staged_invocations_claim_idx
|
|
5319
|
+
ON fabric_platform.staged_invocations (status, available_at, lease_expires_at, created_at);
|
|
5320
|
+
CREATE INDEX IF NOT EXISTS staged_invocations_parent_idx
|
|
5321
|
+
ON fabric_platform.staged_invocations (tenant_id, space_id, parent_invocation_id, created_at);
|
|
5322
|
+
`
|
|
4595
5323
|
}
|
|
4596
5324
|
]);
|
|
4597
5325
|
}
|
|
@@ -4607,11 +5335,11 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4607
5335
|
try {
|
|
4608
5336
|
result = await this.sql.query(`INSERT INTO fabric_platform.action_invocations
|
|
4609
5337
|
(id,tenant_id,space_id,action_id,action_version,actor_id,actor_type,status,
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
5338
|
+
parameters,result,correlation_id,causation_id,idempotency_key,
|
|
5339
|
+
parameter_digest,parameter_digest_algorithm,idempotency_actor_id,
|
|
5340
|
+
idempotency_authorization_binding_id,invocation_provenance,authorization_binding,execution_reason,
|
|
5341
|
+
authorization_binding_id,error,runtime_evidence,${admittedColumn}created_at,updated_at)
|
|
5342
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10::jsonb,$11,$12,$13,$14,$15,$16,$17,$18::jsonb,$19::jsonb,$20,$21,$22,$23::jsonb,${admittedValue}$24,$24)
|
|
4615
5343
|
ON CONFLICT (tenant_id,space_id,action_id,idempotency_key)
|
|
4616
5344
|
WHERE idempotency_key IS NOT NULL DO UPDATE SET id=fabric_platform.action_invocations.id
|
|
4617
5345
|
RETURNING *`, [
|
|
@@ -4647,6 +5375,140 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4647
5375
|
}
|
|
4648
5376
|
return toActionRecord(result.rows[0]);
|
|
4649
5377
|
}
|
|
5378
|
+
async createActionInvocationForStagedLease(input) {
|
|
5379
|
+
const { invocation } = input;
|
|
5380
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
5381
|
+
const admitted = invocation.admittedAt !== void 0;
|
|
5382
|
+
const admittedColumn = admitted ? ",admitted_at" : "";
|
|
5383
|
+
const admittedValue = admitted ? ",$30" : "";
|
|
5384
|
+
const values = [
|
|
5385
|
+
invocation.id,
|
|
5386
|
+
invocation.tenantId,
|
|
5387
|
+
invocation.spaceId,
|
|
5388
|
+
invocation.actionId,
|
|
5389
|
+
invocation.actionVersion,
|
|
5390
|
+
invocation.actorId,
|
|
5391
|
+
invocation.actorType,
|
|
5392
|
+
invocation.status,
|
|
5393
|
+
JSON.stringify(invocation.parameters),
|
|
5394
|
+
JSON.stringify(invocation.result),
|
|
5395
|
+
invocation.correlationId,
|
|
5396
|
+
invocation.causationId ?? null,
|
|
5397
|
+
invocation.idempotencyKey ?? null,
|
|
5398
|
+
invocation.parameterDigest ?? null,
|
|
5399
|
+
invocation.parameterDigestAlgorithm ?? null,
|
|
5400
|
+
invocation.idempotencyActorId ?? null,
|
|
5401
|
+
invocation.idempotencyAuthorizationBindingId ?? null,
|
|
5402
|
+
invocation.provenance ? JSON.stringify(invocation.provenance) : null,
|
|
5403
|
+
invocation.authorizationBinding ? JSON.stringify(invocation.authorizationBinding) : null,
|
|
5404
|
+
invocation.executionReason ?? null,
|
|
5405
|
+
invocation.authorizationBindingId ?? null,
|
|
5406
|
+
invocation.error ?? null,
|
|
5407
|
+
invocation.runtimeEvidence ? JSON.stringify(invocation.runtimeEvidence) : null,
|
|
5408
|
+
invocation.stagedOrigin.parentInvocationId,
|
|
5409
|
+
invocation.stagedOrigin.stageKey,
|
|
5410
|
+
input.stageId,
|
|
5411
|
+
input.workerId,
|
|
5412
|
+
input.leaseToken,
|
|
5413
|
+
current,
|
|
5414
|
+
...admitted ? [invocation.admittedAt] : []
|
|
5415
|
+
];
|
|
5416
|
+
let result;
|
|
5417
|
+
try {
|
|
5418
|
+
result = await this.sql.query(`WITH fence AS (
|
|
5419
|
+
SELECT stage.id
|
|
5420
|
+
FROM fabric_platform.staged_invocations AS stage
|
|
5421
|
+
WHERE stage.id=$26 AND stage.tenant_id=$2 AND stage.space_id=$3
|
|
5422
|
+
AND stage.parent_invocation_id=$24 AND stage.stage_key=$25 AND stage.action_id=$4
|
|
5423
|
+
AND stage.status='pending' AND stage.lease_owner=$27 AND stage.lease_token=$28
|
|
5424
|
+
AND stage.lease_expires_at IS NOT NULL AND stage.lease_expires_at > $29
|
|
5425
|
+
FOR SHARE
|
|
5426
|
+
)
|
|
5427
|
+
INSERT INTO fabric_platform.action_invocations
|
|
5428
|
+
(id,tenant_id,space_id,action_id,action_version,actor_id,actor_type,status,
|
|
5429
|
+
parameters,result,correlation_id,causation_id,idempotency_key,parameter_digest,
|
|
5430
|
+
parameter_digest_algorithm,idempotency_actor_id,idempotency_authorization_binding_id,
|
|
5431
|
+
invocation_provenance,authorization_binding,execution_reason,authorization_binding_id,error,
|
|
5432
|
+
runtime_evidence,staged_parent_invocation_id,staged_stage_key${admittedColumn},created_at,updated_at)
|
|
5433
|
+
SELECT $1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10::jsonb,$11,$12,$13,$14,$15,$16,$17,
|
|
5434
|
+
$18::jsonb,$19::jsonb,$20,$21,$22,$23::jsonb,$24,$25${admittedValue},$29,$29
|
|
5435
|
+
FROM fence
|
|
5436
|
+
ON CONFLICT (tenant_id,space_id,staged_parent_invocation_id,staged_stage_key)
|
|
5437
|
+
WHERE staged_parent_invocation_id IS NOT NULL AND staged_stage_key IS NOT NULL
|
|
5438
|
+
DO UPDATE SET id=fabric_platform.action_invocations.id
|
|
5439
|
+
RETURNING *`, values);
|
|
5440
|
+
} catch (error) {
|
|
5441
|
+
if (error?.code === "42703" || error?.code === "42P01") throw new Error("A staged child needs PostgreSQL migration 11 (staged_descendant_invocations); run ensureSchema() before dispatching staged invocations.", { cause: error });
|
|
5442
|
+
throw error;
|
|
5443
|
+
}
|
|
5444
|
+
if (result.rows[0]) {
|
|
5445
|
+
const record = toActionRecord(result.rows[0]);
|
|
5446
|
+
return {
|
|
5447
|
+
kind: record.id === invocation.id ? "created" : "existing",
|
|
5448
|
+
invocation: record
|
|
5449
|
+
};
|
|
5450
|
+
}
|
|
5451
|
+
const existing = await this.getActionInvocationByStagedOrigin(invocation.tenantId, invocation.spaceId, invocation.stagedOrigin);
|
|
5452
|
+
return existing ? {
|
|
5453
|
+
kind: "existing",
|
|
5454
|
+
invocation: existing
|
|
5455
|
+
} : { kind: "lease_lost" };
|
|
5456
|
+
}
|
|
5457
|
+
async stageInvocations(records) {
|
|
5458
|
+
if (records.length === 0) return;
|
|
5459
|
+
const first = records[0];
|
|
5460
|
+
if (!(await this.sql.query(`SELECT id FROM fabric_platform.action_invocations
|
|
5461
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3 FOR UPDATE`, [
|
|
5462
|
+
first.parentInvocationId,
|
|
5463
|
+
first.tenantId,
|
|
5464
|
+
first.spaceId
|
|
5465
|
+
])).rows[0]) throw new Error(`ActionInvocation not found: ${first.parentInvocationId}`);
|
|
5466
|
+
const current = await this.sql.query(`SELECT * FROM fabric_platform.staged_invocations
|
|
5467
|
+
WHERE tenant_id=$1 AND space_id=$2 AND parent_invocation_id=$3
|
|
5468
|
+
ORDER BY stage_key`, [
|
|
5469
|
+
first.tenantId,
|
|
5470
|
+
first.spaceId,
|
|
5471
|
+
first.parentInvocationId
|
|
5472
|
+
]);
|
|
5473
|
+
if (current.rows.length > 0) {
|
|
5474
|
+
const stored = current.rows.map(toStagedInvocationRecord);
|
|
5475
|
+
if (!(stored.length === records.length && records.every((record) => {
|
|
5476
|
+
const candidate = stored.find((item) => item.key === record.key);
|
|
5477
|
+
return candidate !== void 0 && candidate.batchDigest === record.batchDigest && stagedInvocationRequestIdentity(candidate) === stagedInvocationRequestIdentity(record);
|
|
5478
|
+
}))) throw new ReconciliationRequiredError(`Staged invocation batch drift for parent ${first.parentInvocationId}.`);
|
|
5479
|
+
return;
|
|
5480
|
+
}
|
|
5481
|
+
for (const record of records) {
|
|
5482
|
+
if (record.tenantId !== first.tenantId || record.spaceId !== first.spaceId || record.parentInvocationId !== first.parentInvocationId || record.batchDigest !== first.batchDigest) throw new Error(`Invalid staged invocation batch for parent ${first.parentInvocationId}.`);
|
|
5483
|
+
await this.sql.query(`INSERT INTO fabric_platform.staged_invocations
|
|
5484
|
+
(id,tenant_id,space_id,parent_invocation_id,parent_action_id,stage_key,action_id,
|
|
5485
|
+
expected_action_version,parameters,parameter_digest,batch_digest,status,child_invocation_id,
|
|
5486
|
+
attempt_count,available_at,lease_owner,lease_expires_at,lease_token,last_error,created_at,updated_at)
|
|
5487
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9::jsonb,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21)`, [
|
|
5488
|
+
record.id,
|
|
5489
|
+
record.tenantId,
|
|
5490
|
+
record.spaceId,
|
|
5491
|
+
record.parentInvocationId,
|
|
5492
|
+
record.parentActionId,
|
|
5493
|
+
record.key,
|
|
5494
|
+
record.actionId,
|
|
5495
|
+
record.expectedActionVersion,
|
|
5496
|
+
JSON.stringify(record.parameters),
|
|
5497
|
+
record.parameterDigest,
|
|
5498
|
+
record.batchDigest,
|
|
5499
|
+
record.status,
|
|
5500
|
+
record.childInvocationId ?? null,
|
|
5501
|
+
record.attemptCount,
|
|
5502
|
+
record.availableAt,
|
|
5503
|
+
record.leaseOwner ?? null,
|
|
5504
|
+
record.leaseExpiresAt ?? null,
|
|
5505
|
+
record.leaseToken,
|
|
5506
|
+
record.lastError ?? null,
|
|
5507
|
+
record.createdAt,
|
|
5508
|
+
record.updatedAt
|
|
5509
|
+
]);
|
|
5510
|
+
}
|
|
5511
|
+
}
|
|
4650
5512
|
async getActionInvocation(id, tenantId, spaceId) {
|
|
4651
5513
|
const result = await this.sql.query(`SELECT * FROM fabric_platform.action_invocations
|
|
4652
5514
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`, [
|
|
@@ -4656,6 +5518,17 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
4656
5518
|
]);
|
|
4657
5519
|
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
4658
5520
|
}
|
|
5521
|
+
async getActionInvocationByStagedOrigin(tenantId, spaceId, origin) {
|
|
5522
|
+
const result = await this.sql.query(`SELECT * FROM fabric_platform.action_invocations
|
|
5523
|
+
WHERE tenant_id=$1 AND space_id=$2
|
|
5524
|
+
AND staged_parent_invocation_id=$3 AND staged_stage_key=$4`, [
|
|
5525
|
+
tenantId,
|
|
5526
|
+
spaceId,
|
|
5527
|
+
origin.parentInvocationId,
|
|
5528
|
+
origin.stageKey
|
|
5529
|
+
]);
|
|
5530
|
+
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
5531
|
+
}
|
|
4659
5532
|
async updateActionInvocation(id, tenantId, spaceId, patch) {
|
|
4660
5533
|
await this.sql.query(`UPDATE fabric_platform.action_invocations SET
|
|
4661
5534
|
status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
|
|
@@ -5186,6 +6059,139 @@ var PostgresPlatformHostStore = class PostgresPlatformHostStore {
|
|
|
5186
6059
|
input.outboxId ?? null
|
|
5187
6060
|
])).rows.map((row) => toOutboxRequeueRecord(row, ""));
|
|
5188
6061
|
}
|
|
6062
|
+
async claimStagedInvocations(input) {
|
|
6063
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
6064
|
+
const leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
6065
|
+
const limit = Math.max(1, Math.min(input.limit ?? 100, 1e3));
|
|
6066
|
+
return (await this.sql.query(`WITH claimable AS (
|
|
6067
|
+
SELECT stage.id
|
|
6068
|
+
FROM fabric_platform.staged_invocations AS stage
|
|
6069
|
+
JOIN fabric_platform.action_invocations AS parent
|
|
6070
|
+
ON parent.id=stage.parent_invocation_id
|
|
6071
|
+
AND parent.tenant_id=stage.tenant_id AND parent.space_id=stage.space_id
|
|
6072
|
+
WHERE stage.status='pending' AND parent.status='completed'
|
|
6073
|
+
AND stage.available_at <= $1
|
|
6074
|
+
AND (stage.lease_expires_at IS NULL OR stage.lease_expires_at <= $1)
|
|
6075
|
+
AND ($2::text IS NULL OR stage.tenant_id=$2)
|
|
6076
|
+
AND ($3::text IS NULL OR stage.space_id=$3)
|
|
6077
|
+
ORDER BY stage.created_at,stage.id
|
|
6078
|
+
FOR UPDATE OF stage SKIP LOCKED LIMIT $4
|
|
6079
|
+
)
|
|
6080
|
+
UPDATE fabric_platform.staged_invocations AS stage
|
|
6081
|
+
SET lease_owner=$5, lease_expires_at=$6, lease_token=stage.lease_token+1,
|
|
6082
|
+
attempt_count=stage.attempt_count+1, updated_at=$1
|
|
6083
|
+
FROM claimable WHERE stage.id=claimable.id
|
|
6084
|
+
RETURNING stage.*`, [
|
|
6085
|
+
current,
|
|
6086
|
+
input.tenantId ?? null,
|
|
6087
|
+
input.spaceId ?? null,
|
|
6088
|
+
limit,
|
|
6089
|
+
input.workerId,
|
|
6090
|
+
leaseExpiresAt
|
|
6091
|
+
])).rows.map(toStagedInvocationRecord);
|
|
6092
|
+
}
|
|
6093
|
+
async markStagedInvocationDispatched(input) {
|
|
6094
|
+
return this.settleStagedInvocation(input.id, input.workerId, input.leaseToken, "dispatched", input.now ?? /* @__PURE__ */ new Date(), input.childInvocationId, null);
|
|
6095
|
+
}
|
|
6096
|
+
async markStagedInvocationRejected(input) {
|
|
6097
|
+
return this.settleStagedInvocation(input.id, input.workerId, input.leaseToken, "rejected", input.now ?? /* @__PURE__ */ new Date(), null, input.error);
|
|
6098
|
+
}
|
|
6099
|
+
async markStagedInvocationFailed(input) {
|
|
6100
|
+
return (await this.sql.query(`UPDATE fabric_platform.staged_invocations
|
|
6101
|
+
SET status=$4, last_error=$5, available_at=$6,
|
|
6102
|
+
child_invocation_id=COALESCE($7,child_invocation_id),
|
|
6103
|
+
lease_owner=NULL, lease_expires_at=NULL, updated_at=$8
|
|
6104
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3 RETURNING id`, [
|
|
6105
|
+
input.id,
|
|
6106
|
+
input.workerId,
|
|
6107
|
+
input.leaseToken,
|
|
6108
|
+
input.deadLetter ? "dead_lettered" : "pending",
|
|
6109
|
+
input.error,
|
|
6110
|
+
input.availableAt,
|
|
6111
|
+
input.childInvocationId ?? null,
|
|
6112
|
+
input.now ?? /* @__PURE__ */ new Date()
|
|
6113
|
+
])).rows.length === 1;
|
|
6114
|
+
}
|
|
6115
|
+
async renewStagedInvocationLease(input) {
|
|
6116
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
6117
|
+
return (await this.sql.query(`UPDATE fabric_platform.staged_invocations
|
|
6118
|
+
SET lease_expires_at=$4, updated_at=$5
|
|
6119
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
6120
|
+
AND lease_expires_at IS NOT NULL AND lease_expires_at > $5 RETURNING id`, [
|
|
6121
|
+
input.id,
|
|
6122
|
+
input.workerId,
|
|
6123
|
+
input.leaseToken,
|
|
6124
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
6125
|
+
current
|
|
6126
|
+
])).rows.length === 1;
|
|
6127
|
+
}
|
|
6128
|
+
/**
|
|
6129
|
+
* The UPDATE needs a row lock that conflicts with the `FOR SHARE` lock the
|
|
6130
|
+
* fenced child insert takes on this row, so it waits for an in-flight
|
|
6131
|
+
* insert to commit. A fenced insert under the old token that reaches the
|
|
6132
|
+
* row while this rotation is uncommitted waits for it, rechecks the new row
|
|
6133
|
+
* version and inserts nothing (live tests "orders a token rotation after an
|
|
6134
|
+
* in-flight fenced create ..." and "refuses a fenced create under the old
|
|
6135
|
+
* token once a token rotation is ordered before it").
|
|
6136
|
+
*/
|
|
6137
|
+
async rotateStagedInvocationLease(input) {
|
|
6138
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
6139
|
+
const result = await this.sql.query(`UPDATE fabric_platform.staged_invocations
|
|
6140
|
+
SET lease_token=lease_token+1, lease_expires_at=$4, updated_at=$5
|
|
6141
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
6142
|
+
AND lease_expires_at IS NOT NULL AND lease_expires_at > $5
|
|
6143
|
+
RETURNING lease_token`, [
|
|
6144
|
+
input.id,
|
|
6145
|
+
input.workerId,
|
|
6146
|
+
input.leaseToken,
|
|
6147
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
6148
|
+
current
|
|
6149
|
+
]);
|
|
6150
|
+
return result.rows[0] ? Number(result.rows[0].lease_token) : void 0;
|
|
6151
|
+
}
|
|
6152
|
+
async cancelStagedInvocationsForFailedParents(input = {}) {
|
|
6153
|
+
return (await this.sql.query(`UPDATE fabric_platform.staged_invocations AS stage
|
|
6154
|
+
SET status='cancelled', last_error='Parent invocation ended with ' || parent.status || '.',
|
|
6155
|
+
lease_owner=NULL, lease_expires_at=NULL, updated_at=$3
|
|
6156
|
+
FROM fabric_platform.action_invocations AS parent
|
|
6157
|
+
WHERE parent.id=stage.parent_invocation_id
|
|
6158
|
+
AND parent.tenant_id=stage.tenant_id AND parent.space_id=stage.space_id
|
|
6159
|
+
AND parent.status IN ('failed','blocked_by_policy','validation_failed')
|
|
6160
|
+
AND stage.status='pending'
|
|
6161
|
+
AND ($1::text IS NULL OR stage.tenant_id=$1)
|
|
6162
|
+
AND ($2::text IS NULL OR stage.space_id=$2)
|
|
6163
|
+
RETURNING stage.id`, [
|
|
6164
|
+
input.tenantId ?? null,
|
|
6165
|
+
input.spaceId ?? null,
|
|
6166
|
+
input.now ?? /* @__PURE__ */ new Date()
|
|
6167
|
+
])).rows.length;
|
|
6168
|
+
}
|
|
6169
|
+
async listStagedInvocations(input) {
|
|
6170
|
+
return (await this.sql.query(`SELECT * FROM fabric_platform.staged_invocations
|
|
6171
|
+
WHERE tenant_id=$1 AND space_id=$2
|
|
6172
|
+
AND ($3::text IS NULL OR parent_invocation_id=$3)
|
|
6173
|
+
AND ($4::text[] IS NULL OR status=ANY($4::text[]))
|
|
6174
|
+
ORDER BY created_at,id`, [
|
|
6175
|
+
input.tenantId,
|
|
6176
|
+
input.spaceId,
|
|
6177
|
+
input.parentInvocationId ?? null,
|
|
6178
|
+
input.statuses?.length ? [...input.statuses] : null
|
|
6179
|
+
])).rows.map(toStagedInvocationRecord);
|
|
6180
|
+
}
|
|
6181
|
+
async settleStagedInvocation(id, workerId, leaseToken, status, now, childInvocationId, lastError) {
|
|
6182
|
+
return (await this.sql.query(`UPDATE fabric_platform.staged_invocations
|
|
6183
|
+
SET status=$4, child_invocation_id=COALESCE($5,child_invocation_id), last_error=$6,
|
|
6184
|
+
lease_owner=NULL, lease_expires_at=NULL, updated_at=$7
|
|
6185
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3 RETURNING id`, [
|
|
6186
|
+
id,
|
|
6187
|
+
workerId,
|
|
6188
|
+
leaseToken,
|
|
6189
|
+
status,
|
|
6190
|
+
childInvocationId,
|
|
6191
|
+
lastError,
|
|
6192
|
+
now
|
|
6193
|
+
])).rows.length === 1;
|
|
6194
|
+
}
|
|
5189
6195
|
async nextEventSequence(tenantId, spaceId) {
|
|
5190
6196
|
const result = await this.sql.query(`INSERT INTO fabric_platform.event_sequences (tenant_id,space_id,next_sequence)
|
|
5191
6197
|
VALUES ($1,$2,1) ON CONFLICT (tenant_id,space_id) DO UPDATE
|
|
@@ -5440,6 +6446,10 @@ function toActionRecord(row) {
|
|
|
5440
6446
|
...row.parameter_digest_algorithm ? { parameterDigestAlgorithm: String(row.parameter_digest_algorithm) } : {},
|
|
5441
6447
|
...row.idempotency_actor_id ? { idempotencyActorId: String(row.idempotency_actor_id) } : {},
|
|
5442
6448
|
...row.idempotency_authorization_binding_id ? { idempotencyAuthorizationBindingId: String(row.idempotency_authorization_binding_id) } : {},
|
|
6449
|
+
...row.staged_parent_invocation_id && row.staged_stage_key ? { stagedOrigin: {
|
|
6450
|
+
parentInvocationId: String(row.staged_parent_invocation_id),
|
|
6451
|
+
stageKey: String(row.staged_stage_key)
|
|
6452
|
+
} } : {},
|
|
5443
6453
|
...row.invocation_provenance ? { provenance: row.invocation_provenance } : {},
|
|
5444
6454
|
...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
|
|
5445
6455
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
@@ -5469,6 +6479,31 @@ function toActionRecord(row) {
|
|
|
5469
6479
|
updatedAt: new Date(row.updated_at)
|
|
5470
6480
|
};
|
|
5471
6481
|
}
|
|
6482
|
+
function toStagedInvocationRecord(row) {
|
|
6483
|
+
return {
|
|
6484
|
+
id: String(row.id),
|
|
6485
|
+
tenantId: String(row.tenant_id),
|
|
6486
|
+
spaceId: String(row.space_id),
|
|
6487
|
+
parentInvocationId: String(row.parent_invocation_id),
|
|
6488
|
+
parentActionId: String(row.parent_action_id),
|
|
6489
|
+
key: String(row.stage_key),
|
|
6490
|
+
actionId: String(row.action_id),
|
|
6491
|
+
expectedActionVersion: Number(row.expected_action_version),
|
|
6492
|
+
parameters: row.parameters,
|
|
6493
|
+
parameterDigest: String(row.parameter_digest),
|
|
6494
|
+
batchDigest: String(row.batch_digest),
|
|
6495
|
+
status: String(row.status),
|
|
6496
|
+
...row.child_invocation_id ? { childInvocationId: String(row.child_invocation_id) } : {},
|
|
6497
|
+
attemptCount: Number(row.attempt_count),
|
|
6498
|
+
availableAt: new Date(row.available_at),
|
|
6499
|
+
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
6500
|
+
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
6501
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
6502
|
+
...row.last_error ? { lastError: String(row.last_error) } : {},
|
|
6503
|
+
createdAt: new Date(row.created_at),
|
|
6504
|
+
updatedAt: new Date(row.updated_at)
|
|
6505
|
+
};
|
|
6506
|
+
}
|
|
5472
6507
|
function toObligationRecord(row) {
|
|
5473
6508
|
return {
|
|
5474
6509
|
id: String(row.id),
|
|
@@ -5608,6 +6643,23 @@ function createStoreBackedActionDispatcher() {
|
|
|
5608
6643
|
async function runPlatformActionWorkerCycle(options) {
|
|
5609
6644
|
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
5610
6645
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
6646
|
+
let staged;
|
|
6647
|
+
if ((options.dispatchStagedInvocations === true || options.dispatchStagedInvocations !== false && options.host.hasStagedInvocationActions === true) && typeof options.host.dispatchStagedInvocations === "function") try {
|
|
6648
|
+
staged = await options.host.dispatchStagedInvocations({
|
|
6649
|
+
workerId: options.workerId,
|
|
6650
|
+
...options.stagedInvocationBatchSize !== void 0 ? { limit: options.stagedInvocationBatchSize } : {},
|
|
6651
|
+
...options.stagedInvocationLeaseDurationMs !== void 0 ? { leaseDurationMs: options.stagedInvocationLeaseDurationMs } : {},
|
|
6652
|
+
...options.stagedInvocationLeaseRenewalIntervalMs !== void 0 ? { leaseRenewalIntervalMs: options.stagedInvocationLeaseRenewalIntervalMs } : {},
|
|
6653
|
+
...options.stagedInvocationSubmissionTimeoutMs !== void 0 ? { submissionTimeoutMs: options.stagedInvocationSubmissionTimeoutMs } : {},
|
|
6654
|
+
...options.stagedInvocationMaxAttempts !== void 0 ? { maxAttempts: options.stagedInvocationMaxAttempts } : {},
|
|
6655
|
+
...options.stagedInvocationRetryDelayMs ? { retryDelayMs: options.stagedInvocationRetryDelayMs } : {},
|
|
6656
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
6657
|
+
...options.spaceId ? { spaceId: options.spaceId } : {},
|
|
6658
|
+
...options.onError ? { onError: (error) => options.onError?.(error) } : {}
|
|
6659
|
+
});
|
|
6660
|
+
} catch (error) {
|
|
6661
|
+
options.onError?.(error);
|
|
6662
|
+
}
|
|
5611
6663
|
if (options.sweepOverdueCompletions !== false && typeof options.host.reconcileOverdueCompletions === "function") try {
|
|
5612
6664
|
await options.host.reconcileOverdueCompletions({
|
|
5613
6665
|
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
@@ -5665,9 +6717,20 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
5665
6717
|
completed,
|
|
5666
6718
|
failed,
|
|
5667
6719
|
...waitingForApproval > 0 ? { waitingForApproval } : {},
|
|
5668
|
-
...deferred > 0 ? { deferred } : {}
|
|
6720
|
+
...deferred > 0 ? { deferred } : {},
|
|
6721
|
+
...staged && Object.values(staged).some((value) => value > 0) ? { staged } : {}
|
|
5669
6722
|
};
|
|
5670
|
-
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId,
|
|
6723
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, {
|
|
6724
|
+
claimed: result.claimed,
|
|
6725
|
+
completed: result.completed,
|
|
6726
|
+
failed: result.failed,
|
|
6727
|
+
...result.waitingForApproval !== void 0 ? { waitingForApproval: result.waitingForApproval } : {},
|
|
6728
|
+
...result.deferred !== void 0 ? { deferred: result.deferred } : {},
|
|
6729
|
+
...result.staged ? {
|
|
6730
|
+
stagedClaimed: result.staged.claimed,
|
|
6731
|
+
stagedDispatched: result.staged.dispatched
|
|
6732
|
+
} : {}
|
|
6733
|
+
});
|
|
5671
6734
|
return result;
|
|
5672
6735
|
}
|
|
5673
6736
|
function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
|
|
@@ -5925,15 +6988,16 @@ async function adapterProbe(subject) {
|
|
|
5925
6988
|
};
|
|
5926
6989
|
}
|
|
5927
6990
|
/**
|
|
5928
|
-
* The suite
|
|
6991
|
+
* The suite a deployment of the host must pass before it claims isolation.
|
|
5929
6992
|
*
|
|
5930
|
-
*
|
|
6993
|
+
* Nine checks: an invocation cannot be read, executed, or completed under
|
|
5931
6994
|
* another tenant's scope; an idempotency key is a tenant's own, so the same
|
|
5932
6995
|
* key in two tenants is two invocations; a tenant's event stream carries no
|
|
5933
6996
|
* other tenant's events; a worker scoped to one tenant does not claim another's
|
|
5934
|
-
* work;
|
|
5935
|
-
*
|
|
5936
|
-
* the
|
|
6997
|
+
* work; staged listing and claiming retain tenant and space scope; an adapter
|
|
6998
|
+
* checkpoint cannot be read or written under another tenant, space, or owning
|
|
6999
|
+
* invocation; and, when the subject lends a read, a viewer in the other tenant
|
|
7000
|
+
* sees nothing of ours.
|
|
5937
7001
|
*/
|
|
5938
7002
|
function tenantIsolationChecks() {
|
|
5939
7003
|
return [
|
|
@@ -5977,6 +7041,103 @@ function tenantIsolationChecks() {
|
|
|
5977
7041
|
return ["one idempotency key: one invocation within a tenant, two across tenants"];
|
|
5978
7042
|
}
|
|
5979
7043
|
},
|
|
7044
|
+
{
|
|
7045
|
+
id: "fabric.tenancy.staged-invocation-scoped.v1",
|
|
7046
|
+
async run(subject) {
|
|
7047
|
+
await subject.reset?.();
|
|
7048
|
+
const store = subject.store;
|
|
7049
|
+
if (store.transactionalStagedInvocations !== true || typeof store.transactionWithEvents !== "function" || typeof store.listStagedInvocations !== "function" || typeof store.claimStagedInvocations !== "function") return ["skipped: store does not expose transactional staged invocations"];
|
|
7050
|
+
const suffix = globalThis.crypto.randomUUID();
|
|
7051
|
+
const probeTenant = `iso-stage-tenant-${suffix}`;
|
|
7052
|
+
const otherProbeTenant = `iso-stage-tenant-other-${suffix}`;
|
|
7053
|
+
const probeSpace = `iso-stage-space-${suffix}`;
|
|
7054
|
+
const otherProbeSpace = `iso-stage-space-other-${suffix}`;
|
|
7055
|
+
const parents = [
|
|
7056
|
+
{
|
|
7057
|
+
id: `iso_stage_parent_ours_${suffix}`,
|
|
7058
|
+
tenantId: probeTenant,
|
|
7059
|
+
spaceId: probeSpace,
|
|
7060
|
+
actorId: subject.ours.actorId,
|
|
7061
|
+
actorType: subject.ours.actorType
|
|
7062
|
+
},
|
|
7063
|
+
{
|
|
7064
|
+
id: `iso_stage_parent_theirs_${suffix}`,
|
|
7065
|
+
tenantId: otherProbeTenant,
|
|
7066
|
+
spaceId: probeSpace,
|
|
7067
|
+
actorId: subject.theirs.actorId,
|
|
7068
|
+
actorType: subject.theirs.actorType
|
|
7069
|
+
},
|
|
7070
|
+
{
|
|
7071
|
+
id: `iso_stage_parent_space_${suffix}`,
|
|
7072
|
+
tenantId: probeTenant,
|
|
7073
|
+
spaceId: otherProbeSpace,
|
|
7074
|
+
actorId: subject.ours.actorId,
|
|
7075
|
+
actorType: subject.ours.actorType
|
|
7076
|
+
}
|
|
7077
|
+
];
|
|
7078
|
+
for (const parent of parents) await store.createActionInvocation({
|
|
7079
|
+
...parent,
|
|
7080
|
+
actionId: subject.actionId,
|
|
7081
|
+
actionVersion: 1,
|
|
7082
|
+
status: "completed",
|
|
7083
|
+
parameters: {},
|
|
7084
|
+
result: {},
|
|
7085
|
+
correlationId: suffix
|
|
7086
|
+
});
|
|
7087
|
+
const now = /* @__PURE__ */ new Date();
|
|
7088
|
+
const records = parents.map((parent, index) => ({
|
|
7089
|
+
id: `iso_stage_${index}_${suffix}`,
|
|
7090
|
+
tenantId: parent.tenantId,
|
|
7091
|
+
spaceId: parent.spaceId,
|
|
7092
|
+
parentInvocationId: parent.id,
|
|
7093
|
+
parentActionId: subject.actionId,
|
|
7094
|
+
key: "child",
|
|
7095
|
+
actionId: subject.actionId,
|
|
7096
|
+
expectedActionVersion: 1,
|
|
7097
|
+
parameters: {},
|
|
7098
|
+
parameterDigest: `digest-${index}`,
|
|
7099
|
+
batchDigest: `batch-${index}`,
|
|
7100
|
+
status: "pending",
|
|
7101
|
+
attemptCount: 0,
|
|
7102
|
+
availableAt: now,
|
|
7103
|
+
leaseToken: 0,
|
|
7104
|
+
createdAt: now,
|
|
7105
|
+
updatedAt: now
|
|
7106
|
+
}));
|
|
7107
|
+
try {
|
|
7108
|
+
await store.transactionWithEvents(async (transaction) => {
|
|
7109
|
+
if (!transaction.stageInvocations) fail$1("transactionalStagedInvocations did not supply stageInvocations");
|
|
7110
|
+
for (const record of records) await transaction.stageInvocations([record]);
|
|
7111
|
+
});
|
|
7112
|
+
const ours = await store.listStagedInvocations({
|
|
7113
|
+
tenantId: probeTenant,
|
|
7114
|
+
spaceId: probeSpace
|
|
7115
|
+
});
|
|
7116
|
+
if (!ours.some((record) => record.id === records[0].id)) fail$1("our staged invocation was not listable under our scope, so the check proves nothing");
|
|
7117
|
+
if (ours.some((record) => record.id === records[1].id || record.id === records[2].id)) fail$1("staged invocation listing crossed tenant or space");
|
|
7118
|
+
if ((await store.listStagedInvocations({
|
|
7119
|
+
tenantId: otherProbeTenant,
|
|
7120
|
+
spaceId: probeSpace,
|
|
7121
|
+
parentInvocationId: parents[0].id
|
|
7122
|
+
})).length !== 0) fail$1("a staged invocation was listable under another tenant's scope");
|
|
7123
|
+
const claimed = await store.claimStagedInvocations({
|
|
7124
|
+
workerId: `iso-stage-worker-${suffix}`,
|
|
7125
|
+
tenantId: probeTenant,
|
|
7126
|
+
spaceId: probeSpace,
|
|
7127
|
+
leaseDurationMs: 6e4,
|
|
7128
|
+
now
|
|
7129
|
+
});
|
|
7130
|
+
if (!claimed.some((record) => record.id === records[0].id)) fail$1("our staged invocation was not claimable under our scope, so the check proves nothing");
|
|
7131
|
+
if (claimed.some((record) => record.tenantId !== probeTenant || record.spaceId !== probeSpace)) fail$1("staged invocation claim crossed tenant or space");
|
|
7132
|
+
return ["staged listing and claiming retained both tenant and space scope"];
|
|
7133
|
+
} finally {
|
|
7134
|
+
for (const parent of parents) await store.updateActionInvocation(parent.id, parent.tenantId, parent.spaceId, {
|
|
7135
|
+
status: "failed",
|
|
7136
|
+
error: "tenant isolation probe; retired"
|
|
7137
|
+
});
|
|
7138
|
+
}
|
|
7139
|
+
}
|
|
7140
|
+
},
|
|
5980
7141
|
{
|
|
5981
7142
|
id: "fabric.tenancy.events-scoped.v1",
|
|
5982
7143
|
async run(subject) {
|
|
@@ -6771,6 +7932,6 @@ async function runEventAppendChecks(subject) {
|
|
|
6771
7932
|
};
|
|
6772
7933
|
}
|
|
6773
7934
|
//#endregion
|
|
6774
|
-
export { ActionAdmissionError, ExternalCompletionError, IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, eventAppendChecks, getPlatformHostHealthSnapshot, outboxActorDisclosure, runEventAppendChecks, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, runTenantIsolationChecks, tenantIsolationChecks, toEnterpriseEventEnvelope };
|
|
7935
|
+
export { ActionAdmissionError, ExternalCompletionError, IdempotencyConflictError, MAX_STAGED_INVOCATIONS, MAX_STAGED_INVOCATION_BATCH_BYTES, MAX_STAGED_INVOCATION_BYTES, MAX_STAGED_INVOCATION_KEY_LENGTH, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, STAGED_INVOCATION_KEY_PATTERN, SubmissionRefusalError, applyPostgresMigrations, asStagedInvocationStore, canonicalJson, cloneOutboxRecord, cloneStagedInvocationRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, eventAppendChecks, getPlatformHostHealthSnapshot, outboxActorDisclosure, prepareStagedInvocationBatch, runEventAppendChecks, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, runTenantIsolationChecks, stagedInvocationRequestIdentity, tenantIsolationChecks, toEnterpriseEventEnvelope };
|
|
6775
7936
|
|
|
6776
7937
|
//# sourceMappingURL=index.js.map
|