@fabricorg/platform-host 7.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +51 -0
- package/dist/index.cjs +356 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +115 -7
- package/dist/index.d.ts +115 -7
- package/dist/index.js +356 -26
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -17,6 +17,18 @@ var IdempotencyConflictError = class extends Error {
|
|
|
17
17
|
conflict;
|
|
18
18
|
code = "IDEMPOTENCY_CONFLICT";
|
|
19
19
|
};
|
|
20
|
+
var ExternalCompletionError = class extends Error {
|
|
21
|
+
constructor(refusal, message) {
|
|
22
|
+
super(message);
|
|
23
|
+
this.refusal = refusal;
|
|
24
|
+
this.name = "ExternalCompletionError";
|
|
25
|
+
}
|
|
26
|
+
refusal;
|
|
27
|
+
code = "EXTERNAL_COMPLETION_REFUSED";
|
|
28
|
+
get retryable() {
|
|
29
|
+
return this.refusal === "still_executing";
|
|
30
|
+
}
|
|
31
|
+
};
|
|
20
32
|
var PARAMETER_DIGEST_ALGORITHM = "fabric-canonical-json-sha256-v1";
|
|
21
33
|
function canonicalJson(value) {
|
|
22
34
|
return JSON.stringify(sort(value));
|
|
@@ -431,7 +443,8 @@ function createGovernedActionHost(options) {
|
|
|
431
443
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
432
444
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
433
445
|
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
434
|
-
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
446
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {},
|
|
447
|
+
...durableInvocation.pendingCompletion ? { pendingCompletion: durableInvocation.pendingCompletion } : {}
|
|
435
448
|
}, input.actionId);
|
|
436
449
|
}
|
|
437
450
|
if (options.dispatcher) {
|
|
@@ -476,6 +489,9 @@ function createGovernedActionHost(options) {
|
|
|
476
489
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
477
490
|
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
478
491
|
}
|
|
492
|
+
if (invocation.status === "running" && invocation.pendingCompletion && !invocation.leaseOwner) {
|
|
493
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
494
|
+
}
|
|
479
495
|
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
480
496
|
return {
|
|
481
497
|
...actionResult(invocation),
|
|
@@ -810,6 +826,7 @@ function createGovernedActionHost(options) {
|
|
|
810
826
|
} catch (error) {
|
|
811
827
|
return fail(invocation, "failed", errorMessage(error));
|
|
812
828
|
}
|
|
829
|
+
let pendingHandoff = invocation.pendingCompletion;
|
|
813
830
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
814
831
|
const input = step.getInput(parsed.data, data);
|
|
815
832
|
if (!input) continue;
|
|
@@ -819,6 +836,28 @@ function createGovernedActionHost(options) {
|
|
|
819
836
|
adapterInvocationId
|
|
820
837
|
);
|
|
821
838
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
839
|
+
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
840
|
+
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
841
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
842
|
+
await appendEvent(invocation, {
|
|
843
|
+
eventType: "ExternalOperationAccepted",
|
|
844
|
+
subjectType: subject.subjectType,
|
|
845
|
+
subjectId: subject.subjectId,
|
|
846
|
+
payload: {
|
|
847
|
+
adapterType: step.adapterType,
|
|
848
|
+
operation: step.operation,
|
|
849
|
+
externalReference: pendingHandoff.externalReference,
|
|
850
|
+
...pendingHandoff.dueAt ? { dueAt: pendingHandoff.dueAt.toISOString() } : {}
|
|
851
|
+
}
|
|
852
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
853
|
+
await appendEvent(invocation, {
|
|
854
|
+
eventType: "AdapterInvocationSucceeded",
|
|
855
|
+
subjectType: subject.subjectType,
|
|
856
|
+
subjectId: subject.subjectId,
|
|
857
|
+
payload: { adapterType: step.adapterType, operation: step.operation, recovered: true }
|
|
858
|
+
}, `adapter:${stepIndex}:succeeded`);
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
822
861
|
const adapterEventSubject = options.adapterEventSubject?.(
|
|
823
862
|
action.actionId,
|
|
824
863
|
parsed.data,
|
|
@@ -973,6 +1012,63 @@ function createGovernedActionHost(options) {
|
|
|
973
1012
|
([key]) => key !== "success" && key !== "error"
|
|
974
1013
|
)
|
|
975
1014
|
);
|
|
1015
|
+
const handoff = result2.acceptedExternalOperation;
|
|
1016
|
+
if (handoff) {
|
|
1017
|
+
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1018
|
+
if (reference === "") {
|
|
1019
|
+
const message = `Adapter ${step.adapterType}:${step.operation} accepted an external operation without a reference to complete it by; the external effect may be in flight.`;
|
|
1020
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1021
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message }, action.actionId);
|
|
1022
|
+
}
|
|
1023
|
+
const acceptedAt = now();
|
|
1024
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1025
|
+
const accepted = {
|
|
1026
|
+
provider: adapter.vendor,
|
|
1027
|
+
adapterType: step.adapterType,
|
|
1028
|
+
operation: step.operation,
|
|
1029
|
+
adapterInvocationId,
|
|
1030
|
+
externalReference: reference,
|
|
1031
|
+
acceptedAt,
|
|
1032
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1033
|
+
};
|
|
1034
|
+
const completion = action.execution?.completion ?? "immediate";
|
|
1035
|
+
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1036
|
+
if (refuse) {
|
|
1037
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() });
|
|
1038
|
+
await persistInvocation(invocation, {
|
|
1039
|
+
status: "reconciliation_required",
|
|
1040
|
+
error: refuse,
|
|
1041
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1042
|
+
});
|
|
1043
|
+
let error = refuse;
|
|
1044
|
+
try {
|
|
1045
|
+
await recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1046
|
+
} catch (logError) {
|
|
1047
|
+
error = `${refuse} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1048
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1049
|
+
}
|
|
1050
|
+
return withConsistency({
|
|
1051
|
+
actionInvocationId,
|
|
1052
|
+
status: "reconciliation_required",
|
|
1053
|
+
error,
|
|
1054
|
+
...pendingHandoff ? { pendingCompletion: pendingHandoff } : { pendingCompletion: accepted }
|
|
1055
|
+
}, action.actionId);
|
|
1056
|
+
}
|
|
1057
|
+
pendingHandoff = accepted;
|
|
1058
|
+
await persistInvocation(invocation, { pendingCompletion: accepted });
|
|
1059
|
+
invocation = { ...invocation, pendingCompletion: accepted };
|
|
1060
|
+
await appendEvent(invocation, {
|
|
1061
|
+
eventType: "ExternalOperationAccepted",
|
|
1062
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1063
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1064
|
+
payload: {
|
|
1065
|
+
adapterType: step.adapterType,
|
|
1066
|
+
operation: step.operation,
|
|
1067
|
+
externalReference: reference,
|
|
1068
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1069
|
+
}
|
|
1070
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
1071
|
+
}
|
|
976
1072
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
977
1073
|
status: "succeeded",
|
|
978
1074
|
output,
|
|
@@ -1022,10 +1118,24 @@ function createGovernedActionHost(options) {
|
|
|
1022
1118
|
const obligations = await governanceStore.listPolicyObligations(actionInvocationId, tenantId, spaceId);
|
|
1023
1119
|
const unsatisfied = obligations.filter((obligation) => (obligation.required ?? true) && obligation.status !== "satisfied" && obligation.status !== "waived");
|
|
1024
1120
|
if (unsatisfied.length > 0) {
|
|
1025
|
-
|
|
1121
|
+
const message = `Unsatisfied policy obligations: ${unsatisfied.map((item) => item.id).join(", ")}`;
|
|
1122
|
+
if (pendingHandoff) {
|
|
1123
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1124
|
+
let error = message;
|
|
1125
|
+
try {
|
|
1126
|
+
await recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1127
|
+
} catch (logError) {
|
|
1128
|
+
error = `${message} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1129
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1130
|
+
}
|
|
1131
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
1132
|
+
}
|
|
1133
|
+
return fail(invocation, "failed", message);
|
|
1026
1134
|
}
|
|
1027
1135
|
}
|
|
1028
1136
|
const result = withoutPrivateHostFields(data, eventResultFields);
|
|
1137
|
+
const parked = pendingHandoff ? { ...pendingHandoff, parkedAt: now() } : void 0;
|
|
1138
|
+
const finalPatch = parked ? { status: "running", result, pendingCompletion: parked } : { status: "completed", result };
|
|
1029
1139
|
try {
|
|
1030
1140
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1031
1141
|
if (action.eventPhase === "after_adapters" && atomicStore) {
|
|
@@ -1049,7 +1159,7 @@ function createGovernedActionHost(options) {
|
|
|
1049
1159
|
spaceId,
|
|
1050
1160
|
workerId: invocation.leaseOwner,
|
|
1051
1161
|
leaseToken: invocation.leaseToken,
|
|
1052
|
-
patch:
|
|
1162
|
+
patch: finalPatch
|
|
1053
1163
|
});
|
|
1054
1164
|
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1055
1165
|
} else {
|
|
@@ -1057,26 +1167,25 @@ function createGovernedActionHost(options) {
|
|
|
1057
1167
|
actionInvocationId,
|
|
1058
1168
|
tenantId,
|
|
1059
1169
|
spaceId,
|
|
1060
|
-
|
|
1170
|
+
finalPatch
|
|
1061
1171
|
);
|
|
1062
1172
|
}
|
|
1063
1173
|
});
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1174
|
+
if (!pendingHandoff) {
|
|
1175
|
+
emitInvocationStatusTelemetry(
|
|
1176
|
+
invocation,
|
|
1177
|
+
"completed",
|
|
1178
|
+
options.telemetry,
|
|
1179
|
+
now()
|
|
1180
|
+
);
|
|
1181
|
+
}
|
|
1070
1182
|
} else {
|
|
1071
1183
|
if (action.eventPhase === "after_adapters") {
|
|
1072
1184
|
for (const [index, event] of domainEvents.entries()) {
|
|
1073
1185
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1074
1186
|
}
|
|
1075
1187
|
}
|
|
1076
|
-
await persistInvocation(invocation,
|
|
1077
|
-
status: "completed",
|
|
1078
|
-
result
|
|
1079
|
-
});
|
|
1188
|
+
await persistInvocation(invocation, finalPatch);
|
|
1080
1189
|
}
|
|
1081
1190
|
} catch (error) {
|
|
1082
1191
|
if (action.eventPhase === "after_adapters") {
|
|
@@ -1086,8 +1195,9 @@ function createGovernedActionHost(options) {
|
|
|
1086
1195
|
}
|
|
1087
1196
|
return {
|
|
1088
1197
|
actionInvocationId,
|
|
1089
|
-
status: "completed",
|
|
1198
|
+
status: parked ? "running" : "completed",
|
|
1090
1199
|
result,
|
|
1200
|
+
...parked ? { pendingCompletion: parked } : {},
|
|
1091
1201
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1092
1202
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
1093
1203
|
};
|
|
@@ -1206,6 +1316,163 @@ function createGovernedActionHost(options) {
|
|
|
1206
1316
|
spaceId
|
|
1207
1317
|
});
|
|
1208
1318
|
}
|
|
1319
|
+
async function completeExternalInvocation(actionInvocationId, tenantId, spaceId, completion) {
|
|
1320
|
+
if (typeof completion.externalReference !== "string" || completion.externalReference.trim() === "") {
|
|
1321
|
+
throw new ExternalCompletionError("invalid_completion", "External completion must name the external reference it completes.");
|
|
1322
|
+
}
|
|
1323
|
+
if (completion.outcome !== "completed" && completion.outcome !== "failed") {
|
|
1324
|
+
throw new ExternalCompletionError("invalid_completion", `External completion outcome must be "completed" or "failed", received "${String(completion.outcome)}".`);
|
|
1325
|
+
}
|
|
1326
|
+
const atomicStore = asAtomicMutationStore(options.store);
|
|
1327
|
+
if (atomicStore) {
|
|
1328
|
+
return atomicStore.transactionWithEvents(async (transaction) => {
|
|
1329
|
+
const read2 = transaction.getActionInvocationForUpdate ? await transaction.getActionInvocationForUpdate(actionInvocationId, tenantId, spaceId) : await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1330
|
+
return settleExternalCompletion(read2, actionInvocationId, completion, {
|
|
1331
|
+
update: (patch) => transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch),
|
|
1332
|
+
transaction
|
|
1333
|
+
});
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
const read = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1337
|
+
return settleExternalCompletion(read, actionInvocationId, completion, {
|
|
1338
|
+
update: (patch) => options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch)
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
async function settleExternalCompletion(invocation, actionInvocationId, completion, writer) {
|
|
1342
|
+
if (!invocation) throw new ExternalCompletionError("not_found", `ActionInvocation not found: ${actionInvocationId}`);
|
|
1343
|
+
const recorded = invocation.externalCompletion;
|
|
1344
|
+
const stillExecuting = !recorded && (invocation.status === "pending" || invocation.status === "running" && (invocation.leaseOwner !== void 0 || invocation.pendingCompletion?.parkedAt === void 0));
|
|
1345
|
+
if (stillExecuting) {
|
|
1346
|
+
throw new ExternalCompletionError("still_executing", `ActionInvocation ${actionInvocationId} is still executing; retry the completion after it parks.`);
|
|
1347
|
+
}
|
|
1348
|
+
if (recorded) {
|
|
1349
|
+
if (recorded.externalReference !== completion.externalReference) {
|
|
1350
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.externalReference}", not "${completion.externalReference}".`);
|
|
1351
|
+
}
|
|
1352
|
+
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
1353
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
1354
|
+
}
|
|
1355
|
+
if (recorded.outcome === completion.outcome) {
|
|
1356
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
1357
|
+
}
|
|
1358
|
+
const message = `External operation "${completion.externalReference}" reported ${completion.outcome} after reporting ${recorded.outcome}.`;
|
|
1359
|
+
await recordCompletionReconciliation(invocation, { ...completion, kind: `contradicted:${completion.outcome}` }, message);
|
|
1360
|
+
await appendEvent(invocation, {
|
|
1361
|
+
eventType: "ExternalOperationContradicted",
|
|
1362
|
+
subjectType: "AdapterInvocation",
|
|
1363
|
+
subjectId: invocation.pendingCompletion?.adapterInvocationId ?? actionInvocationId,
|
|
1364
|
+
payload: { externalReference: completion.externalReference, recorded: recorded.outcome, reported: completion.outcome }
|
|
1365
|
+
}, `external:${completion.externalReference}:contradicted:${completion.outcome}`, 1, writer.transaction);
|
|
1366
|
+
await writer.update({ status: "reconciliation_required", error: message });
|
|
1367
|
+
emitInvocationStatusTelemetry(invocation, "reconciliation_required", options.telemetry, now(), invocation.status);
|
|
1368
|
+
return withConsistency(actionResult({ ...invocation, status: "reconciliation_required", error: message }), invocation.actionId);
|
|
1369
|
+
}
|
|
1370
|
+
const pending = invocation.pendingCompletion;
|
|
1371
|
+
if (!pending) {
|
|
1372
|
+
const action = actionResolver(invocation.actionId);
|
|
1373
|
+
if ((action?.execution?.completion ?? "immediate") === "immediate") {
|
|
1374
|
+
throw new ExternalCompletionError("immediate_contract", `Action ${invocation.actionId} declares immediate completion; nothing external completes it.`);
|
|
1375
|
+
}
|
|
1376
|
+
throw new ExternalCompletionError("not_awaiting", `ActionInvocation ${actionInvocationId} is not awaiting an external completion.`);
|
|
1377
|
+
}
|
|
1378
|
+
if (pending.externalReference !== completion.externalReference) {
|
|
1379
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} awaits "${pending.externalReference}", not "${completion.externalReference}".`);
|
|
1380
|
+
}
|
|
1381
|
+
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
1382
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
1383
|
+
}
|
|
1384
|
+
const recordedAt = now();
|
|
1385
|
+
const externalCompletion = {
|
|
1386
|
+
...completion,
|
|
1387
|
+
provider: completion.provider ?? pending.provider,
|
|
1388
|
+
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
1389
|
+
recordedAt
|
|
1390
|
+
};
|
|
1391
|
+
const subject = { subjectType: "AdapterInvocation", subjectId: pending.adapterInvocationId };
|
|
1392
|
+
const settles = invocation.status === "running" && pending.parkedAt !== void 0 || invocation.status === "reconciliation_required" && invocation.error?.includes("did not complete by") === true;
|
|
1393
|
+
if (completion.outcome === "completed") {
|
|
1394
|
+
const action = actionResolver(invocation.actionId);
|
|
1395
|
+
const merged = withoutPrivateHostFields({ ...invocation.result, ...completion.result ?? {} }, eventResultFields);
|
|
1396
|
+
if (action?.resultSchema) {
|
|
1397
|
+
const parsedResult = action.resultSchema.safeParse(merged);
|
|
1398
|
+
if (!parsedResult.success) {
|
|
1399
|
+
throw new ExternalCompletionError("result_invalid", `External completion result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
1403
|
+
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
1404
|
+
updatedAt: recordedAt
|
|
1405
|
+
});
|
|
1406
|
+
await appendEvent(invocation, {
|
|
1407
|
+
eventType: "ExternalOperationCompleted",
|
|
1408
|
+
...subject,
|
|
1409
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, ...completion.result ? { result: withoutPrivateHostFields(completion.result, eventResultFields) } : {} }
|
|
1410
|
+
}, `external:${pending.adapterInvocationId}:completed`, 1, writer.transaction);
|
|
1411
|
+
const patch2 = settles ? { status: "completed", result: merged, externalCompletion, pendingCompletion: void 0, error: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1412
|
+
await writer.update(patch2);
|
|
1413
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "completed", options.telemetry, now(), invocation.status);
|
|
1414
|
+
return withConsistency(actionResult({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
1415
|
+
}
|
|
1416
|
+
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
1417
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
1418
|
+
await appendEvent(invocation, {
|
|
1419
|
+
eventType: "ExternalOperationFailed",
|
|
1420
|
+
...subject,
|
|
1421
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, error }
|
|
1422
|
+
}, `external:${pending.adapterInvocationId}:failed`, 1, writer.transaction);
|
|
1423
|
+
const patch = settles ? { status: "failed", error, externalCompletion, pendingCompletion: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1424
|
+
await writer.update(patch);
|
|
1425
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "failed", options.telemetry, now(), invocation.status);
|
|
1426
|
+
return withConsistency(actionResult({ ...invocation, ...patch }), invocation.actionId);
|
|
1427
|
+
}
|
|
1428
|
+
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
1429
|
+
const governanceStore = asGovernanceStore(options.store);
|
|
1430
|
+
if (!governanceStore) throw new Error("External completion reconciliation requires a governance-capable store.");
|
|
1431
|
+
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
1432
|
+
const seed = crypto.createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
1433
|
+
await governanceStore.appendExternalReconciliation({
|
|
1434
|
+
id: lifecycleId("rec", invocation.id, `ext:${seed}`),
|
|
1435
|
+
actionInvocationId: invocation.id,
|
|
1436
|
+
tenantId: invocation.tenantId,
|
|
1437
|
+
spaceId: invocation.spaceId,
|
|
1438
|
+
status: "pending",
|
|
1439
|
+
provider,
|
|
1440
|
+
externalOperationId: completion.externalReference,
|
|
1441
|
+
attempt: 1,
|
|
1442
|
+
reason,
|
|
1443
|
+
...completion.evidenceReferences ? { evidenceReferences: completion.evidenceReferences } : {},
|
|
1444
|
+
observedAt: now()
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
async function reconcileOverdueCompletions(input = {}) {
|
|
1448
|
+
const recoverable = options.store;
|
|
1449
|
+
if (!recoverable.listActionInvocations) {
|
|
1450
|
+
throw new Error("Overdue completion reconciliation requires a store that can list invocations.");
|
|
1451
|
+
}
|
|
1452
|
+
if (!asGovernanceStore(options.store)) {
|
|
1453
|
+
throw new Error("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
1454
|
+
}
|
|
1455
|
+
const current = input.now ?? now();
|
|
1456
|
+
const candidates = await recoverable.listActionInvocations({
|
|
1457
|
+
statuses: ["running"],
|
|
1458
|
+
completionDueBefore: current,
|
|
1459
|
+
unleased: true,
|
|
1460
|
+
...input.tenantId ? { tenantId: input.tenantId } : {},
|
|
1461
|
+
...input.spaceId ? { spaceId: input.spaceId } : {},
|
|
1462
|
+
limit: Math.max(1, Math.min(input.limit ?? 100, 1e3))
|
|
1463
|
+
});
|
|
1464
|
+
const overdue = [];
|
|
1465
|
+
for (const invocation of candidates) {
|
|
1466
|
+
const pending = invocation.pendingCompletion;
|
|
1467
|
+
if (!pending?.dueAt || pending.dueAt.getTime() > current.getTime()) continue;
|
|
1468
|
+
if (invocation.leaseOwner) continue;
|
|
1469
|
+
const message = `External operation "${pending.externalReference}" from ${pending.provider} did not complete by ${pending.dueAt.toISOString()}.`;
|
|
1470
|
+
await recordCompletionReconciliation(invocation, { externalReference: pending.externalReference, provider: pending.provider, kind: "overdue" }, message);
|
|
1471
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1472
|
+
overdue.push(invocation.id);
|
|
1473
|
+
}
|
|
1474
|
+
return { overdue };
|
|
1475
|
+
}
|
|
1209
1476
|
function declaredConsistency(actionId) {
|
|
1210
1477
|
return actionResolver(actionId)?.execution?.consistency;
|
|
1211
1478
|
}
|
|
@@ -1239,7 +1506,7 @@ function createGovernedActionHost(options) {
|
|
|
1239
1506
|
...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
|
|
1240
1507
|
};
|
|
1241
1508
|
if (options.outbox) {
|
|
1242
|
-
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1509
|
+
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ExternalOperationAccepted", "ExternalOperationCompleted", "ExternalOperationFailed", "ExternalOperationContradicted", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1243
1510
|
const shouldPublish = options.outbox.shouldPublish?.(envelope) ?? !hostLifecycleEvent;
|
|
1244
1511
|
if (!shouldPublish) {
|
|
1245
1512
|
await (transaction ?? options.store).appendEvent(envelope);
|
|
@@ -1302,7 +1569,7 @@ function createGovernedActionHost(options) {
|
|
|
1302
1569
|
);
|
|
1303
1570
|
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1304
1571
|
}
|
|
1305
|
-
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
1572
|
+
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation, completeExternalInvocation, reconcileOverdueCompletions };
|
|
1306
1573
|
}
|
|
1307
1574
|
function actionResult(invocation) {
|
|
1308
1575
|
return {
|
|
@@ -1313,7 +1580,8 @@ function actionResult(invocation) {
|
|
|
1313
1580
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1314
1581
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
1315
1582
|
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1316
|
-
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
1583
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {},
|
|
1584
|
+
...invocation.pendingCompletion ? { pendingCompletion: invocation.pendingCompletion } : {}
|
|
1317
1585
|
};
|
|
1318
1586
|
}
|
|
1319
1587
|
function asApprovalStore(store) {
|
|
@@ -1713,6 +1981,13 @@ var MemoryPlatformHostStore = class {
|
|
|
1713
1981
|
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1714
1982
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
1715
1983
|
Object.assign(record, patch, { updatedAt: /* @__PURE__ */ new Date() });
|
|
1984
|
+
if (Object.hasOwn(patch, "pendingCompletion") && patch.pendingCompletion === void 0) delete record.pendingCompletion;
|
|
1985
|
+
if (Object.hasOwn(patch, "externalCompletion") && patch.externalCompletion === void 0) delete record.externalCompletion;
|
|
1986
|
+
if (Object.hasOwn(patch, "error") && patch.error === void 0) delete record.error;
|
|
1987
|
+
if (patch.pendingCompletion && patch.status === "running") {
|
|
1988
|
+
delete record.leaseOwner;
|
|
1989
|
+
delete record.leaseExpiresAt;
|
|
1990
|
+
}
|
|
1716
1991
|
if (patch.status === "waiting_for_approval" || patch.status === "completed" || patch.status === "failed" || patch.status === "blocked_by_policy" || patch.status === "reconciliation_required" || patch.status === "validation_failed") {
|
|
1717
1992
|
delete record.leaseOwner;
|
|
1718
1993
|
delete record.leaseExpiresAt;
|
|
@@ -1881,7 +2156,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1881
2156
|
}
|
|
1882
2157
|
async listActionInvocations(input = {}) {
|
|
1883
2158
|
return this.invocations.filter(
|
|
1884
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
2159
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore) && (!(input.awaitingCompletion || input.completionDueBefore) || record.pendingCompletion !== void 0) && (!input.completionDueBefore || record.pendingCompletion?.dueAt !== void 0 && record.pendingCompletion.dueAt <= input.completionDueBefore) && (!input.unleased || record.leaseOwner === void 0)
|
|
1885
2160
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1886
2161
|
}
|
|
1887
2162
|
async getHealthCounts(input) {
|
|
@@ -2168,6 +2443,18 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2168
2443
|
ALTER TABLE fabric_platform.event_outbox
|
|
2169
2444
|
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2170
2445
|
`
|
|
2446
|
+
}, {
|
|
2447
|
+
version: 3,
|
|
2448
|
+
name: "external_completion",
|
|
2449
|
+
sql: `
|
|
2450
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2451
|
+
ADD COLUMN pending_completion jsonb,
|
|
2452
|
+
ADD COLUMN external_completion jsonb,
|
|
2453
|
+
ADD COLUMN completion_due_at timestamptz;
|
|
2454
|
+
CREATE INDEX action_invocations_completion_due_idx
|
|
2455
|
+
ON fabric_platform.action_invocations (completion_due_at)
|
|
2456
|
+
WHERE completion_due_at IS NOT NULL;
|
|
2457
|
+
`
|
|
2171
2458
|
}]);
|
|
2172
2459
|
}
|
|
2173
2460
|
async transaction(run) {
|
|
@@ -2230,9 +2517,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2230
2517
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
2231
2518
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2232
2519
|
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
2233
|
-
|
|
2520
|
+
pending_completion=CASE WHEN $12::boolean THEN $13::jsonb ELSE pending_completion END,
|
|
2521
|
+
completion_due_at=CASE WHEN $12::boolean THEN ($13::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
2522
|
+
external_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE external_completion END,
|
|
2523
|
+
lease_owner=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2234
2524
|
THEN NULL ELSE lease_owner END,
|
|
2235
|
-
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2525
|
+
lease_expires_at=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2236
2526
|
THEN NULL ELSE lease_expires_at END,
|
|
2237
2527
|
updated_at=now()
|
|
2238
2528
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
@@ -2247,7 +2537,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2247
2537
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2248
2538
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2249
2539
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2250
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2540
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
2541
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
2542
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
2543
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
2544
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2251
2545
|
]
|
|
2252
2546
|
);
|
|
2253
2547
|
}
|
|
@@ -2600,6 +2894,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2600
2894
|
if (input.spaceId) add("space_id=?", input.spaceId);
|
|
2601
2895
|
if (input.statuses?.length) add("status = ANY(?::text[])", [...input.statuses]);
|
|
2602
2896
|
if (input.updatedBefore) add("updated_at<?", input.updatedBefore);
|
|
2897
|
+
if (input.awaitingCompletion || input.completionDueBefore) conditions.push("pending_completion IS NOT NULL");
|
|
2898
|
+
if (input.completionDueBefore) add("completion_due_at <= ?", input.completionDueBefore);
|
|
2899
|
+
if (input.unleased) conditions.push("lease_owner IS NULL");
|
|
2603
2900
|
values.push(Math.max(1, Math.min(input.limit ?? 100, 1e3)));
|
|
2604
2901
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2605
2902
|
const result = await this.sql.query(
|
|
@@ -2697,9 +2994,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2697
2994
|
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2698
2995
|
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2699
2996
|
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2700
|
-
|
|
2997
|
+
pending_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE pending_completion END,
|
|
2998
|
+
completion_due_at=CASE WHEN $14::boolean THEN ($15::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
2999
|
+
external_completion=CASE WHEN $16::boolean THEN $17::jsonb ELSE external_completion END,
|
|
3000
|
+
lease_owner=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
3001
|
THEN NULL ELSE lease_owner END,
|
|
2702
|
-
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3002
|
+
lease_expires_at=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2703
3003
|
THEN NULL ELSE lease_expires_at END,
|
|
2704
3004
|
updated_at=now()
|
|
2705
3005
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
@@ -2718,7 +3018,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2718
3018
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2719
3019
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2720
3020
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2721
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
3021
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
3022
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
3023
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3024
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
3025
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2722
3026
|
]
|
|
2723
3027
|
);
|
|
2724
3028
|
return result.rows.length === 1;
|
|
@@ -2779,6 +3083,20 @@ function toAdapterRecord(row) {
|
|
|
2779
3083
|
updatedAt: new Date(row.updated_at)
|
|
2780
3084
|
};
|
|
2781
3085
|
}
|
|
3086
|
+
function toPendingCompletion(value) {
|
|
3087
|
+
return {
|
|
3088
|
+
...value,
|
|
3089
|
+
acceptedAt: new Date(value.acceptedAt),
|
|
3090
|
+
...value.dueAt ? { dueAt: new Date(value.dueAt) } : {},
|
|
3091
|
+
...value.parkedAt ? { parkedAt: new Date(value.parkedAt) } : {}
|
|
3092
|
+
};
|
|
3093
|
+
}
|
|
3094
|
+
function toExternalCompletion(value) {
|
|
3095
|
+
return {
|
|
3096
|
+
...value,
|
|
3097
|
+
recordedAt: new Date(value.recordedAt)
|
|
3098
|
+
};
|
|
3099
|
+
}
|
|
2782
3100
|
function toActionRecord(row) {
|
|
2783
3101
|
return {
|
|
2784
3102
|
id: String(row.id),
|
|
@@ -2803,6 +3121,8 @@ function toActionRecord(row) {
|
|
|
2803
3121
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
2804
3122
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2805
3123
|
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
3124
|
+
...row.pending_completion ? { pendingCompletion: toPendingCompletion(row.pending_completion) } : {},
|
|
3125
|
+
...row.external_completion ? { externalCompletion: toExternalCompletion(row.external_completion) } : {},
|
|
2806
3126
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
2807
3127
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2808
3128
|
leaseToken: Number(row.lease_token ?? 0),
|
|
@@ -2933,6 +3253,16 @@ function createStoreBackedActionDispatcher() {
|
|
|
2933
3253
|
async function runPlatformActionWorkerCycle(options) {
|
|
2934
3254
|
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2935
3255
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
3256
|
+
if (options.sweepOverdueCompletions !== false && typeof options.host.reconcileOverdueCompletions === "function") {
|
|
3257
|
+
try {
|
|
3258
|
+
await options.host.reconcileOverdueCompletions({
|
|
3259
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
3260
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
3261
|
+
});
|
|
3262
|
+
} catch (error) {
|
|
3263
|
+
options.onError?.(error);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
2936
3266
|
let claimed;
|
|
2937
3267
|
try {
|
|
2938
3268
|
claimed = await options.store.claimActionInvocations({
|
|
@@ -3195,6 +3525,7 @@ function createDurableSagaParentLifecycle(options) {
|
|
|
3195
3525
|
};
|
|
3196
3526
|
}
|
|
3197
3527
|
|
|
3528
|
+
exports.ExternalCompletionError = ExternalCompletionError;
|
|
3198
3529
|
exports.IdempotencyConflictError = IdempotencyConflictError;
|
|
3199
3530
|
exports.MemoryPlatformHostStore = MemoryPlatformHostStore;
|
|
3200
3531
|
exports.PARAMETER_DIGEST_ALGORITHM = PARAMETER_DIGEST_ALGORITHM;
|