@fabricorg/platform-host 7.0.0 → 7.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +53 -0
- package/dist/index.cjs +368 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -7
- package/dist/index.d.ts +123 -7
- package/dist/index.js +368 -26
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
package/dist/index.cjs
CHANGED
|
@@ -17,6 +17,18 @@ var IdempotencyConflictError = class extends Error {
|
|
|
17
17
|
conflict;
|
|
18
18
|
code = "IDEMPOTENCY_CONFLICT";
|
|
19
19
|
};
|
|
20
|
+
var ExternalCompletionError = class extends Error {
|
|
21
|
+
constructor(refusal, message) {
|
|
22
|
+
super(message);
|
|
23
|
+
this.refusal = refusal;
|
|
24
|
+
this.name = "ExternalCompletionError";
|
|
25
|
+
}
|
|
26
|
+
refusal;
|
|
27
|
+
code = "EXTERNAL_COMPLETION_REFUSED";
|
|
28
|
+
get retryable() {
|
|
29
|
+
return this.refusal === "still_executing";
|
|
30
|
+
}
|
|
31
|
+
};
|
|
20
32
|
var PARAMETER_DIGEST_ALGORITHM = "fabric-canonical-json-sha256-v1";
|
|
21
33
|
function canonicalJson(value) {
|
|
22
34
|
return JSON.stringify(sort(value));
|
|
@@ -431,7 +443,8 @@ function createGovernedActionHost(options) {
|
|
|
431
443
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
432
444
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
433
445
|
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
434
|
-
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
446
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {},
|
|
447
|
+
...durableInvocation.pendingCompletion ? { pendingCompletion: durableInvocation.pendingCompletion } : {}
|
|
435
448
|
}, input.actionId);
|
|
436
449
|
}
|
|
437
450
|
if (options.dispatcher) {
|
|
@@ -476,6 +489,9 @@ function createGovernedActionHost(options) {
|
|
|
476
489
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
477
490
|
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
478
491
|
}
|
|
492
|
+
if (invocation.status === "running" && invocation.pendingCompletion && !invocation.leaseOwner) {
|
|
493
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
494
|
+
}
|
|
479
495
|
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
480
496
|
return {
|
|
481
497
|
...actionResult(invocation),
|
|
@@ -810,6 +826,7 @@ function createGovernedActionHost(options) {
|
|
|
810
826
|
} catch (error) {
|
|
811
827
|
return fail(invocation, "failed", errorMessage(error));
|
|
812
828
|
}
|
|
829
|
+
let pendingHandoff = invocation.pendingCompletion;
|
|
813
830
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
814
831
|
const input = step.getInput(parsed.data, data);
|
|
815
832
|
if (!input) continue;
|
|
@@ -819,6 +836,28 @@ function createGovernedActionHost(options) {
|
|
|
819
836
|
adapterInvocationId
|
|
820
837
|
);
|
|
821
838
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
839
|
+
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
840
|
+
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
841
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
842
|
+
await appendEvent(invocation, {
|
|
843
|
+
eventType: "ExternalOperationAccepted",
|
|
844
|
+
subjectType: subject.subjectType,
|
|
845
|
+
subjectId: subject.subjectId,
|
|
846
|
+
payload: {
|
|
847
|
+
adapterType: step.adapterType,
|
|
848
|
+
operation: step.operation,
|
|
849
|
+
externalReference: pendingHandoff.externalReference,
|
|
850
|
+
...pendingHandoff.dueAt ? { dueAt: pendingHandoff.dueAt.toISOString() } : {}
|
|
851
|
+
}
|
|
852
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
853
|
+
await appendEvent(invocation, {
|
|
854
|
+
eventType: "AdapterInvocationSucceeded",
|
|
855
|
+
subjectType: subject.subjectType,
|
|
856
|
+
subjectId: subject.subjectId,
|
|
857
|
+
payload: { adapterType: step.adapterType, operation: step.operation, recovered: true }
|
|
858
|
+
}, `adapter:${stepIndex}:succeeded`);
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
822
861
|
const adapterEventSubject = options.adapterEventSubject?.(
|
|
823
862
|
action.actionId,
|
|
824
863
|
parsed.data,
|
|
@@ -973,6 +1012,63 @@ function createGovernedActionHost(options) {
|
|
|
973
1012
|
([key]) => key !== "success" && key !== "error"
|
|
974
1013
|
)
|
|
975
1014
|
);
|
|
1015
|
+
const handoff = result2.acceptedExternalOperation;
|
|
1016
|
+
if (handoff) {
|
|
1017
|
+
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1018
|
+
if (reference === "") {
|
|
1019
|
+
const message = `Adapter ${step.adapterType}:${step.operation} accepted an external operation without a reference to complete it by; the external effect may be in flight.`;
|
|
1020
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1021
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message }, action.actionId);
|
|
1022
|
+
}
|
|
1023
|
+
const acceptedAt = now();
|
|
1024
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1025
|
+
const accepted = {
|
|
1026
|
+
provider: adapter.vendor,
|
|
1027
|
+
adapterType: step.adapterType,
|
|
1028
|
+
operation: step.operation,
|
|
1029
|
+
adapterInvocationId,
|
|
1030
|
+
externalReference: reference,
|
|
1031
|
+
acceptedAt,
|
|
1032
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1033
|
+
};
|
|
1034
|
+
const completion = action.execution?.completion ?? "immediate";
|
|
1035
|
+
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1036
|
+
if (refuse) {
|
|
1037
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() });
|
|
1038
|
+
await persistInvocation(invocation, {
|
|
1039
|
+
status: "reconciliation_required",
|
|
1040
|
+
error: refuse,
|
|
1041
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1042
|
+
});
|
|
1043
|
+
let error = refuse;
|
|
1044
|
+
try {
|
|
1045
|
+
await recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1046
|
+
} catch (logError) {
|
|
1047
|
+
error = `${refuse} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1048
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1049
|
+
}
|
|
1050
|
+
return withConsistency({
|
|
1051
|
+
actionInvocationId,
|
|
1052
|
+
status: "reconciliation_required",
|
|
1053
|
+
error,
|
|
1054
|
+
...pendingHandoff ? { pendingCompletion: pendingHandoff } : { pendingCompletion: accepted }
|
|
1055
|
+
}, action.actionId);
|
|
1056
|
+
}
|
|
1057
|
+
pendingHandoff = accepted;
|
|
1058
|
+
await persistInvocation(invocation, { pendingCompletion: accepted });
|
|
1059
|
+
invocation = { ...invocation, pendingCompletion: accepted };
|
|
1060
|
+
await appendEvent(invocation, {
|
|
1061
|
+
eventType: "ExternalOperationAccepted",
|
|
1062
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1063
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1064
|
+
payload: {
|
|
1065
|
+
adapterType: step.adapterType,
|
|
1066
|
+
operation: step.operation,
|
|
1067
|
+
externalReference: reference,
|
|
1068
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1069
|
+
}
|
|
1070
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
1071
|
+
}
|
|
976
1072
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
977
1073
|
status: "succeeded",
|
|
978
1074
|
output,
|
|
@@ -1022,10 +1118,24 @@ function createGovernedActionHost(options) {
|
|
|
1022
1118
|
const obligations = await governanceStore.listPolicyObligations(actionInvocationId, tenantId, spaceId);
|
|
1023
1119
|
const unsatisfied = obligations.filter((obligation) => (obligation.required ?? true) && obligation.status !== "satisfied" && obligation.status !== "waived");
|
|
1024
1120
|
if (unsatisfied.length > 0) {
|
|
1025
|
-
|
|
1121
|
+
const message = `Unsatisfied policy obligations: ${unsatisfied.map((item) => item.id).join(", ")}`;
|
|
1122
|
+
if (pendingHandoff) {
|
|
1123
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1124
|
+
let error = message;
|
|
1125
|
+
try {
|
|
1126
|
+
await recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1127
|
+
} catch (logError) {
|
|
1128
|
+
error = `${message} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1129
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1130
|
+
}
|
|
1131
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
1132
|
+
}
|
|
1133
|
+
return fail(invocation, "failed", message);
|
|
1026
1134
|
}
|
|
1027
1135
|
}
|
|
1028
1136
|
const result = withoutPrivateHostFields(data, eventResultFields);
|
|
1137
|
+
const parked = pendingHandoff ? { ...pendingHandoff, parkedAt: now() } : void 0;
|
|
1138
|
+
const finalPatch = parked ? { status: "running", result, pendingCompletion: parked } : { status: "completed", result };
|
|
1029
1139
|
try {
|
|
1030
1140
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1031
1141
|
if (action.eventPhase === "after_adapters" && atomicStore) {
|
|
@@ -1049,7 +1159,7 @@ function createGovernedActionHost(options) {
|
|
|
1049
1159
|
spaceId,
|
|
1050
1160
|
workerId: invocation.leaseOwner,
|
|
1051
1161
|
leaseToken: invocation.leaseToken,
|
|
1052
|
-
patch:
|
|
1162
|
+
patch: finalPatch
|
|
1053
1163
|
});
|
|
1054
1164
|
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1055
1165
|
} else {
|
|
@@ -1057,26 +1167,25 @@ function createGovernedActionHost(options) {
|
|
|
1057
1167
|
actionInvocationId,
|
|
1058
1168
|
tenantId,
|
|
1059
1169
|
spaceId,
|
|
1060
|
-
|
|
1170
|
+
finalPatch
|
|
1061
1171
|
);
|
|
1062
1172
|
}
|
|
1063
1173
|
});
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1174
|
+
if (!pendingHandoff) {
|
|
1175
|
+
emitInvocationStatusTelemetry(
|
|
1176
|
+
invocation,
|
|
1177
|
+
"completed",
|
|
1178
|
+
options.telemetry,
|
|
1179
|
+
now()
|
|
1180
|
+
);
|
|
1181
|
+
}
|
|
1070
1182
|
} else {
|
|
1071
1183
|
if (action.eventPhase === "after_adapters") {
|
|
1072
1184
|
for (const [index, event] of domainEvents.entries()) {
|
|
1073
1185
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1074
1186
|
}
|
|
1075
1187
|
}
|
|
1076
|
-
await persistInvocation(invocation,
|
|
1077
|
-
status: "completed",
|
|
1078
|
-
result
|
|
1079
|
-
});
|
|
1188
|
+
await persistInvocation(invocation, finalPatch);
|
|
1080
1189
|
}
|
|
1081
1190
|
} catch (error) {
|
|
1082
1191
|
if (action.eventPhase === "after_adapters") {
|
|
@@ -1086,8 +1195,9 @@ function createGovernedActionHost(options) {
|
|
|
1086
1195
|
}
|
|
1087
1196
|
return {
|
|
1088
1197
|
actionInvocationId,
|
|
1089
|
-
status: "completed",
|
|
1198
|
+
status: parked ? "running" : "completed",
|
|
1090
1199
|
result,
|
|
1200
|
+
...parked ? { pendingCompletion: parked } : {},
|
|
1091
1201
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1092
1202
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
1093
1203
|
};
|
|
@@ -1206,6 +1316,175 @@ function createGovernedActionHost(options) {
|
|
|
1206
1316
|
spaceId
|
|
1207
1317
|
});
|
|
1208
1318
|
}
|
|
1319
|
+
async function completeExternalInvocation(actionInvocationId, tenantId, spaceId, completion) {
|
|
1320
|
+
if (typeof completion.externalReference !== "string" || completion.externalReference.trim() === "") {
|
|
1321
|
+
throw new ExternalCompletionError("invalid_completion", "External completion must name the external reference it completes.");
|
|
1322
|
+
}
|
|
1323
|
+
if (completion.outcome !== "completed" && completion.outcome !== "failed") {
|
|
1324
|
+
throw new ExternalCompletionError("invalid_completion", `External completion outcome must be "completed" or "failed", received "${String(completion.outcome)}".`);
|
|
1325
|
+
}
|
|
1326
|
+
const atomicStore = asAtomicMutationStore(options.store);
|
|
1327
|
+
if (atomicStore) {
|
|
1328
|
+
return atomicStore.transactionWithEvents(async (transaction) => {
|
|
1329
|
+
const read2 = transaction.getActionInvocationForUpdate ? await transaction.getActionInvocationForUpdate(actionInvocationId, tenantId, spaceId) : await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1330
|
+
return settleExternalCompletion(read2, actionInvocationId, completion, {
|
|
1331
|
+
update: (patch) => transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch),
|
|
1332
|
+
transaction
|
|
1333
|
+
});
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
const read = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1337
|
+
return settleExternalCompletion(read, actionInvocationId, completion, {
|
|
1338
|
+
update: (patch) => options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch)
|
|
1339
|
+
});
|
|
1340
|
+
}
|
|
1341
|
+
async function settleExternalCompletion(invocation, actionInvocationId, completion, writer) {
|
|
1342
|
+
if (!invocation) throw new ExternalCompletionError("not_found", `ActionInvocation not found: ${actionInvocationId}`);
|
|
1343
|
+
const recorded = invocation.externalCompletion;
|
|
1344
|
+
const stillExecuting = !recorded && (invocation.status === "pending" || invocation.status === "running" && (invocation.leaseOwner !== void 0 || invocation.pendingCompletion?.parkedAt === void 0));
|
|
1345
|
+
if (stillExecuting) {
|
|
1346
|
+
throw new ExternalCompletionError("still_executing", `ActionInvocation ${actionInvocationId} is still executing; retry the completion after it parks.`);
|
|
1347
|
+
}
|
|
1348
|
+
if (recorded) {
|
|
1349
|
+
if (recorded.externalReference !== completion.externalReference) {
|
|
1350
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.externalReference}", not "${completion.externalReference}".`);
|
|
1351
|
+
}
|
|
1352
|
+
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
1353
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
1354
|
+
}
|
|
1355
|
+
const incomingDigest = completionDigest(completion, recorded.provider ?? completion.provider);
|
|
1356
|
+
if (recorded.digest === incomingDigest) {
|
|
1357
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
1358
|
+
}
|
|
1359
|
+
const message = recorded.outcome === completion.outcome ? `External operation "${completion.externalReference}" reported ${completion.outcome} again with different evidence.` : `External operation "${completion.externalReference}" reported ${completion.outcome} after reporting ${recorded.outcome}.`;
|
|
1360
|
+
await recordCompletionReconciliation(invocation, { ...completion, kind: `contradicted:${incomingDigest.slice(0, 16)}` }, message);
|
|
1361
|
+
await appendEvent(invocation, {
|
|
1362
|
+
eventType: "ExternalOperationContradicted",
|
|
1363
|
+
subjectType: "AdapterInvocation",
|
|
1364
|
+
subjectId: invocation.pendingCompletion?.adapterInvocationId ?? actionInvocationId,
|
|
1365
|
+
payload: { externalReference: completion.externalReference, recorded: recorded.outcome, reported: completion.outcome }
|
|
1366
|
+
}, `external:${completion.externalReference}:contradicted:${completion.outcome}`, 1, writer.transaction);
|
|
1367
|
+
await writer.update({ status: "reconciliation_required", error: message });
|
|
1368
|
+
emitInvocationStatusTelemetry(invocation, "reconciliation_required", options.telemetry, now(), invocation.status);
|
|
1369
|
+
return withConsistency(actionResult({ ...invocation, status: "reconciliation_required", error: message }), invocation.actionId);
|
|
1370
|
+
}
|
|
1371
|
+
const pending = invocation.pendingCompletion;
|
|
1372
|
+
if (!pending) {
|
|
1373
|
+
const action = actionResolver(invocation.actionId);
|
|
1374
|
+
if ((action?.execution?.completion ?? "immediate") === "immediate") {
|
|
1375
|
+
throw new ExternalCompletionError("immediate_contract", `Action ${invocation.actionId} declares immediate completion; nothing external completes it.`);
|
|
1376
|
+
}
|
|
1377
|
+
throw new ExternalCompletionError("not_awaiting", `ActionInvocation ${actionInvocationId} is not awaiting an external completion.`);
|
|
1378
|
+
}
|
|
1379
|
+
if (pending.externalReference !== completion.externalReference) {
|
|
1380
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} awaits "${pending.externalReference}", not "${completion.externalReference}".`);
|
|
1381
|
+
}
|
|
1382
|
+
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
1383
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
1384
|
+
}
|
|
1385
|
+
const recordedAt = now();
|
|
1386
|
+
const externalCompletion = {
|
|
1387
|
+
...completion,
|
|
1388
|
+
provider: completion.provider ?? pending.provider,
|
|
1389
|
+
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
1390
|
+
recordedAt,
|
|
1391
|
+
digest: completionDigest(completion, completion.provider ?? pending.provider)
|
|
1392
|
+
};
|
|
1393
|
+
const subject = { subjectType: "AdapterInvocation", subjectId: pending.adapterInvocationId };
|
|
1394
|
+
const settles = invocation.status === "running" && pending.parkedAt !== void 0 || invocation.status === "reconciliation_required" && invocation.error?.includes("did not complete by") === true;
|
|
1395
|
+
if (completion.outcome === "completed") {
|
|
1396
|
+
const action = actionResolver(invocation.actionId);
|
|
1397
|
+
const merged = withoutPrivateHostFields({ ...invocation.result, ...completion.result ?? {} }, eventResultFields);
|
|
1398
|
+
if (action?.resultSchema) {
|
|
1399
|
+
const parsedResult = action.resultSchema.safeParse(merged);
|
|
1400
|
+
if (!parsedResult.success) {
|
|
1401
|
+
throw new ExternalCompletionError("result_invalid", `External completion result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
1405
|
+
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
1406
|
+
updatedAt: recordedAt
|
|
1407
|
+
});
|
|
1408
|
+
await appendEvent(invocation, {
|
|
1409
|
+
eventType: "ExternalOperationCompleted",
|
|
1410
|
+
...subject,
|
|
1411
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, ...completion.result ? { result: withoutPrivateHostFields(completion.result, eventResultFields) } : {} }
|
|
1412
|
+
}, `external:${pending.adapterInvocationId}:completed`, 1, writer.transaction);
|
|
1413
|
+
const patch2 = settles ? { status: "completed", result: merged, externalCompletion, pendingCompletion: void 0, error: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1414
|
+
await writer.update(patch2);
|
|
1415
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "completed", options.telemetry, now(), invocation.status);
|
|
1416
|
+
return withConsistency(actionResult({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
1417
|
+
}
|
|
1418
|
+
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
1419
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
1420
|
+
await appendEvent(invocation, {
|
|
1421
|
+
eventType: "ExternalOperationFailed",
|
|
1422
|
+
...subject,
|
|
1423
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, error }
|
|
1424
|
+
}, `external:${pending.adapterInvocationId}:failed`, 1, writer.transaction);
|
|
1425
|
+
const patch = settles ? { status: "failed", error, externalCompletion, pendingCompletion: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1426
|
+
await writer.update(patch);
|
|
1427
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "failed", options.telemetry, now(), invocation.status);
|
|
1428
|
+
return withConsistency(actionResult({ ...invocation, ...patch }), invocation.actionId);
|
|
1429
|
+
}
|
|
1430
|
+
function completionDigest(completion, provider) {
|
|
1431
|
+
return crypto.createHash("sha256").update(canonicalJson({
|
|
1432
|
+
provider: provider ?? null,
|
|
1433
|
+
externalReference: completion.externalReference,
|
|
1434
|
+
outcome: completion.outcome,
|
|
1435
|
+
result: completion.result ?? null,
|
|
1436
|
+
error: completion.error ?? null,
|
|
1437
|
+
evidenceReferences: completion.evidenceReferences ?? null
|
|
1438
|
+
})).digest("hex");
|
|
1439
|
+
}
|
|
1440
|
+
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
1441
|
+
const governanceStore = asGovernanceStore(options.store);
|
|
1442
|
+
if (!governanceStore) throw new Error("External completion reconciliation requires a governance-capable store.");
|
|
1443
|
+
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
1444
|
+
const seed = crypto.createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
1445
|
+
await governanceStore.appendExternalReconciliation({
|
|
1446
|
+
id: lifecycleId("rec", invocation.id, `ext:${seed}`),
|
|
1447
|
+
actionInvocationId: invocation.id,
|
|
1448
|
+
tenantId: invocation.tenantId,
|
|
1449
|
+
spaceId: invocation.spaceId,
|
|
1450
|
+
status: "pending",
|
|
1451
|
+
provider,
|
|
1452
|
+
externalOperationId: completion.externalReference,
|
|
1453
|
+
attempt: 1,
|
|
1454
|
+
reason,
|
|
1455
|
+
...completion.evidenceReferences ? { evidenceReferences: completion.evidenceReferences } : {},
|
|
1456
|
+
observedAt: now()
|
|
1457
|
+
});
|
|
1458
|
+
}
|
|
1459
|
+
async function reconcileOverdueCompletions(input = {}) {
|
|
1460
|
+
const recoverable = options.store;
|
|
1461
|
+
if (!recoverable.listActionInvocations) {
|
|
1462
|
+
throw new Error("Overdue completion reconciliation requires a store that can list invocations.");
|
|
1463
|
+
}
|
|
1464
|
+
if (!asGovernanceStore(options.store)) {
|
|
1465
|
+
throw new Error("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
1466
|
+
}
|
|
1467
|
+
const current = input.now ?? now();
|
|
1468
|
+
const candidates = await recoverable.listActionInvocations({
|
|
1469
|
+
statuses: ["running"],
|
|
1470
|
+
completionDueBefore: current,
|
|
1471
|
+
unleased: true,
|
|
1472
|
+
...input.tenantId ? { tenantId: input.tenantId } : {},
|
|
1473
|
+
...input.spaceId ? { spaceId: input.spaceId } : {},
|
|
1474
|
+
limit: Math.max(1, Math.min(input.limit ?? 100, 1e3))
|
|
1475
|
+
});
|
|
1476
|
+
const overdue = [];
|
|
1477
|
+
for (const invocation of candidates) {
|
|
1478
|
+
const pending = invocation.pendingCompletion;
|
|
1479
|
+
if (!pending?.dueAt || pending.dueAt.getTime() > current.getTime()) continue;
|
|
1480
|
+
if (invocation.leaseOwner) continue;
|
|
1481
|
+
const message = `External operation "${pending.externalReference}" from ${pending.provider} did not complete by ${pending.dueAt.toISOString()}.`;
|
|
1482
|
+
await recordCompletionReconciliation(invocation, { externalReference: pending.externalReference, provider: pending.provider, kind: "overdue" }, message);
|
|
1483
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1484
|
+
overdue.push(invocation.id);
|
|
1485
|
+
}
|
|
1486
|
+
return { overdue };
|
|
1487
|
+
}
|
|
1209
1488
|
function declaredConsistency(actionId) {
|
|
1210
1489
|
return actionResolver(actionId)?.execution?.consistency;
|
|
1211
1490
|
}
|
|
@@ -1239,7 +1518,7 @@ function createGovernedActionHost(options) {
|
|
|
1239
1518
|
...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
|
|
1240
1519
|
};
|
|
1241
1520
|
if (options.outbox) {
|
|
1242
|
-
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1521
|
+
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ExternalOperationAccepted", "ExternalOperationCompleted", "ExternalOperationFailed", "ExternalOperationContradicted", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1243
1522
|
const shouldPublish = options.outbox.shouldPublish?.(envelope) ?? !hostLifecycleEvent;
|
|
1244
1523
|
if (!shouldPublish) {
|
|
1245
1524
|
await (transaction ?? options.store).appendEvent(envelope);
|
|
@@ -1302,7 +1581,7 @@ function createGovernedActionHost(options) {
|
|
|
1302
1581
|
);
|
|
1303
1582
|
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1304
1583
|
}
|
|
1305
|
-
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
1584
|
+
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation, completeExternalInvocation, reconcileOverdueCompletions };
|
|
1306
1585
|
}
|
|
1307
1586
|
function actionResult(invocation) {
|
|
1308
1587
|
return {
|
|
@@ -1313,7 +1592,8 @@ function actionResult(invocation) {
|
|
|
1313
1592
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1314
1593
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
1315
1594
|
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1316
|
-
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
1595
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {},
|
|
1596
|
+
...invocation.pendingCompletion ? { pendingCompletion: invocation.pendingCompletion } : {}
|
|
1317
1597
|
};
|
|
1318
1598
|
}
|
|
1319
1599
|
function asApprovalStore(store) {
|
|
@@ -1713,6 +1993,13 @@ var MemoryPlatformHostStore = class {
|
|
|
1713
1993
|
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1714
1994
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
1715
1995
|
Object.assign(record, patch, { updatedAt: /* @__PURE__ */ new Date() });
|
|
1996
|
+
if (Object.hasOwn(patch, "pendingCompletion") && patch.pendingCompletion === void 0) delete record.pendingCompletion;
|
|
1997
|
+
if (Object.hasOwn(patch, "externalCompletion") && patch.externalCompletion === void 0) delete record.externalCompletion;
|
|
1998
|
+
if (Object.hasOwn(patch, "error") && patch.error === void 0) delete record.error;
|
|
1999
|
+
if (patch.pendingCompletion && patch.status === "running") {
|
|
2000
|
+
delete record.leaseOwner;
|
|
2001
|
+
delete record.leaseExpiresAt;
|
|
2002
|
+
}
|
|
1716
2003
|
if (patch.status === "waiting_for_approval" || patch.status === "completed" || patch.status === "failed" || patch.status === "blocked_by_policy" || patch.status === "reconciliation_required" || patch.status === "validation_failed") {
|
|
1717
2004
|
delete record.leaseOwner;
|
|
1718
2005
|
delete record.leaseExpiresAt;
|
|
@@ -1881,7 +2168,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1881
2168
|
}
|
|
1882
2169
|
async listActionInvocations(input = {}) {
|
|
1883
2170
|
return this.invocations.filter(
|
|
1884
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
2171
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore) && (!(input.awaitingCompletion || input.completionDueBefore) || record.pendingCompletion !== void 0) && (!input.completionDueBefore || record.pendingCompletion?.dueAt !== void 0 && record.pendingCompletion.dueAt <= input.completionDueBefore) && (!input.unleased || record.leaseOwner === void 0)
|
|
1885
2172
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1886
2173
|
}
|
|
1887
2174
|
async getHealthCounts(input) {
|
|
@@ -2168,6 +2455,18 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2168
2455
|
ALTER TABLE fabric_platform.event_outbox
|
|
2169
2456
|
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2170
2457
|
`
|
|
2458
|
+
}, {
|
|
2459
|
+
version: 3,
|
|
2460
|
+
name: "external_completion",
|
|
2461
|
+
sql: `
|
|
2462
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2463
|
+
ADD COLUMN pending_completion jsonb,
|
|
2464
|
+
ADD COLUMN external_completion jsonb,
|
|
2465
|
+
ADD COLUMN completion_due_at timestamptz;
|
|
2466
|
+
CREATE INDEX action_invocations_completion_due_idx
|
|
2467
|
+
ON fabric_platform.action_invocations (completion_due_at)
|
|
2468
|
+
WHERE completion_due_at IS NOT NULL;
|
|
2469
|
+
`
|
|
2171
2470
|
}]);
|
|
2172
2471
|
}
|
|
2173
2472
|
async transaction(run) {
|
|
@@ -2230,9 +2529,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2230
2529
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
2231
2530
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2232
2531
|
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
2233
|
-
|
|
2532
|
+
pending_completion=CASE WHEN $12::boolean THEN $13::jsonb ELSE pending_completion END,
|
|
2533
|
+
completion_due_at=CASE WHEN $12::boolean THEN ($13::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
2534
|
+
external_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE external_completion END,
|
|
2535
|
+
lease_owner=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2234
2536
|
THEN NULL ELSE lease_owner END,
|
|
2235
|
-
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2537
|
+
lease_expires_at=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2236
2538
|
THEN NULL ELSE lease_expires_at END,
|
|
2237
2539
|
updated_at=now()
|
|
2238
2540
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
@@ -2247,7 +2549,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2247
2549
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2248
2550
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2249
2551
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2250
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2552
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
2553
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
2554
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
2555
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
2556
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2251
2557
|
]
|
|
2252
2558
|
);
|
|
2253
2559
|
}
|
|
@@ -2600,6 +2906,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2600
2906
|
if (input.spaceId) add("space_id=?", input.spaceId);
|
|
2601
2907
|
if (input.statuses?.length) add("status = ANY(?::text[])", [...input.statuses]);
|
|
2602
2908
|
if (input.updatedBefore) add("updated_at<?", input.updatedBefore);
|
|
2909
|
+
if (input.awaitingCompletion || input.completionDueBefore) conditions.push("pending_completion IS NOT NULL");
|
|
2910
|
+
if (input.completionDueBefore) add("completion_due_at <= ?", input.completionDueBefore);
|
|
2911
|
+
if (input.unleased) conditions.push("lease_owner IS NULL");
|
|
2603
2912
|
values.push(Math.max(1, Math.min(input.limit ?? 100, 1e3)));
|
|
2604
2913
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2605
2914
|
const result = await this.sql.query(
|
|
@@ -2697,9 +3006,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2697
3006
|
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2698
3007
|
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2699
3008
|
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2700
|
-
|
|
3009
|
+
pending_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE pending_completion END,
|
|
3010
|
+
completion_due_at=CASE WHEN $14::boolean THEN ($15::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
3011
|
+
external_completion=CASE WHEN $16::boolean THEN $17::jsonb ELSE external_completion END,
|
|
3012
|
+
lease_owner=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
3013
|
THEN NULL ELSE lease_owner END,
|
|
2702
|
-
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3014
|
+
lease_expires_at=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2703
3015
|
THEN NULL ELSE lease_expires_at END,
|
|
2704
3016
|
updated_at=now()
|
|
2705
3017
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
@@ -2718,7 +3030,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2718
3030
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2719
3031
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2720
3032
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2721
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
3033
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
3034
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
3035
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3036
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
3037
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2722
3038
|
]
|
|
2723
3039
|
);
|
|
2724
3040
|
return result.rows.length === 1;
|
|
@@ -2779,6 +3095,20 @@ function toAdapterRecord(row) {
|
|
|
2779
3095
|
updatedAt: new Date(row.updated_at)
|
|
2780
3096
|
};
|
|
2781
3097
|
}
|
|
3098
|
+
function toPendingCompletion(value) {
|
|
3099
|
+
return {
|
|
3100
|
+
...value,
|
|
3101
|
+
acceptedAt: new Date(value.acceptedAt),
|
|
3102
|
+
...value.dueAt ? { dueAt: new Date(value.dueAt) } : {},
|
|
3103
|
+
...value.parkedAt ? { parkedAt: new Date(value.parkedAt) } : {}
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
function toExternalCompletion(value) {
|
|
3107
|
+
return {
|
|
3108
|
+
...value,
|
|
3109
|
+
recordedAt: new Date(value.recordedAt)
|
|
3110
|
+
};
|
|
3111
|
+
}
|
|
2782
3112
|
function toActionRecord(row) {
|
|
2783
3113
|
return {
|
|
2784
3114
|
id: String(row.id),
|
|
@@ -2803,6 +3133,8 @@ function toActionRecord(row) {
|
|
|
2803
3133
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
2804
3134
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2805
3135
|
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
3136
|
+
...row.pending_completion ? { pendingCompletion: toPendingCompletion(row.pending_completion) } : {},
|
|
3137
|
+
...row.external_completion ? { externalCompletion: toExternalCompletion(row.external_completion) } : {},
|
|
2806
3138
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
2807
3139
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2808
3140
|
leaseToken: Number(row.lease_token ?? 0),
|
|
@@ -2933,6 +3265,16 @@ function createStoreBackedActionDispatcher() {
|
|
|
2933
3265
|
async function runPlatformActionWorkerCycle(options) {
|
|
2934
3266
|
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2935
3267
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
3268
|
+
if (options.sweepOverdueCompletions !== false && typeof options.host.reconcileOverdueCompletions === "function") {
|
|
3269
|
+
try {
|
|
3270
|
+
await options.host.reconcileOverdueCompletions({
|
|
3271
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
3272
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
3273
|
+
});
|
|
3274
|
+
} catch (error) {
|
|
3275
|
+
options.onError?.(error);
|
|
3276
|
+
}
|
|
3277
|
+
}
|
|
2936
3278
|
let claimed;
|
|
2937
3279
|
try {
|
|
2938
3280
|
claimed = await options.store.claimActionInvocations({
|
|
@@ -3195,6 +3537,7 @@ function createDurableSagaParentLifecycle(options) {
|
|
|
3195
3537
|
};
|
|
3196
3538
|
}
|
|
3197
3539
|
|
|
3540
|
+
exports.ExternalCompletionError = ExternalCompletionError;
|
|
3198
3541
|
exports.IdempotencyConflictError = IdempotencyConflictError;
|
|
3199
3542
|
exports.MemoryPlatformHostStore = MemoryPlatformHostStore;
|
|
3200
3543
|
exports.PARAMETER_DIGEST_ALGORITHM = PARAMETER_DIGEST_ALGORITHM;
|