@fabricorg/platform-host 6.0.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +51 -0
- package/dist/index.cjs +356 -25
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +115 -7
- package/dist/index.d.ts +115 -7
- package/dist/index.js +356 -26
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -15,6 +15,18 @@ var IdempotencyConflictError = class extends Error {
|
|
|
15
15
|
conflict;
|
|
16
16
|
code = "IDEMPOTENCY_CONFLICT";
|
|
17
17
|
};
|
|
18
|
+
var ExternalCompletionError = class extends Error {
|
|
19
|
+
constructor(refusal, message) {
|
|
20
|
+
super(message);
|
|
21
|
+
this.refusal = refusal;
|
|
22
|
+
this.name = "ExternalCompletionError";
|
|
23
|
+
}
|
|
24
|
+
refusal;
|
|
25
|
+
code = "EXTERNAL_COMPLETION_REFUSED";
|
|
26
|
+
get retryable() {
|
|
27
|
+
return this.refusal === "still_executing";
|
|
28
|
+
}
|
|
29
|
+
};
|
|
18
30
|
var PARAMETER_DIGEST_ALGORITHM = "fabric-canonical-json-sha256-v1";
|
|
19
31
|
function canonicalJson(value) {
|
|
20
32
|
return JSON.stringify(sort(value));
|
|
@@ -429,7 +441,8 @@ function createGovernedActionHost(options) {
|
|
|
429
441
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
430
442
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
431
443
|
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
432
|
-
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
444
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {},
|
|
445
|
+
...durableInvocation.pendingCompletion ? { pendingCompletion: durableInvocation.pendingCompletion } : {}
|
|
433
446
|
}, input.actionId);
|
|
434
447
|
}
|
|
435
448
|
if (options.dispatcher) {
|
|
@@ -474,6 +487,9 @@ function createGovernedActionHost(options) {
|
|
|
474
487
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
475
488
|
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
476
489
|
}
|
|
490
|
+
if (invocation.status === "running" && invocation.pendingCompletion && !invocation.leaseOwner) {
|
|
491
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
492
|
+
}
|
|
477
493
|
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
478
494
|
return {
|
|
479
495
|
...actionResult(invocation),
|
|
@@ -808,6 +824,7 @@ function createGovernedActionHost(options) {
|
|
|
808
824
|
} catch (error) {
|
|
809
825
|
return fail(invocation, "failed", errorMessage(error));
|
|
810
826
|
}
|
|
827
|
+
let pendingHandoff = invocation.pendingCompletion;
|
|
811
828
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
812
829
|
const input = step.getInput(parsed.data, data);
|
|
813
830
|
if (!input) continue;
|
|
@@ -817,6 +834,28 @@ function createGovernedActionHost(options) {
|
|
|
817
834
|
adapterInvocationId
|
|
818
835
|
);
|
|
819
836
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
837
|
+
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
838
|
+
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
839
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
840
|
+
await appendEvent(invocation, {
|
|
841
|
+
eventType: "ExternalOperationAccepted",
|
|
842
|
+
subjectType: subject.subjectType,
|
|
843
|
+
subjectId: subject.subjectId,
|
|
844
|
+
payload: {
|
|
845
|
+
adapterType: step.adapterType,
|
|
846
|
+
operation: step.operation,
|
|
847
|
+
externalReference: pendingHandoff.externalReference,
|
|
848
|
+
...pendingHandoff.dueAt ? { dueAt: pendingHandoff.dueAt.toISOString() } : {}
|
|
849
|
+
}
|
|
850
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
851
|
+
await appendEvent(invocation, {
|
|
852
|
+
eventType: "AdapterInvocationSucceeded",
|
|
853
|
+
subjectType: subject.subjectType,
|
|
854
|
+
subjectId: subject.subjectId,
|
|
855
|
+
payload: { adapterType: step.adapterType, operation: step.operation, recovered: true }
|
|
856
|
+
}, `adapter:${stepIndex}:succeeded`);
|
|
857
|
+
continue;
|
|
858
|
+
}
|
|
820
859
|
const adapterEventSubject = options.adapterEventSubject?.(
|
|
821
860
|
action.actionId,
|
|
822
861
|
parsed.data,
|
|
@@ -971,6 +1010,63 @@ function createGovernedActionHost(options) {
|
|
|
971
1010
|
([key]) => key !== "success" && key !== "error"
|
|
972
1011
|
)
|
|
973
1012
|
);
|
|
1013
|
+
const handoff = result2.acceptedExternalOperation;
|
|
1014
|
+
if (handoff) {
|
|
1015
|
+
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1016
|
+
if (reference === "") {
|
|
1017
|
+
const message = `Adapter ${step.adapterType}:${step.operation} accepted an external operation without a reference to complete it by; the external effect may be in flight.`;
|
|
1018
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1019
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message }, action.actionId);
|
|
1020
|
+
}
|
|
1021
|
+
const acceptedAt = now();
|
|
1022
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1023
|
+
const accepted = {
|
|
1024
|
+
provider: adapter.vendor,
|
|
1025
|
+
adapterType: step.adapterType,
|
|
1026
|
+
operation: step.operation,
|
|
1027
|
+
adapterInvocationId,
|
|
1028
|
+
externalReference: reference,
|
|
1029
|
+
acceptedAt,
|
|
1030
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1031
|
+
};
|
|
1032
|
+
const completion = action.execution?.completion ?? "immediate";
|
|
1033
|
+
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1034
|
+
if (refuse) {
|
|
1035
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() });
|
|
1036
|
+
await persistInvocation(invocation, {
|
|
1037
|
+
status: "reconciliation_required",
|
|
1038
|
+
error: refuse,
|
|
1039
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1040
|
+
});
|
|
1041
|
+
let error = refuse;
|
|
1042
|
+
try {
|
|
1043
|
+
await recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1044
|
+
} catch (logError) {
|
|
1045
|
+
error = `${refuse} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1046
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1047
|
+
}
|
|
1048
|
+
return withConsistency({
|
|
1049
|
+
actionInvocationId,
|
|
1050
|
+
status: "reconciliation_required",
|
|
1051
|
+
error,
|
|
1052
|
+
...pendingHandoff ? { pendingCompletion: pendingHandoff } : { pendingCompletion: accepted }
|
|
1053
|
+
}, action.actionId);
|
|
1054
|
+
}
|
|
1055
|
+
pendingHandoff = accepted;
|
|
1056
|
+
await persistInvocation(invocation, { pendingCompletion: accepted });
|
|
1057
|
+
invocation = { ...invocation, pendingCompletion: accepted };
|
|
1058
|
+
await appendEvent(invocation, {
|
|
1059
|
+
eventType: "ExternalOperationAccepted",
|
|
1060
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1061
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1062
|
+
payload: {
|
|
1063
|
+
adapterType: step.adapterType,
|
|
1064
|
+
operation: step.operation,
|
|
1065
|
+
externalReference: reference,
|
|
1066
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1067
|
+
}
|
|
1068
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
1069
|
+
}
|
|
974
1070
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
975
1071
|
status: "succeeded",
|
|
976
1072
|
output,
|
|
@@ -1020,10 +1116,24 @@ function createGovernedActionHost(options) {
|
|
|
1020
1116
|
const obligations = await governanceStore.listPolicyObligations(actionInvocationId, tenantId, spaceId);
|
|
1021
1117
|
const unsatisfied = obligations.filter((obligation) => (obligation.required ?? true) && obligation.status !== "satisfied" && obligation.status !== "waived");
|
|
1022
1118
|
if (unsatisfied.length > 0) {
|
|
1023
|
-
|
|
1119
|
+
const message = `Unsatisfied policy obligations: ${unsatisfied.map((item) => item.id).join(", ")}`;
|
|
1120
|
+
if (pendingHandoff) {
|
|
1121
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1122
|
+
let error = message;
|
|
1123
|
+
try {
|
|
1124
|
+
await recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1125
|
+
} catch (logError) {
|
|
1126
|
+
error = `${message} Reconciliation log unavailable: ${errorMessage(logError)}`;
|
|
1127
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1128
|
+
}
|
|
1129
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
1130
|
+
}
|
|
1131
|
+
return fail(invocation, "failed", message);
|
|
1024
1132
|
}
|
|
1025
1133
|
}
|
|
1026
1134
|
const result = withoutPrivateHostFields(data, eventResultFields);
|
|
1135
|
+
const parked = pendingHandoff ? { ...pendingHandoff, parkedAt: now() } : void 0;
|
|
1136
|
+
const finalPatch = parked ? { status: "running", result, pendingCompletion: parked } : { status: "completed", result };
|
|
1027
1137
|
try {
|
|
1028
1138
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1029
1139
|
if (action.eventPhase === "after_adapters" && atomicStore) {
|
|
@@ -1047,7 +1157,7 @@ function createGovernedActionHost(options) {
|
|
|
1047
1157
|
spaceId,
|
|
1048
1158
|
workerId: invocation.leaseOwner,
|
|
1049
1159
|
leaseToken: invocation.leaseToken,
|
|
1050
|
-
patch:
|
|
1160
|
+
patch: finalPatch
|
|
1051
1161
|
});
|
|
1052
1162
|
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1053
1163
|
} else {
|
|
@@ -1055,26 +1165,25 @@ function createGovernedActionHost(options) {
|
|
|
1055
1165
|
actionInvocationId,
|
|
1056
1166
|
tenantId,
|
|
1057
1167
|
spaceId,
|
|
1058
|
-
|
|
1168
|
+
finalPatch
|
|
1059
1169
|
);
|
|
1060
1170
|
}
|
|
1061
1171
|
});
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1172
|
+
if (!pendingHandoff) {
|
|
1173
|
+
emitInvocationStatusTelemetry(
|
|
1174
|
+
invocation,
|
|
1175
|
+
"completed",
|
|
1176
|
+
options.telemetry,
|
|
1177
|
+
now()
|
|
1178
|
+
);
|
|
1179
|
+
}
|
|
1068
1180
|
} else {
|
|
1069
1181
|
if (action.eventPhase === "after_adapters") {
|
|
1070
1182
|
for (const [index, event] of domainEvents.entries()) {
|
|
1071
1183
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1072
1184
|
}
|
|
1073
1185
|
}
|
|
1074
|
-
await persistInvocation(invocation,
|
|
1075
|
-
status: "completed",
|
|
1076
|
-
result
|
|
1077
|
-
});
|
|
1186
|
+
await persistInvocation(invocation, finalPatch);
|
|
1078
1187
|
}
|
|
1079
1188
|
} catch (error) {
|
|
1080
1189
|
if (action.eventPhase === "after_adapters") {
|
|
@@ -1084,8 +1193,9 @@ function createGovernedActionHost(options) {
|
|
|
1084
1193
|
}
|
|
1085
1194
|
return {
|
|
1086
1195
|
actionInvocationId,
|
|
1087
|
-
status: "completed",
|
|
1196
|
+
status: parked ? "running" : "completed",
|
|
1088
1197
|
result,
|
|
1198
|
+
...parked ? { pendingCompletion: parked } : {},
|
|
1089
1199
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1090
1200
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
1091
1201
|
};
|
|
@@ -1204,6 +1314,163 @@ function createGovernedActionHost(options) {
|
|
|
1204
1314
|
spaceId
|
|
1205
1315
|
});
|
|
1206
1316
|
}
|
|
1317
|
+
async function completeExternalInvocation(actionInvocationId, tenantId, spaceId, completion) {
|
|
1318
|
+
if (typeof completion.externalReference !== "string" || completion.externalReference.trim() === "") {
|
|
1319
|
+
throw new ExternalCompletionError("invalid_completion", "External completion must name the external reference it completes.");
|
|
1320
|
+
}
|
|
1321
|
+
if (completion.outcome !== "completed" && completion.outcome !== "failed") {
|
|
1322
|
+
throw new ExternalCompletionError("invalid_completion", `External completion outcome must be "completed" or "failed", received "${String(completion.outcome)}".`);
|
|
1323
|
+
}
|
|
1324
|
+
const atomicStore = asAtomicMutationStore(options.store);
|
|
1325
|
+
if (atomicStore) {
|
|
1326
|
+
return atomicStore.transactionWithEvents(async (transaction) => {
|
|
1327
|
+
const read2 = transaction.getActionInvocationForUpdate ? await transaction.getActionInvocationForUpdate(actionInvocationId, tenantId, spaceId) : await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1328
|
+
return settleExternalCompletion(read2, actionInvocationId, completion, {
|
|
1329
|
+
update: (patch) => transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch),
|
|
1330
|
+
transaction
|
|
1331
|
+
});
|
|
1332
|
+
});
|
|
1333
|
+
}
|
|
1334
|
+
const read = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1335
|
+
return settleExternalCompletion(read, actionInvocationId, completion, {
|
|
1336
|
+
update: (patch) => options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch)
|
|
1337
|
+
});
|
|
1338
|
+
}
|
|
1339
|
+
async function settleExternalCompletion(invocation, actionInvocationId, completion, writer) {
|
|
1340
|
+
if (!invocation) throw new ExternalCompletionError("not_found", `ActionInvocation not found: ${actionInvocationId}`);
|
|
1341
|
+
const recorded = invocation.externalCompletion;
|
|
1342
|
+
const stillExecuting = !recorded && (invocation.status === "pending" || invocation.status === "running" && (invocation.leaseOwner !== void 0 || invocation.pendingCompletion?.parkedAt === void 0));
|
|
1343
|
+
if (stillExecuting) {
|
|
1344
|
+
throw new ExternalCompletionError("still_executing", `ActionInvocation ${actionInvocationId} is still executing; retry the completion after it parks.`);
|
|
1345
|
+
}
|
|
1346
|
+
if (recorded) {
|
|
1347
|
+
if (recorded.externalReference !== completion.externalReference) {
|
|
1348
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.externalReference}", not "${completion.externalReference}".`);
|
|
1349
|
+
}
|
|
1350
|
+
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
1351
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
1352
|
+
}
|
|
1353
|
+
if (recorded.outcome === completion.outcome) {
|
|
1354
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
1355
|
+
}
|
|
1356
|
+
const message = `External operation "${completion.externalReference}" reported ${completion.outcome} after reporting ${recorded.outcome}.`;
|
|
1357
|
+
await recordCompletionReconciliation(invocation, { ...completion, kind: `contradicted:${completion.outcome}` }, message);
|
|
1358
|
+
await appendEvent(invocation, {
|
|
1359
|
+
eventType: "ExternalOperationContradicted",
|
|
1360
|
+
subjectType: "AdapterInvocation",
|
|
1361
|
+
subjectId: invocation.pendingCompletion?.adapterInvocationId ?? actionInvocationId,
|
|
1362
|
+
payload: { externalReference: completion.externalReference, recorded: recorded.outcome, reported: completion.outcome }
|
|
1363
|
+
}, `external:${completion.externalReference}:contradicted:${completion.outcome}`, 1, writer.transaction);
|
|
1364
|
+
await writer.update({ status: "reconciliation_required", error: message });
|
|
1365
|
+
emitInvocationStatusTelemetry(invocation, "reconciliation_required", options.telemetry, now(), invocation.status);
|
|
1366
|
+
return withConsistency(actionResult({ ...invocation, status: "reconciliation_required", error: message }), invocation.actionId);
|
|
1367
|
+
}
|
|
1368
|
+
const pending = invocation.pendingCompletion;
|
|
1369
|
+
if (!pending) {
|
|
1370
|
+
const action = actionResolver(invocation.actionId);
|
|
1371
|
+
if ((action?.execution?.completion ?? "immediate") === "immediate") {
|
|
1372
|
+
throw new ExternalCompletionError("immediate_contract", `Action ${invocation.actionId} declares immediate completion; nothing external completes it.`);
|
|
1373
|
+
}
|
|
1374
|
+
throw new ExternalCompletionError("not_awaiting", `ActionInvocation ${actionInvocationId} is not awaiting an external completion.`);
|
|
1375
|
+
}
|
|
1376
|
+
if (pending.externalReference !== completion.externalReference) {
|
|
1377
|
+
throw new ExternalCompletionError("reference_mismatch", `ActionInvocation ${actionInvocationId} awaits "${pending.externalReference}", not "${completion.externalReference}".`);
|
|
1378
|
+
}
|
|
1379
|
+
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
1380
|
+
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
1381
|
+
}
|
|
1382
|
+
const recordedAt = now();
|
|
1383
|
+
const externalCompletion = {
|
|
1384
|
+
...completion,
|
|
1385
|
+
provider: completion.provider ?? pending.provider,
|
|
1386
|
+
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
1387
|
+
recordedAt
|
|
1388
|
+
};
|
|
1389
|
+
const subject = { subjectType: "AdapterInvocation", subjectId: pending.adapterInvocationId };
|
|
1390
|
+
const settles = invocation.status === "running" && pending.parkedAt !== void 0 || invocation.status === "reconciliation_required" && invocation.error?.includes("did not complete by") === true;
|
|
1391
|
+
if (completion.outcome === "completed") {
|
|
1392
|
+
const action = actionResolver(invocation.actionId);
|
|
1393
|
+
const merged = withoutPrivateHostFields({ ...invocation.result, ...completion.result ?? {} }, eventResultFields);
|
|
1394
|
+
if (action?.resultSchema) {
|
|
1395
|
+
const parsedResult = action.resultSchema.safeParse(merged);
|
|
1396
|
+
if (!parsedResult.success) {
|
|
1397
|
+
throw new ExternalCompletionError("result_invalid", `External completion result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
1401
|
+
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
1402
|
+
updatedAt: recordedAt
|
|
1403
|
+
});
|
|
1404
|
+
await appendEvent(invocation, {
|
|
1405
|
+
eventType: "ExternalOperationCompleted",
|
|
1406
|
+
...subject,
|
|
1407
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, ...completion.result ? { result: withoutPrivateHostFields(completion.result, eventResultFields) } : {} }
|
|
1408
|
+
}, `external:${pending.adapterInvocationId}:completed`, 1, writer.transaction);
|
|
1409
|
+
const patch2 = settles ? { status: "completed", result: merged, externalCompletion, pendingCompletion: void 0, error: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1410
|
+
await writer.update(patch2);
|
|
1411
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "completed", options.telemetry, now(), invocation.status);
|
|
1412
|
+
return withConsistency(actionResult({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
1413
|
+
}
|
|
1414
|
+
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
1415
|
+
await options.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
1416
|
+
await appendEvent(invocation, {
|
|
1417
|
+
eventType: "ExternalOperationFailed",
|
|
1418
|
+
...subject,
|
|
1419
|
+
payload: { externalReference: pending.externalReference, provider: pending.provider, error }
|
|
1420
|
+
}, `external:${pending.adapterInvocationId}:failed`, 1, writer.transaction);
|
|
1421
|
+
const patch = settles ? { status: "failed", error, externalCompletion, pendingCompletion: void 0 } : { externalCompletion, pendingCompletion: void 0 };
|
|
1422
|
+
await writer.update(patch);
|
|
1423
|
+
if (settles) emitInvocationStatusTelemetry(invocation, "failed", options.telemetry, now(), invocation.status);
|
|
1424
|
+
return withConsistency(actionResult({ ...invocation, ...patch }), invocation.actionId);
|
|
1425
|
+
}
|
|
1426
|
+
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
1427
|
+
const governanceStore = asGovernanceStore(options.store);
|
|
1428
|
+
if (!governanceStore) throw new Error("External completion reconciliation requires a governance-capable store.");
|
|
1429
|
+
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
1430
|
+
const seed = createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
1431
|
+
await governanceStore.appendExternalReconciliation({
|
|
1432
|
+
id: lifecycleId("rec", invocation.id, `ext:${seed}`),
|
|
1433
|
+
actionInvocationId: invocation.id,
|
|
1434
|
+
tenantId: invocation.tenantId,
|
|
1435
|
+
spaceId: invocation.spaceId,
|
|
1436
|
+
status: "pending",
|
|
1437
|
+
provider,
|
|
1438
|
+
externalOperationId: completion.externalReference,
|
|
1439
|
+
attempt: 1,
|
|
1440
|
+
reason,
|
|
1441
|
+
...completion.evidenceReferences ? { evidenceReferences: completion.evidenceReferences } : {},
|
|
1442
|
+
observedAt: now()
|
|
1443
|
+
});
|
|
1444
|
+
}
|
|
1445
|
+
async function reconcileOverdueCompletions(input = {}) {
|
|
1446
|
+
const recoverable = options.store;
|
|
1447
|
+
if (!recoverable.listActionInvocations) {
|
|
1448
|
+
throw new Error("Overdue completion reconciliation requires a store that can list invocations.");
|
|
1449
|
+
}
|
|
1450
|
+
if (!asGovernanceStore(options.store)) {
|
|
1451
|
+
throw new Error("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
1452
|
+
}
|
|
1453
|
+
const current = input.now ?? now();
|
|
1454
|
+
const candidates = await recoverable.listActionInvocations({
|
|
1455
|
+
statuses: ["running"],
|
|
1456
|
+
completionDueBefore: current,
|
|
1457
|
+
unleased: true,
|
|
1458
|
+
...input.tenantId ? { tenantId: input.tenantId } : {},
|
|
1459
|
+
...input.spaceId ? { spaceId: input.spaceId } : {},
|
|
1460
|
+
limit: Math.max(1, Math.min(input.limit ?? 100, 1e3))
|
|
1461
|
+
});
|
|
1462
|
+
const overdue = [];
|
|
1463
|
+
for (const invocation of candidates) {
|
|
1464
|
+
const pending = invocation.pendingCompletion;
|
|
1465
|
+
if (!pending?.dueAt || pending.dueAt.getTime() > current.getTime()) continue;
|
|
1466
|
+
if (invocation.leaseOwner) continue;
|
|
1467
|
+
const message = `External operation "${pending.externalReference}" from ${pending.provider} did not complete by ${pending.dueAt.toISOString()}.`;
|
|
1468
|
+
await recordCompletionReconciliation(invocation, { externalReference: pending.externalReference, provider: pending.provider, kind: "overdue" }, message);
|
|
1469
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1470
|
+
overdue.push(invocation.id);
|
|
1471
|
+
}
|
|
1472
|
+
return { overdue };
|
|
1473
|
+
}
|
|
1207
1474
|
function declaredConsistency(actionId) {
|
|
1208
1475
|
return actionResolver(actionId)?.execution?.consistency;
|
|
1209
1476
|
}
|
|
@@ -1237,7 +1504,7 @@ function createGovernedActionHost(options) {
|
|
|
1237
1504
|
...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
|
|
1238
1505
|
};
|
|
1239
1506
|
if (options.outbox) {
|
|
1240
|
-
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1507
|
+
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ExternalOperationAccepted", "ExternalOperationCompleted", "ExternalOperationFailed", "ExternalOperationContradicted", "ComplianceBlocked"])).has(envelope.eventType);
|
|
1241
1508
|
const shouldPublish = options.outbox.shouldPublish?.(envelope) ?? !hostLifecycleEvent;
|
|
1242
1509
|
if (!shouldPublish) {
|
|
1243
1510
|
await (transaction ?? options.store).appendEvent(envelope);
|
|
@@ -1300,7 +1567,7 @@ function createGovernedActionHost(options) {
|
|
|
1300
1567
|
);
|
|
1301
1568
|
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1302
1569
|
}
|
|
1303
|
-
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
1570
|
+
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation, completeExternalInvocation, reconcileOverdueCompletions };
|
|
1304
1571
|
}
|
|
1305
1572
|
function actionResult(invocation) {
|
|
1306
1573
|
return {
|
|
@@ -1311,7 +1578,8 @@ function actionResult(invocation) {
|
|
|
1311
1578
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1312
1579
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
1313
1580
|
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1314
|
-
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
1581
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {},
|
|
1582
|
+
...invocation.pendingCompletion ? { pendingCompletion: invocation.pendingCompletion } : {}
|
|
1315
1583
|
};
|
|
1316
1584
|
}
|
|
1317
1585
|
function asApprovalStore(store) {
|
|
@@ -1711,6 +1979,13 @@ var MemoryPlatformHostStore = class {
|
|
|
1711
1979
|
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1712
1980
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
1713
1981
|
Object.assign(record, patch, { updatedAt: /* @__PURE__ */ new Date() });
|
|
1982
|
+
if (Object.hasOwn(patch, "pendingCompletion") && patch.pendingCompletion === void 0) delete record.pendingCompletion;
|
|
1983
|
+
if (Object.hasOwn(patch, "externalCompletion") && patch.externalCompletion === void 0) delete record.externalCompletion;
|
|
1984
|
+
if (Object.hasOwn(patch, "error") && patch.error === void 0) delete record.error;
|
|
1985
|
+
if (patch.pendingCompletion && patch.status === "running") {
|
|
1986
|
+
delete record.leaseOwner;
|
|
1987
|
+
delete record.leaseExpiresAt;
|
|
1988
|
+
}
|
|
1714
1989
|
if (patch.status === "waiting_for_approval" || patch.status === "completed" || patch.status === "failed" || patch.status === "blocked_by_policy" || patch.status === "reconciliation_required" || patch.status === "validation_failed") {
|
|
1715
1990
|
delete record.leaseOwner;
|
|
1716
1991
|
delete record.leaseExpiresAt;
|
|
@@ -1879,7 +2154,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1879
2154
|
}
|
|
1880
2155
|
async listActionInvocations(input = {}) {
|
|
1881
2156
|
return this.invocations.filter(
|
|
1882
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
2157
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore) && (!(input.awaitingCompletion || input.completionDueBefore) || record.pendingCompletion !== void 0) && (!input.completionDueBefore || record.pendingCompletion?.dueAt !== void 0 && record.pendingCompletion.dueAt <= input.completionDueBefore) && (!input.unleased || record.leaseOwner === void 0)
|
|
1883
2158
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1884
2159
|
}
|
|
1885
2160
|
async getHealthCounts(input) {
|
|
@@ -2166,6 +2441,18 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2166
2441
|
ALTER TABLE fabric_platform.event_outbox
|
|
2167
2442
|
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2168
2443
|
`
|
|
2444
|
+
}, {
|
|
2445
|
+
version: 3,
|
|
2446
|
+
name: "external_completion",
|
|
2447
|
+
sql: `
|
|
2448
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2449
|
+
ADD COLUMN pending_completion jsonb,
|
|
2450
|
+
ADD COLUMN external_completion jsonb,
|
|
2451
|
+
ADD COLUMN completion_due_at timestamptz;
|
|
2452
|
+
CREATE INDEX action_invocations_completion_due_idx
|
|
2453
|
+
ON fabric_platform.action_invocations (completion_due_at)
|
|
2454
|
+
WHERE completion_due_at IS NOT NULL;
|
|
2455
|
+
`
|
|
2169
2456
|
}]);
|
|
2170
2457
|
}
|
|
2171
2458
|
async transaction(run) {
|
|
@@ -2228,9 +2515,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2228
2515
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
2229
2516
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2230
2517
|
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
2231
|
-
|
|
2518
|
+
pending_completion=CASE WHEN $12::boolean THEN $13::jsonb ELSE pending_completion END,
|
|
2519
|
+
completion_due_at=CASE WHEN $12::boolean THEN ($13::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
2520
|
+
external_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE external_completion END,
|
|
2521
|
+
lease_owner=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2232
2522
|
THEN NULL ELSE lease_owner END,
|
|
2233
|
-
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2523
|
+
lease_expires_at=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2234
2524
|
THEN NULL ELSE lease_expires_at END,
|
|
2235
2525
|
updated_at=now()
|
|
2236
2526
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
@@ -2245,7 +2535,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2245
2535
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2246
2536
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2247
2537
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2248
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2538
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
2539
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
2540
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
2541
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
2542
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2249
2543
|
]
|
|
2250
2544
|
);
|
|
2251
2545
|
}
|
|
@@ -2598,6 +2892,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2598
2892
|
if (input.spaceId) add("space_id=?", input.spaceId);
|
|
2599
2893
|
if (input.statuses?.length) add("status = ANY(?::text[])", [...input.statuses]);
|
|
2600
2894
|
if (input.updatedBefore) add("updated_at<?", input.updatedBefore);
|
|
2895
|
+
if (input.awaitingCompletion || input.completionDueBefore) conditions.push("pending_completion IS NOT NULL");
|
|
2896
|
+
if (input.completionDueBefore) add("completion_due_at <= ?", input.completionDueBefore);
|
|
2897
|
+
if (input.unleased) conditions.push("lease_owner IS NULL");
|
|
2601
2898
|
values.push(Math.max(1, Math.min(input.limit ?? 100, 1e3)));
|
|
2602
2899
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
2603
2900
|
const result = await this.sql.query(
|
|
@@ -2695,9 +2992,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2695
2992
|
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2696
2993
|
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2697
2994
|
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2698
|
-
|
|
2995
|
+
pending_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE pending_completion END,
|
|
2996
|
+
completion_due_at=CASE WHEN $14::boolean THEN ($15::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
2997
|
+
external_completion=CASE WHEN $16::boolean THEN $17::jsonb ELSE external_completion END,
|
|
2998
|
+
lease_owner=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2699
2999
|
THEN NULL ELSE lease_owner END,
|
|
2700
|
-
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3000
|
+
lease_expires_at=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
3001
|
THEN NULL ELSE lease_expires_at END,
|
|
2702
3002
|
updated_at=now()
|
|
2703
3003
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
@@ -2716,7 +3016,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2716
3016
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2717
3017
|
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2718
3018
|
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2719
|
-
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
3019
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null,
|
|
3020
|
+
Object.hasOwn(patch, "pendingCompletion"),
|
|
3021
|
+
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3022
|
+
Object.hasOwn(patch, "externalCompletion"),
|
|
3023
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
2720
3024
|
]
|
|
2721
3025
|
);
|
|
2722
3026
|
return result.rows.length === 1;
|
|
@@ -2777,6 +3081,20 @@ function toAdapterRecord(row) {
|
|
|
2777
3081
|
updatedAt: new Date(row.updated_at)
|
|
2778
3082
|
};
|
|
2779
3083
|
}
|
|
3084
|
+
function toPendingCompletion(value) {
|
|
3085
|
+
return {
|
|
3086
|
+
...value,
|
|
3087
|
+
acceptedAt: new Date(value.acceptedAt),
|
|
3088
|
+
...value.dueAt ? { dueAt: new Date(value.dueAt) } : {},
|
|
3089
|
+
...value.parkedAt ? { parkedAt: new Date(value.parkedAt) } : {}
|
|
3090
|
+
};
|
|
3091
|
+
}
|
|
3092
|
+
function toExternalCompletion(value) {
|
|
3093
|
+
return {
|
|
3094
|
+
...value,
|
|
3095
|
+
recordedAt: new Date(value.recordedAt)
|
|
3096
|
+
};
|
|
3097
|
+
}
|
|
2780
3098
|
function toActionRecord(row) {
|
|
2781
3099
|
return {
|
|
2782
3100
|
id: String(row.id),
|
|
@@ -2801,6 +3119,8 @@ function toActionRecord(row) {
|
|
|
2801
3119
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
2802
3120
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2803
3121
|
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
3122
|
+
...row.pending_completion ? { pendingCompletion: toPendingCompletion(row.pending_completion) } : {},
|
|
3123
|
+
...row.external_completion ? { externalCompletion: toExternalCompletion(row.external_completion) } : {},
|
|
2804
3124
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
2805
3125
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2806
3126
|
leaseToken: Number(row.lease_token ?? 0),
|
|
@@ -2931,6 +3251,16 @@ function createStoreBackedActionDispatcher() {
|
|
|
2931
3251
|
async function runPlatformActionWorkerCycle(options) {
|
|
2932
3252
|
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2933
3253
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
3254
|
+
if (options.sweepOverdueCompletions !== false && typeof options.host.reconcileOverdueCompletions === "function") {
|
|
3255
|
+
try {
|
|
3256
|
+
await options.host.reconcileOverdueCompletions({
|
|
3257
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
3258
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
3259
|
+
});
|
|
3260
|
+
} catch (error) {
|
|
3261
|
+
options.onError?.(error);
|
|
3262
|
+
}
|
|
3263
|
+
}
|
|
2934
3264
|
let claimed;
|
|
2935
3265
|
try {
|
|
2936
3266
|
claimed = await options.store.claimActionInvocations({
|
|
@@ -3193,6 +3523,6 @@ function createDurableSagaParentLifecycle(options) {
|
|
|
3193
3523
|
};
|
|
3194
3524
|
}
|
|
3195
3525
|
|
|
3196
|
-
export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, getPlatformHostHealthSnapshot, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
3526
|
+
export { ExternalCompletionError, IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, getPlatformHostHealthSnapshot, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
3197
3527
|
//# sourceMappingURL=index.js.map
|
|
3198
3528
|
//# sourceMappingURL=index.js.map
|