@fabricorg/platform-host 7.4.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +295 -15
- package/dist/index.cjs +540 -73
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +150 -17
- package/dist/index.d.ts +150 -17
- package/dist/index.js +541 -74
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -309,6 +309,24 @@ function withoutPrivateHostFields(data, eventResultFields) {
|
|
|
309
309
|
function errorMessage(error) {
|
|
310
310
|
return error instanceof Error ? error.message : String(error);
|
|
311
311
|
}
|
|
312
|
+
var HostError = class extends Error {
|
|
313
|
+
};
|
|
314
|
+
function isRecord(value) {
|
|
315
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
316
|
+
}
|
|
317
|
+
function tryRedactErrorText(hook, context, message) {
|
|
318
|
+
if (!hook || message === "") return { success: true, text: message };
|
|
319
|
+
try {
|
|
320
|
+
const text = hook(context, message);
|
|
321
|
+
if (typeof text === "string" && text.length > 0) return { success: true, text };
|
|
322
|
+
} catch {
|
|
323
|
+
}
|
|
324
|
+
return { success: false };
|
|
325
|
+
}
|
|
326
|
+
function redactErrorText(hook, context, message) {
|
|
327
|
+
const result = tryRedactErrorText(hook, context, message);
|
|
328
|
+
return result.success ? result.text : "Error text withheld: the error-text redactor failed.";
|
|
329
|
+
}
|
|
312
330
|
function lifecycleId(prefix, invocationId, key) {
|
|
313
331
|
const safeKey = key.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 96);
|
|
314
332
|
return `${prefix}_${invocationId}_${safeKey}`;
|
|
@@ -371,7 +389,11 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
371
389
|
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
372
390
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
373
391
|
}
|
|
374
|
-
const
|
|
392
|
+
const provider = recorded.provider ?? completion.provider;
|
|
393
|
+
const rawDigest = completionDigest(completion, provider);
|
|
394
|
+
if (recorded.digest === rawDigest) return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
395
|
+
completion = redactCompletion(invocation, completion, provider ?? "external");
|
|
396
|
+
const incomingDigest = completionDigest(completion, provider);
|
|
375
397
|
if (recorded.digest === incomingDigest) {
|
|
376
398
|
return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
377
399
|
}
|
|
@@ -401,8 +423,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
401
423
|
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
402
424
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
403
425
|
}
|
|
426
|
+
completion = redactCompletion(invocation, completion, completion.provider ?? pending.provider);
|
|
404
427
|
const recordedAt = now();
|
|
405
428
|
const externalCompletion = {
|
|
429
|
+
// completion has passed the hooks; retain the callback shape for compatibility.
|
|
406
430
|
...completion,
|
|
407
431
|
provider: completion.provider ?? pending.provider,
|
|
408
432
|
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
@@ -423,7 +447,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
423
447
|
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
424
448
|
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
425
449
|
updatedAt: recordedAt
|
|
426
|
-
});
|
|
450
|
+
}, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
427
451
|
await appendEvent(invocation, {
|
|
428
452
|
eventType: "ExternalOperationCompleted",
|
|
429
453
|
...subject,
|
|
@@ -435,7 +459,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
435
459
|
return withConsistency(actionResult2({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
436
460
|
}
|
|
437
461
|
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
438
|
-
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
462
|
+
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt }, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
439
463
|
await appendEvent(invocation, {
|
|
440
464
|
eventType: "ExternalOperationFailed",
|
|
441
465
|
...subject,
|
|
@@ -446,6 +470,30 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
446
470
|
if (settles) emitInvocationStatusTelemetry(invocation, "failed", deps.telemetry, now(), invocation.status);
|
|
447
471
|
return withConsistency(actionResult2({ ...invocation, ...patch }), invocation.actionId);
|
|
448
472
|
}
|
|
473
|
+
function redactCompletion(invocation, completion, provider) {
|
|
474
|
+
let result = completion.result;
|
|
475
|
+
if (result && deps.redactExternalCompletionResult) {
|
|
476
|
+
try {
|
|
477
|
+
result = deps.redactExternalCompletionResult(invocation.actionId, provider, result);
|
|
478
|
+
if (!isRecord(result)) throw new HostError("Completion result redactor must return a non-null, non-array object.");
|
|
479
|
+
} catch {
|
|
480
|
+
throw new ExternalCompletionError("redaction_failed", "External completion result withheld: the result redactor failed; nothing was recorded.");
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
const error = completion.error == null ? void 0 : tryRedactErrorText(
|
|
484
|
+
deps.redactErrorText,
|
|
485
|
+
{ actionId: invocation.actionId, source: "external_completion", provider },
|
|
486
|
+
completion.error
|
|
487
|
+
);
|
|
488
|
+
if (error && !error.success) {
|
|
489
|
+
throw new ExternalCompletionError("redaction_failed", "External completion error withheld: the error-text redactor failed; nothing was recorded.");
|
|
490
|
+
}
|
|
491
|
+
return {
|
|
492
|
+
...completion,
|
|
493
|
+
...completion.result !== void 0 ? { result } : {},
|
|
494
|
+
...error?.success ? { error: error.text } : {}
|
|
495
|
+
};
|
|
496
|
+
}
|
|
449
497
|
function completionDigest(completion, provider) {
|
|
450
498
|
return crypto.createHash("sha256").update(platform.canonicalJson({
|
|
451
499
|
provider: provider ?? null,
|
|
@@ -458,7 +506,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
458
506
|
}
|
|
459
507
|
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
460
508
|
const governanceStore = asGovernanceStore(deps.store);
|
|
461
|
-
if (!governanceStore) throw new
|
|
509
|
+
if (!governanceStore) throw new HostError("External completion reconciliation requires a governance-capable store.");
|
|
462
510
|
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
463
511
|
const seed = crypto.createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
464
512
|
await governanceStore.appendExternalReconciliation({
|
|
@@ -478,10 +526,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
478
526
|
async function reconcileOverdueCompletions(input = {}) {
|
|
479
527
|
const recoverable = deps.store;
|
|
480
528
|
if (!recoverable.listActionInvocations) {
|
|
481
|
-
throw new
|
|
529
|
+
throw new HostError("Overdue completion reconciliation requires a store that can list invocations.");
|
|
482
530
|
}
|
|
483
531
|
if (!asGovernanceStore(deps.store)) {
|
|
484
|
-
throw new
|
|
532
|
+
throw new HostError("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
485
533
|
}
|
|
486
534
|
const current = input.now ?? now();
|
|
487
535
|
const candidates = await recoverable.listActionInvocations({
|
|
@@ -856,6 +904,10 @@ var ADMISSION_FAILED = Object.freeze({ kind: "failed" });
|
|
|
856
904
|
var ADMISSION_TIMED_OUT = Object.freeze({ kind: "timed_out" });
|
|
857
905
|
var ADMISSION_REFUSAL_CODE = /^[a-z][A-Za-z0-9_.:-]{0,127}$/;
|
|
858
906
|
function createGovernedActionHost(options) {
|
|
907
|
+
const retryableExecutionMaxAttempts = options.retryableExecutionMaxAttempts ?? 5;
|
|
908
|
+
if (!Number.isInteger(retryableExecutionMaxAttempts) || retryableExecutionMaxAttempts <= 0) {
|
|
909
|
+
throw new Error("retryableExecutionMaxAttempts must be a positive integer.");
|
|
910
|
+
}
|
|
859
911
|
if (options.composition) {
|
|
860
912
|
if (!options.registry?.orderedModules) {
|
|
861
913
|
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
@@ -892,6 +944,9 @@ function createGovernedActionHost(options) {
|
|
|
892
944
|
if (typeof admissionTimeoutMs !== "number" || !Number.isInteger(admissionTimeoutMs) || admissionTimeoutMs < 1 || admissionTimeoutMs > MAX_TIMER_MS) {
|
|
893
945
|
throw new Error(`admissionTimeoutMs must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}.`);
|
|
894
946
|
}
|
|
947
|
+
function caughtError(context, error) {
|
|
948
|
+
return error instanceof HostError ? errorMessage(error) : redactErrorText(options.redactErrorText, context, errorMessage(error));
|
|
949
|
+
}
|
|
895
950
|
async function submitAction(input) {
|
|
896
951
|
const read = snapshotSubmission(input);
|
|
897
952
|
if (read.executionReason && !["initial", "offline_replay"].includes(read.executionReason)) {
|
|
@@ -1189,6 +1244,7 @@ function createGovernedActionHost(options) {
|
|
|
1189
1244
|
spaceId,
|
|
1190
1245
|
attributes: { actionVersion: invocation.actionVersion }
|
|
1191
1246
|
});
|
|
1247
|
+
let retryableGovernance = true;
|
|
1192
1248
|
try {
|
|
1193
1249
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
1194
1250
|
if (!parsed.success) {
|
|
@@ -1341,6 +1397,7 @@ function createGovernedActionHost(options) {
|
|
|
1341
1397
|
mode: "execute",
|
|
1342
1398
|
now: now()
|
|
1343
1399
|
});
|
|
1400
|
+
retryableGovernance = false;
|
|
1344
1401
|
for (const outcome of outcomes) {
|
|
1345
1402
|
await options.store.appendPolicyEvaluation({
|
|
1346
1403
|
id: lifecycleId("pol", actionInvocationId, outcome.policyId),
|
|
@@ -1384,12 +1441,16 @@ function createGovernedActionHost(options) {
|
|
|
1384
1441
|
}
|
|
1385
1442
|
let data;
|
|
1386
1443
|
let domainEvents = [];
|
|
1444
|
+
let pendingTransitionEvent;
|
|
1387
1445
|
try {
|
|
1388
1446
|
const runHandler = async (db, transaction) => {
|
|
1389
1447
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
1390
|
-
throw new
|
|
1448
|
+
throw new HostError("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
1391
1449
|
}
|
|
1392
1450
|
const binding = action.stateMachine;
|
|
1451
|
+
let replayedTransition;
|
|
1452
|
+
let pendingStateTransition;
|
|
1453
|
+
let validatedTransition;
|
|
1393
1454
|
if (binding) {
|
|
1394
1455
|
const machine = stateMachineResolver(binding.entityType);
|
|
1395
1456
|
const entityId = binding.getEntityId(parsed.data);
|
|
@@ -1419,9 +1480,45 @@ function createGovernedActionHost(options) {
|
|
|
1419
1480
|
db
|
|
1420
1481
|
}
|
|
1421
1482
|
);
|
|
1422
|
-
|
|
1483
|
+
validatedTransition = { entityId, currentState, targetState, valid: transition.valid };
|
|
1484
|
+
let replayingAppliedTransition = false;
|
|
1485
|
+
if (action.idempotent && currentState === targetState && !transition.valid && entityId) {
|
|
1486
|
+
let source;
|
|
1487
|
+
if (transaction?.getEntityStateSource) {
|
|
1488
|
+
source = await transaction.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1489
|
+
} else if (options.store.getEntityStateSource) {
|
|
1490
|
+
source = await options.store.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1491
|
+
} else {
|
|
1492
|
+
const eventStore = options.store;
|
|
1493
|
+
const listEvents = transaction?.listEvents?.bind(transaction) ?? eventStore.listEvents?.bind(eventStore);
|
|
1494
|
+
const events2 = listEvents ? await listEvents(tenantId, spaceId) : [];
|
|
1495
|
+
let latest;
|
|
1496
|
+
for (const event of events2) {
|
|
1497
|
+
if (event.tenantId === tenantId && event.spaceId === spaceId && event.subjectType === binding.entityType && event.subjectId === entityId && typeof event.payload?.toState === "string" && (!latest || event.sequence > latest.sequence)) latest = event;
|
|
1498
|
+
}
|
|
1499
|
+
if (latest && typeof latest.actionInvocationId === "string") {
|
|
1500
|
+
source = { state: latest.payload.toState, actionInvocationId: latest.actionInvocationId };
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
replayingAppliedTransition = source?.actionInvocationId === actionInvocationId && source.state === currentState;
|
|
1504
|
+
}
|
|
1505
|
+
const pending = invocation.pendingStateTransition;
|
|
1506
|
+
if (replayingAppliedTransition && pending && pending.entityType === binding.entityType && pending.entityId === entityId && pending.toState === currentState) {
|
|
1507
|
+
replayedTransition = pending;
|
|
1508
|
+
}
|
|
1509
|
+
if (action.idempotent && transition.valid && entityId && action.eventPhase === "after_adapters") {
|
|
1510
|
+
pendingStateTransition = {
|
|
1511
|
+
entityType: binding.entityType,
|
|
1512
|
+
entityId,
|
|
1513
|
+
fromState: currentState,
|
|
1514
|
+
toState: targetState
|
|
1515
|
+
};
|
|
1516
|
+
if (!transaction) {
|
|
1517
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { pendingStateTransition });
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1423
1520
|
if (!transition.valid && !replayingAppliedTransition) {
|
|
1424
|
-
throw new Error(transition.error
|
|
1521
|
+
throw transition.error != null ? new Error(transition.error) : new HostError("Invalid state transition");
|
|
1425
1522
|
}
|
|
1426
1523
|
}
|
|
1427
1524
|
}
|
|
@@ -1442,14 +1539,14 @@ function createGovernedActionHost(options) {
|
|
|
1442
1539
|
parsed.data
|
|
1443
1540
|
) : { success: true, data: {} };
|
|
1444
1541
|
if (!handlerResult.success) {
|
|
1445
|
-
throw new Error(handlerResult.error
|
|
1542
|
+
throw handlerResult.error != null ? new Error(handlerResult.error) : new HostError("Action handler failed");
|
|
1446
1543
|
}
|
|
1447
1544
|
const handlerData = handlerResult.data ?? {};
|
|
1448
1545
|
if (action.resultSchema) {
|
|
1449
1546
|
const publicResult = withoutPrivateHostFields(handlerData, eventResultFields);
|
|
1450
1547
|
const parsedResult = action.resultSchema.safeParse(publicResult);
|
|
1451
1548
|
if (!parsedResult.success) {
|
|
1452
|
-
throw new
|
|
1549
|
+
throw new HostError(`Action result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1453
1550
|
}
|
|
1454
1551
|
}
|
|
1455
1552
|
const events = extractEvents(handlerData);
|
|
@@ -1457,7 +1554,7 @@ function createGovernedActionHost(options) {
|
|
|
1457
1554
|
(event) => !action.emitsEvents.includes(event.eventType)
|
|
1458
1555
|
);
|
|
1459
1556
|
if (undeclaredEvent) {
|
|
1460
|
-
throw new
|
|
1557
|
+
throw new HostError(
|
|
1461
1558
|
`${action.actionId} emitted undeclared event type ${undeclaredEvent.eventType}`
|
|
1462
1559
|
);
|
|
1463
1560
|
}
|
|
@@ -1472,7 +1569,32 @@ function createGovernedActionHost(options) {
|
|
|
1472
1569
|
);
|
|
1473
1570
|
}
|
|
1474
1571
|
}
|
|
1475
|
-
|
|
1572
|
+
let transitionEvent;
|
|
1573
|
+
if (binding && validatedTransition?.entityId && (validatedTransition.valid || replayedTransition)) {
|
|
1574
|
+
const { entityId, currentState, targetState } = validatedTransition;
|
|
1575
|
+
const handlerEmittedTransition = events.some((event) => event.eventType === "StateTransitioned" && event.subjectType === binding.entityType && event.subjectId === entityId);
|
|
1576
|
+
if (!handlerEmittedTransition) {
|
|
1577
|
+
transitionEvent = {
|
|
1578
|
+
eventType: "StateTransitioned",
|
|
1579
|
+
subjectType: binding.entityType,
|
|
1580
|
+
subjectId: entityId,
|
|
1581
|
+
payload: {
|
|
1582
|
+
entityType: binding.entityType,
|
|
1583
|
+
entityId,
|
|
1584
|
+
fromState: replayedTransition?.fromState ?? currentState,
|
|
1585
|
+
toState: targetState,
|
|
1586
|
+
causedByAction: action.actionId
|
|
1587
|
+
}
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
if (transitionEvent && action.eventPhase !== "after_adapters") {
|
|
1592
|
+
await appendEvent(invocation, transitionEvent, "state:transition", 1, transaction);
|
|
1593
|
+
}
|
|
1594
|
+
if (transaction && pendingStateTransition) {
|
|
1595
|
+
await transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, { pendingStateTransition });
|
|
1596
|
+
}
|
|
1597
|
+
return { data: handlerData, domainEvents: events, pendingTransitionEvent: transitionEvent };
|
|
1476
1598
|
};
|
|
1477
1599
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1478
1600
|
const executed = atomicStore ? await atomicStore.transactionWithEvents(
|
|
@@ -1480,9 +1602,22 @@ function createGovernedActionHost(options) {
|
|
|
1480
1602
|
) : await options.store.transaction((db) => runHandler(db));
|
|
1481
1603
|
data = executed.data;
|
|
1482
1604
|
domainEvents = executed.domainEvents;
|
|
1605
|
+
pendingTransitionEvent = executed.pendingTransitionEvent;
|
|
1483
1606
|
} catch (error) {
|
|
1484
|
-
|
|
1607
|
+
if (platform.isReconciliationRequiredError(error)) {
|
|
1608
|
+
const message = caughtError({ actionId: invocation.actionId, source: "handler" }, error);
|
|
1609
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1610
|
+
return withConsistency({
|
|
1611
|
+
actionInvocationId,
|
|
1612
|
+
status: "reconciliation_required",
|
|
1613
|
+
error: message,
|
|
1614
|
+
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1615
|
+
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
1616
|
+
}, action.actionId);
|
|
1617
|
+
}
|
|
1618
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "handler" }, error));
|
|
1485
1619
|
}
|
|
1620
|
+
const adapterInvocationScope = { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id };
|
|
1486
1621
|
let pendingHandoff = invocation.pendingCompletion;
|
|
1487
1622
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
1488
1623
|
const input = step.getInput(parsed.data, data);
|
|
@@ -1490,12 +1625,13 @@ function createGovernedActionHost(options) {
|
|
|
1490
1625
|
const adapter = adapters.require(step.adapterType, step.operation);
|
|
1491
1626
|
const adapterInvocationId = lifecycleId("adp", actionInvocationId, String(stepIndex));
|
|
1492
1627
|
const previousAdapterInvocation = await options.store.getAdapterInvocation(
|
|
1493
|
-
adapterInvocationId
|
|
1628
|
+
adapterInvocationId,
|
|
1629
|
+
adapterInvocationScope
|
|
1494
1630
|
);
|
|
1495
1631
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
1496
1632
|
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
1497
1633
|
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
1498
|
-
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
1634
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1499
1635
|
await appendEvent(invocation, {
|
|
1500
1636
|
eventType: "ExternalOperationAccepted",
|
|
1501
1637
|
subjectType: subject.subjectType,
|
|
@@ -1529,9 +1665,7 @@ function createGovernedActionHost(options) {
|
|
|
1529
1665
|
const startedAt = now();
|
|
1530
1666
|
const record = {
|
|
1531
1667
|
id: adapterInvocationId,
|
|
1532
|
-
|
|
1533
|
-
tenantId,
|
|
1534
|
-
spaceId,
|
|
1668
|
+
...adapterInvocationScope,
|
|
1535
1669
|
adapterType: step.adapterType,
|
|
1536
1670
|
operation: step.operation,
|
|
1537
1671
|
vendor: adapter.vendor,
|
|
@@ -1548,6 +1682,96 @@ function createGovernedActionHost(options) {
|
|
|
1548
1682
|
subjectId: adapterEventSubject.subjectId,
|
|
1549
1683
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1550
1684
|
}, `adapter:${stepIndex}:started`);
|
|
1685
|
+
async function appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference, transaction) {
|
|
1686
|
+
if (accepted) {
|
|
1687
|
+
await appendEvent(invocation, {
|
|
1688
|
+
eventType: "ExternalOperationAccepted",
|
|
1689
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1690
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1691
|
+
payload: {
|
|
1692
|
+
adapterType: step.adapterType,
|
|
1693
|
+
operation: step.operation,
|
|
1694
|
+
externalReference: accepted.externalReference,
|
|
1695
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1696
|
+
}
|
|
1697
|
+
}, `adapter:${stepIndex}:accepted`, 1, transaction);
|
|
1698
|
+
}
|
|
1699
|
+
await appendEvent(invocation, {
|
|
1700
|
+
eventType: "AdapterInvocationSucceeded",
|
|
1701
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1702
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1703
|
+
payload: {
|
|
1704
|
+
adapterType: step.adapterType,
|
|
1705
|
+
operation: step.operation,
|
|
1706
|
+
input: recordedInput,
|
|
1707
|
+
...outputWithheld ? { outputWithheld: true } : { output }
|
|
1708
|
+
}
|
|
1709
|
+
}, `adapter:${stepIndex}:succeeded`, 1, transaction);
|
|
1710
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
1711
|
+
if (outputWithheld && governanceStore2) {
|
|
1712
|
+
await (transaction ?? governanceStore2).appendExternalReconciliation({
|
|
1713
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
1714
|
+
actionInvocationId,
|
|
1715
|
+
tenantId,
|
|
1716
|
+
spaceId,
|
|
1717
|
+
status: "pending",
|
|
1718
|
+
provider: adapter.vendor,
|
|
1719
|
+
...externalReference ?? accepted?.externalReference ? { externalOperationId: externalReference ?? accepted?.externalReference } : {},
|
|
1720
|
+
attempt: 1,
|
|
1721
|
+
reason: message,
|
|
1722
|
+
observedAt: now()
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
async function finalizeDegradedAdapterAtomically(patch, outputWithheld, message, accepted, output, externalReference) {
|
|
1727
|
+
const atomicStore = asAtomicMutationStore(options.store);
|
|
1728
|
+
if (!atomicStore) return false;
|
|
1729
|
+
const needsReconciliation = outputWithheld && !!asGovernanceStore(options.store);
|
|
1730
|
+
const previousStatus = invocation.status;
|
|
1731
|
+
const finalize = () => atomicStore.transactionWithEvents(async (transaction) => {
|
|
1732
|
+
if (!transaction.updateAdapterInvocation || needsReconciliation && !transaction.appendExternalReconciliation || invocation.leaseOwner && invocation.leaseToken !== void 0 && !transaction.updateLeasedActionInvocation) return false;
|
|
1733
|
+
await appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference, transaction);
|
|
1734
|
+
await transaction.updateAdapterInvocation(adapterInvocationId, {
|
|
1735
|
+
status: "succeeded",
|
|
1736
|
+
...outputWithheld ? {} : { output },
|
|
1737
|
+
updatedAt: now()
|
|
1738
|
+
}, adapterInvocationScope);
|
|
1739
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1740
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1741
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1742
|
+
}
|
|
1743
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1744
|
+
id: actionInvocationId,
|
|
1745
|
+
tenantId,
|
|
1746
|
+
spaceId,
|
|
1747
|
+
workerId: invocation.leaseOwner,
|
|
1748
|
+
leaseToken: invocation.leaseToken,
|
|
1749
|
+
patch
|
|
1750
|
+
});
|
|
1751
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1752
|
+
} else {
|
|
1753
|
+
await transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch);
|
|
1754
|
+
}
|
|
1755
|
+
return true;
|
|
1756
|
+
});
|
|
1757
|
+
let committed;
|
|
1758
|
+
try {
|
|
1759
|
+
committed = await finalize();
|
|
1760
|
+
} catch (error) {
|
|
1761
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1762
|
+
try {
|
|
1763
|
+
committed = await finalize();
|
|
1764
|
+
} catch (retryError) {
|
|
1765
|
+
if (retryError instanceof RecoverableFinalizationError) throw retryError;
|
|
1766
|
+
return false;
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
if (committed) emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1770
|
+
return committed;
|
|
1771
|
+
}
|
|
1772
|
+
const adapterErrorContext = { actionId: action.actionId, source: "adapter", adapterType: step.adapterType, operation: step.operation, provider: adapter.vendor };
|
|
1773
|
+
const executionErrorContext = { actionId: action.actionId, source: "execution" };
|
|
1774
|
+
let errorContext = executionErrorContext;
|
|
1551
1775
|
let adapterDeadlineTimer;
|
|
1552
1776
|
try {
|
|
1553
1777
|
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
@@ -1555,15 +1779,15 @@ function createGovernedActionHost(options) {
|
|
|
1555
1779
|
if (adapterController && deadlineMs) {
|
|
1556
1780
|
const remaining = deadlineMs - now().getTime();
|
|
1557
1781
|
if (remaining <= 0) {
|
|
1558
|
-
throw new
|
|
1782
|
+
throw new HostError(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
1559
1783
|
}
|
|
1560
1784
|
adapterDeadlineTimer = setTimeout(
|
|
1561
|
-
() => adapterController.abort(new
|
|
1785
|
+
() => adapterController.abort(new HostError("Adapter deadline exceeded")),
|
|
1562
1786
|
remaining
|
|
1563
1787
|
);
|
|
1564
1788
|
adapterDeadlineTimer.unref?.();
|
|
1565
1789
|
}
|
|
1566
|
-
const circuitBreaker = circuitBreakers.forAdapter(adapter);
|
|
1790
|
+
const circuitBreaker = circuitBreakers.forAdapter(adapter, { tenantId, spaceId });
|
|
1567
1791
|
const result2 = await platform.executeWithAdapterRetry({
|
|
1568
1792
|
...circuitBreaker ? { circuitBreaker } : {},
|
|
1569
1793
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
@@ -1571,13 +1795,15 @@ function createGovernedActionHost(options) {
|
|
|
1571
1795
|
// Every await before the call happens here, before breaker
|
|
1572
1796
|
// admission, so nothing awaits between admission and the adapter.
|
|
1573
1797
|
prepareAttempt: async (attempt) => {
|
|
1798
|
+
errorContext = executionErrorContext;
|
|
1574
1799
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1575
1800
|
attempt,
|
|
1576
1801
|
updatedAt: now()
|
|
1577
|
-
});
|
|
1802
|
+
}, adapterInvocationScope);
|
|
1578
1803
|
},
|
|
1579
|
-
execute: (attempt, maxAttempts) => {
|
|
1580
|
-
|
|
1804
|
+
execute: async (attempt, maxAttempts) => {
|
|
1805
|
+
errorContext = adapterErrorContext;
|
|
1806
|
+
const adapterResult = await adapter.execute(input, {
|
|
1581
1807
|
tenantId,
|
|
1582
1808
|
spaceId,
|
|
1583
1809
|
actionInvocationId,
|
|
@@ -1589,6 +1815,8 @@ function createGovernedActionHost(options) {
|
|
|
1589
1815
|
...deadlineMs ? { deadlineMs } : {},
|
|
1590
1816
|
...adapterController ? { signal: adapterController.signal } : {}
|
|
1591
1817
|
});
|
|
1818
|
+
errorContext = executionErrorContext;
|
|
1819
|
+
return adapterResult;
|
|
1592
1820
|
},
|
|
1593
1821
|
isSuccessful: (result3) => result3.success,
|
|
1594
1822
|
getError: (result3) => result3.error,
|
|
@@ -1601,19 +1829,23 @@ function createGovernedActionHost(options) {
|
|
|
1601
1829
|
return "transient_failure";
|
|
1602
1830
|
}
|
|
1603
1831
|
});
|
|
1832
|
+
errorContext = adapterErrorContext;
|
|
1604
1833
|
platform.assertAdapterOutcomeConsistency(result2);
|
|
1834
|
+
errorContext = executionErrorContext;
|
|
1605
1835
|
if (!result2.success) {
|
|
1836
|
+
const resultError = result2.error == null ? result2.error : redactErrorText(options.redactErrorText, adapterErrorContext, result2.error);
|
|
1606
1837
|
const outcome = result2.outcome ?? "transient_failure";
|
|
1607
1838
|
if (outcome === "ambiguous") {
|
|
1608
1839
|
const evidence = result2.outcomeEvidence;
|
|
1609
|
-
const
|
|
1840
|
+
const reason = evidence?.reason == null ? resultError : redactErrorText(options.redactErrorText, adapterErrorContext, evidence.reason);
|
|
1841
|
+
const message = resultError ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
1610
1842
|
const adapterReconciliation = {
|
|
1611
1843
|
kind: "adapter_outcome_ambiguous",
|
|
1612
1844
|
adapterType: step.adapterType,
|
|
1613
1845
|
operation: step.operation,
|
|
1614
1846
|
vendor: adapter.vendor,
|
|
1615
1847
|
adapterInvocationId,
|
|
1616
|
-
...evidence?.reason ?? result2.error ? { reason
|
|
1848
|
+
...evidence?.reason ?? result2.error ? { reason } : {},
|
|
1617
1849
|
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
1618
1850
|
message
|
|
1619
1851
|
};
|
|
@@ -1621,7 +1853,7 @@ function createGovernedActionHost(options) {
|
|
|
1621
1853
|
status: "ambiguous",
|
|
1622
1854
|
error: message,
|
|
1623
1855
|
updatedAt: now()
|
|
1624
|
-
});
|
|
1856
|
+
}, adapterInvocationScope);
|
|
1625
1857
|
await appendEvent(invocation, {
|
|
1626
1858
|
eventType: "AdapterInvocationAmbiguous",
|
|
1627
1859
|
subjectType: adapterEventSubject.subjectType,
|
|
@@ -1659,24 +1891,56 @@ function createGovernedActionHost(options) {
|
|
|
1659
1891
|
}
|
|
1660
1892
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1661
1893
|
status: "failed",
|
|
1662
|
-
error:
|
|
1894
|
+
error: resultError ?? "Adapter failed",
|
|
1663
1895
|
updatedAt: now()
|
|
1664
|
-
});
|
|
1896
|
+
}, adapterInvocationScope);
|
|
1665
1897
|
await appendEvent(invocation, {
|
|
1666
1898
|
eventType: "AdapterInvocationFailed",
|
|
1667
1899
|
subjectType: adapterEventSubject.subjectType,
|
|
1668
1900
|
subjectId: adapterEventSubject.subjectId,
|
|
1669
1901
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1670
1902
|
}, `adapter:${stepIndex}:failed`);
|
|
1671
|
-
return fail2(invocation, "failed",
|
|
1903
|
+
return fail2(invocation, "failed", resultError ?? "Adapter failed");
|
|
1672
1904
|
}
|
|
1673
1905
|
const resultRecord = result2;
|
|
1674
|
-
|
|
1906
|
+
let output = isRecord(resultRecord.output) ? resultRecord.output : Object.fromEntries(
|
|
1675
1907
|
Object.entries(resultRecord).filter(
|
|
1676
1908
|
([key]) => key !== "success" && key !== "error"
|
|
1677
1909
|
)
|
|
1678
1910
|
);
|
|
1679
1911
|
const handoff = result2.acceptedExternalOperation;
|
|
1912
|
+
if (options.redactAdapterOutput) {
|
|
1913
|
+
try {
|
|
1914
|
+
output = options.redactAdapterOutput(action.actionId, step.adapterType, step.operation, output);
|
|
1915
|
+
if (!isRecord(output)) throw new HostError("Adapter output redactor must return a non-null, non-array object.");
|
|
1916
|
+
} catch {
|
|
1917
|
+
const message = "Adapter output withheld: the output redactor failed after the external effect; reconciliation required.";
|
|
1918
|
+
const reference = typeof handoff?.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1919
|
+
const acceptedAt = now();
|
|
1920
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1921
|
+
const pendingCompletion = pendingHandoff ?? (reference ? {
|
|
1922
|
+
provider: adapter.vendor,
|
|
1923
|
+
adapterType: step.adapterType,
|
|
1924
|
+
operation: step.operation,
|
|
1925
|
+
adapterInvocationId,
|
|
1926
|
+
externalReference: reference,
|
|
1927
|
+
acceptedAt,
|
|
1928
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1929
|
+
} : void 0);
|
|
1930
|
+
try {
|
|
1931
|
+
const patch = { status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} };
|
|
1932
|
+
const accepted = pendingCompletion?.adapterInvocationId === adapterInvocationId ? pendingCompletion : void 0;
|
|
1933
|
+
if (!await finalizeDegradedAdapterAtomically(patch, true, message, accepted, void 0, reference || void 0)) {
|
|
1934
|
+
await persistReconciliationVerdict(invocation, patch);
|
|
1935
|
+
await appendDegradedAdapterEvidence(true, message, accepted, void 0, reference || void 0);
|
|
1936
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1937
|
+
}
|
|
1938
|
+
} catch {
|
|
1939
|
+
throw new RecoverableFinalizationError(message);
|
|
1940
|
+
}
|
|
1941
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} }, action.actionId);
|
|
1942
|
+
}
|
|
1943
|
+
}
|
|
1680
1944
|
if (handoff) {
|
|
1681
1945
|
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1682
1946
|
if (reference === "") {
|
|
@@ -1698,19 +1962,33 @@ function createGovernedActionHost(options) {
|
|
|
1698
1962
|
const completion = action.execution?.completion ?? "immediate";
|
|
1699
1963
|
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1700
1964
|
if (refuse) {
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1965
|
+
let adapterFinalizedAtomically;
|
|
1966
|
+
try {
|
|
1967
|
+
const patch = {
|
|
1968
|
+
status: "reconciliation_required",
|
|
1969
|
+
error: refuse,
|
|
1970
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1971
|
+
};
|
|
1972
|
+
adapterFinalizedAtomically = await finalizeDegradedAdapterAtomically(patch, false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1973
|
+
if (!adapterFinalizedAtomically) {
|
|
1974
|
+
await persistReconciliationVerdict(invocation, patch);
|
|
1975
|
+
await appendDegradedAdapterEvidence(false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1976
|
+
}
|
|
1977
|
+
} catch {
|
|
1978
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1979
|
+
}
|
|
1707
1980
|
let error = refuse;
|
|
1708
1981
|
try {
|
|
1709
1982
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1710
1983
|
} catch (logError) {
|
|
1711
|
-
error = `${refuse} Reconciliation log unavailable: ${
|
|
1984
|
+
error = `${refuse} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1712
1985
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1713
1986
|
}
|
|
1987
|
+
try {
|
|
1988
|
+
if (!adapterFinalizedAtomically) await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() }, adapterInvocationScope);
|
|
1989
|
+
} catch {
|
|
1990
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1991
|
+
}
|
|
1714
1992
|
return withConsistency({
|
|
1715
1993
|
actionInvocationId,
|
|
1716
1994
|
status: "reconciliation_required",
|
|
@@ -1737,7 +2015,7 @@ function createGovernedActionHost(options) {
|
|
|
1737
2015
|
status: "succeeded",
|
|
1738
2016
|
output,
|
|
1739
2017
|
updatedAt: now()
|
|
1740
|
-
});
|
|
2018
|
+
}, adapterInvocationScope);
|
|
1741
2019
|
if (options.extractExecutionAttestation) {
|
|
1742
2020
|
const attestation = await options.extractExecutionAttestation({
|
|
1743
2021
|
actionInvocationId,
|
|
@@ -1766,12 +2044,13 @@ function createGovernedActionHost(options) {
|
|
|
1766
2044
|
}
|
|
1767
2045
|
}, `adapter:${stepIndex}:succeeded`);
|
|
1768
2046
|
} catch (error) {
|
|
1769
|
-
|
|
2047
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2048
|
+
const message = caughtError(platform.isAdapterCircuitOpenError(error) ? adapterErrorContext : errorContext, error);
|
|
1770
2049
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1771
2050
|
status: "failed",
|
|
1772
2051
|
error: message,
|
|
1773
2052
|
updatedAt: now()
|
|
1774
|
-
});
|
|
2053
|
+
}, adapterInvocationScope);
|
|
1775
2054
|
if (platform.isAdapterCircuitOpenError(error)) {
|
|
1776
2055
|
await appendEvent(invocation, {
|
|
1777
2056
|
eventType: "AdapterInvocationFailed",
|
|
@@ -1802,7 +2081,7 @@ function createGovernedActionHost(options) {
|
|
|
1802
2081
|
try {
|
|
1803
2082
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1804
2083
|
} catch (logError) {
|
|
1805
|
-
error = `${message} Reconciliation log unavailable: ${
|
|
2084
|
+
error = `${message} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1806
2085
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1807
2086
|
}
|
|
1808
2087
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
@@ -1826,9 +2105,12 @@ function createGovernedActionHost(options) {
|
|
|
1826
2105
|
transaction
|
|
1827
2106
|
);
|
|
1828
2107
|
}
|
|
2108
|
+
if (pendingTransitionEvent) {
|
|
2109
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1, transaction);
|
|
2110
|
+
}
|
|
1829
2111
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1830
2112
|
if (!transaction.updateLeasedActionInvocation) {
|
|
1831
|
-
throw new
|
|
2113
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1832
2114
|
}
|
|
1833
2115
|
const updated = await transaction.updateLeasedActionInvocation({
|
|
1834
2116
|
id: actionInvocationId,
|
|
@@ -1861,6 +2143,9 @@ function createGovernedActionHost(options) {
|
|
|
1861
2143
|
for (const [index, event] of domainEvents.entries()) {
|
|
1862
2144
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1863
2145
|
}
|
|
2146
|
+
if (pendingTransitionEvent) {
|
|
2147
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1);
|
|
2148
|
+
}
|
|
1864
2149
|
}
|
|
1865
2150
|
await persistInvocation(invocation, finalPatch);
|
|
1866
2151
|
}
|
|
@@ -1880,7 +2165,15 @@ function createGovernedActionHost(options) {
|
|
|
1880
2165
|
};
|
|
1881
2166
|
} catch (error) {
|
|
1882
2167
|
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1883
|
-
|
|
2168
|
+
if (retryableGovernance && platform.isRetryableExecutionError(error)) {
|
|
2169
|
+
const reason = executionReasonOverride === "approval_resume" ? "approval resume" : !invocation.leaseOwner || invocation.leaseToken === void 0 ? "not leased" : !action.idempotent ? "action not idempotent" : invocation.attemptCount >= retryableExecutionMaxAttempts ? `attempts exhausted (${invocation.attemptCount}/${retryableExecutionMaxAttempts})` : void 0;
|
|
2170
|
+
if (!reason) throw error;
|
|
2171
|
+
return fail2(invocation, "failed", caughtError(
|
|
2172
|
+
{ actionId: invocation.actionId, source: "execution" },
|
|
2173
|
+
new Error(`Retryable execution not retried: ${reason}; ${errorMessage(error)}`)
|
|
2174
|
+
));
|
|
2175
|
+
}
|
|
2176
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "execution" }, error));
|
|
1884
2177
|
}
|
|
1885
2178
|
}
|
|
1886
2179
|
async function resumeApprovedInvocation(actionInvocationId, tenantId, spaceId, decision) {
|
|
@@ -1963,9 +2256,9 @@ function createGovernedActionHost(options) {
|
|
|
1963
2256
|
}
|
|
1964
2257
|
async function recordExecutionAttestation(actionInvocationId, tenantId, spaceId, attestation, adapterInvocationId) {
|
|
1965
2258
|
const governanceStore = asGovernanceStore(options.store);
|
|
1966
|
-
if (!governanceStore) throw new
|
|
2259
|
+
if (!governanceStore) throw new HostError("Execution attestations require a governance-capable store.");
|
|
1967
2260
|
const invocation = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1968
|
-
if (!invocation) throw new
|
|
2261
|
+
if (!invocation) throw new HostError(`ActionInvocation not found: ${actionInvocationId}`);
|
|
1969
2262
|
const id = lifecycleId("att", actionInvocationId, adapterInvocationId ?? `${attestation.provider}:${attestation.operation}:${attestation.externalOperationId ?? "external"}`);
|
|
1970
2263
|
await governanceStore.appendExecutionAttestation({
|
|
1971
2264
|
...attestation,
|
|
@@ -2043,11 +2336,11 @@ function createGovernedActionHost(options) {
|
|
|
2043
2336
|
...traceContext ? { traceContext } : {}
|
|
2044
2337
|
};
|
|
2045
2338
|
if (transaction) {
|
|
2046
|
-
if (!transaction.appendEventWithOutbox) throw new
|
|
2339
|
+
if (!transaction.appendEventWithOutbox) throw new HostError("Outbox egress requires an atomic transaction with appendEventWithOutbox.");
|
|
2047
2340
|
await transaction.appendEventWithOutbox(envelope, metadata);
|
|
2048
2341
|
} else {
|
|
2049
2342
|
const outboxStore = asOutboxStore(options.store);
|
|
2050
|
-
if (!outboxStore) throw new
|
|
2343
|
+
if (!outboxStore) throw new HostError("Outbox egress requires an outbox-capable Platform Host store.");
|
|
2051
2344
|
await outboxStore.appendEventWithOutbox(envelope, metadata);
|
|
2052
2345
|
}
|
|
2053
2346
|
} else {
|
|
@@ -2064,12 +2357,20 @@ function createGovernedActionHost(options) {
|
|
|
2064
2357
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
2065
2358
|
};
|
|
2066
2359
|
}
|
|
2360
|
+
async function persistReconciliationVerdict(invocation, patch) {
|
|
2361
|
+
try {
|
|
2362
|
+
await persistInvocation(invocation, patch);
|
|
2363
|
+
} catch (error) {
|
|
2364
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2365
|
+
await persistInvocation(invocation, patch);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2067
2368
|
async function persistInvocation(invocation, patch) {
|
|
2068
2369
|
const previousStatus = invocation.status;
|
|
2069
2370
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
2070
2371
|
const recoverable = options.store;
|
|
2071
2372
|
if (!recoverable.updateLeasedActionInvocation) {
|
|
2072
|
-
throw new
|
|
2373
|
+
throw new HostError("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
2073
2374
|
}
|
|
2074
2375
|
const updated = await recoverable.updateLeasedActionInvocation({
|
|
2075
2376
|
id: invocation.id,
|
|
@@ -2094,6 +2395,8 @@ function createGovernedActionHost(options) {
|
|
|
2094
2395
|
const externalCompletion = createExternalCompletionLifecycle({
|
|
2095
2396
|
store: options.store,
|
|
2096
2397
|
telemetry: options.telemetry,
|
|
2398
|
+
redactErrorText: options.redactErrorText,
|
|
2399
|
+
redactExternalCompletionResult: options.redactExternalCompletionResult,
|
|
2097
2400
|
eventResultFields,
|
|
2098
2401
|
now,
|
|
2099
2402
|
actionResolver,
|
|
@@ -2174,9 +2477,6 @@ function initialState(machine) {
|
|
|
2174
2477
|
function isTerminal(status) {
|
|
2175
2478
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
2176
2479
|
}
|
|
2177
|
-
function isRecord(value) {
|
|
2178
|
-
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
2179
|
-
}
|
|
2180
2480
|
function isAdmissionTime(value) {
|
|
2181
2481
|
return value instanceof Date && Number.isFinite(value.getTime());
|
|
2182
2482
|
}
|
|
@@ -2459,11 +2759,14 @@ var MemoryPlatformHostStore = class {
|
|
|
2459
2759
|
await previous;
|
|
2460
2760
|
let domainSnapshot;
|
|
2461
2761
|
let snapshotCreated = false;
|
|
2762
|
+
const undoLedgerWrites = [];
|
|
2462
2763
|
try {
|
|
2463
2764
|
domainSnapshot = transactionProvider.snapshot(this.db);
|
|
2464
2765
|
snapshotCreated = true;
|
|
2465
2766
|
const pendingEvents = [];
|
|
2466
2767
|
const pendingUpdates = [];
|
|
2768
|
+
const pendingAdapterUpdates = [];
|
|
2769
|
+
const pendingReconciliations = [];
|
|
2467
2770
|
const appendPending = async (event, metadata) => {
|
|
2468
2771
|
if (this.events.some((candidate) => candidate.id === event.id) || pendingEvents.some((candidate) => candidate.event.id === event.id)) return;
|
|
2469
2772
|
pendingEvents.push({ event, ...metadata ? { metadata } : {} });
|
|
@@ -2479,6 +2782,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2479
2782
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
2480
2783
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
2481
2784
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
2785
|
+
getEntityStateSource: async (tenantId, spaceId, entityType, entityId) => {
|
|
2786
|
+
let source = await this.getEntityStateSource(tenantId, spaceId, entityType, entityId);
|
|
2787
|
+
for (const { event } of pendingEvents) {
|
|
2788
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2789
|
+
const state = event.payload?.toState;
|
|
2790
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2791
|
+
}
|
|
2792
|
+
return source;
|
|
2793
|
+
},
|
|
2482
2794
|
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
2483
2795
|
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
2484
2796
|
for (const pending of pendingEvents) {
|
|
@@ -2489,26 +2801,62 @@ var MemoryPlatformHostStore = class {
|
|
|
2489
2801
|
}
|
|
2490
2802
|
return state;
|
|
2491
2803
|
},
|
|
2804
|
+
updateAdapterInvocation: async (id, patch, scope) => {
|
|
2805
|
+
if (!await this.getAdapterInvocation(id, scope)) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2806
|
+
pendingAdapterUpdates.push([id, patch, scope]);
|
|
2807
|
+
},
|
|
2808
|
+
appendExternalReconciliation: async (record) => {
|
|
2809
|
+
pendingReconciliations.push(record);
|
|
2810
|
+
},
|
|
2492
2811
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
2493
|
-
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
2812
|
+
pendingUpdates.push({ id, tenantId, spaceId, patch: copyInvocationPatch(patch) });
|
|
2494
2813
|
},
|
|
2495
2814
|
updateLeasedActionInvocation: async (input) => {
|
|
2496
2815
|
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
2497
2816
|
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
2498
|
-
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
2817
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: copyInvocationPatch(input.patch) });
|
|
2499
2818
|
return true;
|
|
2500
2819
|
}
|
|
2501
2820
|
});
|
|
2502
2821
|
for (const update of pendingUpdates) {
|
|
2503
2822
|
if (!await this.getActionInvocation(update.id, update.tenantId, update.spaceId)) throw new Error(`ActionInvocation not found: ${update.id}`);
|
|
2504
2823
|
}
|
|
2824
|
+
const rememberRecord = (record) => {
|
|
2825
|
+
const before = { ...record };
|
|
2826
|
+
undoLedgerWrites.push(() => {
|
|
2827
|
+
for (const key of Object.keys(record)) Reflect.deleteProperty(record, key);
|
|
2828
|
+
Object.assign(record, before);
|
|
2829
|
+
});
|
|
2830
|
+
};
|
|
2831
|
+
const rememberAppend = (rows, id) => {
|
|
2832
|
+
if (!rows.some((row) => row.id === id)) undoLedgerWrites.push(() => {
|
|
2833
|
+
const index = rows.findIndex((row) => row.id === id);
|
|
2834
|
+
if (index !== -1) rows.splice(index, 1);
|
|
2835
|
+
});
|
|
2836
|
+
};
|
|
2505
2837
|
for (const pending of pendingEvents) {
|
|
2838
|
+
rememberAppend(this.events, pending.event.id);
|
|
2839
|
+
rememberAppend(this.outbox, pending.event.id);
|
|
2506
2840
|
if (pending.metadata) await this.appendEventWithOutbox(pending.event, pending.metadata);
|
|
2507
2841
|
else await this.appendEvent(pending.event);
|
|
2508
2842
|
}
|
|
2509
|
-
for (const
|
|
2843
|
+
for (const record of pendingReconciliations) {
|
|
2844
|
+
rememberAppend(this.externalReconciliations, record.id);
|
|
2845
|
+
await this.appendExternalReconciliation(record);
|
|
2846
|
+
}
|
|
2847
|
+
for (const [id, patch, scope] of pendingAdapterUpdates) {
|
|
2848
|
+
const record = await this.getAdapterInvocation(id, scope);
|
|
2849
|
+
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2850
|
+
rememberRecord(record);
|
|
2851
|
+
await this.updateAdapterInvocation(id, patch, scope);
|
|
2852
|
+
}
|
|
2853
|
+
for (const update of pendingUpdates) {
|
|
2854
|
+
rememberRecord(await this.getActionInvocation(update.id, update.tenantId, update.spaceId));
|
|
2855
|
+
await this.updateActionInvocation(update.id, update.tenantId, update.spaceId, update.patch);
|
|
2856
|
+
}
|
|
2510
2857
|
return result;
|
|
2511
2858
|
} catch (error) {
|
|
2859
|
+
for (const undo of undoLedgerWrites.reverse()) undo();
|
|
2512
2860
|
if (snapshotCreated) transactionProvider.restore(this.db, domainSnapshot);
|
|
2513
2861
|
throw error;
|
|
2514
2862
|
} finally {
|
|
@@ -2536,6 +2884,10 @@ var MemoryPlatformHostStore = class {
|
|
|
2536
2884
|
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
2537
2885
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
2538
2886
|
Object.assign(record, patch, { updatedAt: /* @__PURE__ */ new Date() });
|
|
2887
|
+
if (Object.hasOwn(patch, "pendingStateTransition")) {
|
|
2888
|
+
if (patch.pendingStateTransition) record.pendingStateTransition = { ...patch.pendingStateTransition };
|
|
2889
|
+
else delete record.pendingStateTransition;
|
|
2890
|
+
}
|
|
2539
2891
|
if (Object.hasOwn(patch, "pendingCompletion") && patch.pendingCompletion === void 0) delete record.pendingCompletion;
|
|
2540
2892
|
if (Object.hasOwn(patch, "externalCompletion") && patch.externalCompletion === void 0) delete record.externalCompletion;
|
|
2541
2893
|
if (Object.hasOwn(patch, "error") && patch.error === void 0) delete record.error;
|
|
@@ -2626,11 +2978,11 @@ var MemoryPlatformHostStore = class {
|
|
|
2626
2978
|
if (this.adapterInvocations.some((candidate) => candidate.id === record.id)) return;
|
|
2627
2979
|
this.adapterInvocations.push(record);
|
|
2628
2980
|
}
|
|
2629
|
-
async getAdapterInvocation(id) {
|
|
2630
|
-
return this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2981
|
+
async getAdapterInvocation(id, scope) {
|
|
2982
|
+
return this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2631
2983
|
}
|
|
2632
|
-
async updateAdapterInvocation(id, patch) {
|
|
2633
|
-
const record = this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2984
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
2985
|
+
const record = this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2634
2986
|
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2635
2987
|
Object.assign(record, patch);
|
|
2636
2988
|
}
|
|
@@ -2735,6 +3087,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2735
3087
|
async nextEventSequence(tenantId, spaceId) {
|
|
2736
3088
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).length + 1;
|
|
2737
3089
|
}
|
|
3090
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3091
|
+
let source;
|
|
3092
|
+
for (const event of this.events) {
|
|
3093
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
3094
|
+
const state = event.payload?.toState;
|
|
3095
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
3096
|
+
}
|
|
3097
|
+
return source;
|
|
3098
|
+
}
|
|
2738
3099
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
2739
3100
|
let state;
|
|
2740
3101
|
for (const event of this.events) {
|
|
@@ -2814,6 +3175,9 @@ function cloneOutboxRequeue(requeue) {
|
|
|
2814
3175
|
priorAvailableAt: new Date(requeue.priorAvailableAt)
|
|
2815
3176
|
};
|
|
2816
3177
|
}
|
|
3178
|
+
function copyInvocationPatch(patch) {
|
|
3179
|
+
return { ...patch, ...patch.pendingStateTransition ? { pendingStateTransition: { ...patch.pendingStateTransition } } : {} };
|
|
3180
|
+
}
|
|
2817
3181
|
async function applyPostgresMigrations(client, migrations) {
|
|
2818
3182
|
const ordered = validateMigrations(migrations);
|
|
2819
3183
|
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
@@ -2907,9 +3271,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2907
3271
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
2908
3272
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
2909
3273
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
3274
|
+
getEntityStateSource: (tenantId, spaceId, entityType, entityId) => scoped.getEntityStateSource(tenantId, spaceId, entityType, entityId),
|
|
2910
3275
|
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2911
3276
|
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2912
|
-
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
3277
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input),
|
|
3278
|
+
updateAdapterInvocation: (id, patch, scope) => scoped.updateAdapterInvocation(id, patch, scope),
|
|
3279
|
+
appendExternalReconciliation: (record) => scoped.appendExternalReconciliation(record)
|
|
2913
3280
|
});
|
|
2914
3281
|
});
|
|
2915
3282
|
}
|
|
@@ -3092,6 +3459,13 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3092
3459
|
ALTER TABLE fabric_platform.action_invocations
|
|
3093
3460
|
ADD COLUMN admitted_at timestamptz;
|
|
3094
3461
|
`
|
|
3462
|
+
}, {
|
|
3463
|
+
version: 7,
|
|
3464
|
+
name: "pending_state_transition",
|
|
3465
|
+
sql: `
|
|
3466
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
3467
|
+
ADD COLUMN IF NOT EXISTS pending_state_transition jsonb;
|
|
3468
|
+
`
|
|
3095
3469
|
}]);
|
|
3096
3470
|
}
|
|
3097
3471
|
async transaction(run) {
|
|
@@ -3172,6 +3546,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3172
3546
|
pending_completion=CASE WHEN $12::boolean THEN $13::jsonb ELSE pending_completion END,
|
|
3173
3547
|
completion_due_at=CASE WHEN $12::boolean THEN ($13::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
3174
3548
|
external_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE external_completion END,
|
|
3549
|
+
pending_state_transition=CASE WHEN $16::boolean THEN $17::jsonb ELSE pending_state_transition END,
|
|
3175
3550
|
lease_owner=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3176
3551
|
THEN NULL ELSE lease_owner END,
|
|
3177
3552
|
lease_expires_at=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -3193,7 +3568,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3193
3568
|
Object.hasOwn(patch, "pendingCompletion"),
|
|
3194
3569
|
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3195
3570
|
Object.hasOwn(patch, "externalCompletion"),
|
|
3196
|
-
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
3571
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null,
|
|
3572
|
+
Object.hasOwn(patch, "pendingStateTransition"),
|
|
3573
|
+
patch.pendingStateTransition ? JSON.stringify(patch.pendingStateTransition) : null
|
|
3197
3574
|
]
|
|
3198
3575
|
);
|
|
3199
3576
|
}
|
|
@@ -3360,19 +3737,19 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3360
3737
|
]
|
|
3361
3738
|
);
|
|
3362
3739
|
}
|
|
3363
|
-
async getAdapterInvocation(id) {
|
|
3740
|
+
async getAdapterInvocation(id, scope) {
|
|
3364
3741
|
const result = await this.sql.query(
|
|
3365
|
-
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1`,
|
|
3366
|
-
[id]
|
|
3742
|
+
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1${scope ? " AND tenant_id=$2 AND space_id=$3 AND action_invocation_id=$4" : ""}`,
|
|
3743
|
+
[id, ...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []]
|
|
3367
3744
|
);
|
|
3368
3745
|
return result.rows[0] ? toAdapterRecord(result.rows[0]) : void 0;
|
|
3369
3746
|
}
|
|
3370
|
-
async updateAdapterInvocation(id, patch) {
|
|
3747
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
3371
3748
|
await this.sql.query(
|
|
3372
3749
|
`UPDATE fabric_platform.adapter_invocations SET
|
|
3373
3750
|
status=COALESCE($2,status), output=COALESCE($3::jsonb,output),
|
|
3374
3751
|
error=CASE WHEN $4::boolean THEN $5 ELSE error END,
|
|
3375
|
-
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1`,
|
|
3752
|
+
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1${scope ? " AND tenant_id=$8 AND space_id=$9 AND action_invocation_id=$10" : ""}`,
|
|
3376
3753
|
[
|
|
3377
3754
|
id,
|
|
3378
3755
|
patch.status ?? null,
|
|
@@ -3380,7 +3757,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3380
3757
|
Object.hasOwn(patch, "error"),
|
|
3381
3758
|
patch.error ?? null,
|
|
3382
3759
|
patch.attempt ?? null,
|
|
3383
|
-
patch.updatedAt ?? null
|
|
3760
|
+
patch.updatedAt ?? null,
|
|
3761
|
+
...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []
|
|
3384
3762
|
]
|
|
3385
3763
|
);
|
|
3386
3764
|
}
|
|
@@ -3611,6 +3989,16 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3611
3989
|
);
|
|
3612
3990
|
return Number(result.rows[0].next_sequence);
|
|
3613
3991
|
}
|
|
3992
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3993
|
+
const result = await this.sql.query(
|
|
3994
|
+
`SELECT payload->>'toState' AS state, action_invocation_id FROM fabric_platform.asset_events
|
|
3995
|
+
WHERE tenant_id=$1 AND space_id=$2 AND subject_type=$3 AND subject_id=$4
|
|
3996
|
+
AND payload ? 'toState' ORDER BY sequence DESC LIMIT 1`,
|
|
3997
|
+
[tenantId, spaceId, entityType, entityId]
|
|
3998
|
+
);
|
|
3999
|
+
const row = result.rows[0];
|
|
4000
|
+
return typeof row?.state === "string" && typeof row.action_invocation_id === "string" ? { state: row.state, actionInvocationId: row.action_invocation_id } : void 0;
|
|
4001
|
+
}
|
|
3614
4002
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
3615
4003
|
const result = await this.sql.query(
|
|
3616
4004
|
`SELECT payload->>'toState' AS state FROM fabric_platform.asset_events
|
|
@@ -3734,6 +4122,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3734
4122
|
pending_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE pending_completion END,
|
|
3735
4123
|
completion_due_at=CASE WHEN $14::boolean THEN ($15::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
3736
4124
|
external_completion=CASE WHEN $16::boolean THEN $17::jsonb ELSE external_completion END,
|
|
4125
|
+
pending_state_transition=CASE WHEN $18::boolean THEN $19::jsonb ELSE pending_state_transition END,
|
|
3737
4126
|
lease_owner=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3738
4127
|
THEN NULL ELSE lease_owner END,
|
|
3739
4128
|
lease_expires_at=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -3759,7 +4148,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3759
4148
|
Object.hasOwn(patch, "pendingCompletion"),
|
|
3760
4149
|
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3761
4150
|
Object.hasOwn(patch, "externalCompletion"),
|
|
3762
|
-
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
4151
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null,
|
|
4152
|
+
Object.hasOwn(patch, "pendingStateTransition"),
|
|
4153
|
+
patch.pendingStateTransition ? JSON.stringify(patch.pendingStateTransition) : null
|
|
3763
4154
|
]
|
|
3764
4155
|
);
|
|
3765
4156
|
return result.rows.length === 1;
|
|
@@ -3858,6 +4249,7 @@ function toActionRecord(row) {
|
|
|
3858
4249
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
3859
4250
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
3860
4251
|
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
4252
|
+
...row.pending_state_transition ? { pendingStateTransition: row.pending_state_transition } : {},
|
|
3861
4253
|
...row.pending_completion ? { pendingCompletion: toPendingCompletion(row.pending_completion) } : {},
|
|
3862
4254
|
...row.external_completion ? { externalCompletion: toExternalCompletion(row.external_completion) } : {},
|
|
3863
4255
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
@@ -4003,8 +4395,6 @@ function toOutboxRequeueRecord(row, prefix) {
|
|
|
4003
4395
|
priorLeaseToken: Number(field("prior_lease_token"))
|
|
4004
4396
|
};
|
|
4005
4397
|
}
|
|
4006
|
-
|
|
4007
|
-
// src/worker.ts
|
|
4008
4398
|
var DEFAULT_BATCH_SIZE = 10;
|
|
4009
4399
|
var DEFAULT_LEASE_DURATION_MS = 5 * 6e4;
|
|
4010
4400
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
@@ -4053,6 +4443,7 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4053
4443
|
let completed = 0;
|
|
4054
4444
|
let failed = 0;
|
|
4055
4445
|
let waitingForApproval = 0;
|
|
4446
|
+
let deferred = 0;
|
|
4056
4447
|
for (const invocation of claimed) {
|
|
4057
4448
|
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
4058
4449
|
try {
|
|
@@ -4066,7 +4457,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4066
4457
|
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
4067
4458
|
else failed += 1;
|
|
4068
4459
|
} catch (error) {
|
|
4069
|
-
|
|
4460
|
+
if (platform.isRetryableExecutionError(error)) deferred += 1;
|
|
4461
|
+
else failed += 1;
|
|
4070
4462
|
options.onError?.(error, invocation);
|
|
4071
4463
|
} finally {
|
|
4072
4464
|
await stopHeartbeat();
|
|
@@ -4076,7 +4468,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4076
4468
|
claimed: claimed.length,
|
|
4077
4469
|
completed,
|
|
4078
4470
|
failed,
|
|
4079
|
-
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
4471
|
+
...waitingForApproval > 0 ? { waitingForApproval } : {},
|
|
4472
|
+
...deferred > 0 ? { deferred } : {}
|
|
4080
4473
|
};
|
|
4081
4474
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
4082
4475
|
return result;
|
|
@@ -4303,6 +4696,32 @@ function submitInput(subject, scope, idempotencyKey) {
|
|
|
4303
4696
|
idempotencyKey
|
|
4304
4697
|
};
|
|
4305
4698
|
}
|
|
4699
|
+
async function adapterProbe(subject) {
|
|
4700
|
+
await subject.reset?.();
|
|
4701
|
+
if (subject.ours.tenantId === subject.theirs.tenantId) fail("the two scopes must be in different tenants");
|
|
4702
|
+
const suffix = globalThis.crypto.randomUUID();
|
|
4703
|
+
const scope = {
|
|
4704
|
+
tenantId: subject.ours.tenantId,
|
|
4705
|
+
spaceId: `iso-probe-${suffix}`,
|
|
4706
|
+
actionInvocationId: `iso_act_${suffix}`
|
|
4707
|
+
};
|
|
4708
|
+
const id = `iso_adp_${suffix}`;
|
|
4709
|
+
const now = /* @__PURE__ */ new Date();
|
|
4710
|
+
await subject.store.createAdapterInvocation({
|
|
4711
|
+
id,
|
|
4712
|
+
...scope,
|
|
4713
|
+
adapterType: "iso",
|
|
4714
|
+
operation: "probe",
|
|
4715
|
+
vendor: "iso",
|
|
4716
|
+
status: "running",
|
|
4717
|
+
input: {},
|
|
4718
|
+
output: { probe: true },
|
|
4719
|
+
attempt: 1,
|
|
4720
|
+
createdAt: now,
|
|
4721
|
+
updatedAt: now
|
|
4722
|
+
});
|
|
4723
|
+
return { id, scope };
|
|
4724
|
+
}
|
|
4306
4725
|
function tenantIsolationChecks() {
|
|
4307
4726
|
return [
|
|
4308
4727
|
{
|
|
@@ -4447,6 +4866,54 @@ function tenantIsolationChecks() {
|
|
|
4447
4866
|
if (leaked.length > 0) fail(`a viewer in the other tenant saw ${leaked.length} of our rows`);
|
|
4448
4867
|
return [theirs.allowed ? "other tenant's viewer sees none of our rows" : "other tenant's viewer is denied"];
|
|
4449
4868
|
}
|
|
4869
|
+
},
|
|
4870
|
+
{
|
|
4871
|
+
id: "fabric.tenancy.adapter-invocation-read-scoped.v1",
|
|
4872
|
+
async run(subject) {
|
|
4873
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4874
|
+
try {
|
|
4875
|
+
const theirs = { ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId };
|
|
4876
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, tenantId: subject.theirs.tenantId })) fail("an adapter invocation was readable under another tenant's scope");
|
|
4877
|
+
if (await subject.store.getAdapterInvocation(id, theirs)) fail("an adapter invocation was readable under another tenant's scope");
|
|
4878
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, spaceId: `${scope.spaceId}-other` })) fail("an adapter invocation was readable under another space's scope");
|
|
4879
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, actionInvocationId: `${scope.actionInvocationId}-other` })) fail("an adapter invocation was readable under another action invocation's scope");
|
|
4880
|
+
const ours = await subject.store.getAdapterInvocation(id, scope);
|
|
4881
|
+
if (!ours || ours.id !== id || ours.tenantId !== scope.tenantId || ours.spaceId !== scope.spaceId || ours.actionInvocationId !== scope.actionInvocationId) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4882
|
+
} finally {
|
|
4883
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4884
|
+
}
|
|
4885
|
+
return ["adapter invocation readable under our scope and invisible outside it"];
|
|
4886
|
+
}
|
|
4887
|
+
},
|
|
4888
|
+
{
|
|
4889
|
+
id: "fabric.tenancy.adapter-invocation-write-scoped.v1",
|
|
4890
|
+
async run(subject) {
|
|
4891
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4892
|
+
const shape = (record) => JSON.stringify({ status: record.status, error: record.error, output: record.output, attempt: record.attempt });
|
|
4893
|
+
try {
|
|
4894
|
+
const before = await subject.store.getAdapterInvocation(id, scope);
|
|
4895
|
+
if (!before) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4896
|
+
const untouched = shape(before);
|
|
4897
|
+
for (const other of [
|
|
4898
|
+
{ ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId },
|
|
4899
|
+
{ ...scope, tenantId: subject.theirs.tenantId },
|
|
4900
|
+
{ ...scope, spaceId: `${scope.spaceId}-other` },
|
|
4901
|
+
{ ...scope, actionInvocationId: `${scope.actionInvocationId}-other` }
|
|
4902
|
+
]) {
|
|
4903
|
+
try {
|
|
4904
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "iso probe", updatedAt: /* @__PURE__ */ new Date() }, other);
|
|
4905
|
+
} catch {
|
|
4906
|
+
}
|
|
4907
|
+
const after = await subject.store.getAdapterInvocation(id, scope);
|
|
4908
|
+
if (!after || shape(after) !== untouched) fail("an adapter invocation changed under another scope");
|
|
4909
|
+
}
|
|
4910
|
+
await subject.store.updateAdapterInvocation(id, { status: "succeeded", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4911
|
+
if ((await subject.store.getAdapterInvocation(id, scope))?.status !== "succeeded") fail("our own adapter invocation was not updated under our scope, so the check proves nothing");
|
|
4912
|
+
} finally {
|
|
4913
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4914
|
+
}
|
|
4915
|
+
return ["adapter invocation untouched outside our scope and writable under ours"];
|
|
4916
|
+
}
|
|
4450
4917
|
}
|
|
4451
4918
|
];
|
|
4452
4919
|
}
|