@fabricorg/platform-host 7.4.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +295 -15
- package/dist/index.cjs +540 -73
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +150 -17
- package/dist/index.d.ts +150 -17
- package/dist/index.js +541 -74
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { canonicalJson, AdapterRegistry, assertAdapterCircuitBreakerConfig, AdapterCircuitBreakerRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, isAdapterCircuitOpenError, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
1
|
+
import { canonicalJson, AdapterRegistry, assertAdapterCircuitBreakerConfig, AdapterCircuitBreakerRegistry, isRetryableExecutionError, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, isReconciliationRequiredError, executeWithAdapterRetry, isAdapterCircuitOpenError, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
2
2
|
export { canonicalJson } from '@fabricorg/platform';
|
|
3
3
|
import { performance } from 'perf_hooks';
|
|
4
4
|
import { assertAssemblyRuntimeCompatible } from '@fabricorg/assembly';
|
|
@@ -308,6 +308,24 @@ function withoutPrivateHostFields(data, eventResultFields) {
|
|
|
308
308
|
function errorMessage(error) {
|
|
309
309
|
return error instanceof Error ? error.message : String(error);
|
|
310
310
|
}
|
|
311
|
+
var HostError = class extends Error {
|
|
312
|
+
};
|
|
313
|
+
function isRecord(value) {
|
|
314
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
315
|
+
}
|
|
316
|
+
function tryRedactErrorText(hook, context, message) {
|
|
317
|
+
if (!hook || message === "") return { success: true, text: message };
|
|
318
|
+
try {
|
|
319
|
+
const text = hook(context, message);
|
|
320
|
+
if (typeof text === "string" && text.length > 0) return { success: true, text };
|
|
321
|
+
} catch {
|
|
322
|
+
}
|
|
323
|
+
return { success: false };
|
|
324
|
+
}
|
|
325
|
+
function redactErrorText(hook, context, message) {
|
|
326
|
+
const result = tryRedactErrorText(hook, context, message);
|
|
327
|
+
return result.success ? result.text : "Error text withheld: the error-text redactor failed.";
|
|
328
|
+
}
|
|
311
329
|
function lifecycleId(prefix, invocationId, key) {
|
|
312
330
|
const safeKey = key.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 96);
|
|
313
331
|
return `${prefix}_${invocationId}_${safeKey}`;
|
|
@@ -370,7 +388,11 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
370
388
|
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
371
389
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
372
390
|
}
|
|
373
|
-
const
|
|
391
|
+
const provider = recorded.provider ?? completion.provider;
|
|
392
|
+
const rawDigest = completionDigest(completion, provider);
|
|
393
|
+
if (recorded.digest === rawDigest) return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
394
|
+
completion = redactCompletion(invocation, completion, provider ?? "external");
|
|
395
|
+
const incomingDigest = completionDigest(completion, provider);
|
|
374
396
|
if (recorded.digest === incomingDigest) {
|
|
375
397
|
return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
376
398
|
}
|
|
@@ -400,8 +422,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
400
422
|
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
401
423
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
402
424
|
}
|
|
425
|
+
completion = redactCompletion(invocation, completion, completion.provider ?? pending.provider);
|
|
403
426
|
const recordedAt = now();
|
|
404
427
|
const externalCompletion = {
|
|
428
|
+
// completion has passed the hooks; retain the callback shape for compatibility.
|
|
405
429
|
...completion,
|
|
406
430
|
provider: completion.provider ?? pending.provider,
|
|
407
431
|
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
@@ -422,7 +446,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
422
446
|
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
423
447
|
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
424
448
|
updatedAt: recordedAt
|
|
425
|
-
});
|
|
449
|
+
}, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
426
450
|
await appendEvent(invocation, {
|
|
427
451
|
eventType: "ExternalOperationCompleted",
|
|
428
452
|
...subject,
|
|
@@ -434,7 +458,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
434
458
|
return withConsistency(actionResult2({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
435
459
|
}
|
|
436
460
|
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
437
|
-
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
461
|
+
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt }, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
438
462
|
await appendEvent(invocation, {
|
|
439
463
|
eventType: "ExternalOperationFailed",
|
|
440
464
|
...subject,
|
|
@@ -445,6 +469,30 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
445
469
|
if (settles) emitInvocationStatusTelemetry(invocation, "failed", deps.telemetry, now(), invocation.status);
|
|
446
470
|
return withConsistency(actionResult2({ ...invocation, ...patch }), invocation.actionId);
|
|
447
471
|
}
|
|
472
|
+
function redactCompletion(invocation, completion, provider) {
|
|
473
|
+
let result = completion.result;
|
|
474
|
+
if (result && deps.redactExternalCompletionResult) {
|
|
475
|
+
try {
|
|
476
|
+
result = deps.redactExternalCompletionResult(invocation.actionId, provider, result);
|
|
477
|
+
if (!isRecord(result)) throw new HostError("Completion result redactor must return a non-null, non-array object.");
|
|
478
|
+
} catch {
|
|
479
|
+
throw new ExternalCompletionError("redaction_failed", "External completion result withheld: the result redactor failed; nothing was recorded.");
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
const error = completion.error == null ? void 0 : tryRedactErrorText(
|
|
483
|
+
deps.redactErrorText,
|
|
484
|
+
{ actionId: invocation.actionId, source: "external_completion", provider },
|
|
485
|
+
completion.error
|
|
486
|
+
);
|
|
487
|
+
if (error && !error.success) {
|
|
488
|
+
throw new ExternalCompletionError("redaction_failed", "External completion error withheld: the error-text redactor failed; nothing was recorded.");
|
|
489
|
+
}
|
|
490
|
+
return {
|
|
491
|
+
...completion,
|
|
492
|
+
...completion.result !== void 0 ? { result } : {},
|
|
493
|
+
...error?.success ? { error: error.text } : {}
|
|
494
|
+
};
|
|
495
|
+
}
|
|
448
496
|
function completionDigest(completion, provider) {
|
|
449
497
|
return createHash("sha256").update(canonicalJson({
|
|
450
498
|
provider: provider ?? null,
|
|
@@ -457,7 +505,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
457
505
|
}
|
|
458
506
|
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
459
507
|
const governanceStore = asGovernanceStore(deps.store);
|
|
460
|
-
if (!governanceStore) throw new
|
|
508
|
+
if (!governanceStore) throw new HostError("External completion reconciliation requires a governance-capable store.");
|
|
461
509
|
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
462
510
|
const seed = createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
463
511
|
await governanceStore.appendExternalReconciliation({
|
|
@@ -477,10 +525,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
477
525
|
async function reconcileOverdueCompletions(input = {}) {
|
|
478
526
|
const recoverable = deps.store;
|
|
479
527
|
if (!recoverable.listActionInvocations) {
|
|
480
|
-
throw new
|
|
528
|
+
throw new HostError("Overdue completion reconciliation requires a store that can list invocations.");
|
|
481
529
|
}
|
|
482
530
|
if (!asGovernanceStore(deps.store)) {
|
|
483
|
-
throw new
|
|
531
|
+
throw new HostError("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
484
532
|
}
|
|
485
533
|
const current = input.now ?? now();
|
|
486
534
|
const candidates = await recoverable.listActionInvocations({
|
|
@@ -855,6 +903,10 @@ var ADMISSION_FAILED = Object.freeze({ kind: "failed" });
|
|
|
855
903
|
var ADMISSION_TIMED_OUT = Object.freeze({ kind: "timed_out" });
|
|
856
904
|
var ADMISSION_REFUSAL_CODE = /^[a-z][A-Za-z0-9_.:-]{0,127}$/;
|
|
857
905
|
function createGovernedActionHost(options) {
|
|
906
|
+
const retryableExecutionMaxAttempts = options.retryableExecutionMaxAttempts ?? 5;
|
|
907
|
+
if (!Number.isInteger(retryableExecutionMaxAttempts) || retryableExecutionMaxAttempts <= 0) {
|
|
908
|
+
throw new Error("retryableExecutionMaxAttempts must be a positive integer.");
|
|
909
|
+
}
|
|
858
910
|
if (options.composition) {
|
|
859
911
|
if (!options.registry?.orderedModules) {
|
|
860
912
|
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
@@ -891,6 +943,9 @@ function createGovernedActionHost(options) {
|
|
|
891
943
|
if (typeof admissionTimeoutMs !== "number" || !Number.isInteger(admissionTimeoutMs) || admissionTimeoutMs < 1 || admissionTimeoutMs > MAX_TIMER_MS) {
|
|
892
944
|
throw new Error(`admissionTimeoutMs must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}.`);
|
|
893
945
|
}
|
|
946
|
+
function caughtError(context, error) {
|
|
947
|
+
return error instanceof HostError ? errorMessage(error) : redactErrorText(options.redactErrorText, context, errorMessage(error));
|
|
948
|
+
}
|
|
894
949
|
async function submitAction(input) {
|
|
895
950
|
const read = snapshotSubmission(input);
|
|
896
951
|
if (read.executionReason && !["initial", "offline_replay"].includes(read.executionReason)) {
|
|
@@ -1188,6 +1243,7 @@ function createGovernedActionHost(options) {
|
|
|
1188
1243
|
spaceId,
|
|
1189
1244
|
attributes: { actionVersion: invocation.actionVersion }
|
|
1190
1245
|
});
|
|
1246
|
+
let retryableGovernance = true;
|
|
1191
1247
|
try {
|
|
1192
1248
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
1193
1249
|
if (!parsed.success) {
|
|
@@ -1340,6 +1396,7 @@ function createGovernedActionHost(options) {
|
|
|
1340
1396
|
mode: "execute",
|
|
1341
1397
|
now: now()
|
|
1342
1398
|
});
|
|
1399
|
+
retryableGovernance = false;
|
|
1343
1400
|
for (const outcome of outcomes) {
|
|
1344
1401
|
await options.store.appendPolicyEvaluation({
|
|
1345
1402
|
id: lifecycleId("pol", actionInvocationId, outcome.policyId),
|
|
@@ -1383,12 +1440,16 @@ function createGovernedActionHost(options) {
|
|
|
1383
1440
|
}
|
|
1384
1441
|
let data;
|
|
1385
1442
|
let domainEvents = [];
|
|
1443
|
+
let pendingTransitionEvent;
|
|
1386
1444
|
try {
|
|
1387
1445
|
const runHandler = async (db, transaction) => {
|
|
1388
1446
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
1389
|
-
throw new
|
|
1447
|
+
throw new HostError("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
1390
1448
|
}
|
|
1391
1449
|
const binding = action.stateMachine;
|
|
1450
|
+
let replayedTransition;
|
|
1451
|
+
let pendingStateTransition;
|
|
1452
|
+
let validatedTransition;
|
|
1392
1453
|
if (binding) {
|
|
1393
1454
|
const machine = stateMachineResolver(binding.entityType);
|
|
1394
1455
|
const entityId = binding.getEntityId(parsed.data);
|
|
@@ -1418,9 +1479,45 @@ function createGovernedActionHost(options) {
|
|
|
1418
1479
|
db
|
|
1419
1480
|
}
|
|
1420
1481
|
);
|
|
1421
|
-
|
|
1482
|
+
validatedTransition = { entityId, currentState, targetState, valid: transition.valid };
|
|
1483
|
+
let replayingAppliedTransition = false;
|
|
1484
|
+
if (action.idempotent && currentState === targetState && !transition.valid && entityId) {
|
|
1485
|
+
let source;
|
|
1486
|
+
if (transaction?.getEntityStateSource) {
|
|
1487
|
+
source = await transaction.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1488
|
+
} else if (options.store.getEntityStateSource) {
|
|
1489
|
+
source = await options.store.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1490
|
+
} else {
|
|
1491
|
+
const eventStore = options.store;
|
|
1492
|
+
const listEvents = transaction?.listEvents?.bind(transaction) ?? eventStore.listEvents?.bind(eventStore);
|
|
1493
|
+
const events2 = listEvents ? await listEvents(tenantId, spaceId) : [];
|
|
1494
|
+
let latest;
|
|
1495
|
+
for (const event of events2) {
|
|
1496
|
+
if (event.tenantId === tenantId && event.spaceId === spaceId && event.subjectType === binding.entityType && event.subjectId === entityId && typeof event.payload?.toState === "string" && (!latest || event.sequence > latest.sequence)) latest = event;
|
|
1497
|
+
}
|
|
1498
|
+
if (latest && typeof latest.actionInvocationId === "string") {
|
|
1499
|
+
source = { state: latest.payload.toState, actionInvocationId: latest.actionInvocationId };
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
replayingAppliedTransition = source?.actionInvocationId === actionInvocationId && source.state === currentState;
|
|
1503
|
+
}
|
|
1504
|
+
const pending = invocation.pendingStateTransition;
|
|
1505
|
+
if (replayingAppliedTransition && pending && pending.entityType === binding.entityType && pending.entityId === entityId && pending.toState === currentState) {
|
|
1506
|
+
replayedTransition = pending;
|
|
1507
|
+
}
|
|
1508
|
+
if (action.idempotent && transition.valid && entityId && action.eventPhase === "after_adapters") {
|
|
1509
|
+
pendingStateTransition = {
|
|
1510
|
+
entityType: binding.entityType,
|
|
1511
|
+
entityId,
|
|
1512
|
+
fromState: currentState,
|
|
1513
|
+
toState: targetState
|
|
1514
|
+
};
|
|
1515
|
+
if (!transaction) {
|
|
1516
|
+
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { pendingStateTransition });
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1422
1519
|
if (!transition.valid && !replayingAppliedTransition) {
|
|
1423
|
-
throw new Error(transition.error
|
|
1520
|
+
throw transition.error != null ? new Error(transition.error) : new HostError("Invalid state transition");
|
|
1424
1521
|
}
|
|
1425
1522
|
}
|
|
1426
1523
|
}
|
|
@@ -1441,14 +1538,14 @@ function createGovernedActionHost(options) {
|
|
|
1441
1538
|
parsed.data
|
|
1442
1539
|
) : { success: true, data: {} };
|
|
1443
1540
|
if (!handlerResult.success) {
|
|
1444
|
-
throw new Error(handlerResult.error
|
|
1541
|
+
throw handlerResult.error != null ? new Error(handlerResult.error) : new HostError("Action handler failed");
|
|
1445
1542
|
}
|
|
1446
1543
|
const handlerData = handlerResult.data ?? {};
|
|
1447
1544
|
if (action.resultSchema) {
|
|
1448
1545
|
const publicResult = withoutPrivateHostFields(handlerData, eventResultFields);
|
|
1449
1546
|
const parsedResult = action.resultSchema.safeParse(publicResult);
|
|
1450
1547
|
if (!parsedResult.success) {
|
|
1451
|
-
throw new
|
|
1548
|
+
throw new HostError(`Action result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1452
1549
|
}
|
|
1453
1550
|
}
|
|
1454
1551
|
const events = extractEvents(handlerData);
|
|
@@ -1456,7 +1553,7 @@ function createGovernedActionHost(options) {
|
|
|
1456
1553
|
(event) => !action.emitsEvents.includes(event.eventType)
|
|
1457
1554
|
);
|
|
1458
1555
|
if (undeclaredEvent) {
|
|
1459
|
-
throw new
|
|
1556
|
+
throw new HostError(
|
|
1460
1557
|
`${action.actionId} emitted undeclared event type ${undeclaredEvent.eventType}`
|
|
1461
1558
|
);
|
|
1462
1559
|
}
|
|
@@ -1471,7 +1568,32 @@ function createGovernedActionHost(options) {
|
|
|
1471
1568
|
);
|
|
1472
1569
|
}
|
|
1473
1570
|
}
|
|
1474
|
-
|
|
1571
|
+
let transitionEvent;
|
|
1572
|
+
if (binding && validatedTransition?.entityId && (validatedTransition.valid || replayedTransition)) {
|
|
1573
|
+
const { entityId, currentState, targetState } = validatedTransition;
|
|
1574
|
+
const handlerEmittedTransition = events.some((event) => event.eventType === "StateTransitioned" && event.subjectType === binding.entityType && event.subjectId === entityId);
|
|
1575
|
+
if (!handlerEmittedTransition) {
|
|
1576
|
+
transitionEvent = {
|
|
1577
|
+
eventType: "StateTransitioned",
|
|
1578
|
+
subjectType: binding.entityType,
|
|
1579
|
+
subjectId: entityId,
|
|
1580
|
+
payload: {
|
|
1581
|
+
entityType: binding.entityType,
|
|
1582
|
+
entityId,
|
|
1583
|
+
fromState: replayedTransition?.fromState ?? currentState,
|
|
1584
|
+
toState: targetState,
|
|
1585
|
+
causedByAction: action.actionId
|
|
1586
|
+
}
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
if (transitionEvent && action.eventPhase !== "after_adapters") {
|
|
1591
|
+
await appendEvent(invocation, transitionEvent, "state:transition", 1, transaction);
|
|
1592
|
+
}
|
|
1593
|
+
if (transaction && pendingStateTransition) {
|
|
1594
|
+
await transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, { pendingStateTransition });
|
|
1595
|
+
}
|
|
1596
|
+
return { data: handlerData, domainEvents: events, pendingTransitionEvent: transitionEvent };
|
|
1475
1597
|
};
|
|
1476
1598
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1477
1599
|
const executed = atomicStore ? await atomicStore.transactionWithEvents(
|
|
@@ -1479,9 +1601,22 @@ function createGovernedActionHost(options) {
|
|
|
1479
1601
|
) : await options.store.transaction((db) => runHandler(db));
|
|
1480
1602
|
data = executed.data;
|
|
1481
1603
|
domainEvents = executed.domainEvents;
|
|
1604
|
+
pendingTransitionEvent = executed.pendingTransitionEvent;
|
|
1482
1605
|
} catch (error) {
|
|
1483
|
-
|
|
1606
|
+
if (isReconciliationRequiredError(error)) {
|
|
1607
|
+
const message = caughtError({ actionId: invocation.actionId, source: "handler" }, error);
|
|
1608
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: message });
|
|
1609
|
+
return withConsistency({
|
|
1610
|
+
actionInvocationId,
|
|
1611
|
+
status: "reconciliation_required",
|
|
1612
|
+
error: message,
|
|
1613
|
+
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
1614
|
+
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
1615
|
+
}, action.actionId);
|
|
1616
|
+
}
|
|
1617
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "handler" }, error));
|
|
1484
1618
|
}
|
|
1619
|
+
const adapterInvocationScope = { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id };
|
|
1485
1620
|
let pendingHandoff = invocation.pendingCompletion;
|
|
1486
1621
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
1487
1622
|
const input = step.getInput(parsed.data, data);
|
|
@@ -1489,12 +1624,13 @@ function createGovernedActionHost(options) {
|
|
|
1489
1624
|
const adapter = adapters.require(step.adapterType, step.operation);
|
|
1490
1625
|
const adapterInvocationId = lifecycleId("adp", actionInvocationId, String(stepIndex));
|
|
1491
1626
|
const previousAdapterInvocation = await options.store.getAdapterInvocation(
|
|
1492
|
-
adapterInvocationId
|
|
1627
|
+
adapterInvocationId,
|
|
1628
|
+
adapterInvocationScope
|
|
1493
1629
|
);
|
|
1494
1630
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
1495
1631
|
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
1496
1632
|
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
1497
|
-
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
1633
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1498
1634
|
await appendEvent(invocation, {
|
|
1499
1635
|
eventType: "ExternalOperationAccepted",
|
|
1500
1636
|
subjectType: subject.subjectType,
|
|
@@ -1528,9 +1664,7 @@ function createGovernedActionHost(options) {
|
|
|
1528
1664
|
const startedAt = now();
|
|
1529
1665
|
const record = {
|
|
1530
1666
|
id: adapterInvocationId,
|
|
1531
|
-
|
|
1532
|
-
tenantId,
|
|
1533
|
-
spaceId,
|
|
1667
|
+
...adapterInvocationScope,
|
|
1534
1668
|
adapterType: step.adapterType,
|
|
1535
1669
|
operation: step.operation,
|
|
1536
1670
|
vendor: adapter.vendor,
|
|
@@ -1547,6 +1681,96 @@ function createGovernedActionHost(options) {
|
|
|
1547
1681
|
subjectId: adapterEventSubject.subjectId,
|
|
1548
1682
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1549
1683
|
}, `adapter:${stepIndex}:started`);
|
|
1684
|
+
async function appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference, transaction) {
|
|
1685
|
+
if (accepted) {
|
|
1686
|
+
await appendEvent(invocation, {
|
|
1687
|
+
eventType: "ExternalOperationAccepted",
|
|
1688
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1689
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1690
|
+
payload: {
|
|
1691
|
+
adapterType: step.adapterType,
|
|
1692
|
+
operation: step.operation,
|
|
1693
|
+
externalReference: accepted.externalReference,
|
|
1694
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1695
|
+
}
|
|
1696
|
+
}, `adapter:${stepIndex}:accepted`, 1, transaction);
|
|
1697
|
+
}
|
|
1698
|
+
await appendEvent(invocation, {
|
|
1699
|
+
eventType: "AdapterInvocationSucceeded",
|
|
1700
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1701
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1702
|
+
payload: {
|
|
1703
|
+
adapterType: step.adapterType,
|
|
1704
|
+
operation: step.operation,
|
|
1705
|
+
input: recordedInput,
|
|
1706
|
+
...outputWithheld ? { outputWithheld: true } : { output }
|
|
1707
|
+
}
|
|
1708
|
+
}, `adapter:${stepIndex}:succeeded`, 1, transaction);
|
|
1709
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
1710
|
+
if (outputWithheld && governanceStore2) {
|
|
1711
|
+
await (transaction ?? governanceStore2).appendExternalReconciliation({
|
|
1712
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
1713
|
+
actionInvocationId,
|
|
1714
|
+
tenantId,
|
|
1715
|
+
spaceId,
|
|
1716
|
+
status: "pending",
|
|
1717
|
+
provider: adapter.vendor,
|
|
1718
|
+
...externalReference ?? accepted?.externalReference ? { externalOperationId: externalReference ?? accepted?.externalReference } : {},
|
|
1719
|
+
attempt: 1,
|
|
1720
|
+
reason: message,
|
|
1721
|
+
observedAt: now()
|
|
1722
|
+
});
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
async function finalizeDegradedAdapterAtomically(patch, outputWithheld, message, accepted, output, externalReference) {
|
|
1726
|
+
const atomicStore = asAtomicMutationStore(options.store);
|
|
1727
|
+
if (!atomicStore) return false;
|
|
1728
|
+
const needsReconciliation = outputWithheld && !!asGovernanceStore(options.store);
|
|
1729
|
+
const previousStatus = invocation.status;
|
|
1730
|
+
const finalize = () => atomicStore.transactionWithEvents(async (transaction) => {
|
|
1731
|
+
if (!transaction.updateAdapterInvocation || needsReconciliation && !transaction.appendExternalReconciliation || invocation.leaseOwner && invocation.leaseToken !== void 0 && !transaction.updateLeasedActionInvocation) return false;
|
|
1732
|
+
await appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference, transaction);
|
|
1733
|
+
await transaction.updateAdapterInvocation(adapterInvocationId, {
|
|
1734
|
+
status: "succeeded",
|
|
1735
|
+
...outputWithheld ? {} : { output },
|
|
1736
|
+
updatedAt: now()
|
|
1737
|
+
}, adapterInvocationScope);
|
|
1738
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1739
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1740
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1741
|
+
}
|
|
1742
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1743
|
+
id: actionInvocationId,
|
|
1744
|
+
tenantId,
|
|
1745
|
+
spaceId,
|
|
1746
|
+
workerId: invocation.leaseOwner,
|
|
1747
|
+
leaseToken: invocation.leaseToken,
|
|
1748
|
+
patch
|
|
1749
|
+
});
|
|
1750
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1751
|
+
} else {
|
|
1752
|
+
await transaction.updateActionInvocation(actionInvocationId, tenantId, spaceId, patch);
|
|
1753
|
+
}
|
|
1754
|
+
return true;
|
|
1755
|
+
});
|
|
1756
|
+
let committed;
|
|
1757
|
+
try {
|
|
1758
|
+
committed = await finalize();
|
|
1759
|
+
} catch (error) {
|
|
1760
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1761
|
+
try {
|
|
1762
|
+
committed = await finalize();
|
|
1763
|
+
} catch (retryError) {
|
|
1764
|
+
if (retryError instanceof RecoverableFinalizationError) throw retryError;
|
|
1765
|
+
return false;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
if (committed) emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1769
|
+
return committed;
|
|
1770
|
+
}
|
|
1771
|
+
const adapterErrorContext = { actionId: action.actionId, source: "adapter", adapterType: step.adapterType, operation: step.operation, provider: adapter.vendor };
|
|
1772
|
+
const executionErrorContext = { actionId: action.actionId, source: "execution" };
|
|
1773
|
+
let errorContext = executionErrorContext;
|
|
1550
1774
|
let adapterDeadlineTimer;
|
|
1551
1775
|
try {
|
|
1552
1776
|
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
@@ -1554,15 +1778,15 @@ function createGovernedActionHost(options) {
|
|
|
1554
1778
|
if (adapterController && deadlineMs) {
|
|
1555
1779
|
const remaining = deadlineMs - now().getTime();
|
|
1556
1780
|
if (remaining <= 0) {
|
|
1557
|
-
throw new
|
|
1781
|
+
throw new HostError(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
1558
1782
|
}
|
|
1559
1783
|
adapterDeadlineTimer = setTimeout(
|
|
1560
|
-
() => adapterController.abort(new
|
|
1784
|
+
() => adapterController.abort(new HostError("Adapter deadline exceeded")),
|
|
1561
1785
|
remaining
|
|
1562
1786
|
);
|
|
1563
1787
|
adapterDeadlineTimer.unref?.();
|
|
1564
1788
|
}
|
|
1565
|
-
const circuitBreaker = circuitBreakers.forAdapter(adapter);
|
|
1789
|
+
const circuitBreaker = circuitBreakers.forAdapter(adapter, { tenantId, spaceId });
|
|
1566
1790
|
const result2 = await executeWithAdapterRetry({
|
|
1567
1791
|
...circuitBreaker ? { circuitBreaker } : {},
|
|
1568
1792
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
@@ -1570,13 +1794,15 @@ function createGovernedActionHost(options) {
|
|
|
1570
1794
|
// Every await before the call happens here, before breaker
|
|
1571
1795
|
// admission, so nothing awaits between admission and the adapter.
|
|
1572
1796
|
prepareAttempt: async (attempt) => {
|
|
1797
|
+
errorContext = executionErrorContext;
|
|
1573
1798
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1574
1799
|
attempt,
|
|
1575
1800
|
updatedAt: now()
|
|
1576
|
-
});
|
|
1801
|
+
}, adapterInvocationScope);
|
|
1577
1802
|
},
|
|
1578
|
-
execute: (attempt, maxAttempts) => {
|
|
1579
|
-
|
|
1803
|
+
execute: async (attempt, maxAttempts) => {
|
|
1804
|
+
errorContext = adapterErrorContext;
|
|
1805
|
+
const adapterResult = await adapter.execute(input, {
|
|
1580
1806
|
tenantId,
|
|
1581
1807
|
spaceId,
|
|
1582
1808
|
actionInvocationId,
|
|
@@ -1588,6 +1814,8 @@ function createGovernedActionHost(options) {
|
|
|
1588
1814
|
...deadlineMs ? { deadlineMs } : {},
|
|
1589
1815
|
...adapterController ? { signal: adapterController.signal } : {}
|
|
1590
1816
|
});
|
|
1817
|
+
errorContext = executionErrorContext;
|
|
1818
|
+
return adapterResult;
|
|
1591
1819
|
},
|
|
1592
1820
|
isSuccessful: (result3) => result3.success,
|
|
1593
1821
|
getError: (result3) => result3.error,
|
|
@@ -1600,19 +1828,23 @@ function createGovernedActionHost(options) {
|
|
|
1600
1828
|
return "transient_failure";
|
|
1601
1829
|
}
|
|
1602
1830
|
});
|
|
1831
|
+
errorContext = adapterErrorContext;
|
|
1603
1832
|
assertAdapterOutcomeConsistency(result2);
|
|
1833
|
+
errorContext = executionErrorContext;
|
|
1604
1834
|
if (!result2.success) {
|
|
1835
|
+
const resultError = result2.error == null ? result2.error : redactErrorText(options.redactErrorText, adapterErrorContext, result2.error);
|
|
1605
1836
|
const outcome = result2.outcome ?? "transient_failure";
|
|
1606
1837
|
if (outcome === "ambiguous") {
|
|
1607
1838
|
const evidence = result2.outcomeEvidence;
|
|
1608
|
-
const
|
|
1839
|
+
const reason = evidence?.reason == null ? resultError : redactErrorText(options.redactErrorText, adapterErrorContext, evidence.reason);
|
|
1840
|
+
const message = resultError ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
1609
1841
|
const adapterReconciliation = {
|
|
1610
1842
|
kind: "adapter_outcome_ambiguous",
|
|
1611
1843
|
adapterType: step.adapterType,
|
|
1612
1844
|
operation: step.operation,
|
|
1613
1845
|
vendor: adapter.vendor,
|
|
1614
1846
|
adapterInvocationId,
|
|
1615
|
-
...evidence?.reason ?? result2.error ? { reason
|
|
1847
|
+
...evidence?.reason ?? result2.error ? { reason } : {},
|
|
1616
1848
|
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
1617
1849
|
message
|
|
1618
1850
|
};
|
|
@@ -1620,7 +1852,7 @@ function createGovernedActionHost(options) {
|
|
|
1620
1852
|
status: "ambiguous",
|
|
1621
1853
|
error: message,
|
|
1622
1854
|
updatedAt: now()
|
|
1623
|
-
});
|
|
1855
|
+
}, adapterInvocationScope);
|
|
1624
1856
|
await appendEvent(invocation, {
|
|
1625
1857
|
eventType: "AdapterInvocationAmbiguous",
|
|
1626
1858
|
subjectType: adapterEventSubject.subjectType,
|
|
@@ -1658,24 +1890,56 @@ function createGovernedActionHost(options) {
|
|
|
1658
1890
|
}
|
|
1659
1891
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1660
1892
|
status: "failed",
|
|
1661
|
-
error:
|
|
1893
|
+
error: resultError ?? "Adapter failed",
|
|
1662
1894
|
updatedAt: now()
|
|
1663
|
-
});
|
|
1895
|
+
}, adapterInvocationScope);
|
|
1664
1896
|
await appendEvent(invocation, {
|
|
1665
1897
|
eventType: "AdapterInvocationFailed",
|
|
1666
1898
|
subjectType: adapterEventSubject.subjectType,
|
|
1667
1899
|
subjectId: adapterEventSubject.subjectId,
|
|
1668
1900
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1669
1901
|
}, `adapter:${stepIndex}:failed`);
|
|
1670
|
-
return fail2(invocation, "failed",
|
|
1902
|
+
return fail2(invocation, "failed", resultError ?? "Adapter failed");
|
|
1671
1903
|
}
|
|
1672
1904
|
const resultRecord = result2;
|
|
1673
|
-
|
|
1905
|
+
let output = isRecord(resultRecord.output) ? resultRecord.output : Object.fromEntries(
|
|
1674
1906
|
Object.entries(resultRecord).filter(
|
|
1675
1907
|
([key]) => key !== "success" && key !== "error"
|
|
1676
1908
|
)
|
|
1677
1909
|
);
|
|
1678
1910
|
const handoff = result2.acceptedExternalOperation;
|
|
1911
|
+
if (options.redactAdapterOutput) {
|
|
1912
|
+
try {
|
|
1913
|
+
output = options.redactAdapterOutput(action.actionId, step.adapterType, step.operation, output);
|
|
1914
|
+
if (!isRecord(output)) throw new HostError("Adapter output redactor must return a non-null, non-array object.");
|
|
1915
|
+
} catch {
|
|
1916
|
+
const message = "Adapter output withheld: the output redactor failed after the external effect; reconciliation required.";
|
|
1917
|
+
const reference = typeof handoff?.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1918
|
+
const acceptedAt = now();
|
|
1919
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1920
|
+
const pendingCompletion = pendingHandoff ?? (reference ? {
|
|
1921
|
+
provider: adapter.vendor,
|
|
1922
|
+
adapterType: step.adapterType,
|
|
1923
|
+
operation: step.operation,
|
|
1924
|
+
adapterInvocationId,
|
|
1925
|
+
externalReference: reference,
|
|
1926
|
+
acceptedAt,
|
|
1927
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1928
|
+
} : void 0);
|
|
1929
|
+
try {
|
|
1930
|
+
const patch = { status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} };
|
|
1931
|
+
const accepted = pendingCompletion?.adapterInvocationId === adapterInvocationId ? pendingCompletion : void 0;
|
|
1932
|
+
if (!await finalizeDegradedAdapterAtomically(patch, true, message, accepted, void 0, reference || void 0)) {
|
|
1933
|
+
await persistReconciliationVerdict(invocation, patch);
|
|
1934
|
+
await appendDegradedAdapterEvidence(true, message, accepted, void 0, reference || void 0);
|
|
1935
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1936
|
+
}
|
|
1937
|
+
} catch {
|
|
1938
|
+
throw new RecoverableFinalizationError(message);
|
|
1939
|
+
}
|
|
1940
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} }, action.actionId);
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1679
1943
|
if (handoff) {
|
|
1680
1944
|
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1681
1945
|
if (reference === "") {
|
|
@@ -1697,19 +1961,33 @@ function createGovernedActionHost(options) {
|
|
|
1697
1961
|
const completion = action.execution?.completion ?? "immediate";
|
|
1698
1962
|
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1699
1963
|
if (refuse) {
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1964
|
+
let adapterFinalizedAtomically;
|
|
1965
|
+
try {
|
|
1966
|
+
const patch = {
|
|
1967
|
+
status: "reconciliation_required",
|
|
1968
|
+
error: refuse,
|
|
1969
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1970
|
+
};
|
|
1971
|
+
adapterFinalizedAtomically = await finalizeDegradedAdapterAtomically(patch, false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1972
|
+
if (!adapterFinalizedAtomically) {
|
|
1973
|
+
await persistReconciliationVerdict(invocation, patch);
|
|
1974
|
+
await appendDegradedAdapterEvidence(false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1975
|
+
}
|
|
1976
|
+
} catch {
|
|
1977
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1978
|
+
}
|
|
1706
1979
|
let error = refuse;
|
|
1707
1980
|
try {
|
|
1708
1981
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1709
1982
|
} catch (logError) {
|
|
1710
|
-
error = `${refuse} Reconciliation log unavailable: ${
|
|
1983
|
+
error = `${refuse} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1711
1984
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1712
1985
|
}
|
|
1986
|
+
try {
|
|
1987
|
+
if (!adapterFinalizedAtomically) await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() }, adapterInvocationScope);
|
|
1988
|
+
} catch {
|
|
1989
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1990
|
+
}
|
|
1713
1991
|
return withConsistency({
|
|
1714
1992
|
actionInvocationId,
|
|
1715
1993
|
status: "reconciliation_required",
|
|
@@ -1736,7 +2014,7 @@ function createGovernedActionHost(options) {
|
|
|
1736
2014
|
status: "succeeded",
|
|
1737
2015
|
output,
|
|
1738
2016
|
updatedAt: now()
|
|
1739
|
-
});
|
|
2017
|
+
}, adapterInvocationScope);
|
|
1740
2018
|
if (options.extractExecutionAttestation) {
|
|
1741
2019
|
const attestation = await options.extractExecutionAttestation({
|
|
1742
2020
|
actionInvocationId,
|
|
@@ -1765,12 +2043,13 @@ function createGovernedActionHost(options) {
|
|
|
1765
2043
|
}
|
|
1766
2044
|
}, `adapter:${stepIndex}:succeeded`);
|
|
1767
2045
|
} catch (error) {
|
|
1768
|
-
|
|
2046
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2047
|
+
const message = caughtError(isAdapterCircuitOpenError(error) ? adapterErrorContext : errorContext, error);
|
|
1769
2048
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1770
2049
|
status: "failed",
|
|
1771
2050
|
error: message,
|
|
1772
2051
|
updatedAt: now()
|
|
1773
|
-
});
|
|
2052
|
+
}, adapterInvocationScope);
|
|
1774
2053
|
if (isAdapterCircuitOpenError(error)) {
|
|
1775
2054
|
await appendEvent(invocation, {
|
|
1776
2055
|
eventType: "AdapterInvocationFailed",
|
|
@@ -1801,7 +2080,7 @@ function createGovernedActionHost(options) {
|
|
|
1801
2080
|
try {
|
|
1802
2081
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1803
2082
|
} catch (logError) {
|
|
1804
|
-
error = `${message} Reconciliation log unavailable: ${
|
|
2083
|
+
error = `${message} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1805
2084
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1806
2085
|
}
|
|
1807
2086
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
@@ -1825,9 +2104,12 @@ function createGovernedActionHost(options) {
|
|
|
1825
2104
|
transaction
|
|
1826
2105
|
);
|
|
1827
2106
|
}
|
|
2107
|
+
if (pendingTransitionEvent) {
|
|
2108
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1, transaction);
|
|
2109
|
+
}
|
|
1828
2110
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1829
2111
|
if (!transaction.updateLeasedActionInvocation) {
|
|
1830
|
-
throw new
|
|
2112
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1831
2113
|
}
|
|
1832
2114
|
const updated = await transaction.updateLeasedActionInvocation({
|
|
1833
2115
|
id: actionInvocationId,
|
|
@@ -1860,6 +2142,9 @@ function createGovernedActionHost(options) {
|
|
|
1860
2142
|
for (const [index, event] of domainEvents.entries()) {
|
|
1861
2143
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1862
2144
|
}
|
|
2145
|
+
if (pendingTransitionEvent) {
|
|
2146
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1);
|
|
2147
|
+
}
|
|
1863
2148
|
}
|
|
1864
2149
|
await persistInvocation(invocation, finalPatch);
|
|
1865
2150
|
}
|
|
@@ -1879,7 +2164,15 @@ function createGovernedActionHost(options) {
|
|
|
1879
2164
|
};
|
|
1880
2165
|
} catch (error) {
|
|
1881
2166
|
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1882
|
-
|
|
2167
|
+
if (retryableGovernance && isRetryableExecutionError(error)) {
|
|
2168
|
+
const reason = executionReasonOverride === "approval_resume" ? "approval resume" : !invocation.leaseOwner || invocation.leaseToken === void 0 ? "not leased" : !action.idempotent ? "action not idempotent" : invocation.attemptCount >= retryableExecutionMaxAttempts ? `attempts exhausted (${invocation.attemptCount}/${retryableExecutionMaxAttempts})` : void 0;
|
|
2169
|
+
if (!reason) throw error;
|
|
2170
|
+
return fail2(invocation, "failed", caughtError(
|
|
2171
|
+
{ actionId: invocation.actionId, source: "execution" },
|
|
2172
|
+
new Error(`Retryable execution not retried: ${reason}; ${errorMessage(error)}`)
|
|
2173
|
+
));
|
|
2174
|
+
}
|
|
2175
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "execution" }, error));
|
|
1883
2176
|
}
|
|
1884
2177
|
}
|
|
1885
2178
|
async function resumeApprovedInvocation(actionInvocationId, tenantId, spaceId, decision) {
|
|
@@ -1962,9 +2255,9 @@ function createGovernedActionHost(options) {
|
|
|
1962
2255
|
}
|
|
1963
2256
|
async function recordExecutionAttestation(actionInvocationId, tenantId, spaceId, attestation, adapterInvocationId) {
|
|
1964
2257
|
const governanceStore = asGovernanceStore(options.store);
|
|
1965
|
-
if (!governanceStore) throw new
|
|
2258
|
+
if (!governanceStore) throw new HostError("Execution attestations require a governance-capable store.");
|
|
1966
2259
|
const invocation = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1967
|
-
if (!invocation) throw new
|
|
2260
|
+
if (!invocation) throw new HostError(`ActionInvocation not found: ${actionInvocationId}`);
|
|
1968
2261
|
const id = lifecycleId("att", actionInvocationId, adapterInvocationId ?? `${attestation.provider}:${attestation.operation}:${attestation.externalOperationId ?? "external"}`);
|
|
1969
2262
|
await governanceStore.appendExecutionAttestation({
|
|
1970
2263
|
...attestation,
|
|
@@ -2042,11 +2335,11 @@ function createGovernedActionHost(options) {
|
|
|
2042
2335
|
...traceContext ? { traceContext } : {}
|
|
2043
2336
|
};
|
|
2044
2337
|
if (transaction) {
|
|
2045
|
-
if (!transaction.appendEventWithOutbox) throw new
|
|
2338
|
+
if (!transaction.appendEventWithOutbox) throw new HostError("Outbox egress requires an atomic transaction with appendEventWithOutbox.");
|
|
2046
2339
|
await transaction.appendEventWithOutbox(envelope, metadata);
|
|
2047
2340
|
} else {
|
|
2048
2341
|
const outboxStore = asOutboxStore(options.store);
|
|
2049
|
-
if (!outboxStore) throw new
|
|
2342
|
+
if (!outboxStore) throw new HostError("Outbox egress requires an outbox-capable Platform Host store.");
|
|
2050
2343
|
await outboxStore.appendEventWithOutbox(envelope, metadata);
|
|
2051
2344
|
}
|
|
2052
2345
|
} else {
|
|
@@ -2063,12 +2356,20 @@ function createGovernedActionHost(options) {
|
|
|
2063
2356
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
2064
2357
|
};
|
|
2065
2358
|
}
|
|
2359
|
+
async function persistReconciliationVerdict(invocation, patch) {
|
|
2360
|
+
try {
|
|
2361
|
+
await persistInvocation(invocation, patch);
|
|
2362
|
+
} catch (error) {
|
|
2363
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2364
|
+
await persistInvocation(invocation, patch);
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2066
2367
|
async function persistInvocation(invocation, patch) {
|
|
2067
2368
|
const previousStatus = invocation.status;
|
|
2068
2369
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
2069
2370
|
const recoverable = options.store;
|
|
2070
2371
|
if (!recoverable.updateLeasedActionInvocation) {
|
|
2071
|
-
throw new
|
|
2372
|
+
throw new HostError("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
2072
2373
|
}
|
|
2073
2374
|
const updated = await recoverable.updateLeasedActionInvocation({
|
|
2074
2375
|
id: invocation.id,
|
|
@@ -2093,6 +2394,8 @@ function createGovernedActionHost(options) {
|
|
|
2093
2394
|
const externalCompletion = createExternalCompletionLifecycle({
|
|
2094
2395
|
store: options.store,
|
|
2095
2396
|
telemetry: options.telemetry,
|
|
2397
|
+
redactErrorText: options.redactErrorText,
|
|
2398
|
+
redactExternalCompletionResult: options.redactExternalCompletionResult,
|
|
2096
2399
|
eventResultFields,
|
|
2097
2400
|
now,
|
|
2098
2401
|
actionResolver,
|
|
@@ -2173,9 +2476,6 @@ function initialState(machine) {
|
|
|
2173
2476
|
function isTerminal(status) {
|
|
2174
2477
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
2175
2478
|
}
|
|
2176
|
-
function isRecord(value) {
|
|
2177
|
-
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
2178
|
-
}
|
|
2179
2479
|
function isAdmissionTime(value) {
|
|
2180
2480
|
return value instanceof Date && Number.isFinite(value.getTime());
|
|
2181
2481
|
}
|
|
@@ -2458,11 +2758,14 @@ var MemoryPlatformHostStore = class {
|
|
|
2458
2758
|
await previous;
|
|
2459
2759
|
let domainSnapshot;
|
|
2460
2760
|
let snapshotCreated = false;
|
|
2761
|
+
const undoLedgerWrites = [];
|
|
2461
2762
|
try {
|
|
2462
2763
|
domainSnapshot = transactionProvider.snapshot(this.db);
|
|
2463
2764
|
snapshotCreated = true;
|
|
2464
2765
|
const pendingEvents = [];
|
|
2465
2766
|
const pendingUpdates = [];
|
|
2767
|
+
const pendingAdapterUpdates = [];
|
|
2768
|
+
const pendingReconciliations = [];
|
|
2466
2769
|
const appendPending = async (event, metadata) => {
|
|
2467
2770
|
if (this.events.some((candidate) => candidate.id === event.id) || pendingEvents.some((candidate) => candidate.event.id === event.id)) return;
|
|
2468
2771
|
pendingEvents.push({ event, ...metadata ? { metadata } : {} });
|
|
@@ -2478,6 +2781,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2478
2781
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
2479
2782
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
2480
2783
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
2784
|
+
getEntityStateSource: async (tenantId, spaceId, entityType, entityId) => {
|
|
2785
|
+
let source = await this.getEntityStateSource(tenantId, spaceId, entityType, entityId);
|
|
2786
|
+
for (const { event } of pendingEvents) {
|
|
2787
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2788
|
+
const state = event.payload?.toState;
|
|
2789
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2790
|
+
}
|
|
2791
|
+
return source;
|
|
2792
|
+
},
|
|
2481
2793
|
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
2482
2794
|
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
2483
2795
|
for (const pending of pendingEvents) {
|
|
@@ -2488,26 +2800,62 @@ var MemoryPlatformHostStore = class {
|
|
|
2488
2800
|
}
|
|
2489
2801
|
return state;
|
|
2490
2802
|
},
|
|
2803
|
+
updateAdapterInvocation: async (id, patch, scope) => {
|
|
2804
|
+
if (!await this.getAdapterInvocation(id, scope)) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2805
|
+
pendingAdapterUpdates.push([id, patch, scope]);
|
|
2806
|
+
},
|
|
2807
|
+
appendExternalReconciliation: async (record) => {
|
|
2808
|
+
pendingReconciliations.push(record);
|
|
2809
|
+
},
|
|
2491
2810
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
2492
|
-
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
2811
|
+
pendingUpdates.push({ id, tenantId, spaceId, patch: copyInvocationPatch(patch) });
|
|
2493
2812
|
},
|
|
2494
2813
|
updateLeasedActionInvocation: async (input) => {
|
|
2495
2814
|
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
2496
2815
|
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
2497
|
-
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
2816
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: copyInvocationPatch(input.patch) });
|
|
2498
2817
|
return true;
|
|
2499
2818
|
}
|
|
2500
2819
|
});
|
|
2501
2820
|
for (const update of pendingUpdates) {
|
|
2502
2821
|
if (!await this.getActionInvocation(update.id, update.tenantId, update.spaceId)) throw new Error(`ActionInvocation not found: ${update.id}`);
|
|
2503
2822
|
}
|
|
2823
|
+
const rememberRecord = (record) => {
|
|
2824
|
+
const before = { ...record };
|
|
2825
|
+
undoLedgerWrites.push(() => {
|
|
2826
|
+
for (const key of Object.keys(record)) Reflect.deleteProperty(record, key);
|
|
2827
|
+
Object.assign(record, before);
|
|
2828
|
+
});
|
|
2829
|
+
};
|
|
2830
|
+
const rememberAppend = (rows, id) => {
|
|
2831
|
+
if (!rows.some((row) => row.id === id)) undoLedgerWrites.push(() => {
|
|
2832
|
+
const index = rows.findIndex((row) => row.id === id);
|
|
2833
|
+
if (index !== -1) rows.splice(index, 1);
|
|
2834
|
+
});
|
|
2835
|
+
};
|
|
2504
2836
|
for (const pending of pendingEvents) {
|
|
2837
|
+
rememberAppend(this.events, pending.event.id);
|
|
2838
|
+
rememberAppend(this.outbox, pending.event.id);
|
|
2505
2839
|
if (pending.metadata) await this.appendEventWithOutbox(pending.event, pending.metadata);
|
|
2506
2840
|
else await this.appendEvent(pending.event);
|
|
2507
2841
|
}
|
|
2508
|
-
for (const
|
|
2842
|
+
for (const record of pendingReconciliations) {
|
|
2843
|
+
rememberAppend(this.externalReconciliations, record.id);
|
|
2844
|
+
await this.appendExternalReconciliation(record);
|
|
2845
|
+
}
|
|
2846
|
+
for (const [id, patch, scope] of pendingAdapterUpdates) {
|
|
2847
|
+
const record = await this.getAdapterInvocation(id, scope);
|
|
2848
|
+
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2849
|
+
rememberRecord(record);
|
|
2850
|
+
await this.updateAdapterInvocation(id, patch, scope);
|
|
2851
|
+
}
|
|
2852
|
+
for (const update of pendingUpdates) {
|
|
2853
|
+
rememberRecord(await this.getActionInvocation(update.id, update.tenantId, update.spaceId));
|
|
2854
|
+
await this.updateActionInvocation(update.id, update.tenantId, update.spaceId, update.patch);
|
|
2855
|
+
}
|
|
2509
2856
|
return result;
|
|
2510
2857
|
} catch (error) {
|
|
2858
|
+
for (const undo of undoLedgerWrites.reverse()) undo();
|
|
2511
2859
|
if (snapshotCreated) transactionProvider.restore(this.db, domainSnapshot);
|
|
2512
2860
|
throw error;
|
|
2513
2861
|
} finally {
|
|
@@ -2535,6 +2883,10 @@ var MemoryPlatformHostStore = class {
|
|
|
2535
2883
|
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
2536
2884
|
if (!record) throw new Error(`ActionInvocation not found: ${id}`);
|
|
2537
2885
|
Object.assign(record, patch, { updatedAt: /* @__PURE__ */ new Date() });
|
|
2886
|
+
if (Object.hasOwn(patch, "pendingStateTransition")) {
|
|
2887
|
+
if (patch.pendingStateTransition) record.pendingStateTransition = { ...patch.pendingStateTransition };
|
|
2888
|
+
else delete record.pendingStateTransition;
|
|
2889
|
+
}
|
|
2538
2890
|
if (Object.hasOwn(patch, "pendingCompletion") && patch.pendingCompletion === void 0) delete record.pendingCompletion;
|
|
2539
2891
|
if (Object.hasOwn(patch, "externalCompletion") && patch.externalCompletion === void 0) delete record.externalCompletion;
|
|
2540
2892
|
if (Object.hasOwn(patch, "error") && patch.error === void 0) delete record.error;
|
|
@@ -2625,11 +2977,11 @@ var MemoryPlatformHostStore = class {
|
|
|
2625
2977
|
if (this.adapterInvocations.some((candidate) => candidate.id === record.id)) return;
|
|
2626
2978
|
this.adapterInvocations.push(record);
|
|
2627
2979
|
}
|
|
2628
|
-
async getAdapterInvocation(id) {
|
|
2629
|
-
return this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2980
|
+
async getAdapterInvocation(id, scope) {
|
|
2981
|
+
return this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2630
2982
|
}
|
|
2631
|
-
async updateAdapterInvocation(id, patch) {
|
|
2632
|
-
const record = this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2983
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
2984
|
+
const record = this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2633
2985
|
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2634
2986
|
Object.assign(record, patch);
|
|
2635
2987
|
}
|
|
@@ -2734,6 +3086,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2734
3086
|
async nextEventSequence(tenantId, spaceId) {
|
|
2735
3087
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).length + 1;
|
|
2736
3088
|
}
|
|
3089
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3090
|
+
let source;
|
|
3091
|
+
for (const event of this.events) {
|
|
3092
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
3093
|
+
const state = event.payload?.toState;
|
|
3094
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
3095
|
+
}
|
|
3096
|
+
return source;
|
|
3097
|
+
}
|
|
2737
3098
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
2738
3099
|
let state;
|
|
2739
3100
|
for (const event of this.events) {
|
|
@@ -2813,6 +3174,9 @@ function cloneOutboxRequeue(requeue) {
|
|
|
2813
3174
|
priorAvailableAt: new Date(requeue.priorAvailableAt)
|
|
2814
3175
|
};
|
|
2815
3176
|
}
|
|
3177
|
+
function copyInvocationPatch(patch) {
|
|
3178
|
+
return { ...patch, ...patch.pendingStateTransition ? { pendingStateTransition: { ...patch.pendingStateTransition } } : {} };
|
|
3179
|
+
}
|
|
2816
3180
|
async function applyPostgresMigrations(client, migrations) {
|
|
2817
3181
|
const ordered = validateMigrations(migrations);
|
|
2818
3182
|
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
@@ -2906,9 +3270,12 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2906
3270
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
2907
3271
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
2908
3272
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
3273
|
+
getEntityStateSource: (tenantId, spaceId, entityType, entityId) => scoped.getEntityStateSource(tenantId, spaceId, entityType, entityId),
|
|
2909
3274
|
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2910
3275
|
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2911
|
-
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
3276
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input),
|
|
3277
|
+
updateAdapterInvocation: (id, patch, scope) => scoped.updateAdapterInvocation(id, patch, scope),
|
|
3278
|
+
appendExternalReconciliation: (record) => scoped.appendExternalReconciliation(record)
|
|
2912
3279
|
});
|
|
2913
3280
|
});
|
|
2914
3281
|
}
|
|
@@ -3091,6 +3458,13 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3091
3458
|
ALTER TABLE fabric_platform.action_invocations
|
|
3092
3459
|
ADD COLUMN admitted_at timestamptz;
|
|
3093
3460
|
`
|
|
3461
|
+
}, {
|
|
3462
|
+
version: 7,
|
|
3463
|
+
name: "pending_state_transition",
|
|
3464
|
+
sql: `
|
|
3465
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
3466
|
+
ADD COLUMN IF NOT EXISTS pending_state_transition jsonb;
|
|
3467
|
+
`
|
|
3094
3468
|
}]);
|
|
3095
3469
|
}
|
|
3096
3470
|
async transaction(run) {
|
|
@@ -3171,6 +3545,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3171
3545
|
pending_completion=CASE WHEN $12::boolean THEN $13::jsonb ELSE pending_completion END,
|
|
3172
3546
|
completion_due_at=CASE WHEN $12::boolean THEN ($13::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
3173
3547
|
external_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE external_completion END,
|
|
3548
|
+
pending_state_transition=CASE WHEN $16::boolean THEN $17::jsonb ELSE pending_state_transition END,
|
|
3174
3549
|
lease_owner=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3175
3550
|
THEN NULL ELSE lease_owner END,
|
|
3176
3551
|
lease_expires_at=CASE WHEN ($13::jsonb IS NOT NULL AND $4='running') OR $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -3192,7 +3567,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3192
3567
|
Object.hasOwn(patch, "pendingCompletion"),
|
|
3193
3568
|
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3194
3569
|
Object.hasOwn(patch, "externalCompletion"),
|
|
3195
|
-
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
3570
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null,
|
|
3571
|
+
Object.hasOwn(patch, "pendingStateTransition"),
|
|
3572
|
+
patch.pendingStateTransition ? JSON.stringify(patch.pendingStateTransition) : null
|
|
3196
3573
|
]
|
|
3197
3574
|
);
|
|
3198
3575
|
}
|
|
@@ -3359,19 +3736,19 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3359
3736
|
]
|
|
3360
3737
|
);
|
|
3361
3738
|
}
|
|
3362
|
-
async getAdapterInvocation(id) {
|
|
3739
|
+
async getAdapterInvocation(id, scope) {
|
|
3363
3740
|
const result = await this.sql.query(
|
|
3364
|
-
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1`,
|
|
3365
|
-
[id]
|
|
3741
|
+
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1${scope ? " AND tenant_id=$2 AND space_id=$3 AND action_invocation_id=$4" : ""}`,
|
|
3742
|
+
[id, ...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []]
|
|
3366
3743
|
);
|
|
3367
3744
|
return result.rows[0] ? toAdapterRecord(result.rows[0]) : void 0;
|
|
3368
3745
|
}
|
|
3369
|
-
async updateAdapterInvocation(id, patch) {
|
|
3746
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
3370
3747
|
await this.sql.query(
|
|
3371
3748
|
`UPDATE fabric_platform.adapter_invocations SET
|
|
3372
3749
|
status=COALESCE($2,status), output=COALESCE($3::jsonb,output),
|
|
3373
3750
|
error=CASE WHEN $4::boolean THEN $5 ELSE error END,
|
|
3374
|
-
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1`,
|
|
3751
|
+
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1${scope ? " AND tenant_id=$8 AND space_id=$9 AND action_invocation_id=$10" : ""}`,
|
|
3375
3752
|
[
|
|
3376
3753
|
id,
|
|
3377
3754
|
patch.status ?? null,
|
|
@@ -3379,7 +3756,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3379
3756
|
Object.hasOwn(patch, "error"),
|
|
3380
3757
|
patch.error ?? null,
|
|
3381
3758
|
patch.attempt ?? null,
|
|
3382
|
-
patch.updatedAt ?? null
|
|
3759
|
+
patch.updatedAt ?? null,
|
|
3760
|
+
...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []
|
|
3383
3761
|
]
|
|
3384
3762
|
);
|
|
3385
3763
|
}
|
|
@@ -3610,6 +3988,16 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3610
3988
|
);
|
|
3611
3989
|
return Number(result.rows[0].next_sequence);
|
|
3612
3990
|
}
|
|
3991
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3992
|
+
const result = await this.sql.query(
|
|
3993
|
+
`SELECT payload->>'toState' AS state, action_invocation_id FROM fabric_platform.asset_events
|
|
3994
|
+
WHERE tenant_id=$1 AND space_id=$2 AND subject_type=$3 AND subject_id=$4
|
|
3995
|
+
AND payload ? 'toState' ORDER BY sequence DESC LIMIT 1`,
|
|
3996
|
+
[tenantId, spaceId, entityType, entityId]
|
|
3997
|
+
);
|
|
3998
|
+
const row = result.rows[0];
|
|
3999
|
+
return typeof row?.state === "string" && typeof row.action_invocation_id === "string" ? { state: row.state, actionInvocationId: row.action_invocation_id } : void 0;
|
|
4000
|
+
}
|
|
3613
4001
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
3614
4002
|
const result = await this.sql.query(
|
|
3615
4003
|
`SELECT payload->>'toState' AS state FROM fabric_platform.asset_events
|
|
@@ -3733,6 +4121,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3733
4121
|
pending_completion=CASE WHEN $14::boolean THEN $15::jsonb ELSE pending_completion END,
|
|
3734
4122
|
completion_due_at=CASE WHEN $14::boolean THEN ($15::jsonb->>'dueAt')::timestamptz ELSE completion_due_at END,
|
|
3735
4123
|
external_completion=CASE WHEN $16::boolean THEN $17::jsonb ELSE external_completion END,
|
|
4124
|
+
pending_state_transition=CASE WHEN $18::boolean THEN $19::jsonb ELSE pending_state_transition END,
|
|
3736
4125
|
lease_owner=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
3737
4126
|
THEN NULL ELSE lease_owner END,
|
|
3738
4127
|
lease_expires_at=CASE WHEN ($15::jsonb IS NOT NULL AND $6='running') OR $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -3758,7 +4147,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3758
4147
|
Object.hasOwn(patch, "pendingCompletion"),
|
|
3759
4148
|
patch.pendingCompletion ? JSON.stringify(patch.pendingCompletion) : null,
|
|
3760
4149
|
Object.hasOwn(patch, "externalCompletion"),
|
|
3761
|
-
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null
|
|
4150
|
+
patch.externalCompletion ? JSON.stringify(patch.externalCompletion) : null,
|
|
4151
|
+
Object.hasOwn(patch, "pendingStateTransition"),
|
|
4152
|
+
patch.pendingStateTransition ? JSON.stringify(patch.pendingStateTransition) : null
|
|
3762
4153
|
]
|
|
3763
4154
|
);
|
|
3764
4155
|
return result.rows.length === 1;
|
|
@@ -3857,6 +4248,7 @@ function toActionRecord(row) {
|
|
|
3857
4248
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
3858
4249
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
3859
4250
|
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
4251
|
+
...row.pending_state_transition ? { pendingStateTransition: row.pending_state_transition } : {},
|
|
3860
4252
|
...row.pending_completion ? { pendingCompletion: toPendingCompletion(row.pending_completion) } : {},
|
|
3861
4253
|
...row.external_completion ? { externalCompletion: toExternalCompletion(row.external_completion) } : {},
|
|
3862
4254
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
@@ -4002,8 +4394,6 @@ function toOutboxRequeueRecord(row, prefix) {
|
|
|
4002
4394
|
priorLeaseToken: Number(field("prior_lease_token"))
|
|
4003
4395
|
};
|
|
4004
4396
|
}
|
|
4005
|
-
|
|
4006
|
-
// src/worker.ts
|
|
4007
4397
|
var DEFAULT_BATCH_SIZE = 10;
|
|
4008
4398
|
var DEFAULT_LEASE_DURATION_MS = 5 * 6e4;
|
|
4009
4399
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
@@ -4052,6 +4442,7 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4052
4442
|
let completed = 0;
|
|
4053
4443
|
let failed = 0;
|
|
4054
4444
|
let waitingForApproval = 0;
|
|
4445
|
+
let deferred = 0;
|
|
4055
4446
|
for (const invocation of claimed) {
|
|
4056
4447
|
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
4057
4448
|
try {
|
|
@@ -4065,7 +4456,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4065
4456
|
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
4066
4457
|
else failed += 1;
|
|
4067
4458
|
} catch (error) {
|
|
4068
|
-
|
|
4459
|
+
if (isRetryableExecutionError(error)) deferred += 1;
|
|
4460
|
+
else failed += 1;
|
|
4069
4461
|
options.onError?.(error, invocation);
|
|
4070
4462
|
} finally {
|
|
4071
4463
|
await stopHeartbeat();
|
|
@@ -4075,7 +4467,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4075
4467
|
claimed: claimed.length,
|
|
4076
4468
|
completed,
|
|
4077
4469
|
failed,
|
|
4078
|
-
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
4470
|
+
...waitingForApproval > 0 ? { waitingForApproval } : {},
|
|
4471
|
+
...deferred > 0 ? { deferred } : {}
|
|
4079
4472
|
};
|
|
4080
4473
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
4081
4474
|
return result;
|
|
@@ -4302,6 +4695,32 @@ function submitInput(subject, scope, idempotencyKey) {
|
|
|
4302
4695
|
idempotencyKey
|
|
4303
4696
|
};
|
|
4304
4697
|
}
|
|
4698
|
+
async function adapterProbe(subject) {
|
|
4699
|
+
await subject.reset?.();
|
|
4700
|
+
if (subject.ours.tenantId === subject.theirs.tenantId) fail("the two scopes must be in different tenants");
|
|
4701
|
+
const suffix = globalThis.crypto.randomUUID();
|
|
4702
|
+
const scope = {
|
|
4703
|
+
tenantId: subject.ours.tenantId,
|
|
4704
|
+
spaceId: `iso-probe-${suffix}`,
|
|
4705
|
+
actionInvocationId: `iso_act_${suffix}`
|
|
4706
|
+
};
|
|
4707
|
+
const id = `iso_adp_${suffix}`;
|
|
4708
|
+
const now = /* @__PURE__ */ new Date();
|
|
4709
|
+
await subject.store.createAdapterInvocation({
|
|
4710
|
+
id,
|
|
4711
|
+
...scope,
|
|
4712
|
+
adapterType: "iso",
|
|
4713
|
+
operation: "probe",
|
|
4714
|
+
vendor: "iso",
|
|
4715
|
+
status: "running",
|
|
4716
|
+
input: {},
|
|
4717
|
+
output: { probe: true },
|
|
4718
|
+
attempt: 1,
|
|
4719
|
+
createdAt: now,
|
|
4720
|
+
updatedAt: now
|
|
4721
|
+
});
|
|
4722
|
+
return { id, scope };
|
|
4723
|
+
}
|
|
4305
4724
|
function tenantIsolationChecks() {
|
|
4306
4725
|
return [
|
|
4307
4726
|
{
|
|
@@ -4446,6 +4865,54 @@ function tenantIsolationChecks() {
|
|
|
4446
4865
|
if (leaked.length > 0) fail(`a viewer in the other tenant saw ${leaked.length} of our rows`);
|
|
4447
4866
|
return [theirs.allowed ? "other tenant's viewer sees none of our rows" : "other tenant's viewer is denied"];
|
|
4448
4867
|
}
|
|
4868
|
+
},
|
|
4869
|
+
{
|
|
4870
|
+
id: "fabric.tenancy.adapter-invocation-read-scoped.v1",
|
|
4871
|
+
async run(subject) {
|
|
4872
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4873
|
+
try {
|
|
4874
|
+
const theirs = { ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId };
|
|
4875
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, tenantId: subject.theirs.tenantId })) fail("an adapter invocation was readable under another tenant's scope");
|
|
4876
|
+
if (await subject.store.getAdapterInvocation(id, theirs)) fail("an adapter invocation was readable under another tenant's scope");
|
|
4877
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, spaceId: `${scope.spaceId}-other` })) fail("an adapter invocation was readable under another space's scope");
|
|
4878
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, actionInvocationId: `${scope.actionInvocationId}-other` })) fail("an adapter invocation was readable under another action invocation's scope");
|
|
4879
|
+
const ours = await subject.store.getAdapterInvocation(id, scope);
|
|
4880
|
+
if (!ours || ours.id !== id || ours.tenantId !== scope.tenantId || ours.spaceId !== scope.spaceId || ours.actionInvocationId !== scope.actionInvocationId) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4881
|
+
} finally {
|
|
4882
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4883
|
+
}
|
|
4884
|
+
return ["adapter invocation readable under our scope and invisible outside it"];
|
|
4885
|
+
}
|
|
4886
|
+
},
|
|
4887
|
+
{
|
|
4888
|
+
id: "fabric.tenancy.adapter-invocation-write-scoped.v1",
|
|
4889
|
+
async run(subject) {
|
|
4890
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4891
|
+
const shape = (record) => JSON.stringify({ status: record.status, error: record.error, output: record.output, attempt: record.attempt });
|
|
4892
|
+
try {
|
|
4893
|
+
const before = await subject.store.getAdapterInvocation(id, scope);
|
|
4894
|
+
if (!before) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4895
|
+
const untouched = shape(before);
|
|
4896
|
+
for (const other of [
|
|
4897
|
+
{ ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId },
|
|
4898
|
+
{ ...scope, tenantId: subject.theirs.tenantId },
|
|
4899
|
+
{ ...scope, spaceId: `${scope.spaceId}-other` },
|
|
4900
|
+
{ ...scope, actionInvocationId: `${scope.actionInvocationId}-other` }
|
|
4901
|
+
]) {
|
|
4902
|
+
try {
|
|
4903
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "iso probe", updatedAt: /* @__PURE__ */ new Date() }, other);
|
|
4904
|
+
} catch {
|
|
4905
|
+
}
|
|
4906
|
+
const after = await subject.store.getAdapterInvocation(id, scope);
|
|
4907
|
+
if (!after || shape(after) !== untouched) fail("an adapter invocation changed under another scope");
|
|
4908
|
+
}
|
|
4909
|
+
await subject.store.updateAdapterInvocation(id, { status: "succeeded", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4910
|
+
if ((await subject.store.getAdapterInvocation(id, scope))?.status !== "succeeded") fail("our own adapter invocation was not updated under our scope, so the check proves nothing");
|
|
4911
|
+
} finally {
|
|
4912
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4913
|
+
}
|
|
4914
|
+
return ["adapter invocation untouched outside our scope and writable under ours"];
|
|
4915
|
+
}
|
|
4449
4916
|
}
|
|
4450
4917
|
];
|
|
4451
4918
|
}
|