@fabricorg/platform-host 7.4.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +214 -15
- package/dist/index.cjs +386 -67
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -15
- package/dist/index.d.ts +123 -15
- package/dist/index.js +387 -68
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -309,6 +309,24 @@ function withoutPrivateHostFields(data, eventResultFields) {
|
|
|
309
309
|
function errorMessage(error) {
|
|
310
310
|
return error instanceof Error ? error.message : String(error);
|
|
311
311
|
}
|
|
312
|
+
var HostError = class extends Error {
|
|
313
|
+
};
|
|
314
|
+
function isRecord(value) {
|
|
315
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
316
|
+
}
|
|
317
|
+
function tryRedactErrorText(hook, context, message) {
|
|
318
|
+
if (!hook || message === "") return { success: true, text: message };
|
|
319
|
+
try {
|
|
320
|
+
const text = hook(context, message);
|
|
321
|
+
if (typeof text === "string" && text.length > 0) return { success: true, text };
|
|
322
|
+
} catch {
|
|
323
|
+
}
|
|
324
|
+
return { success: false };
|
|
325
|
+
}
|
|
326
|
+
function redactErrorText(hook, context, message) {
|
|
327
|
+
const result = tryRedactErrorText(hook, context, message);
|
|
328
|
+
return result.success ? result.text : "Error text withheld: the error-text redactor failed.";
|
|
329
|
+
}
|
|
312
330
|
function lifecycleId(prefix, invocationId, key) {
|
|
313
331
|
const safeKey = key.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 96);
|
|
314
332
|
return `${prefix}_${invocationId}_${safeKey}`;
|
|
@@ -371,7 +389,11 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
371
389
|
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
372
390
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
373
391
|
}
|
|
374
|
-
const
|
|
392
|
+
const provider = recorded.provider ?? completion.provider;
|
|
393
|
+
const rawDigest = completionDigest(completion, provider);
|
|
394
|
+
if (recorded.digest === rawDigest) return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
395
|
+
completion = redactCompletion(invocation, completion, provider ?? "external");
|
|
396
|
+
const incomingDigest = completionDigest(completion, provider);
|
|
375
397
|
if (recorded.digest === incomingDigest) {
|
|
376
398
|
return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
377
399
|
}
|
|
@@ -401,8 +423,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
401
423
|
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
402
424
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
403
425
|
}
|
|
426
|
+
completion = redactCompletion(invocation, completion, completion.provider ?? pending.provider);
|
|
404
427
|
const recordedAt = now();
|
|
405
428
|
const externalCompletion = {
|
|
429
|
+
// completion has passed the hooks; retain the callback shape for compatibility.
|
|
406
430
|
...completion,
|
|
407
431
|
provider: completion.provider ?? pending.provider,
|
|
408
432
|
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
@@ -423,7 +447,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
423
447
|
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
424
448
|
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
425
449
|
updatedAt: recordedAt
|
|
426
|
-
});
|
|
450
|
+
}, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
427
451
|
await appendEvent(invocation, {
|
|
428
452
|
eventType: "ExternalOperationCompleted",
|
|
429
453
|
...subject,
|
|
@@ -435,7 +459,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
435
459
|
return withConsistency(actionResult2({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
436
460
|
}
|
|
437
461
|
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
438
|
-
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
462
|
+
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt }, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
439
463
|
await appendEvent(invocation, {
|
|
440
464
|
eventType: "ExternalOperationFailed",
|
|
441
465
|
...subject,
|
|
@@ -446,6 +470,30 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
446
470
|
if (settles) emitInvocationStatusTelemetry(invocation, "failed", deps.telemetry, now(), invocation.status);
|
|
447
471
|
return withConsistency(actionResult2({ ...invocation, ...patch }), invocation.actionId);
|
|
448
472
|
}
|
|
473
|
+
function redactCompletion(invocation, completion, provider) {
|
|
474
|
+
let result = completion.result;
|
|
475
|
+
if (result && deps.redactExternalCompletionResult) {
|
|
476
|
+
try {
|
|
477
|
+
result = deps.redactExternalCompletionResult(invocation.actionId, provider, result);
|
|
478
|
+
if (!isRecord(result)) throw new HostError("Completion result redactor must return a non-null, non-array object.");
|
|
479
|
+
} catch {
|
|
480
|
+
throw new ExternalCompletionError("redaction_failed", "External completion result withheld: the result redactor failed; nothing was recorded.");
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
const error = completion.error == null ? void 0 : tryRedactErrorText(
|
|
484
|
+
deps.redactErrorText,
|
|
485
|
+
{ actionId: invocation.actionId, source: "external_completion", provider },
|
|
486
|
+
completion.error
|
|
487
|
+
);
|
|
488
|
+
if (error && !error.success) {
|
|
489
|
+
throw new ExternalCompletionError("redaction_failed", "External completion error withheld: the error-text redactor failed; nothing was recorded.");
|
|
490
|
+
}
|
|
491
|
+
return {
|
|
492
|
+
...completion,
|
|
493
|
+
...completion.result !== void 0 ? { result } : {},
|
|
494
|
+
...error?.success ? { error: error.text } : {}
|
|
495
|
+
};
|
|
496
|
+
}
|
|
449
497
|
function completionDigest(completion, provider) {
|
|
450
498
|
return crypto.createHash("sha256").update(platform.canonicalJson({
|
|
451
499
|
provider: provider ?? null,
|
|
@@ -458,7 +506,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
458
506
|
}
|
|
459
507
|
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
460
508
|
const governanceStore = asGovernanceStore(deps.store);
|
|
461
|
-
if (!governanceStore) throw new
|
|
509
|
+
if (!governanceStore) throw new HostError("External completion reconciliation requires a governance-capable store.");
|
|
462
510
|
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
463
511
|
const seed = crypto.createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
464
512
|
await governanceStore.appendExternalReconciliation({
|
|
@@ -478,10 +526,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
478
526
|
async function reconcileOverdueCompletions(input = {}) {
|
|
479
527
|
const recoverable = deps.store;
|
|
480
528
|
if (!recoverable.listActionInvocations) {
|
|
481
|
-
throw new
|
|
529
|
+
throw new HostError("Overdue completion reconciliation requires a store that can list invocations.");
|
|
482
530
|
}
|
|
483
531
|
if (!asGovernanceStore(deps.store)) {
|
|
484
|
-
throw new
|
|
532
|
+
throw new HostError("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
485
533
|
}
|
|
486
534
|
const current = input.now ?? now();
|
|
487
535
|
const candidates = await recoverable.listActionInvocations({
|
|
@@ -856,6 +904,10 @@ var ADMISSION_FAILED = Object.freeze({ kind: "failed" });
|
|
|
856
904
|
var ADMISSION_TIMED_OUT = Object.freeze({ kind: "timed_out" });
|
|
857
905
|
var ADMISSION_REFUSAL_CODE = /^[a-z][A-Za-z0-9_.:-]{0,127}$/;
|
|
858
906
|
function createGovernedActionHost(options) {
|
|
907
|
+
const retryableExecutionMaxAttempts = options.retryableExecutionMaxAttempts ?? 5;
|
|
908
|
+
if (!Number.isInteger(retryableExecutionMaxAttempts) || retryableExecutionMaxAttempts <= 0) {
|
|
909
|
+
throw new Error("retryableExecutionMaxAttempts must be a positive integer.");
|
|
910
|
+
}
|
|
859
911
|
if (options.composition) {
|
|
860
912
|
if (!options.registry?.orderedModules) {
|
|
861
913
|
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
@@ -892,6 +944,9 @@ function createGovernedActionHost(options) {
|
|
|
892
944
|
if (typeof admissionTimeoutMs !== "number" || !Number.isInteger(admissionTimeoutMs) || admissionTimeoutMs < 1 || admissionTimeoutMs > MAX_TIMER_MS) {
|
|
893
945
|
throw new Error(`admissionTimeoutMs must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}.`);
|
|
894
946
|
}
|
|
947
|
+
function caughtError(context, error) {
|
|
948
|
+
return error instanceof HostError ? errorMessage(error) : redactErrorText(options.redactErrorText, context, errorMessage(error));
|
|
949
|
+
}
|
|
895
950
|
async function submitAction(input) {
|
|
896
951
|
const read = snapshotSubmission(input);
|
|
897
952
|
if (read.executionReason && !["initial", "offline_replay"].includes(read.executionReason)) {
|
|
@@ -1189,6 +1244,7 @@ function createGovernedActionHost(options) {
|
|
|
1189
1244
|
spaceId,
|
|
1190
1245
|
attributes: { actionVersion: invocation.actionVersion }
|
|
1191
1246
|
});
|
|
1247
|
+
let retryableGovernance = true;
|
|
1192
1248
|
try {
|
|
1193
1249
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
1194
1250
|
if (!parsed.success) {
|
|
@@ -1341,6 +1397,7 @@ function createGovernedActionHost(options) {
|
|
|
1341
1397
|
mode: "execute",
|
|
1342
1398
|
now: now()
|
|
1343
1399
|
});
|
|
1400
|
+
retryableGovernance = false;
|
|
1344
1401
|
for (const outcome of outcomes) {
|
|
1345
1402
|
await options.store.appendPolicyEvaluation({
|
|
1346
1403
|
id: lifecycleId("pol", actionInvocationId, outcome.policyId),
|
|
@@ -1384,12 +1441,14 @@ function createGovernedActionHost(options) {
|
|
|
1384
1441
|
}
|
|
1385
1442
|
let data;
|
|
1386
1443
|
let domainEvents = [];
|
|
1444
|
+
let pendingTransitionEvent;
|
|
1387
1445
|
try {
|
|
1388
1446
|
const runHandler = async (db, transaction) => {
|
|
1389
1447
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
1390
|
-
throw new
|
|
1448
|
+
throw new HostError("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
1391
1449
|
}
|
|
1392
1450
|
const binding = action.stateMachine;
|
|
1451
|
+
let validatedTransition;
|
|
1393
1452
|
if (binding) {
|
|
1394
1453
|
const machine = stateMachineResolver(binding.entityType);
|
|
1395
1454
|
const entityId = binding.getEntityId(parsed.data);
|
|
@@ -1419,9 +1478,30 @@ function createGovernedActionHost(options) {
|
|
|
1419
1478
|
db
|
|
1420
1479
|
}
|
|
1421
1480
|
);
|
|
1422
|
-
|
|
1481
|
+
validatedTransition = { entityId, currentState, targetState, valid: transition.valid };
|
|
1482
|
+
let replayingAppliedTransition = false;
|
|
1483
|
+
if (action.idempotent && currentState === targetState && !transition.valid && entityId) {
|
|
1484
|
+
let source;
|
|
1485
|
+
if (transaction?.getEntityStateSource) {
|
|
1486
|
+
source = await transaction.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1487
|
+
} else if (options.store.getEntityStateSource) {
|
|
1488
|
+
source = await options.store.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1489
|
+
} else {
|
|
1490
|
+
const eventStore = options.store;
|
|
1491
|
+
const listEvents = transaction?.listEvents?.bind(transaction) ?? eventStore.listEvents?.bind(eventStore);
|
|
1492
|
+
const events2 = listEvents ? await listEvents(tenantId, spaceId) : [];
|
|
1493
|
+
let latest;
|
|
1494
|
+
for (const event of events2) {
|
|
1495
|
+
if (event.tenantId === tenantId && event.spaceId === spaceId && event.subjectType === binding.entityType && event.subjectId === entityId && typeof event.payload?.toState === "string" && (!latest || event.sequence > latest.sequence)) latest = event;
|
|
1496
|
+
}
|
|
1497
|
+
if (latest && typeof latest.actionInvocationId === "string") {
|
|
1498
|
+
source = { state: latest.payload.toState, actionInvocationId: latest.actionInvocationId };
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
replayingAppliedTransition = source?.actionInvocationId === actionInvocationId && source.state === currentState;
|
|
1502
|
+
}
|
|
1423
1503
|
if (!transition.valid && !replayingAppliedTransition) {
|
|
1424
|
-
throw new Error(transition.error
|
|
1504
|
+
throw transition.error != null ? new Error(transition.error) : new HostError("Invalid state transition");
|
|
1425
1505
|
}
|
|
1426
1506
|
}
|
|
1427
1507
|
}
|
|
@@ -1442,14 +1522,14 @@ function createGovernedActionHost(options) {
|
|
|
1442
1522
|
parsed.data
|
|
1443
1523
|
) : { success: true, data: {} };
|
|
1444
1524
|
if (!handlerResult.success) {
|
|
1445
|
-
throw new Error(handlerResult.error
|
|
1525
|
+
throw handlerResult.error != null ? new Error(handlerResult.error) : new HostError("Action handler failed");
|
|
1446
1526
|
}
|
|
1447
1527
|
const handlerData = handlerResult.data ?? {};
|
|
1448
1528
|
if (action.resultSchema) {
|
|
1449
1529
|
const publicResult = withoutPrivateHostFields(handlerData, eventResultFields);
|
|
1450
1530
|
const parsedResult = action.resultSchema.safeParse(publicResult);
|
|
1451
1531
|
if (!parsedResult.success) {
|
|
1452
|
-
throw new
|
|
1532
|
+
throw new HostError(`Action result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1453
1533
|
}
|
|
1454
1534
|
}
|
|
1455
1535
|
const events = extractEvents(handlerData);
|
|
@@ -1457,7 +1537,7 @@ function createGovernedActionHost(options) {
|
|
|
1457
1537
|
(event) => !action.emitsEvents.includes(event.eventType)
|
|
1458
1538
|
);
|
|
1459
1539
|
if (undeclaredEvent) {
|
|
1460
|
-
throw new
|
|
1540
|
+
throw new HostError(
|
|
1461
1541
|
`${action.actionId} emitted undeclared event type ${undeclaredEvent.eventType}`
|
|
1462
1542
|
);
|
|
1463
1543
|
}
|
|
@@ -1472,7 +1552,29 @@ function createGovernedActionHost(options) {
|
|
|
1472
1552
|
);
|
|
1473
1553
|
}
|
|
1474
1554
|
}
|
|
1475
|
-
|
|
1555
|
+
let transitionEvent;
|
|
1556
|
+
if (binding && validatedTransition?.valid && validatedTransition.entityId) {
|
|
1557
|
+
const { entityId, currentState, targetState } = validatedTransition;
|
|
1558
|
+
const handlerEmittedTransition = events.some((event) => event.eventType === "StateTransitioned" && event.subjectType === binding.entityType && event.subjectId === entityId);
|
|
1559
|
+
if (!handlerEmittedTransition) {
|
|
1560
|
+
transitionEvent = {
|
|
1561
|
+
eventType: "StateTransitioned",
|
|
1562
|
+
subjectType: binding.entityType,
|
|
1563
|
+
subjectId: entityId,
|
|
1564
|
+
payload: {
|
|
1565
|
+
entityType: binding.entityType,
|
|
1566
|
+
entityId,
|
|
1567
|
+
fromState: currentState,
|
|
1568
|
+
toState: targetState,
|
|
1569
|
+
causedByAction: action.actionId
|
|
1570
|
+
}
|
|
1571
|
+
};
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
if (transitionEvent && action.eventPhase !== "after_adapters") {
|
|
1575
|
+
await appendEvent(invocation, transitionEvent, "state:transition", 1, transaction);
|
|
1576
|
+
}
|
|
1577
|
+
return { data: handlerData, domainEvents: events, pendingTransitionEvent: transitionEvent };
|
|
1476
1578
|
};
|
|
1477
1579
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1478
1580
|
const executed = atomicStore ? await atomicStore.transactionWithEvents(
|
|
@@ -1480,9 +1582,11 @@ function createGovernedActionHost(options) {
|
|
|
1480
1582
|
) : await options.store.transaction((db) => runHandler(db));
|
|
1481
1583
|
data = executed.data;
|
|
1482
1584
|
domainEvents = executed.domainEvents;
|
|
1585
|
+
pendingTransitionEvent = executed.pendingTransitionEvent;
|
|
1483
1586
|
} catch (error) {
|
|
1484
|
-
return fail2(invocation, "failed",
|
|
1587
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "handler" }, error));
|
|
1485
1588
|
}
|
|
1589
|
+
const adapterInvocationScope = { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id };
|
|
1486
1590
|
let pendingHandoff = invocation.pendingCompletion;
|
|
1487
1591
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
1488
1592
|
const input = step.getInput(parsed.data, data);
|
|
@@ -1490,12 +1594,13 @@ function createGovernedActionHost(options) {
|
|
|
1490
1594
|
const adapter = adapters.require(step.adapterType, step.operation);
|
|
1491
1595
|
const adapterInvocationId = lifecycleId("adp", actionInvocationId, String(stepIndex));
|
|
1492
1596
|
const previousAdapterInvocation = await options.store.getAdapterInvocation(
|
|
1493
|
-
adapterInvocationId
|
|
1597
|
+
adapterInvocationId,
|
|
1598
|
+
adapterInvocationScope
|
|
1494
1599
|
);
|
|
1495
1600
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
1496
1601
|
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
1497
1602
|
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
1498
|
-
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
1603
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1499
1604
|
await appendEvent(invocation, {
|
|
1500
1605
|
eventType: "ExternalOperationAccepted",
|
|
1501
1606
|
subjectType: subject.subjectType,
|
|
@@ -1529,9 +1634,7 @@ function createGovernedActionHost(options) {
|
|
|
1529
1634
|
const startedAt = now();
|
|
1530
1635
|
const record = {
|
|
1531
1636
|
id: adapterInvocationId,
|
|
1532
|
-
|
|
1533
|
-
tenantId,
|
|
1534
|
-
spaceId,
|
|
1637
|
+
...adapterInvocationScope,
|
|
1535
1638
|
adapterType: step.adapterType,
|
|
1536
1639
|
operation: step.operation,
|
|
1537
1640
|
vendor: adapter.vendor,
|
|
@@ -1548,6 +1651,50 @@ function createGovernedActionHost(options) {
|
|
|
1548
1651
|
subjectId: adapterEventSubject.subjectId,
|
|
1549
1652
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1550
1653
|
}, `adapter:${stepIndex}:started`);
|
|
1654
|
+
async function appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference) {
|
|
1655
|
+
if (accepted) {
|
|
1656
|
+
await appendEvent(invocation, {
|
|
1657
|
+
eventType: "ExternalOperationAccepted",
|
|
1658
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1659
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1660
|
+
payload: {
|
|
1661
|
+
adapterType: step.adapterType,
|
|
1662
|
+
operation: step.operation,
|
|
1663
|
+
externalReference: accepted.externalReference,
|
|
1664
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1665
|
+
}
|
|
1666
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
1667
|
+
}
|
|
1668
|
+
await appendEvent(invocation, {
|
|
1669
|
+
eventType: "AdapterInvocationSucceeded",
|
|
1670
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1671
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1672
|
+
payload: {
|
|
1673
|
+
adapterType: step.adapterType,
|
|
1674
|
+
operation: step.operation,
|
|
1675
|
+
input: recordedInput,
|
|
1676
|
+
...outputWithheld ? { outputWithheld: true } : { output }
|
|
1677
|
+
}
|
|
1678
|
+
}, `adapter:${stepIndex}:succeeded`);
|
|
1679
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
1680
|
+
if (outputWithheld && governanceStore2) {
|
|
1681
|
+
await governanceStore2.appendExternalReconciliation({
|
|
1682
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
1683
|
+
actionInvocationId,
|
|
1684
|
+
tenantId,
|
|
1685
|
+
spaceId,
|
|
1686
|
+
status: "pending",
|
|
1687
|
+
provider: adapter.vendor,
|
|
1688
|
+
...externalReference ?? accepted?.externalReference ? { externalOperationId: externalReference ?? accepted?.externalReference } : {},
|
|
1689
|
+
attempt: 1,
|
|
1690
|
+
reason: message,
|
|
1691
|
+
observedAt: now()
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
const adapterErrorContext = { actionId: action.actionId, source: "adapter", adapterType: step.adapterType, operation: step.operation, provider: adapter.vendor };
|
|
1696
|
+
const executionErrorContext = { actionId: action.actionId, source: "execution" };
|
|
1697
|
+
let errorContext = executionErrorContext;
|
|
1551
1698
|
let adapterDeadlineTimer;
|
|
1552
1699
|
try {
|
|
1553
1700
|
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
@@ -1555,15 +1702,15 @@ function createGovernedActionHost(options) {
|
|
|
1555
1702
|
if (adapterController && deadlineMs) {
|
|
1556
1703
|
const remaining = deadlineMs - now().getTime();
|
|
1557
1704
|
if (remaining <= 0) {
|
|
1558
|
-
throw new
|
|
1705
|
+
throw new HostError(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
1559
1706
|
}
|
|
1560
1707
|
adapterDeadlineTimer = setTimeout(
|
|
1561
|
-
() => adapterController.abort(new
|
|
1708
|
+
() => adapterController.abort(new HostError("Adapter deadline exceeded")),
|
|
1562
1709
|
remaining
|
|
1563
1710
|
);
|
|
1564
1711
|
adapterDeadlineTimer.unref?.();
|
|
1565
1712
|
}
|
|
1566
|
-
const circuitBreaker = circuitBreakers.forAdapter(adapter);
|
|
1713
|
+
const circuitBreaker = circuitBreakers.forAdapter(adapter, { tenantId, spaceId });
|
|
1567
1714
|
const result2 = await platform.executeWithAdapterRetry({
|
|
1568
1715
|
...circuitBreaker ? { circuitBreaker } : {},
|
|
1569
1716
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
@@ -1571,13 +1718,15 @@ function createGovernedActionHost(options) {
|
|
|
1571
1718
|
// Every await before the call happens here, before breaker
|
|
1572
1719
|
// admission, so nothing awaits between admission and the adapter.
|
|
1573
1720
|
prepareAttempt: async (attempt) => {
|
|
1721
|
+
errorContext = executionErrorContext;
|
|
1574
1722
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1575
1723
|
attempt,
|
|
1576
1724
|
updatedAt: now()
|
|
1577
|
-
});
|
|
1725
|
+
}, adapterInvocationScope);
|
|
1578
1726
|
},
|
|
1579
|
-
execute: (attempt, maxAttempts) => {
|
|
1580
|
-
|
|
1727
|
+
execute: async (attempt, maxAttempts) => {
|
|
1728
|
+
errorContext = adapterErrorContext;
|
|
1729
|
+
const adapterResult = await adapter.execute(input, {
|
|
1581
1730
|
tenantId,
|
|
1582
1731
|
spaceId,
|
|
1583
1732
|
actionInvocationId,
|
|
@@ -1589,6 +1738,8 @@ function createGovernedActionHost(options) {
|
|
|
1589
1738
|
...deadlineMs ? { deadlineMs } : {},
|
|
1590
1739
|
...adapterController ? { signal: adapterController.signal } : {}
|
|
1591
1740
|
});
|
|
1741
|
+
errorContext = executionErrorContext;
|
|
1742
|
+
return adapterResult;
|
|
1592
1743
|
},
|
|
1593
1744
|
isSuccessful: (result3) => result3.success,
|
|
1594
1745
|
getError: (result3) => result3.error,
|
|
@@ -1601,19 +1752,23 @@ function createGovernedActionHost(options) {
|
|
|
1601
1752
|
return "transient_failure";
|
|
1602
1753
|
}
|
|
1603
1754
|
});
|
|
1755
|
+
errorContext = adapterErrorContext;
|
|
1604
1756
|
platform.assertAdapterOutcomeConsistency(result2);
|
|
1757
|
+
errorContext = executionErrorContext;
|
|
1605
1758
|
if (!result2.success) {
|
|
1759
|
+
const resultError = result2.error == null ? result2.error : redactErrorText(options.redactErrorText, adapterErrorContext, result2.error);
|
|
1606
1760
|
const outcome = result2.outcome ?? "transient_failure";
|
|
1607
1761
|
if (outcome === "ambiguous") {
|
|
1608
1762
|
const evidence = result2.outcomeEvidence;
|
|
1609
|
-
const
|
|
1763
|
+
const reason = evidence?.reason == null ? resultError : redactErrorText(options.redactErrorText, adapterErrorContext, evidence.reason);
|
|
1764
|
+
const message = resultError ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
1610
1765
|
const adapterReconciliation = {
|
|
1611
1766
|
kind: "adapter_outcome_ambiguous",
|
|
1612
1767
|
adapterType: step.adapterType,
|
|
1613
1768
|
operation: step.operation,
|
|
1614
1769
|
vendor: adapter.vendor,
|
|
1615
1770
|
adapterInvocationId,
|
|
1616
|
-
...evidence?.reason ?? result2.error ? { reason
|
|
1771
|
+
...evidence?.reason ?? result2.error ? { reason } : {},
|
|
1617
1772
|
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
1618
1773
|
message
|
|
1619
1774
|
};
|
|
@@ -1621,7 +1776,7 @@ function createGovernedActionHost(options) {
|
|
|
1621
1776
|
status: "ambiguous",
|
|
1622
1777
|
error: message,
|
|
1623
1778
|
updatedAt: now()
|
|
1624
|
-
});
|
|
1779
|
+
}, adapterInvocationScope);
|
|
1625
1780
|
await appendEvent(invocation, {
|
|
1626
1781
|
eventType: "AdapterInvocationAmbiguous",
|
|
1627
1782
|
subjectType: adapterEventSubject.subjectType,
|
|
@@ -1659,24 +1814,52 @@ function createGovernedActionHost(options) {
|
|
|
1659
1814
|
}
|
|
1660
1815
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1661
1816
|
status: "failed",
|
|
1662
|
-
error:
|
|
1817
|
+
error: resultError ?? "Adapter failed",
|
|
1663
1818
|
updatedAt: now()
|
|
1664
|
-
});
|
|
1819
|
+
}, adapterInvocationScope);
|
|
1665
1820
|
await appendEvent(invocation, {
|
|
1666
1821
|
eventType: "AdapterInvocationFailed",
|
|
1667
1822
|
subjectType: adapterEventSubject.subjectType,
|
|
1668
1823
|
subjectId: adapterEventSubject.subjectId,
|
|
1669
1824
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1670
1825
|
}, `adapter:${stepIndex}:failed`);
|
|
1671
|
-
return fail2(invocation, "failed",
|
|
1826
|
+
return fail2(invocation, "failed", resultError ?? "Adapter failed");
|
|
1672
1827
|
}
|
|
1673
1828
|
const resultRecord = result2;
|
|
1674
|
-
|
|
1829
|
+
let output = isRecord(resultRecord.output) ? resultRecord.output : Object.fromEntries(
|
|
1675
1830
|
Object.entries(resultRecord).filter(
|
|
1676
1831
|
([key]) => key !== "success" && key !== "error"
|
|
1677
1832
|
)
|
|
1678
1833
|
);
|
|
1679
1834
|
const handoff = result2.acceptedExternalOperation;
|
|
1835
|
+
if (options.redactAdapterOutput) {
|
|
1836
|
+
try {
|
|
1837
|
+
output = options.redactAdapterOutput(action.actionId, step.adapterType, step.operation, output);
|
|
1838
|
+
if (!isRecord(output)) throw new HostError("Adapter output redactor must return a non-null, non-array object.");
|
|
1839
|
+
} catch {
|
|
1840
|
+
const message = "Adapter output withheld: the output redactor failed after the external effect; reconciliation required.";
|
|
1841
|
+
const reference = typeof handoff?.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1842
|
+
const acceptedAt = now();
|
|
1843
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1844
|
+
const pendingCompletion = pendingHandoff ?? (reference ? {
|
|
1845
|
+
provider: adapter.vendor,
|
|
1846
|
+
adapterType: step.adapterType,
|
|
1847
|
+
operation: step.operation,
|
|
1848
|
+
adapterInvocationId,
|
|
1849
|
+
externalReference: reference,
|
|
1850
|
+
acceptedAt,
|
|
1851
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1852
|
+
} : void 0);
|
|
1853
|
+
try {
|
|
1854
|
+
await persistReconciliationVerdict(invocation, { status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} });
|
|
1855
|
+
await appendDegradedAdapterEvidence(true, message, pendingCompletion?.adapterInvocationId === adapterInvocationId ? pendingCompletion : void 0, void 0, reference || void 0);
|
|
1856
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1857
|
+
} catch {
|
|
1858
|
+
throw new RecoverableFinalizationError(message);
|
|
1859
|
+
}
|
|
1860
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} }, action.actionId);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1680
1863
|
if (handoff) {
|
|
1681
1864
|
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1682
1865
|
if (reference === "") {
|
|
@@ -1698,19 +1881,28 @@ function createGovernedActionHost(options) {
|
|
|
1698
1881
|
const completion = action.execution?.completion ?? "immediate";
|
|
1699
1882
|
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1700
1883
|
if (refuse) {
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1884
|
+
try {
|
|
1885
|
+
await persistReconciliationVerdict(invocation, {
|
|
1886
|
+
status: "reconciliation_required",
|
|
1887
|
+
error: refuse,
|
|
1888
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1889
|
+
});
|
|
1890
|
+
await appendDegradedAdapterEvidence(false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1891
|
+
} catch {
|
|
1892
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1893
|
+
}
|
|
1707
1894
|
let error = refuse;
|
|
1708
1895
|
try {
|
|
1709
1896
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1710
1897
|
} catch (logError) {
|
|
1711
|
-
error = `${refuse} Reconciliation log unavailable: ${
|
|
1898
|
+
error = `${refuse} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1712
1899
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1713
1900
|
}
|
|
1901
|
+
try {
|
|
1902
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() }, adapterInvocationScope);
|
|
1903
|
+
} catch {
|
|
1904
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1905
|
+
}
|
|
1714
1906
|
return withConsistency({
|
|
1715
1907
|
actionInvocationId,
|
|
1716
1908
|
status: "reconciliation_required",
|
|
@@ -1737,7 +1929,7 @@ function createGovernedActionHost(options) {
|
|
|
1737
1929
|
status: "succeeded",
|
|
1738
1930
|
output,
|
|
1739
1931
|
updatedAt: now()
|
|
1740
|
-
});
|
|
1932
|
+
}, adapterInvocationScope);
|
|
1741
1933
|
if (options.extractExecutionAttestation) {
|
|
1742
1934
|
const attestation = await options.extractExecutionAttestation({
|
|
1743
1935
|
actionInvocationId,
|
|
@@ -1766,12 +1958,13 @@ function createGovernedActionHost(options) {
|
|
|
1766
1958
|
}
|
|
1767
1959
|
}, `adapter:${stepIndex}:succeeded`);
|
|
1768
1960
|
} catch (error) {
|
|
1769
|
-
|
|
1961
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1962
|
+
const message = caughtError(platform.isAdapterCircuitOpenError(error) ? adapterErrorContext : errorContext, error);
|
|
1770
1963
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1771
1964
|
status: "failed",
|
|
1772
1965
|
error: message,
|
|
1773
1966
|
updatedAt: now()
|
|
1774
|
-
});
|
|
1967
|
+
}, adapterInvocationScope);
|
|
1775
1968
|
if (platform.isAdapterCircuitOpenError(error)) {
|
|
1776
1969
|
await appendEvent(invocation, {
|
|
1777
1970
|
eventType: "AdapterInvocationFailed",
|
|
@@ -1802,7 +1995,7 @@ function createGovernedActionHost(options) {
|
|
|
1802
1995
|
try {
|
|
1803
1996
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1804
1997
|
} catch (logError) {
|
|
1805
|
-
error = `${message} Reconciliation log unavailable: ${
|
|
1998
|
+
error = `${message} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1806
1999
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1807
2000
|
}
|
|
1808
2001
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
@@ -1826,9 +2019,12 @@ function createGovernedActionHost(options) {
|
|
|
1826
2019
|
transaction
|
|
1827
2020
|
);
|
|
1828
2021
|
}
|
|
2022
|
+
if (pendingTransitionEvent) {
|
|
2023
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1, transaction);
|
|
2024
|
+
}
|
|
1829
2025
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1830
2026
|
if (!transaction.updateLeasedActionInvocation) {
|
|
1831
|
-
throw new
|
|
2027
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1832
2028
|
}
|
|
1833
2029
|
const updated = await transaction.updateLeasedActionInvocation({
|
|
1834
2030
|
id: actionInvocationId,
|
|
@@ -1861,6 +2057,9 @@ function createGovernedActionHost(options) {
|
|
|
1861
2057
|
for (const [index, event] of domainEvents.entries()) {
|
|
1862
2058
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1863
2059
|
}
|
|
2060
|
+
if (pendingTransitionEvent) {
|
|
2061
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1);
|
|
2062
|
+
}
|
|
1864
2063
|
}
|
|
1865
2064
|
await persistInvocation(invocation, finalPatch);
|
|
1866
2065
|
}
|
|
@@ -1880,7 +2079,15 @@ function createGovernedActionHost(options) {
|
|
|
1880
2079
|
};
|
|
1881
2080
|
} catch (error) {
|
|
1882
2081
|
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1883
|
-
|
|
2082
|
+
if (retryableGovernance && platform.isRetryableExecutionError(error)) {
|
|
2083
|
+
const reason = executionReasonOverride === "approval_resume" ? "approval resume" : !invocation.leaseOwner || invocation.leaseToken === void 0 ? "not leased" : !action.idempotent ? "action not idempotent" : invocation.attemptCount >= retryableExecutionMaxAttempts ? `attempts exhausted (${invocation.attemptCount}/${retryableExecutionMaxAttempts})` : void 0;
|
|
2084
|
+
if (!reason) throw error;
|
|
2085
|
+
return fail2(invocation, "failed", caughtError(
|
|
2086
|
+
{ actionId: invocation.actionId, source: "execution" },
|
|
2087
|
+
new Error(`Retryable execution not retried: ${reason}; ${errorMessage(error)}`)
|
|
2088
|
+
));
|
|
2089
|
+
}
|
|
2090
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "execution" }, error));
|
|
1884
2091
|
}
|
|
1885
2092
|
}
|
|
1886
2093
|
async function resumeApprovedInvocation(actionInvocationId, tenantId, spaceId, decision) {
|
|
@@ -1963,9 +2170,9 @@ function createGovernedActionHost(options) {
|
|
|
1963
2170
|
}
|
|
1964
2171
|
async function recordExecutionAttestation(actionInvocationId, tenantId, spaceId, attestation, adapterInvocationId) {
|
|
1965
2172
|
const governanceStore = asGovernanceStore(options.store);
|
|
1966
|
-
if (!governanceStore) throw new
|
|
2173
|
+
if (!governanceStore) throw new HostError("Execution attestations require a governance-capable store.");
|
|
1967
2174
|
const invocation = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1968
|
-
if (!invocation) throw new
|
|
2175
|
+
if (!invocation) throw new HostError(`ActionInvocation not found: ${actionInvocationId}`);
|
|
1969
2176
|
const id = lifecycleId("att", actionInvocationId, adapterInvocationId ?? `${attestation.provider}:${attestation.operation}:${attestation.externalOperationId ?? "external"}`);
|
|
1970
2177
|
await governanceStore.appendExecutionAttestation({
|
|
1971
2178
|
...attestation,
|
|
@@ -2043,11 +2250,11 @@ function createGovernedActionHost(options) {
|
|
|
2043
2250
|
...traceContext ? { traceContext } : {}
|
|
2044
2251
|
};
|
|
2045
2252
|
if (transaction) {
|
|
2046
|
-
if (!transaction.appendEventWithOutbox) throw new
|
|
2253
|
+
if (!transaction.appendEventWithOutbox) throw new HostError("Outbox egress requires an atomic transaction with appendEventWithOutbox.");
|
|
2047
2254
|
await transaction.appendEventWithOutbox(envelope, metadata);
|
|
2048
2255
|
} else {
|
|
2049
2256
|
const outboxStore = asOutboxStore(options.store);
|
|
2050
|
-
if (!outboxStore) throw new
|
|
2257
|
+
if (!outboxStore) throw new HostError("Outbox egress requires an outbox-capable Platform Host store.");
|
|
2051
2258
|
await outboxStore.appendEventWithOutbox(envelope, metadata);
|
|
2052
2259
|
}
|
|
2053
2260
|
} else {
|
|
@@ -2064,12 +2271,20 @@ function createGovernedActionHost(options) {
|
|
|
2064
2271
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
2065
2272
|
};
|
|
2066
2273
|
}
|
|
2274
|
+
async function persistReconciliationVerdict(invocation, patch) {
|
|
2275
|
+
try {
|
|
2276
|
+
await persistInvocation(invocation, patch);
|
|
2277
|
+
} catch (error) {
|
|
2278
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2279
|
+
await persistInvocation(invocation, patch);
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2067
2282
|
async function persistInvocation(invocation, patch) {
|
|
2068
2283
|
const previousStatus = invocation.status;
|
|
2069
2284
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
2070
2285
|
const recoverable = options.store;
|
|
2071
2286
|
if (!recoverable.updateLeasedActionInvocation) {
|
|
2072
|
-
throw new
|
|
2287
|
+
throw new HostError("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
2073
2288
|
}
|
|
2074
2289
|
const updated = await recoverable.updateLeasedActionInvocation({
|
|
2075
2290
|
id: invocation.id,
|
|
@@ -2094,6 +2309,8 @@ function createGovernedActionHost(options) {
|
|
|
2094
2309
|
const externalCompletion = createExternalCompletionLifecycle({
|
|
2095
2310
|
store: options.store,
|
|
2096
2311
|
telemetry: options.telemetry,
|
|
2312
|
+
redactErrorText: options.redactErrorText,
|
|
2313
|
+
redactExternalCompletionResult: options.redactExternalCompletionResult,
|
|
2097
2314
|
eventResultFields,
|
|
2098
2315
|
now,
|
|
2099
2316
|
actionResolver,
|
|
@@ -2174,9 +2391,6 @@ function initialState(machine) {
|
|
|
2174
2391
|
function isTerminal(status) {
|
|
2175
2392
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
2176
2393
|
}
|
|
2177
|
-
function isRecord(value) {
|
|
2178
|
-
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
2179
|
-
}
|
|
2180
2394
|
function isAdmissionTime(value) {
|
|
2181
2395
|
return value instanceof Date && Number.isFinite(value.getTime());
|
|
2182
2396
|
}
|
|
@@ -2479,6 +2693,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2479
2693
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
2480
2694
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
2481
2695
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
2696
|
+
getEntityStateSource: async (tenantId, spaceId, entityType, entityId) => {
|
|
2697
|
+
let source = await this.getEntityStateSource(tenantId, spaceId, entityType, entityId);
|
|
2698
|
+
for (const { event } of pendingEvents) {
|
|
2699
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2700
|
+
const state = event.payload?.toState;
|
|
2701
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2702
|
+
}
|
|
2703
|
+
return source;
|
|
2704
|
+
},
|
|
2482
2705
|
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
2483
2706
|
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
2484
2707
|
for (const pending of pendingEvents) {
|
|
@@ -2626,11 +2849,11 @@ var MemoryPlatformHostStore = class {
|
|
|
2626
2849
|
if (this.adapterInvocations.some((candidate) => candidate.id === record.id)) return;
|
|
2627
2850
|
this.adapterInvocations.push(record);
|
|
2628
2851
|
}
|
|
2629
|
-
async getAdapterInvocation(id) {
|
|
2630
|
-
return this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2852
|
+
async getAdapterInvocation(id, scope) {
|
|
2853
|
+
return this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2631
2854
|
}
|
|
2632
|
-
async updateAdapterInvocation(id, patch) {
|
|
2633
|
-
const record = this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2855
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
2856
|
+
const record = this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2634
2857
|
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2635
2858
|
Object.assign(record, patch);
|
|
2636
2859
|
}
|
|
@@ -2735,6 +2958,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2735
2958
|
async nextEventSequence(tenantId, spaceId) {
|
|
2736
2959
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).length + 1;
|
|
2737
2960
|
}
|
|
2961
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
2962
|
+
let source;
|
|
2963
|
+
for (const event of this.events) {
|
|
2964
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2965
|
+
const state = event.payload?.toState;
|
|
2966
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2967
|
+
}
|
|
2968
|
+
return source;
|
|
2969
|
+
}
|
|
2738
2970
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
2739
2971
|
let state;
|
|
2740
2972
|
for (const event of this.events) {
|
|
@@ -2907,6 +3139,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2907
3139
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
2908
3140
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
2909
3141
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
3142
|
+
getEntityStateSource: (tenantId, spaceId, entityType, entityId) => scoped.getEntityStateSource(tenantId, spaceId, entityType, entityId),
|
|
2910
3143
|
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2911
3144
|
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2912
3145
|
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
@@ -3360,19 +3593,19 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3360
3593
|
]
|
|
3361
3594
|
);
|
|
3362
3595
|
}
|
|
3363
|
-
async getAdapterInvocation(id) {
|
|
3596
|
+
async getAdapterInvocation(id, scope) {
|
|
3364
3597
|
const result = await this.sql.query(
|
|
3365
|
-
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1`,
|
|
3366
|
-
[id]
|
|
3598
|
+
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1${scope ? " AND tenant_id=$2 AND space_id=$3 AND action_invocation_id=$4" : ""}`,
|
|
3599
|
+
[id, ...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []]
|
|
3367
3600
|
);
|
|
3368
3601
|
return result.rows[0] ? toAdapterRecord(result.rows[0]) : void 0;
|
|
3369
3602
|
}
|
|
3370
|
-
async updateAdapterInvocation(id, patch) {
|
|
3603
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
3371
3604
|
await this.sql.query(
|
|
3372
3605
|
`UPDATE fabric_platform.adapter_invocations SET
|
|
3373
3606
|
status=COALESCE($2,status), output=COALESCE($3::jsonb,output),
|
|
3374
3607
|
error=CASE WHEN $4::boolean THEN $5 ELSE error END,
|
|
3375
|
-
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1`,
|
|
3608
|
+
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1${scope ? " AND tenant_id=$8 AND space_id=$9 AND action_invocation_id=$10" : ""}`,
|
|
3376
3609
|
[
|
|
3377
3610
|
id,
|
|
3378
3611
|
patch.status ?? null,
|
|
@@ -3380,7 +3613,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3380
3613
|
Object.hasOwn(patch, "error"),
|
|
3381
3614
|
patch.error ?? null,
|
|
3382
3615
|
patch.attempt ?? null,
|
|
3383
|
-
patch.updatedAt ?? null
|
|
3616
|
+
patch.updatedAt ?? null,
|
|
3617
|
+
...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []
|
|
3384
3618
|
]
|
|
3385
3619
|
);
|
|
3386
3620
|
}
|
|
@@ -3611,6 +3845,16 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3611
3845
|
);
|
|
3612
3846
|
return Number(result.rows[0].next_sequence);
|
|
3613
3847
|
}
|
|
3848
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3849
|
+
const result = await this.sql.query(
|
|
3850
|
+
`SELECT payload->>'toState' AS state, action_invocation_id FROM fabric_platform.asset_events
|
|
3851
|
+
WHERE tenant_id=$1 AND space_id=$2 AND subject_type=$3 AND subject_id=$4
|
|
3852
|
+
AND payload ? 'toState' ORDER BY sequence DESC LIMIT 1`,
|
|
3853
|
+
[tenantId, spaceId, entityType, entityId]
|
|
3854
|
+
);
|
|
3855
|
+
const row = result.rows[0];
|
|
3856
|
+
return typeof row?.state === "string" && typeof row.action_invocation_id === "string" ? { state: row.state, actionInvocationId: row.action_invocation_id } : void 0;
|
|
3857
|
+
}
|
|
3614
3858
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
3615
3859
|
const result = await this.sql.query(
|
|
3616
3860
|
`SELECT payload->>'toState' AS state FROM fabric_platform.asset_events
|
|
@@ -4003,8 +4247,6 @@ function toOutboxRequeueRecord(row, prefix) {
|
|
|
4003
4247
|
priorLeaseToken: Number(field("prior_lease_token"))
|
|
4004
4248
|
};
|
|
4005
4249
|
}
|
|
4006
|
-
|
|
4007
|
-
// src/worker.ts
|
|
4008
4250
|
var DEFAULT_BATCH_SIZE = 10;
|
|
4009
4251
|
var DEFAULT_LEASE_DURATION_MS = 5 * 6e4;
|
|
4010
4252
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
@@ -4053,6 +4295,7 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4053
4295
|
let completed = 0;
|
|
4054
4296
|
let failed = 0;
|
|
4055
4297
|
let waitingForApproval = 0;
|
|
4298
|
+
let deferred = 0;
|
|
4056
4299
|
for (const invocation of claimed) {
|
|
4057
4300
|
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
4058
4301
|
try {
|
|
@@ -4066,7 +4309,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4066
4309
|
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
4067
4310
|
else failed += 1;
|
|
4068
4311
|
} catch (error) {
|
|
4069
|
-
|
|
4312
|
+
if (platform.isRetryableExecutionError(error)) deferred += 1;
|
|
4313
|
+
else failed += 1;
|
|
4070
4314
|
options.onError?.(error, invocation);
|
|
4071
4315
|
} finally {
|
|
4072
4316
|
await stopHeartbeat();
|
|
@@ -4076,7 +4320,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4076
4320
|
claimed: claimed.length,
|
|
4077
4321
|
completed,
|
|
4078
4322
|
failed,
|
|
4079
|
-
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
4323
|
+
...waitingForApproval > 0 ? { waitingForApproval } : {},
|
|
4324
|
+
...deferred > 0 ? { deferred } : {}
|
|
4080
4325
|
};
|
|
4081
4326
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
4082
4327
|
return result;
|
|
@@ -4303,6 +4548,32 @@ function submitInput(subject, scope, idempotencyKey) {
|
|
|
4303
4548
|
idempotencyKey
|
|
4304
4549
|
};
|
|
4305
4550
|
}
|
|
4551
|
+
async function adapterProbe(subject) {
|
|
4552
|
+
await subject.reset?.();
|
|
4553
|
+
if (subject.ours.tenantId === subject.theirs.tenantId) fail("the two scopes must be in different tenants");
|
|
4554
|
+
const suffix = globalThis.crypto.randomUUID();
|
|
4555
|
+
const scope = {
|
|
4556
|
+
tenantId: subject.ours.tenantId,
|
|
4557
|
+
spaceId: `iso-probe-${suffix}`,
|
|
4558
|
+
actionInvocationId: `iso_act_${suffix}`
|
|
4559
|
+
};
|
|
4560
|
+
const id = `iso_adp_${suffix}`;
|
|
4561
|
+
const now = /* @__PURE__ */ new Date();
|
|
4562
|
+
await subject.store.createAdapterInvocation({
|
|
4563
|
+
id,
|
|
4564
|
+
...scope,
|
|
4565
|
+
adapterType: "iso",
|
|
4566
|
+
operation: "probe",
|
|
4567
|
+
vendor: "iso",
|
|
4568
|
+
status: "running",
|
|
4569
|
+
input: {},
|
|
4570
|
+
output: { probe: true },
|
|
4571
|
+
attempt: 1,
|
|
4572
|
+
createdAt: now,
|
|
4573
|
+
updatedAt: now
|
|
4574
|
+
});
|
|
4575
|
+
return { id, scope };
|
|
4576
|
+
}
|
|
4306
4577
|
function tenantIsolationChecks() {
|
|
4307
4578
|
return [
|
|
4308
4579
|
{
|
|
@@ -4447,6 +4718,54 @@ function tenantIsolationChecks() {
|
|
|
4447
4718
|
if (leaked.length > 0) fail(`a viewer in the other tenant saw ${leaked.length} of our rows`);
|
|
4448
4719
|
return [theirs.allowed ? "other tenant's viewer sees none of our rows" : "other tenant's viewer is denied"];
|
|
4449
4720
|
}
|
|
4721
|
+
},
|
|
4722
|
+
{
|
|
4723
|
+
id: "fabric.tenancy.adapter-invocation-read-scoped.v1",
|
|
4724
|
+
async run(subject) {
|
|
4725
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4726
|
+
try {
|
|
4727
|
+
const theirs = { ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId };
|
|
4728
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, tenantId: subject.theirs.tenantId })) fail("an adapter invocation was readable under another tenant's scope");
|
|
4729
|
+
if (await subject.store.getAdapterInvocation(id, theirs)) fail("an adapter invocation was readable under another tenant's scope");
|
|
4730
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, spaceId: `${scope.spaceId}-other` })) fail("an adapter invocation was readable under another space's scope");
|
|
4731
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, actionInvocationId: `${scope.actionInvocationId}-other` })) fail("an adapter invocation was readable under another action invocation's scope");
|
|
4732
|
+
const ours = await subject.store.getAdapterInvocation(id, scope);
|
|
4733
|
+
if (!ours || ours.id !== id || ours.tenantId !== scope.tenantId || ours.spaceId !== scope.spaceId || ours.actionInvocationId !== scope.actionInvocationId) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4734
|
+
} finally {
|
|
4735
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4736
|
+
}
|
|
4737
|
+
return ["adapter invocation readable under our scope and invisible outside it"];
|
|
4738
|
+
}
|
|
4739
|
+
},
|
|
4740
|
+
{
|
|
4741
|
+
id: "fabric.tenancy.adapter-invocation-write-scoped.v1",
|
|
4742
|
+
async run(subject) {
|
|
4743
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4744
|
+
const shape = (record) => JSON.stringify({ status: record.status, error: record.error, output: record.output, attempt: record.attempt });
|
|
4745
|
+
try {
|
|
4746
|
+
const before = await subject.store.getAdapterInvocation(id, scope);
|
|
4747
|
+
if (!before) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4748
|
+
const untouched = shape(before);
|
|
4749
|
+
for (const other of [
|
|
4750
|
+
{ ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId },
|
|
4751
|
+
{ ...scope, tenantId: subject.theirs.tenantId },
|
|
4752
|
+
{ ...scope, spaceId: `${scope.spaceId}-other` },
|
|
4753
|
+
{ ...scope, actionInvocationId: `${scope.actionInvocationId}-other` }
|
|
4754
|
+
]) {
|
|
4755
|
+
try {
|
|
4756
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "iso probe", updatedAt: /* @__PURE__ */ new Date() }, other);
|
|
4757
|
+
} catch {
|
|
4758
|
+
}
|
|
4759
|
+
const after = await subject.store.getAdapterInvocation(id, scope);
|
|
4760
|
+
if (!after || shape(after) !== untouched) fail("an adapter invocation changed under another scope");
|
|
4761
|
+
}
|
|
4762
|
+
await subject.store.updateAdapterInvocation(id, { status: "succeeded", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4763
|
+
if ((await subject.store.getAdapterInvocation(id, scope))?.status !== "succeeded") fail("our own adapter invocation was not updated under our scope, so the check proves nothing");
|
|
4764
|
+
} finally {
|
|
4765
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4766
|
+
}
|
|
4767
|
+
return ["adapter invocation untouched outside our scope and writable under ours"];
|
|
4768
|
+
}
|
|
4450
4769
|
}
|
|
4451
4770
|
];
|
|
4452
4771
|
}
|