@fabricorg/platform-host 7.4.0 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +214 -15
- package/dist/index.cjs +386 -67
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +123 -15
- package/dist/index.d.ts +123 -15
- package/dist/index.js +387 -68
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { canonicalJson, AdapterRegistry, assertAdapterCircuitBreakerConfig, AdapterCircuitBreakerRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, isAdapterCircuitOpenError, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
1
|
+
import { canonicalJson, AdapterRegistry, assertAdapterCircuitBreakerConfig, AdapterCircuitBreakerRegistry, isRetryableExecutionError, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, isAdapterCircuitOpenError, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
2
2
|
export { canonicalJson } from '@fabricorg/platform';
|
|
3
3
|
import { performance } from 'perf_hooks';
|
|
4
4
|
import { assertAssemblyRuntimeCompatible } from '@fabricorg/assembly';
|
|
@@ -308,6 +308,24 @@ function withoutPrivateHostFields(data, eventResultFields) {
|
|
|
308
308
|
function errorMessage(error) {
|
|
309
309
|
return error instanceof Error ? error.message : String(error);
|
|
310
310
|
}
|
|
311
|
+
var HostError = class extends Error {
|
|
312
|
+
};
|
|
313
|
+
function isRecord(value) {
|
|
314
|
+
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
315
|
+
}
|
|
316
|
+
function tryRedactErrorText(hook, context, message) {
|
|
317
|
+
if (!hook || message === "") return { success: true, text: message };
|
|
318
|
+
try {
|
|
319
|
+
const text = hook(context, message);
|
|
320
|
+
if (typeof text === "string" && text.length > 0) return { success: true, text };
|
|
321
|
+
} catch {
|
|
322
|
+
}
|
|
323
|
+
return { success: false };
|
|
324
|
+
}
|
|
325
|
+
function redactErrorText(hook, context, message) {
|
|
326
|
+
const result = tryRedactErrorText(hook, context, message);
|
|
327
|
+
return result.success ? result.text : "Error text withheld: the error-text redactor failed.";
|
|
328
|
+
}
|
|
311
329
|
function lifecycleId(prefix, invocationId, key) {
|
|
312
330
|
const safeKey = key.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 96);
|
|
313
331
|
return `${prefix}_${invocationId}_${safeKey}`;
|
|
@@ -370,7 +388,11 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
370
388
|
if (completion.provider !== void 0 && recorded.provider !== void 0 && completion.provider !== recorded.provider) {
|
|
371
389
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} was completed by "${recorded.provider}", not "${completion.provider}".`);
|
|
372
390
|
}
|
|
373
|
-
const
|
|
391
|
+
const provider = recorded.provider ?? completion.provider;
|
|
392
|
+
const rawDigest = completionDigest(completion, provider);
|
|
393
|
+
if (recorded.digest === rawDigest) return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
394
|
+
completion = redactCompletion(invocation, completion, provider ?? "external");
|
|
395
|
+
const incomingDigest = completionDigest(completion, provider);
|
|
374
396
|
if (recorded.digest === incomingDigest) {
|
|
375
397
|
return withConsistency(actionResult2(invocation), invocation.actionId);
|
|
376
398
|
}
|
|
@@ -400,8 +422,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
400
422
|
if (completion.provider !== void 0 && completion.provider !== pending.provider) {
|
|
401
423
|
throw new ExternalCompletionError("provider_mismatch", `ActionInvocation ${actionInvocationId} awaits completion from "${pending.provider}", not "${completion.provider}".`);
|
|
402
424
|
}
|
|
425
|
+
completion = redactCompletion(invocation, completion, completion.provider ?? pending.provider);
|
|
403
426
|
const recordedAt = now();
|
|
404
427
|
const externalCompletion = {
|
|
428
|
+
// completion has passed the hooks; retain the callback shape for compatibility.
|
|
405
429
|
...completion,
|
|
406
430
|
provider: completion.provider ?? pending.provider,
|
|
407
431
|
observedAt: completion.observedAt instanceof Date ? completion.observedAt.toISOString() : completion.observedAt,
|
|
@@ -422,7 +446,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
422
446
|
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, {
|
|
423
447
|
...completion.result ? { output: withoutPrivateHostFields(completion.result, eventResultFields) } : {},
|
|
424
448
|
updatedAt: recordedAt
|
|
425
|
-
});
|
|
449
|
+
}, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
426
450
|
await appendEvent(invocation, {
|
|
427
451
|
eventType: "ExternalOperationCompleted",
|
|
428
452
|
...subject,
|
|
@@ -434,7 +458,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
434
458
|
return withConsistency(actionResult2({ ...invocation, ...patch2, result: settles ? merged : invocation.result, ...settles ? { error: void 0 } : {} }), invocation.actionId);
|
|
435
459
|
}
|
|
436
460
|
const error = completion.error ?? `External operation "${pending.externalReference}" failed.`;
|
|
437
|
-
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt });
|
|
461
|
+
await deps.store.updateAdapterInvocation(pending.adapterInvocationId, { status: "failed", error, updatedAt: recordedAt }, { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id });
|
|
438
462
|
await appendEvent(invocation, {
|
|
439
463
|
eventType: "ExternalOperationFailed",
|
|
440
464
|
...subject,
|
|
@@ -445,6 +469,30 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
445
469
|
if (settles) emitInvocationStatusTelemetry(invocation, "failed", deps.telemetry, now(), invocation.status);
|
|
446
470
|
return withConsistency(actionResult2({ ...invocation, ...patch }), invocation.actionId);
|
|
447
471
|
}
|
|
472
|
+
function redactCompletion(invocation, completion, provider) {
|
|
473
|
+
let result = completion.result;
|
|
474
|
+
if (result && deps.redactExternalCompletionResult) {
|
|
475
|
+
try {
|
|
476
|
+
result = deps.redactExternalCompletionResult(invocation.actionId, provider, result);
|
|
477
|
+
if (!isRecord(result)) throw new HostError("Completion result redactor must return a non-null, non-array object.");
|
|
478
|
+
} catch {
|
|
479
|
+
throw new ExternalCompletionError("redaction_failed", "External completion result withheld: the result redactor failed; nothing was recorded.");
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
const error = completion.error == null ? void 0 : tryRedactErrorText(
|
|
483
|
+
deps.redactErrorText,
|
|
484
|
+
{ actionId: invocation.actionId, source: "external_completion", provider },
|
|
485
|
+
completion.error
|
|
486
|
+
);
|
|
487
|
+
if (error && !error.success) {
|
|
488
|
+
throw new ExternalCompletionError("redaction_failed", "External completion error withheld: the error-text redactor failed; nothing was recorded.");
|
|
489
|
+
}
|
|
490
|
+
return {
|
|
491
|
+
...completion,
|
|
492
|
+
...completion.result !== void 0 ? { result } : {},
|
|
493
|
+
...error?.success ? { error: error.text } : {}
|
|
494
|
+
};
|
|
495
|
+
}
|
|
448
496
|
function completionDigest(completion, provider) {
|
|
449
497
|
return createHash("sha256").update(canonicalJson({
|
|
450
498
|
provider: provider ?? null,
|
|
@@ -457,7 +505,7 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
457
505
|
}
|
|
458
506
|
async function recordCompletionReconciliation(invocation, completion, reason) {
|
|
459
507
|
const governanceStore = asGovernanceStore(deps.store);
|
|
460
|
-
if (!governanceStore) throw new
|
|
508
|
+
if (!governanceStore) throw new HostError("External completion reconciliation requires a governance-capable store.");
|
|
461
509
|
const provider = completion.provider ?? invocation.pendingCompletion?.provider ?? invocation.externalCompletion?.provider ?? "external";
|
|
462
510
|
const seed = createHash("sha256").update(`${provider}\0${completion.externalReference}\0${completion.kind ?? "refused"}`).digest("hex").slice(0, 32);
|
|
463
511
|
await governanceStore.appendExternalReconciliation({
|
|
@@ -477,10 +525,10 @@ function createExternalCompletionLifecycle(deps) {
|
|
|
477
525
|
async function reconcileOverdueCompletions(input = {}) {
|
|
478
526
|
const recoverable = deps.store;
|
|
479
527
|
if (!recoverable.listActionInvocations) {
|
|
480
|
-
throw new
|
|
528
|
+
throw new HostError("Overdue completion reconciliation requires a store that can list invocations.");
|
|
481
529
|
}
|
|
482
530
|
if (!asGovernanceStore(deps.store)) {
|
|
483
|
-
throw new
|
|
531
|
+
throw new HostError("Overdue completion reconciliation requires a governance-capable store to record its findings.");
|
|
484
532
|
}
|
|
485
533
|
const current = input.now ?? now();
|
|
486
534
|
const candidates = await recoverable.listActionInvocations({
|
|
@@ -855,6 +903,10 @@ var ADMISSION_FAILED = Object.freeze({ kind: "failed" });
|
|
|
855
903
|
var ADMISSION_TIMED_OUT = Object.freeze({ kind: "timed_out" });
|
|
856
904
|
var ADMISSION_REFUSAL_CODE = /^[a-z][A-Za-z0-9_.:-]{0,127}$/;
|
|
857
905
|
function createGovernedActionHost(options) {
|
|
906
|
+
const retryableExecutionMaxAttempts = options.retryableExecutionMaxAttempts ?? 5;
|
|
907
|
+
if (!Number.isInteger(retryableExecutionMaxAttempts) || retryableExecutionMaxAttempts <= 0) {
|
|
908
|
+
throw new Error("retryableExecutionMaxAttempts must be a positive integer.");
|
|
909
|
+
}
|
|
858
910
|
if (options.composition) {
|
|
859
911
|
if (!options.registry?.orderedModules) {
|
|
860
912
|
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
@@ -891,6 +943,9 @@ function createGovernedActionHost(options) {
|
|
|
891
943
|
if (typeof admissionTimeoutMs !== "number" || !Number.isInteger(admissionTimeoutMs) || admissionTimeoutMs < 1 || admissionTimeoutMs > MAX_TIMER_MS) {
|
|
892
944
|
throw new Error(`admissionTimeoutMs must be a whole number of milliseconds from 1 to ${MAX_TIMER_MS}.`);
|
|
893
945
|
}
|
|
946
|
+
function caughtError(context, error) {
|
|
947
|
+
return error instanceof HostError ? errorMessage(error) : redactErrorText(options.redactErrorText, context, errorMessage(error));
|
|
948
|
+
}
|
|
894
949
|
async function submitAction(input) {
|
|
895
950
|
const read = snapshotSubmission(input);
|
|
896
951
|
if (read.executionReason && !["initial", "offline_replay"].includes(read.executionReason)) {
|
|
@@ -1188,6 +1243,7 @@ function createGovernedActionHost(options) {
|
|
|
1188
1243
|
spaceId,
|
|
1189
1244
|
attributes: { actionVersion: invocation.actionVersion }
|
|
1190
1245
|
});
|
|
1246
|
+
let retryableGovernance = true;
|
|
1191
1247
|
try {
|
|
1192
1248
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
1193
1249
|
if (!parsed.success) {
|
|
@@ -1340,6 +1396,7 @@ function createGovernedActionHost(options) {
|
|
|
1340
1396
|
mode: "execute",
|
|
1341
1397
|
now: now()
|
|
1342
1398
|
});
|
|
1399
|
+
retryableGovernance = false;
|
|
1343
1400
|
for (const outcome of outcomes) {
|
|
1344
1401
|
await options.store.appendPolicyEvaluation({
|
|
1345
1402
|
id: lifecycleId("pol", actionInvocationId, outcome.policyId),
|
|
@@ -1383,12 +1440,14 @@ function createGovernedActionHost(options) {
|
|
|
1383
1440
|
}
|
|
1384
1441
|
let data;
|
|
1385
1442
|
let domainEvents = [];
|
|
1443
|
+
let pendingTransitionEvent;
|
|
1386
1444
|
try {
|
|
1387
1445
|
const runHandler = async (db, transaction) => {
|
|
1388
1446
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
1389
|
-
throw new
|
|
1447
|
+
throw new HostError("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
1390
1448
|
}
|
|
1391
1449
|
const binding = action.stateMachine;
|
|
1450
|
+
let validatedTransition;
|
|
1392
1451
|
if (binding) {
|
|
1393
1452
|
const machine = stateMachineResolver(binding.entityType);
|
|
1394
1453
|
const entityId = binding.getEntityId(parsed.data);
|
|
@@ -1418,9 +1477,30 @@ function createGovernedActionHost(options) {
|
|
|
1418
1477
|
db
|
|
1419
1478
|
}
|
|
1420
1479
|
);
|
|
1421
|
-
|
|
1480
|
+
validatedTransition = { entityId, currentState, targetState, valid: transition.valid };
|
|
1481
|
+
let replayingAppliedTransition = false;
|
|
1482
|
+
if (action.idempotent && currentState === targetState && !transition.valid && entityId) {
|
|
1483
|
+
let source;
|
|
1484
|
+
if (transaction?.getEntityStateSource) {
|
|
1485
|
+
source = await transaction.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1486
|
+
} else if (options.store.getEntityStateSource) {
|
|
1487
|
+
source = await options.store.getEntityStateSource(tenantId, spaceId, binding.entityType, entityId);
|
|
1488
|
+
} else {
|
|
1489
|
+
const eventStore = options.store;
|
|
1490
|
+
const listEvents = transaction?.listEvents?.bind(transaction) ?? eventStore.listEvents?.bind(eventStore);
|
|
1491
|
+
const events2 = listEvents ? await listEvents(tenantId, spaceId) : [];
|
|
1492
|
+
let latest;
|
|
1493
|
+
for (const event of events2) {
|
|
1494
|
+
if (event.tenantId === tenantId && event.spaceId === spaceId && event.subjectType === binding.entityType && event.subjectId === entityId && typeof event.payload?.toState === "string" && (!latest || event.sequence > latest.sequence)) latest = event;
|
|
1495
|
+
}
|
|
1496
|
+
if (latest && typeof latest.actionInvocationId === "string") {
|
|
1497
|
+
source = { state: latest.payload.toState, actionInvocationId: latest.actionInvocationId };
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
replayingAppliedTransition = source?.actionInvocationId === actionInvocationId && source.state === currentState;
|
|
1501
|
+
}
|
|
1422
1502
|
if (!transition.valid && !replayingAppliedTransition) {
|
|
1423
|
-
throw new Error(transition.error
|
|
1503
|
+
throw transition.error != null ? new Error(transition.error) : new HostError("Invalid state transition");
|
|
1424
1504
|
}
|
|
1425
1505
|
}
|
|
1426
1506
|
}
|
|
@@ -1441,14 +1521,14 @@ function createGovernedActionHost(options) {
|
|
|
1441
1521
|
parsed.data
|
|
1442
1522
|
) : { success: true, data: {} };
|
|
1443
1523
|
if (!handlerResult.success) {
|
|
1444
|
-
throw new Error(handlerResult.error
|
|
1524
|
+
throw handlerResult.error != null ? new Error(handlerResult.error) : new HostError("Action handler failed");
|
|
1445
1525
|
}
|
|
1446
1526
|
const handlerData = handlerResult.data ?? {};
|
|
1447
1527
|
if (action.resultSchema) {
|
|
1448
1528
|
const publicResult = withoutPrivateHostFields(handlerData, eventResultFields);
|
|
1449
1529
|
const parsedResult = action.resultSchema.safeParse(publicResult);
|
|
1450
1530
|
if (!parsedResult.success) {
|
|
1451
|
-
throw new
|
|
1531
|
+
throw new HostError(`Action result validation failed at ${parsedResult.error.issues.map((issue) => issue.path.map(String).join(".") || "result").join(", ")}`);
|
|
1452
1532
|
}
|
|
1453
1533
|
}
|
|
1454
1534
|
const events = extractEvents(handlerData);
|
|
@@ -1456,7 +1536,7 @@ function createGovernedActionHost(options) {
|
|
|
1456
1536
|
(event) => !action.emitsEvents.includes(event.eventType)
|
|
1457
1537
|
);
|
|
1458
1538
|
if (undeclaredEvent) {
|
|
1459
|
-
throw new
|
|
1539
|
+
throw new HostError(
|
|
1460
1540
|
`${action.actionId} emitted undeclared event type ${undeclaredEvent.eventType}`
|
|
1461
1541
|
);
|
|
1462
1542
|
}
|
|
@@ -1471,7 +1551,29 @@ function createGovernedActionHost(options) {
|
|
|
1471
1551
|
);
|
|
1472
1552
|
}
|
|
1473
1553
|
}
|
|
1474
|
-
|
|
1554
|
+
let transitionEvent;
|
|
1555
|
+
if (binding && validatedTransition?.valid && validatedTransition.entityId) {
|
|
1556
|
+
const { entityId, currentState, targetState } = validatedTransition;
|
|
1557
|
+
const handlerEmittedTransition = events.some((event) => event.eventType === "StateTransitioned" && event.subjectType === binding.entityType && event.subjectId === entityId);
|
|
1558
|
+
if (!handlerEmittedTransition) {
|
|
1559
|
+
transitionEvent = {
|
|
1560
|
+
eventType: "StateTransitioned",
|
|
1561
|
+
subjectType: binding.entityType,
|
|
1562
|
+
subjectId: entityId,
|
|
1563
|
+
payload: {
|
|
1564
|
+
entityType: binding.entityType,
|
|
1565
|
+
entityId,
|
|
1566
|
+
fromState: currentState,
|
|
1567
|
+
toState: targetState,
|
|
1568
|
+
causedByAction: action.actionId
|
|
1569
|
+
}
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1573
|
+
if (transitionEvent && action.eventPhase !== "after_adapters") {
|
|
1574
|
+
await appendEvent(invocation, transitionEvent, "state:transition", 1, transaction);
|
|
1575
|
+
}
|
|
1576
|
+
return { data: handlerData, domainEvents: events, pendingTransitionEvent: transitionEvent };
|
|
1475
1577
|
};
|
|
1476
1578
|
const atomicStore = asAtomicMutationStore(options.store);
|
|
1477
1579
|
const executed = atomicStore ? await atomicStore.transactionWithEvents(
|
|
@@ -1479,9 +1581,11 @@ function createGovernedActionHost(options) {
|
|
|
1479
1581
|
) : await options.store.transaction((db) => runHandler(db));
|
|
1480
1582
|
data = executed.data;
|
|
1481
1583
|
domainEvents = executed.domainEvents;
|
|
1584
|
+
pendingTransitionEvent = executed.pendingTransitionEvent;
|
|
1482
1585
|
} catch (error) {
|
|
1483
|
-
return fail2(invocation, "failed",
|
|
1586
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "handler" }, error));
|
|
1484
1587
|
}
|
|
1588
|
+
const adapterInvocationScope = { tenantId: invocation.tenantId, spaceId: invocation.spaceId, actionInvocationId: invocation.id };
|
|
1485
1589
|
let pendingHandoff = invocation.pendingCompletion;
|
|
1486
1590
|
for (const [stepIndex, step] of (action.adapterSteps ?? []).entries()) {
|
|
1487
1591
|
const input = step.getInput(parsed.data, data);
|
|
@@ -1489,12 +1593,13 @@ function createGovernedActionHost(options) {
|
|
|
1489
1593
|
const adapter = adapters.require(step.adapterType, step.operation);
|
|
1490
1594
|
const adapterInvocationId = lifecycleId("adp", actionInvocationId, String(stepIndex));
|
|
1491
1595
|
const previousAdapterInvocation = await options.store.getAdapterInvocation(
|
|
1492
|
-
adapterInvocationId
|
|
1596
|
+
adapterInvocationId,
|
|
1597
|
+
adapterInvocationScope
|
|
1493
1598
|
);
|
|
1494
1599
|
if (previousAdapterInvocation?.status === "succeeded") continue;
|
|
1495
1600
|
if (pendingHandoff?.adapterInvocationId === adapterInvocationId) {
|
|
1496
1601
|
const subject = options.adapterEventSubject?.(action.actionId, parsed.data, data) ?? { subjectType: "AdapterInvocation", subjectId: adapterInvocationId };
|
|
1497
|
-
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() });
|
|
1602
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1498
1603
|
await appendEvent(invocation, {
|
|
1499
1604
|
eventType: "ExternalOperationAccepted",
|
|
1500
1605
|
subjectType: subject.subjectType,
|
|
@@ -1528,9 +1633,7 @@ function createGovernedActionHost(options) {
|
|
|
1528
1633
|
const startedAt = now();
|
|
1529
1634
|
const record = {
|
|
1530
1635
|
id: adapterInvocationId,
|
|
1531
|
-
|
|
1532
|
-
tenantId,
|
|
1533
|
-
spaceId,
|
|
1636
|
+
...adapterInvocationScope,
|
|
1534
1637
|
adapterType: step.adapterType,
|
|
1535
1638
|
operation: step.operation,
|
|
1536
1639
|
vendor: adapter.vendor,
|
|
@@ -1547,6 +1650,50 @@ function createGovernedActionHost(options) {
|
|
|
1547
1650
|
subjectId: adapterEventSubject.subjectId,
|
|
1548
1651
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1549
1652
|
}, `adapter:${stepIndex}:started`);
|
|
1653
|
+
async function appendDegradedAdapterEvidence(outputWithheld, message, accepted, output, externalReference) {
|
|
1654
|
+
if (accepted) {
|
|
1655
|
+
await appendEvent(invocation, {
|
|
1656
|
+
eventType: "ExternalOperationAccepted",
|
|
1657
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1658
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1659
|
+
payload: {
|
|
1660
|
+
adapterType: step.adapterType,
|
|
1661
|
+
operation: step.operation,
|
|
1662
|
+
externalReference: accepted.externalReference,
|
|
1663
|
+
...accepted.dueAt ? { dueAt: accepted.dueAt.toISOString() } : {}
|
|
1664
|
+
}
|
|
1665
|
+
}, `adapter:${stepIndex}:accepted`);
|
|
1666
|
+
}
|
|
1667
|
+
await appendEvent(invocation, {
|
|
1668
|
+
eventType: "AdapterInvocationSucceeded",
|
|
1669
|
+
subjectType: adapterEventSubject.subjectType,
|
|
1670
|
+
subjectId: adapterEventSubject.subjectId,
|
|
1671
|
+
payload: {
|
|
1672
|
+
adapterType: step.adapterType,
|
|
1673
|
+
operation: step.operation,
|
|
1674
|
+
input: recordedInput,
|
|
1675
|
+
...outputWithheld ? { outputWithheld: true } : { output }
|
|
1676
|
+
}
|
|
1677
|
+
}, `adapter:${stepIndex}:succeeded`);
|
|
1678
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
1679
|
+
if (outputWithheld && governanceStore2) {
|
|
1680
|
+
await governanceStore2.appendExternalReconciliation({
|
|
1681
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
1682
|
+
actionInvocationId,
|
|
1683
|
+
tenantId,
|
|
1684
|
+
spaceId,
|
|
1685
|
+
status: "pending",
|
|
1686
|
+
provider: adapter.vendor,
|
|
1687
|
+
...externalReference ?? accepted?.externalReference ? { externalOperationId: externalReference ?? accepted?.externalReference } : {},
|
|
1688
|
+
attempt: 1,
|
|
1689
|
+
reason: message,
|
|
1690
|
+
observedAt: now()
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
const adapterErrorContext = { actionId: action.actionId, source: "adapter", adapterType: step.adapterType, operation: step.operation, provider: adapter.vendor };
|
|
1695
|
+
const executionErrorContext = { actionId: action.actionId, source: "execution" };
|
|
1696
|
+
let errorContext = executionErrorContext;
|
|
1550
1697
|
let adapterDeadlineTimer;
|
|
1551
1698
|
try {
|
|
1552
1699
|
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
@@ -1554,15 +1701,15 @@ function createGovernedActionHost(options) {
|
|
|
1554
1701
|
if (adapterController && deadlineMs) {
|
|
1555
1702
|
const remaining = deadlineMs - now().getTime();
|
|
1556
1703
|
if (remaining <= 0) {
|
|
1557
|
-
throw new
|
|
1704
|
+
throw new HostError(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
1558
1705
|
}
|
|
1559
1706
|
adapterDeadlineTimer = setTimeout(
|
|
1560
|
-
() => adapterController.abort(new
|
|
1707
|
+
() => adapterController.abort(new HostError("Adapter deadline exceeded")),
|
|
1561
1708
|
remaining
|
|
1562
1709
|
);
|
|
1563
1710
|
adapterDeadlineTimer.unref?.();
|
|
1564
1711
|
}
|
|
1565
|
-
const circuitBreaker = circuitBreakers.forAdapter(adapter);
|
|
1712
|
+
const circuitBreaker = circuitBreakers.forAdapter(adapter, { tenantId, spaceId });
|
|
1566
1713
|
const result2 = await executeWithAdapterRetry({
|
|
1567
1714
|
...circuitBreaker ? { circuitBreaker } : {},
|
|
1568
1715
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
@@ -1570,13 +1717,15 @@ function createGovernedActionHost(options) {
|
|
|
1570
1717
|
// Every await before the call happens here, before breaker
|
|
1571
1718
|
// admission, so nothing awaits between admission and the adapter.
|
|
1572
1719
|
prepareAttempt: async (attempt) => {
|
|
1720
|
+
errorContext = executionErrorContext;
|
|
1573
1721
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1574
1722
|
attempt,
|
|
1575
1723
|
updatedAt: now()
|
|
1576
|
-
});
|
|
1724
|
+
}, adapterInvocationScope);
|
|
1577
1725
|
},
|
|
1578
|
-
execute: (attempt, maxAttempts) => {
|
|
1579
|
-
|
|
1726
|
+
execute: async (attempt, maxAttempts) => {
|
|
1727
|
+
errorContext = adapterErrorContext;
|
|
1728
|
+
const adapterResult = await adapter.execute(input, {
|
|
1580
1729
|
tenantId,
|
|
1581
1730
|
spaceId,
|
|
1582
1731
|
actionInvocationId,
|
|
@@ -1588,6 +1737,8 @@ function createGovernedActionHost(options) {
|
|
|
1588
1737
|
...deadlineMs ? { deadlineMs } : {},
|
|
1589
1738
|
...adapterController ? { signal: adapterController.signal } : {}
|
|
1590
1739
|
});
|
|
1740
|
+
errorContext = executionErrorContext;
|
|
1741
|
+
return adapterResult;
|
|
1591
1742
|
},
|
|
1592
1743
|
isSuccessful: (result3) => result3.success,
|
|
1593
1744
|
getError: (result3) => result3.error,
|
|
@@ -1600,19 +1751,23 @@ function createGovernedActionHost(options) {
|
|
|
1600
1751
|
return "transient_failure";
|
|
1601
1752
|
}
|
|
1602
1753
|
});
|
|
1754
|
+
errorContext = adapterErrorContext;
|
|
1603
1755
|
assertAdapterOutcomeConsistency(result2);
|
|
1756
|
+
errorContext = executionErrorContext;
|
|
1604
1757
|
if (!result2.success) {
|
|
1758
|
+
const resultError = result2.error == null ? result2.error : redactErrorText(options.redactErrorText, adapterErrorContext, result2.error);
|
|
1605
1759
|
const outcome = result2.outcome ?? "transient_failure";
|
|
1606
1760
|
if (outcome === "ambiguous") {
|
|
1607
1761
|
const evidence = result2.outcomeEvidence;
|
|
1608
|
-
const
|
|
1762
|
+
const reason = evidence?.reason == null ? resultError : redactErrorText(options.redactErrorText, adapterErrorContext, evidence.reason);
|
|
1763
|
+
const message = resultError ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
1609
1764
|
const adapterReconciliation = {
|
|
1610
1765
|
kind: "adapter_outcome_ambiguous",
|
|
1611
1766
|
adapterType: step.adapterType,
|
|
1612
1767
|
operation: step.operation,
|
|
1613
1768
|
vendor: adapter.vendor,
|
|
1614
1769
|
adapterInvocationId,
|
|
1615
|
-
...evidence?.reason ?? result2.error ? { reason
|
|
1770
|
+
...evidence?.reason ?? result2.error ? { reason } : {},
|
|
1616
1771
|
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
1617
1772
|
message
|
|
1618
1773
|
};
|
|
@@ -1620,7 +1775,7 @@ function createGovernedActionHost(options) {
|
|
|
1620
1775
|
status: "ambiguous",
|
|
1621
1776
|
error: message,
|
|
1622
1777
|
updatedAt: now()
|
|
1623
|
-
});
|
|
1778
|
+
}, adapterInvocationScope);
|
|
1624
1779
|
await appendEvent(invocation, {
|
|
1625
1780
|
eventType: "AdapterInvocationAmbiguous",
|
|
1626
1781
|
subjectType: adapterEventSubject.subjectType,
|
|
@@ -1658,24 +1813,52 @@ function createGovernedActionHost(options) {
|
|
|
1658
1813
|
}
|
|
1659
1814
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1660
1815
|
status: "failed",
|
|
1661
|
-
error:
|
|
1816
|
+
error: resultError ?? "Adapter failed",
|
|
1662
1817
|
updatedAt: now()
|
|
1663
|
-
});
|
|
1818
|
+
}, adapterInvocationScope);
|
|
1664
1819
|
await appendEvent(invocation, {
|
|
1665
1820
|
eventType: "AdapterInvocationFailed",
|
|
1666
1821
|
subjectType: adapterEventSubject.subjectType,
|
|
1667
1822
|
subjectId: adapterEventSubject.subjectId,
|
|
1668
1823
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
1669
1824
|
}, `adapter:${stepIndex}:failed`);
|
|
1670
|
-
return fail2(invocation, "failed",
|
|
1825
|
+
return fail2(invocation, "failed", resultError ?? "Adapter failed");
|
|
1671
1826
|
}
|
|
1672
1827
|
const resultRecord = result2;
|
|
1673
|
-
|
|
1828
|
+
let output = isRecord(resultRecord.output) ? resultRecord.output : Object.fromEntries(
|
|
1674
1829
|
Object.entries(resultRecord).filter(
|
|
1675
1830
|
([key]) => key !== "success" && key !== "error"
|
|
1676
1831
|
)
|
|
1677
1832
|
);
|
|
1678
1833
|
const handoff = result2.acceptedExternalOperation;
|
|
1834
|
+
if (options.redactAdapterOutput) {
|
|
1835
|
+
try {
|
|
1836
|
+
output = options.redactAdapterOutput(action.actionId, step.adapterType, step.operation, output);
|
|
1837
|
+
if (!isRecord(output)) throw new HostError("Adapter output redactor must return a non-null, non-array object.");
|
|
1838
|
+
} catch {
|
|
1839
|
+
const message = "Adapter output withheld: the output redactor failed after the external effect; reconciliation required.";
|
|
1840
|
+
const reference = typeof handoff?.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1841
|
+
const acceptedAt = now();
|
|
1842
|
+
const deadlineMs2 = action.execution?.completionDeadlineMs;
|
|
1843
|
+
const pendingCompletion = pendingHandoff ?? (reference ? {
|
|
1844
|
+
provider: adapter.vendor,
|
|
1845
|
+
adapterType: step.adapterType,
|
|
1846
|
+
operation: step.operation,
|
|
1847
|
+
adapterInvocationId,
|
|
1848
|
+
externalReference: reference,
|
|
1849
|
+
acceptedAt,
|
|
1850
|
+
...deadlineMs2 ? { dueAt: new Date(acceptedAt.getTime() + deadlineMs2) } : {}
|
|
1851
|
+
} : void 0);
|
|
1852
|
+
try {
|
|
1853
|
+
await persistReconciliationVerdict(invocation, { status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} });
|
|
1854
|
+
await appendDegradedAdapterEvidence(true, message, pendingCompletion?.adapterInvocationId === adapterInvocationId ? pendingCompletion : void 0, void 0, reference || void 0);
|
|
1855
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", updatedAt: now() }, adapterInvocationScope);
|
|
1856
|
+
} catch {
|
|
1857
|
+
throw new RecoverableFinalizationError(message);
|
|
1858
|
+
}
|
|
1859
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: message, ...pendingCompletion ? { pendingCompletion } : {} }, action.actionId);
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1679
1862
|
if (handoff) {
|
|
1680
1863
|
const reference = typeof handoff.externalReference === "string" ? handoff.externalReference.trim() : "";
|
|
1681
1864
|
if (reference === "") {
|
|
@@ -1697,19 +1880,28 @@ function createGovernedActionHost(options) {
|
|
|
1697
1880
|
const completion = action.execution?.completion ?? "immediate";
|
|
1698
1881
|
const refuse = completion === "immediate" ? `Adapter ${step.adapterType}:${step.operation} handed off external operation "${reference}" but ${action.actionId} declares immediate completion; the external effect may be in flight.` : pendingHandoff ? `Adapter ${step.adapterType}:${step.operation} handed off a second external operation "${reference}" while "${pendingHandoff.externalReference}" is already pending; one invocation waits on one completion.` : void 0;
|
|
1699
1882
|
if (refuse) {
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1883
|
+
try {
|
|
1884
|
+
await persistReconciliationVerdict(invocation, {
|
|
1885
|
+
status: "reconciliation_required",
|
|
1886
|
+
error: refuse,
|
|
1887
|
+
...pendingHandoff ? {} : { pendingCompletion: accepted }
|
|
1888
|
+
});
|
|
1889
|
+
await appendDegradedAdapterEvidence(false, refuse, pendingHandoff ? void 0 : accepted, output);
|
|
1890
|
+
} catch {
|
|
1891
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1892
|
+
}
|
|
1706
1893
|
let error = refuse;
|
|
1707
1894
|
try {
|
|
1708
1895
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: reference, provider: adapter.vendor }, refuse);
|
|
1709
1896
|
} catch (logError) {
|
|
1710
|
-
error = `${refuse} Reconciliation log unavailable: ${
|
|
1897
|
+
error = `${refuse} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1711
1898
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1712
1899
|
}
|
|
1900
|
+
try {
|
|
1901
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, { status: "succeeded", output, updatedAt: now() }, adapterInvocationScope);
|
|
1902
|
+
} catch {
|
|
1903
|
+
throw new RecoverableFinalizationError(refuse);
|
|
1904
|
+
}
|
|
1713
1905
|
return withConsistency({
|
|
1714
1906
|
actionInvocationId,
|
|
1715
1907
|
status: "reconciliation_required",
|
|
@@ -1736,7 +1928,7 @@ function createGovernedActionHost(options) {
|
|
|
1736
1928
|
status: "succeeded",
|
|
1737
1929
|
output,
|
|
1738
1930
|
updatedAt: now()
|
|
1739
|
-
});
|
|
1931
|
+
}, adapterInvocationScope);
|
|
1740
1932
|
if (options.extractExecutionAttestation) {
|
|
1741
1933
|
const attestation = await options.extractExecutionAttestation({
|
|
1742
1934
|
actionInvocationId,
|
|
@@ -1765,12 +1957,13 @@ function createGovernedActionHost(options) {
|
|
|
1765
1957
|
}
|
|
1766
1958
|
}, `adapter:${stepIndex}:succeeded`);
|
|
1767
1959
|
} catch (error) {
|
|
1768
|
-
|
|
1960
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1961
|
+
const message = caughtError(isAdapterCircuitOpenError(error) ? adapterErrorContext : errorContext, error);
|
|
1769
1962
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
1770
1963
|
status: "failed",
|
|
1771
1964
|
error: message,
|
|
1772
1965
|
updatedAt: now()
|
|
1773
|
-
});
|
|
1966
|
+
}, adapterInvocationScope);
|
|
1774
1967
|
if (isAdapterCircuitOpenError(error)) {
|
|
1775
1968
|
await appendEvent(invocation, {
|
|
1776
1969
|
eventType: "AdapterInvocationFailed",
|
|
@@ -1801,7 +1994,7 @@ function createGovernedActionHost(options) {
|
|
|
1801
1994
|
try {
|
|
1802
1995
|
await externalCompletion.recordCompletionReconciliation(invocation, { externalReference: pendingHandoff.externalReference, provider: pendingHandoff.provider, kind: "obligations" }, message);
|
|
1803
1996
|
} catch (logError) {
|
|
1804
|
-
error = `${message} Reconciliation log unavailable: ${
|
|
1997
|
+
error = `${message} Reconciliation log unavailable: ${caughtError({ actionId: invocation.actionId, source: "execution" }, logError)}`;
|
|
1805
1998
|
await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { error });
|
|
1806
1999
|
}
|
|
1807
2000
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error, pendingCompletion: pendingHandoff }, action.actionId);
|
|
@@ -1825,9 +2018,12 @@ function createGovernedActionHost(options) {
|
|
|
1825
2018
|
transaction
|
|
1826
2019
|
);
|
|
1827
2020
|
}
|
|
2021
|
+
if (pendingTransitionEvent) {
|
|
2022
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1, transaction);
|
|
2023
|
+
}
|
|
1828
2024
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1829
2025
|
if (!transaction.updateLeasedActionInvocation) {
|
|
1830
|
-
throw new
|
|
2026
|
+
throw new HostError("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1831
2027
|
}
|
|
1832
2028
|
const updated = await transaction.updateLeasedActionInvocation({
|
|
1833
2029
|
id: actionInvocationId,
|
|
@@ -1860,6 +2056,9 @@ function createGovernedActionHost(options) {
|
|
|
1860
2056
|
for (const [index, event] of domainEvents.entries()) {
|
|
1861
2057
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
1862
2058
|
}
|
|
2059
|
+
if (pendingTransitionEvent) {
|
|
2060
|
+
await appendEvent(invocation, pendingTransitionEvent, "state:transition", 1);
|
|
2061
|
+
}
|
|
1863
2062
|
}
|
|
1864
2063
|
await persistInvocation(invocation, finalPatch);
|
|
1865
2064
|
}
|
|
@@ -1879,7 +2078,15 @@ function createGovernedActionHost(options) {
|
|
|
1879
2078
|
};
|
|
1880
2079
|
} catch (error) {
|
|
1881
2080
|
if (error instanceof RecoverableFinalizationError) throw error;
|
|
1882
|
-
|
|
2081
|
+
if (retryableGovernance && isRetryableExecutionError(error)) {
|
|
2082
|
+
const reason = executionReasonOverride === "approval_resume" ? "approval resume" : !invocation.leaseOwner || invocation.leaseToken === void 0 ? "not leased" : !action.idempotent ? "action not idempotent" : invocation.attemptCount >= retryableExecutionMaxAttempts ? `attempts exhausted (${invocation.attemptCount}/${retryableExecutionMaxAttempts})` : void 0;
|
|
2083
|
+
if (!reason) throw error;
|
|
2084
|
+
return fail2(invocation, "failed", caughtError(
|
|
2085
|
+
{ actionId: invocation.actionId, source: "execution" },
|
|
2086
|
+
new Error(`Retryable execution not retried: ${reason}; ${errorMessage(error)}`)
|
|
2087
|
+
));
|
|
2088
|
+
}
|
|
2089
|
+
return fail2(invocation, "failed", caughtError({ actionId: invocation.actionId, source: "execution" }, error));
|
|
1883
2090
|
}
|
|
1884
2091
|
}
|
|
1885
2092
|
async function resumeApprovedInvocation(actionInvocationId, tenantId, spaceId, decision) {
|
|
@@ -1962,9 +2169,9 @@ function createGovernedActionHost(options) {
|
|
|
1962
2169
|
}
|
|
1963
2170
|
async function recordExecutionAttestation(actionInvocationId, tenantId, spaceId, attestation, adapterInvocationId) {
|
|
1964
2171
|
const governanceStore = asGovernanceStore(options.store);
|
|
1965
|
-
if (!governanceStore) throw new
|
|
2172
|
+
if (!governanceStore) throw new HostError("Execution attestations require a governance-capable store.");
|
|
1966
2173
|
const invocation = await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
1967
|
-
if (!invocation) throw new
|
|
2174
|
+
if (!invocation) throw new HostError(`ActionInvocation not found: ${actionInvocationId}`);
|
|
1968
2175
|
const id = lifecycleId("att", actionInvocationId, adapterInvocationId ?? `${attestation.provider}:${attestation.operation}:${attestation.externalOperationId ?? "external"}`);
|
|
1969
2176
|
await governanceStore.appendExecutionAttestation({
|
|
1970
2177
|
...attestation,
|
|
@@ -2042,11 +2249,11 @@ function createGovernedActionHost(options) {
|
|
|
2042
2249
|
...traceContext ? { traceContext } : {}
|
|
2043
2250
|
};
|
|
2044
2251
|
if (transaction) {
|
|
2045
|
-
if (!transaction.appendEventWithOutbox) throw new
|
|
2252
|
+
if (!transaction.appendEventWithOutbox) throw new HostError("Outbox egress requires an atomic transaction with appendEventWithOutbox.");
|
|
2046
2253
|
await transaction.appendEventWithOutbox(envelope, metadata);
|
|
2047
2254
|
} else {
|
|
2048
2255
|
const outboxStore = asOutboxStore(options.store);
|
|
2049
|
-
if (!outboxStore) throw new
|
|
2256
|
+
if (!outboxStore) throw new HostError("Outbox egress requires an outbox-capable Platform Host store.");
|
|
2050
2257
|
await outboxStore.appendEventWithOutbox(envelope, metadata);
|
|
2051
2258
|
}
|
|
2052
2259
|
} else {
|
|
@@ -2063,12 +2270,20 @@ function createGovernedActionHost(options) {
|
|
|
2063
2270
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
2064
2271
|
};
|
|
2065
2272
|
}
|
|
2273
|
+
async function persistReconciliationVerdict(invocation, patch) {
|
|
2274
|
+
try {
|
|
2275
|
+
await persistInvocation(invocation, patch);
|
|
2276
|
+
} catch (error) {
|
|
2277
|
+
if (error instanceof RecoverableFinalizationError) throw error;
|
|
2278
|
+
await persistInvocation(invocation, patch);
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2066
2281
|
async function persistInvocation(invocation, patch) {
|
|
2067
2282
|
const previousStatus = invocation.status;
|
|
2068
2283
|
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
2069
2284
|
const recoverable = options.store;
|
|
2070
2285
|
if (!recoverable.updateLeasedActionInvocation) {
|
|
2071
|
-
throw new
|
|
2286
|
+
throw new HostError("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
2072
2287
|
}
|
|
2073
2288
|
const updated = await recoverable.updateLeasedActionInvocation({
|
|
2074
2289
|
id: invocation.id,
|
|
@@ -2093,6 +2308,8 @@ function createGovernedActionHost(options) {
|
|
|
2093
2308
|
const externalCompletion = createExternalCompletionLifecycle({
|
|
2094
2309
|
store: options.store,
|
|
2095
2310
|
telemetry: options.telemetry,
|
|
2311
|
+
redactErrorText: options.redactErrorText,
|
|
2312
|
+
redactExternalCompletionResult: options.redactExternalCompletionResult,
|
|
2096
2313
|
eventResultFields,
|
|
2097
2314
|
now,
|
|
2098
2315
|
actionResolver,
|
|
@@ -2173,9 +2390,6 @@ function initialState(machine) {
|
|
|
2173
2390
|
function isTerminal(status) {
|
|
2174
2391
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
2175
2392
|
}
|
|
2176
|
-
function isRecord(value) {
|
|
2177
|
-
return Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
2178
|
-
}
|
|
2179
2393
|
function isAdmissionTime(value) {
|
|
2180
2394
|
return value instanceof Date && Number.isFinite(value.getTime());
|
|
2181
2395
|
}
|
|
@@ -2478,6 +2692,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2478
2692
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
2479
2693
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
2480
2694
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
2695
|
+
getEntityStateSource: async (tenantId, spaceId, entityType, entityId) => {
|
|
2696
|
+
let source = await this.getEntityStateSource(tenantId, spaceId, entityType, entityId);
|
|
2697
|
+
for (const { event } of pendingEvents) {
|
|
2698
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2699
|
+
const state = event.payload?.toState;
|
|
2700
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2701
|
+
}
|
|
2702
|
+
return source;
|
|
2703
|
+
},
|
|
2481
2704
|
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
2482
2705
|
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
2483
2706
|
for (const pending of pendingEvents) {
|
|
@@ -2625,11 +2848,11 @@ var MemoryPlatformHostStore = class {
|
|
|
2625
2848
|
if (this.adapterInvocations.some((candidate) => candidate.id === record.id)) return;
|
|
2626
2849
|
this.adapterInvocations.push(record);
|
|
2627
2850
|
}
|
|
2628
|
-
async getAdapterInvocation(id) {
|
|
2629
|
-
return this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2851
|
+
async getAdapterInvocation(id, scope) {
|
|
2852
|
+
return this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2630
2853
|
}
|
|
2631
|
-
async updateAdapterInvocation(id, patch) {
|
|
2632
|
-
const record = this.adapterInvocations.find((candidate) => candidate.id === id);
|
|
2854
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
2855
|
+
const record = this.adapterInvocations.find((candidate) => candidate.id === id && (scope === void 0 || candidate.tenantId === scope.tenantId && candidate.spaceId === scope.spaceId && candidate.actionInvocationId === scope.actionInvocationId));
|
|
2633
2856
|
if (!record) throw new Error(`AdapterInvocation not found: ${id}`);
|
|
2634
2857
|
Object.assign(record, patch);
|
|
2635
2858
|
}
|
|
@@ -2734,6 +2957,15 @@ var MemoryPlatformHostStore = class {
|
|
|
2734
2957
|
async nextEventSequence(tenantId, spaceId) {
|
|
2735
2958
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).length + 1;
|
|
2736
2959
|
}
|
|
2960
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
2961
|
+
let source;
|
|
2962
|
+
for (const event of this.events) {
|
|
2963
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
2964
|
+
const state = event.payload?.toState;
|
|
2965
|
+
if (typeof state === "string") source = typeof event.actionInvocationId === "string" ? { state, actionInvocationId: event.actionInvocationId } : void 0;
|
|
2966
|
+
}
|
|
2967
|
+
return source;
|
|
2968
|
+
}
|
|
2737
2969
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
2738
2970
|
let state;
|
|
2739
2971
|
for (const event of this.events) {
|
|
@@ -2906,6 +3138,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
2906
3138
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
2907
3139
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
2908
3140
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
3141
|
+
getEntityStateSource: (tenantId, spaceId, entityType, entityId) => scoped.getEntityStateSource(tenantId, spaceId, entityType, entityId),
|
|
2909
3142
|
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2910
3143
|
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2911
3144
|
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
@@ -3359,19 +3592,19 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3359
3592
|
]
|
|
3360
3593
|
);
|
|
3361
3594
|
}
|
|
3362
|
-
async getAdapterInvocation(id) {
|
|
3595
|
+
async getAdapterInvocation(id, scope) {
|
|
3363
3596
|
const result = await this.sql.query(
|
|
3364
|
-
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1`,
|
|
3365
|
-
[id]
|
|
3597
|
+
`SELECT * FROM fabric_platform.adapter_invocations WHERE id=$1${scope ? " AND tenant_id=$2 AND space_id=$3 AND action_invocation_id=$4" : ""}`,
|
|
3598
|
+
[id, ...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []]
|
|
3366
3599
|
);
|
|
3367
3600
|
return result.rows[0] ? toAdapterRecord(result.rows[0]) : void 0;
|
|
3368
3601
|
}
|
|
3369
|
-
async updateAdapterInvocation(id, patch) {
|
|
3602
|
+
async updateAdapterInvocation(id, patch, scope) {
|
|
3370
3603
|
await this.sql.query(
|
|
3371
3604
|
`UPDATE fabric_platform.adapter_invocations SET
|
|
3372
3605
|
status=COALESCE($2,status), output=COALESCE($3::jsonb,output),
|
|
3373
3606
|
error=CASE WHEN $4::boolean THEN $5 ELSE error END,
|
|
3374
|
-
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1`,
|
|
3607
|
+
attempt=COALESCE($6,attempt), updated_at=COALESCE($7,now()) WHERE id=$1${scope ? " AND tenant_id=$8 AND space_id=$9 AND action_invocation_id=$10" : ""}`,
|
|
3375
3608
|
[
|
|
3376
3609
|
id,
|
|
3377
3610
|
patch.status ?? null,
|
|
@@ -3379,7 +3612,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3379
3612
|
Object.hasOwn(patch, "error"),
|
|
3380
3613
|
patch.error ?? null,
|
|
3381
3614
|
patch.attempt ?? null,
|
|
3382
|
-
patch.updatedAt ?? null
|
|
3615
|
+
patch.updatedAt ?? null,
|
|
3616
|
+
...scope ? [scope.tenantId, scope.spaceId, scope.actionInvocationId] : []
|
|
3383
3617
|
]
|
|
3384
3618
|
);
|
|
3385
3619
|
}
|
|
@@ -3610,6 +3844,16 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
3610
3844
|
);
|
|
3611
3845
|
return Number(result.rows[0].next_sequence);
|
|
3612
3846
|
}
|
|
3847
|
+
async getEntityStateSource(tenantId, spaceId, entityType, entityId) {
|
|
3848
|
+
const result = await this.sql.query(
|
|
3849
|
+
`SELECT payload->>'toState' AS state, action_invocation_id FROM fabric_platform.asset_events
|
|
3850
|
+
WHERE tenant_id=$1 AND space_id=$2 AND subject_type=$3 AND subject_id=$4
|
|
3851
|
+
AND payload ? 'toState' ORDER BY sequence DESC LIMIT 1`,
|
|
3852
|
+
[tenantId, spaceId, entityType, entityId]
|
|
3853
|
+
);
|
|
3854
|
+
const row = result.rows[0];
|
|
3855
|
+
return typeof row?.state === "string" && typeof row.action_invocation_id === "string" ? { state: row.state, actionInvocationId: row.action_invocation_id } : void 0;
|
|
3856
|
+
}
|
|
3613
3857
|
async getEntityState(tenantId, spaceId, entityType, entityId) {
|
|
3614
3858
|
const result = await this.sql.query(
|
|
3615
3859
|
`SELECT payload->>'toState' AS state FROM fabric_platform.asset_events
|
|
@@ -4002,8 +4246,6 @@ function toOutboxRequeueRecord(row, prefix) {
|
|
|
4002
4246
|
priorLeaseToken: Number(field("prior_lease_token"))
|
|
4003
4247
|
};
|
|
4004
4248
|
}
|
|
4005
|
-
|
|
4006
|
-
// src/worker.ts
|
|
4007
4249
|
var DEFAULT_BATCH_SIZE = 10;
|
|
4008
4250
|
var DEFAULT_LEASE_DURATION_MS = 5 * 6e4;
|
|
4009
4251
|
var DEFAULT_POLL_INTERVAL_MS = 1e3;
|
|
@@ -4052,6 +4294,7 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4052
4294
|
let completed = 0;
|
|
4053
4295
|
let failed = 0;
|
|
4054
4296
|
let waitingForApproval = 0;
|
|
4297
|
+
let deferred = 0;
|
|
4055
4298
|
for (const invocation of claimed) {
|
|
4056
4299
|
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
4057
4300
|
try {
|
|
@@ -4065,7 +4308,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4065
4308
|
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
4066
4309
|
else failed += 1;
|
|
4067
4310
|
} catch (error) {
|
|
4068
|
-
|
|
4311
|
+
if (isRetryableExecutionError(error)) deferred += 1;
|
|
4312
|
+
else failed += 1;
|
|
4069
4313
|
options.onError?.(error, invocation);
|
|
4070
4314
|
} finally {
|
|
4071
4315
|
await stopHeartbeat();
|
|
@@ -4075,7 +4319,8 @@ async function runPlatformActionWorkerCycle(options) {
|
|
|
4075
4319
|
claimed: claimed.length,
|
|
4076
4320
|
completed,
|
|
4077
4321
|
failed,
|
|
4078
|
-
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
4322
|
+
...waitingForApproval > 0 ? { waitingForApproval } : {},
|
|
4323
|
+
...deferred > 0 ? { deferred } : {}
|
|
4079
4324
|
};
|
|
4080
4325
|
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
4081
4326
|
return result;
|
|
@@ -4302,6 +4547,32 @@ function submitInput(subject, scope, idempotencyKey) {
|
|
|
4302
4547
|
idempotencyKey
|
|
4303
4548
|
};
|
|
4304
4549
|
}
|
|
4550
|
+
async function adapterProbe(subject) {
|
|
4551
|
+
await subject.reset?.();
|
|
4552
|
+
if (subject.ours.tenantId === subject.theirs.tenantId) fail("the two scopes must be in different tenants");
|
|
4553
|
+
const suffix = globalThis.crypto.randomUUID();
|
|
4554
|
+
const scope = {
|
|
4555
|
+
tenantId: subject.ours.tenantId,
|
|
4556
|
+
spaceId: `iso-probe-${suffix}`,
|
|
4557
|
+
actionInvocationId: `iso_act_${suffix}`
|
|
4558
|
+
};
|
|
4559
|
+
const id = `iso_adp_${suffix}`;
|
|
4560
|
+
const now = /* @__PURE__ */ new Date();
|
|
4561
|
+
await subject.store.createAdapterInvocation({
|
|
4562
|
+
id,
|
|
4563
|
+
...scope,
|
|
4564
|
+
adapterType: "iso",
|
|
4565
|
+
operation: "probe",
|
|
4566
|
+
vendor: "iso",
|
|
4567
|
+
status: "running",
|
|
4568
|
+
input: {},
|
|
4569
|
+
output: { probe: true },
|
|
4570
|
+
attempt: 1,
|
|
4571
|
+
createdAt: now,
|
|
4572
|
+
updatedAt: now
|
|
4573
|
+
});
|
|
4574
|
+
return { id, scope };
|
|
4575
|
+
}
|
|
4305
4576
|
function tenantIsolationChecks() {
|
|
4306
4577
|
return [
|
|
4307
4578
|
{
|
|
@@ -4446,6 +4717,54 @@ function tenantIsolationChecks() {
|
|
|
4446
4717
|
if (leaked.length > 0) fail(`a viewer in the other tenant saw ${leaked.length} of our rows`);
|
|
4447
4718
|
return [theirs.allowed ? "other tenant's viewer sees none of our rows" : "other tenant's viewer is denied"];
|
|
4448
4719
|
}
|
|
4720
|
+
},
|
|
4721
|
+
{
|
|
4722
|
+
id: "fabric.tenancy.adapter-invocation-read-scoped.v1",
|
|
4723
|
+
async run(subject) {
|
|
4724
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4725
|
+
try {
|
|
4726
|
+
const theirs = { ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId };
|
|
4727
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, tenantId: subject.theirs.tenantId })) fail("an adapter invocation was readable under another tenant's scope");
|
|
4728
|
+
if (await subject.store.getAdapterInvocation(id, theirs)) fail("an adapter invocation was readable under another tenant's scope");
|
|
4729
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, spaceId: `${scope.spaceId}-other` })) fail("an adapter invocation was readable under another space's scope");
|
|
4730
|
+
if (await subject.store.getAdapterInvocation(id, { ...scope, actionInvocationId: `${scope.actionInvocationId}-other` })) fail("an adapter invocation was readable under another action invocation's scope");
|
|
4731
|
+
const ours = await subject.store.getAdapterInvocation(id, scope);
|
|
4732
|
+
if (!ours || ours.id !== id || ours.tenantId !== scope.tenantId || ours.spaceId !== scope.spaceId || ours.actionInvocationId !== scope.actionInvocationId) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4733
|
+
} finally {
|
|
4734
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4735
|
+
}
|
|
4736
|
+
return ["adapter invocation readable under our scope and invisible outside it"];
|
|
4737
|
+
}
|
|
4738
|
+
},
|
|
4739
|
+
{
|
|
4740
|
+
id: "fabric.tenancy.adapter-invocation-write-scoped.v1",
|
|
4741
|
+
async run(subject) {
|
|
4742
|
+
const { id, scope } = await adapterProbe(subject);
|
|
4743
|
+
const shape = (record) => JSON.stringify({ status: record.status, error: record.error, output: record.output, attempt: record.attempt });
|
|
4744
|
+
try {
|
|
4745
|
+
const before = await subject.store.getAdapterInvocation(id, scope);
|
|
4746
|
+
if (!before) fail("our own adapter invocation was not readable under our scope, so the check proves nothing");
|
|
4747
|
+
const untouched = shape(before);
|
|
4748
|
+
for (const other of [
|
|
4749
|
+
{ ...scope, tenantId: subject.theirs.tenantId, spaceId: subject.theirs.spaceId },
|
|
4750
|
+
{ ...scope, tenantId: subject.theirs.tenantId },
|
|
4751
|
+
{ ...scope, spaceId: `${scope.spaceId}-other` },
|
|
4752
|
+
{ ...scope, actionInvocationId: `${scope.actionInvocationId}-other` }
|
|
4753
|
+
]) {
|
|
4754
|
+
try {
|
|
4755
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "iso probe", updatedAt: /* @__PURE__ */ new Date() }, other);
|
|
4756
|
+
} catch {
|
|
4757
|
+
}
|
|
4758
|
+
const after = await subject.store.getAdapterInvocation(id, scope);
|
|
4759
|
+
if (!after || shape(after) !== untouched) fail("an adapter invocation changed under another scope");
|
|
4760
|
+
}
|
|
4761
|
+
await subject.store.updateAdapterInvocation(id, { status: "succeeded", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4762
|
+
if ((await subject.store.getAdapterInvocation(id, scope))?.status !== "succeeded") fail("our own adapter invocation was not updated under our scope, so the check proves nothing");
|
|
4763
|
+
} finally {
|
|
4764
|
+
await subject.store.updateAdapterInvocation(id, { status: "failed", error: "tenant isolation probe; retired", updatedAt: /* @__PURE__ */ new Date() }, scope);
|
|
4765
|
+
}
|
|
4766
|
+
return ["adapter invocation untouched outside our scope and writable under ours"];
|
|
4767
|
+
}
|
|
4449
4768
|
}
|
|
4450
4769
|
];
|
|
4451
4770
|
}
|