@fabricorg/platform-host 5.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,6 +1,7 @@
1
1
  'use strict';
2
2
 
3
3
  var platform = require('@fabricorg/platform');
4
+ var assembly = require('@fabricorg/assembly');
4
5
  var crypto = require('crypto');
5
6
 
6
7
  // src/host.ts
@@ -64,6 +65,213 @@ function normalizeProvenance(input, options) {
64
65
  };
65
66
  }
66
67
 
68
+ // src/observability.ts
69
+ var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
70
+ var PLATFORM_HOST_METRIC_NAMES = {
71
+ workerCyclesStarted: "fabric.platform.worker.cycles.started",
72
+ workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
73
+ workerCyclesFailed: "fabric.platform.worker.cycles.failed",
74
+ invocationSubmitted: "fabric.platform.invocation.submitted",
75
+ invocationExecutionStarted: "fabric.platform.invocation.execution_started",
76
+ invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
77
+ invocationCompleted: "fabric.platform.invocation.completed",
78
+ invocationFailed: "fabric.platform.invocation.failed",
79
+ invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
80
+ invocationValidationFailed: "fabric.platform.invocation.validation_failed",
81
+ invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
82
+ invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
83
+ outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
84
+ outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
85
+ outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
86
+ outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
87
+ outboxPublished: "fabric.platform.outbox.published",
88
+ outboxFailed: "fabric.platform.outbox.failed",
89
+ outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
90
+ invocationBacklog: "fabric.platform.invocation.backlog",
91
+ invocationRunning: "fabric.platform.invocation.running",
92
+ invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
93
+ invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
94
+ invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
95
+ outboxBacklog: "fabric.platform.outbox.backlog",
96
+ outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
97
+ outboxDeadLetters: "fabric.platform.outbox.dead_letters"
98
+ };
99
+ function emitPlatformHostTelemetry(telemetry, input) {
100
+ if (!telemetry) return;
101
+ const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
102
+ try {
103
+ const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
104
+ if (isPromiseLike(result)) void result.catch(() => void 0);
105
+ } catch {
106
+ }
107
+ }
108
+ async function getPlatformHostHealthSnapshot(options) {
109
+ const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
110
+ const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
111
+ const scope = {
112
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
113
+ ...options.spaceId ? { spaceId: options.spaceId } : {}
114
+ };
115
+ try {
116
+ const counts = await readHealthCounts(options.store, {
117
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
118
+ ...options.spaceId ? { spaceId: options.spaceId } : {},
119
+ now: generatedAt
120
+ });
121
+ const snapshot = createHealthSnapshot(
122
+ generatedAt,
123
+ scope,
124
+ counts,
125
+ workerSummary,
126
+ true
127
+ );
128
+ emitHealthMetrics(options.telemetry, snapshot);
129
+ return snapshot;
130
+ } catch {
131
+ const snapshot = createHealthSnapshot(
132
+ generatedAt,
133
+ scope,
134
+ zeroHealthCounts(),
135
+ workerSummary,
136
+ false
137
+ );
138
+ emitHealthMetrics(options.telemetry, snapshot);
139
+ return snapshot;
140
+ }
141
+ }
142
+ async function readHealthCounts(store, query) {
143
+ const candidate = store;
144
+ if (typeof candidate.getHealthCounts === "function") {
145
+ return assertHealthCounts(await candidate.getHealthCounts(query));
146
+ }
147
+ const recoverable = store;
148
+ const outbox = store;
149
+ if (typeof recoverable.listActionInvocations !== "function") {
150
+ throw new Error("Health counts are not supported by this store.");
151
+ }
152
+ const invocations = await recoverable.listActionInvocations({
153
+ ...query.tenantId ? { tenantId: query.tenantId } : {},
154
+ ...query.spaceId ? { spaceId: query.spaceId } : {},
155
+ limit: Number.MAX_SAFE_INTEGER
156
+ });
157
+ const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
158
+ ...query.tenantId ? { tenantId: query.tenantId } : {},
159
+ ...query.spaceId ? { spaceId: query.spaceId } : {}
160
+ }) : [];
161
+ return countsFromRecords(invocations, outboxRecords, query.now);
162
+ }
163
+ function countsFromRecords(invocations, outbox, now) {
164
+ return {
165
+ invocationBacklog: invocations.filter((item) => item.status === "pending").length,
166
+ invocationRunning: invocations.filter((item) => item.status === "running").length,
167
+ invocationExpiredLeases: invocations.filter(
168
+ (item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
169
+ ).length,
170
+ invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
171
+ invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
172
+ outboxBacklog: outbox.filter((item) => item.status === "pending").length,
173
+ outboxExpiredLeases: outbox.filter(
174
+ (item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
175
+ ).length,
176
+ outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
177
+ };
178
+ }
179
+ function assertHealthCounts(value) {
180
+ for (const [name, count] of Object.entries(value)) {
181
+ if (!Number.isSafeInteger(count) || count < 0) {
182
+ throw new Error(`Invalid health count: ${name}`);
183
+ }
184
+ }
185
+ return value;
186
+ }
187
+ function zeroHealthCounts() {
188
+ return {
189
+ invocationBacklog: 0,
190
+ invocationRunning: 0,
191
+ invocationExpiredLeases: 0,
192
+ invocationApprovalWaits: 0,
193
+ invocationReconciliationRequired: 0,
194
+ outboxBacklog: 0,
195
+ outboxExpiredLeases: 0,
196
+ outboxDeadLetters: 0
197
+ };
198
+ }
199
+ function summarizeWorkers(workers, now) {
200
+ let healthy = 0;
201
+ let stale = 0;
202
+ for (const worker of workers) {
203
+ if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
204
+ else stale += 1;
205
+ }
206
+ return { reported: workers.length, healthy, stale };
207
+ }
208
+ function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
209
+ const reasonCodes = [];
210
+ if (!dataAvailable) reasonCodes.push("health_source_unavailable");
211
+ if (workers.stale > 0) reasonCodes.push("workers_stale");
212
+ if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
213
+ if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
214
+ if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
215
+ if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
216
+ const unhealthy = !dataAvailable;
217
+ const degraded = unhealthy || reasonCodes.length > 0;
218
+ return {
219
+ contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
220
+ generatedAt,
221
+ scope,
222
+ dataAvailable,
223
+ status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
224
+ readiness: {
225
+ ready: dataAvailable && workers.stale === 0,
226
+ reasonCodes
227
+ },
228
+ workers,
229
+ invocations: {
230
+ backlog: counts.invocationBacklog,
231
+ running: counts.invocationRunning,
232
+ expiredLeases: counts.invocationExpiredLeases,
233
+ approvalWaits: counts.invocationApprovalWaits,
234
+ reconciliationRequired: counts.invocationReconciliationRequired
235
+ },
236
+ outbox: {
237
+ backlog: counts.outboxBacklog,
238
+ expiredLeases: counts.outboxExpiredLeases,
239
+ deadLetters: counts.outboxDeadLetters
240
+ }
241
+ };
242
+ }
243
+ function emitHealthMetrics(telemetry, snapshot) {
244
+ const scope = {
245
+ ...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
246
+ ...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
247
+ };
248
+ const metrics = [
249
+ [PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
250
+ [PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
251
+ [PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
252
+ [PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
253
+ [PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
254
+ [PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
255
+ [PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
256
+ [PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
257
+ ];
258
+ for (const [name, value] of metrics) {
259
+ emitPlatformHostTelemetry(telemetry, {
260
+ kind: "metric",
261
+ name,
262
+ metricType: "gauge",
263
+ value,
264
+ ...scope,
265
+ observedAt: snapshot.generatedAt
266
+ });
267
+ }
268
+ }
269
+ function isPromiseLike(value) {
270
+ return Boolean(
271
+ value && typeof value === "object" && "then" in value && typeof value.then === "function"
272
+ );
273
+ }
274
+
67
275
  // src/host.ts
68
276
  var DEFAULT_EXTRACT_EVENTS = (data) => {
69
277
  const value = data._events;
@@ -73,6 +281,19 @@ var RecoverableFinalizationError = class extends Error {
73
281
  name = "RecoverableFinalizationError";
74
282
  };
75
283
  function createGovernedActionHost(options) {
284
+ if (options.composition) {
285
+ if (!options.registry?.orderedModules) {
286
+ throw new Error("Composition-bound Platform Host requires an explicit module registry.");
287
+ }
288
+ assembly.assertAssemblyRuntimeCompatible(
289
+ options.composition.assembly,
290
+ options.registry.orderedModules.map((module) => ({
291
+ namespace: module.namespace,
292
+ version: module.version ?? "",
293
+ manifestDigest: module.manifestDigest ?? ""
294
+ }))
295
+ );
296
+ }
76
297
  if (options.outbox) {
77
298
  const outboxStore = asOutboxStore(options.store);
78
299
  if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
@@ -118,10 +339,13 @@ function createGovernedActionHost(options) {
118
339
  input.provenance,
119
340
  options.provenance
120
341
  );
342
+ const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
121
343
  const runtimeEvidence = {
122
344
  governanceContractVersion: platform.FABRIC_GOVERNANCE_CONTRACT_VERSION,
123
345
  hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
124
- ...options.runtimeEvidence
346
+ ...options.runtimeEvidence,
347
+ ...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
348
+ ...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
125
349
  };
126
350
  platform.assertGovernanceRuntimeEvidence(runtimeEvidence);
127
351
  const durableInvocation = await options.store.createActionInvocation({
@@ -150,6 +374,15 @@ function createGovernedActionHost(options) {
150
374
  ...input.executionReason ? { executionReason: input.executionReason } : {},
151
375
  ...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
152
376
  });
377
+ emitPlatformHostTelemetry(options.telemetry, {
378
+ kind: "event",
379
+ name: "invocation.submitted",
380
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
381
+ occurredAt: now(),
382
+ tenantId: input.tenantId,
383
+ spaceId: input.spaceId,
384
+ attributes: { actionVersion: action.version }
385
+ });
153
386
  if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
154
387
  const conflict = idempotencyConflict(durableInvocation, {
155
388
  actorId: input.actorId,
@@ -172,6 +405,23 @@ function createGovernedActionHost(options) {
172
405
  }
173
406
  const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
174
407
  if (durableInvocation.id !== actionInvocationId) {
408
+ if (options.dispatcher && durableInvocation.status === "pending") {
409
+ const dispatched = await options.dispatcher.dispatch({
410
+ actionInvocationId: durableInvocation.id,
411
+ actionId: durableInvocation.actionId,
412
+ tenantId: input.tenantId,
413
+ spaceId: input.spaceId,
414
+ workflowId: durableWorkflowId
415
+ });
416
+ return withConsistency({
417
+ actionInvocationId: durableInvocation.id,
418
+ status: durableInvocation.status,
419
+ workflowId: dispatched.workflowId,
420
+ ...dispatched.runId ? { runId: dispatched.runId } : {},
421
+ ...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
422
+ ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
423
+ }, input.actionId);
424
+ }
175
425
  return withConsistency({
176
426
  actionInvocationId: durableInvocation.id,
177
427
  status: durableInvocation.status,
@@ -180,13 +430,15 @@ function createGovernedActionHost(options) {
180
430
  ...durableInvocation.error ? { error: durableInvocation.error } : {},
181
431
  ...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
182
432
  ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
183
- ...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
433
+ ...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
434
+ ...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
184
435
  }, input.actionId);
185
436
  }
186
437
  if (options.dispatcher) {
187
438
  try {
188
439
  const dispatched = await options.dispatcher.dispatch({
189
440
  actionInvocationId: durableInvocation.id,
441
+ actionId: durableInvocation.actionId,
190
442
  tenantId: input.tenantId,
191
443
  spaceId: input.spaceId,
192
444
  workflowId: durableWorkflowId
@@ -200,13 +452,6 @@ function createGovernedActionHost(options) {
200
452
  ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
201
453
  }, input.actionId);
202
454
  } catch (error) {
203
- const message = errorMessage(error);
204
- await options.store.updateActionInvocation(
205
- durableInvocation.id,
206
- input.tenantId,
207
- input.spaceId,
208
- { status: "failed", error: message }
209
- );
210
455
  throw error;
211
456
  }
212
457
  }
@@ -231,16 +476,30 @@ function createGovernedActionHost(options) {
231
476
  if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
232
477
  return withConsistency(actionResult(invocation), invocation.actionId);
233
478
  }
234
- if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
479
+ if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
235
480
  return {
236
481
  ...actionResult(invocation),
237
- error: `Invocation is leased by ${invocation.leaseOwner}`
482
+ error: `Invocation lease is not owned by the supplied worker generation`
238
483
  };
239
484
  }
240
485
  const action = actionResolver(invocation.actionId);
241
486
  if (!action) {
242
487
  return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
243
488
  }
489
+ if (action.kind === "saga") {
490
+ return fail(
491
+ invocation,
492
+ "failed",
493
+ `Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
494
+ );
495
+ }
496
+ if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
497
+ return fail(
498
+ invocation,
499
+ "failed",
500
+ `Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
501
+ );
502
+ }
244
503
  if (invocation.attemptCount > 1 && !action.idempotent) {
245
504
  return fail(
246
505
  invocation,
@@ -253,6 +512,15 @@ function createGovernedActionHost(options) {
253
512
  status: "running"
254
513
  });
255
514
  }
515
+ emitPlatformHostTelemetry(options.telemetry, {
516
+ kind: "event",
517
+ name: "invocation.execution_started",
518
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
519
+ occurredAt: now(),
520
+ tenantId,
521
+ spaceId,
522
+ attributes: { actionVersion: invocation.actionVersion }
523
+ });
256
524
  try {
257
525
  const parsed = action.schema.safeParse(invocation.parameters);
258
526
  if (!parsed.success) {
@@ -318,7 +586,7 @@ function createGovernedActionHost(options) {
318
586
  );
319
587
  }
320
588
  if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
321
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, {
589
+ await persistInvocation(invocation, {
322
590
  status: "waiting_for_approval"
323
591
  });
324
592
  return {
@@ -345,7 +613,7 @@ function createGovernedActionHost(options) {
345
613
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
346
614
  message: `Action ${action.actionId} requires durable capture-time authorization evidence`
347
615
  };
348
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
616
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
349
617
  return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
350
618
  }
351
619
  const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
@@ -358,7 +626,7 @@ function createGovernedActionHost(options) {
358
626
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
359
627
  message: `Authorization binding for action ${action.actionId} expired before execution`
360
628
  };
361
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
629
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
362
630
  return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
363
631
  }
364
632
  const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
@@ -380,7 +648,7 @@ function createGovernedActionHost(options) {
380
648
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
381
649
  message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
382
650
  };
383
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
651
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
384
652
  return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
385
653
  }
386
654
  const definitions = options.resolvePolicies ? await options.resolvePolicies({
@@ -442,30 +710,6 @@ function createGovernedActionHost(options) {
442
710
  })));
443
711
  }
444
712
  }
445
- const binding = action.stateMachine;
446
- if (binding) {
447
- const entityId = binding.getEntityId(parsed.data);
448
- const currentState = entityId ? await options.store.getEntityState(
449
- tenantId,
450
- spaceId,
451
- binding.entityType,
452
- entityId
453
- ) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
454
- const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
455
- if (targetState !== "") {
456
- const transition = platform.validateStateMachineTransition(
457
- stateMachineResolver(binding.entityType),
458
- binding.entityType,
459
- currentState,
460
- targetState,
461
- action.actionId
462
- );
463
- const replayingAppliedTransition = action.idempotent && currentState === targetState;
464
- if (!transition.valid && !replayingAppliedTransition) {
465
- return fail(invocation, "failed", transition.error ?? "Invalid state transition");
466
- }
467
- }
468
- }
469
713
  let data;
470
714
  let domainEvents = [];
471
715
  try {
@@ -473,6 +717,42 @@ function createGovernedActionHost(options) {
473
717
  if (options.outbox && !transaction?.appendEventWithOutbox) {
474
718
  throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
475
719
  }
720
+ const binding = action.stateMachine;
721
+ if (binding) {
722
+ const machine = stateMachineResolver(binding.entityType);
723
+ const entityId = binding.getEntityId(parsed.data);
724
+ const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
725
+ const currentState = entityId ? await getEntityState(
726
+ tenantId,
727
+ spaceId,
728
+ binding.entityType,
729
+ entityId
730
+ ) ?? initialState(machine) : initialState(machine);
731
+ const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
732
+ if (targetState !== "") {
733
+ const transition = await platform.evaluateStateMachineTransition(
734
+ machine,
735
+ binding.entityType,
736
+ currentState,
737
+ targetState,
738
+ action.actionId,
739
+ {
740
+ entity: {
741
+ ...entityId ? { id: entityId } : {},
742
+ state: currentState
743
+ },
744
+ actionInvocationId,
745
+ actorId: invocation.actorId,
746
+ parameters: parsed.data,
747
+ db
748
+ }
749
+ );
750
+ const replayingAppliedTransition = action.idempotent && currentState === targetState;
751
+ if (!transition.valid && !replayingAppliedTransition) {
752
+ throw new Error(transition.error ?? "Invalid state transition");
753
+ }
754
+ }
755
+ }
476
756
  const handlerResult = action.handler ? await action.handler(
477
757
  {
478
758
  actionInvocationId,
@@ -572,7 +852,21 @@ function createGovernedActionHost(options) {
572
852
  subjectId: adapterEventSubject.subjectId,
573
853
  payload: { adapterType: step.adapterType, operation: step.operation }
574
854
  }, `adapter:${stepIndex}:started`);
855
+ let adapterDeadlineTimer;
575
856
  try {
857
+ const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
858
+ const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
859
+ if (adapterController && deadlineMs) {
860
+ const remaining = deadlineMs - now().getTime();
861
+ if (remaining <= 0) {
862
+ throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
863
+ }
864
+ adapterDeadlineTimer = setTimeout(
865
+ () => adapterController.abort(new Error("Adapter deadline exceeded")),
866
+ remaining
867
+ );
868
+ adapterDeadlineTimer.unref?.();
869
+ }
576
870
  const result2 = await platform.executeWithAdapterRetry({
577
871
  policy: step.retryPolicy ?? adapter.retryPolicy,
578
872
  defaultIdempotent: adapter.idempotent,
@@ -589,13 +883,77 @@ function createGovernedActionHost(options) {
589
883
  correlationId: invocation.correlationId,
590
884
  ...invocation.causationId ? { causationId: invocation.causationId } : {},
591
885
  attempt,
592
- maxAttempts
886
+ maxAttempts,
887
+ ...deadlineMs ? { deadlineMs } : {},
888
+ ...adapterController ? { signal: adapterController.signal } : {}
593
889
  });
594
890
  },
595
891
  isSuccessful: (result3) => result3.success,
596
- getError: (result3) => result3.error
892
+ getError: (result3) => result3.error,
893
+ classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
894
+ classifyThrownError: (error) => {
895
+ if (adapterController?.signal.aborted) return "timeout";
896
+ const message = error instanceof Error ? error.message : String(error);
897
+ if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
898
+ return "transient_failure";
899
+ }
597
900
  });
901
+ platform.assertAdapterOutcomeConsistency(result2);
598
902
  if (!result2.success) {
903
+ const outcome = result2.outcome ?? "transient_failure";
904
+ if (outcome === "ambiguous") {
905
+ const evidence = result2.outcomeEvidence;
906
+ const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
907
+ const adapterReconciliation = {
908
+ kind: "adapter_outcome_ambiguous",
909
+ adapterType: step.adapterType,
910
+ operation: step.operation,
911
+ vendor: adapter.vendor,
912
+ adapterInvocationId,
913
+ ...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
914
+ ...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
915
+ message
916
+ };
917
+ await options.store.updateAdapterInvocation(adapterInvocationId, {
918
+ status: "ambiguous",
919
+ error: message,
920
+ updatedAt: now()
921
+ });
922
+ await appendEvent(invocation, {
923
+ eventType: "AdapterInvocationAmbiguous",
924
+ subjectType: adapterEventSubject.subjectType,
925
+ subjectId: adapterEventSubject.subjectId,
926
+ payload: {
927
+ adapterType: step.adapterType,
928
+ operation: step.operation,
929
+ ...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
930
+ }
931
+ }, `adapter:${stepIndex}:ambiguous`);
932
+ const governanceStore2 = asGovernanceStore(options.store);
933
+ if (governanceStore2 && evidence) {
934
+ await governanceStore2.appendExternalReconciliation({
935
+ id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
936
+ actionInvocationId,
937
+ tenantId,
938
+ spaceId,
939
+ status: "pending",
940
+ provider: adapter.vendor,
941
+ ...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
942
+ attempt: 1,
943
+ reason: message,
944
+ observedAt: now()
945
+ });
946
+ }
947
+ await persistInvocation(invocation, {
948
+ status: "reconciliation_required",
949
+ error: message,
950
+ adapterReconciliation
951
+ });
952
+ return withConsistency(
953
+ { actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
954
+ action.actionId
955
+ );
956
+ }
599
957
  await options.store.updateAdapterInvocation(adapterInvocationId, {
600
958
  status: "failed",
601
959
  error: result2.error ?? "Adapter failed",
@@ -655,6 +1013,8 @@ function createGovernedActionHost(options) {
655
1013
  updatedAt: now()
656
1014
  });
657
1015
  return fail(invocation, "failed", message);
1016
+ } finally {
1017
+ if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
658
1018
  }
659
1019
  }
660
1020
  const governanceStore = asGovernanceStore(options.store);
@@ -679,20 +1039,41 @@ function createGovernedActionHost(options) {
679
1039
  transaction
680
1040
  );
681
1041
  }
682
- await transaction.updateActionInvocation(
683
- actionInvocationId,
684
- tenantId,
685
- spaceId,
686
- { status: "completed", result }
687
- );
1042
+ if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
1043
+ if (!transaction.updateLeasedActionInvocation) {
1044
+ throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
1045
+ }
1046
+ const updated = await transaction.updateLeasedActionInvocation({
1047
+ id: actionInvocationId,
1048
+ tenantId,
1049
+ spaceId,
1050
+ workerId: invocation.leaseOwner,
1051
+ leaseToken: invocation.leaseToken,
1052
+ patch: { status: "completed", result }
1053
+ });
1054
+ if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
1055
+ } else {
1056
+ await transaction.updateActionInvocation(
1057
+ actionInvocationId,
1058
+ tenantId,
1059
+ spaceId,
1060
+ { status: "completed", result }
1061
+ );
1062
+ }
688
1063
  });
1064
+ emitInvocationStatusTelemetry(
1065
+ invocation,
1066
+ "completed",
1067
+ options.telemetry,
1068
+ now()
1069
+ );
689
1070
  } else {
690
1071
  if (action.eventPhase === "after_adapters") {
691
1072
  for (const [index, event] of domainEvents.entries()) {
692
1073
  await appendEvent(invocation, event, `domain:${index}`, action.version);
693
1074
  }
694
1075
  }
695
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, {
1076
+ await persistInvocation(invocation, {
696
1077
  status: "completed",
697
1078
  result
698
1079
  });
@@ -775,12 +1156,21 @@ function createGovernedActionHost(options) {
775
1156
  if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
776
1157
  return withConsistency(actionResult(transitioned), transitioned.actionId);
777
1158
  }
778
- if (!decision.approved) return withConsistency(actionResult(transitioned), transitioned.actionId);
1159
+ if (!decision.approved) {
1160
+ emitInvocationStatusTelemetry(
1161
+ transitioned,
1162
+ "failed",
1163
+ options.telemetry,
1164
+ now(),
1165
+ "waiting_for_approval"
1166
+ );
1167
+ return withConsistency(actionResult(transitioned), transitioned.actionId);
1168
+ }
779
1169
  return executeInvocation(
780
1170
  actionInvocationId,
781
1171
  tenantId,
782
1172
  spaceId,
783
- { leaseOwner },
1173
+ { leaseOwner, leaseToken: transitioned.leaseToken },
784
1174
  "approval_resume"
785
1175
  );
786
1176
  }
@@ -876,12 +1266,7 @@ function createGovernedActionHost(options) {
876
1266
  }
877
1267
  }
878
1268
  async function fail(invocation, status, error) {
879
- await options.store.updateActionInvocation(
880
- invocation.id,
881
- invocation.tenantId,
882
- invocation.spaceId,
883
- { status, error }
884
- );
1269
+ await persistInvocation(invocation, { status, error });
885
1270
  return {
886
1271
  actionInvocationId: invocation.id,
887
1272
  status,
@@ -890,6 +1275,33 @@ function createGovernedActionHost(options) {
890
1275
  ...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
891
1276
  };
892
1277
  }
1278
+ async function persistInvocation(invocation, patch) {
1279
+ const previousStatus = invocation.status;
1280
+ if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
1281
+ const recoverable = options.store;
1282
+ if (!recoverable.updateLeasedActionInvocation) {
1283
+ throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
1284
+ }
1285
+ const updated = await recoverable.updateLeasedActionInvocation({
1286
+ id: invocation.id,
1287
+ tenantId: invocation.tenantId,
1288
+ spaceId: invocation.spaceId,
1289
+ workerId: invocation.leaseOwner,
1290
+ leaseToken: invocation.leaseToken,
1291
+ patch
1292
+ });
1293
+ if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
1294
+ emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
1295
+ return;
1296
+ }
1297
+ await options.store.updateActionInvocation(
1298
+ invocation.id,
1299
+ invocation.tenantId,
1300
+ invocation.spaceId,
1301
+ patch
1302
+ );
1303
+ emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
1304
+ }
893
1305
  return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
894
1306
  }
895
1307
  function actionResult(invocation) {
@@ -900,7 +1312,8 @@ function actionResult(invocation) {
900
1312
  ...invocation.error ? { error: invocation.error } : {},
901
1313
  ...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
902
1314
  ...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
903
- ...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
1315
+ ...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
1316
+ ...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
904
1317
  };
905
1318
  }
906
1319
  function asApprovalStore(store) {
@@ -954,6 +1367,24 @@ function initialState(machine) {
954
1367
  function isTerminal(status) {
955
1368
  return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
956
1369
  }
1370
+ function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
1371
+ if (!status) return;
1372
+ const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
1373
+ if (!event) return;
1374
+ emitPlatformHostTelemetry(telemetry, {
1375
+ kind: "event",
1376
+ name: event.name,
1377
+ metricName: event.metricName,
1378
+ occurredAt,
1379
+ tenantId: invocation.tenantId,
1380
+ spaceId: invocation.spaceId,
1381
+ attributes: {
1382
+ ...invocation.actionId ? { actionId: invocation.actionId } : {},
1383
+ ...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
1384
+ ...previousStatus ? { fromStatus: previousStatus } : {}
1385
+ }
1386
+ });
1387
+ }
957
1388
  function withoutPrivateHostFields(data, eventResultFields) {
958
1389
  return Object.fromEntries(
959
1390
  Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
@@ -1022,29 +1453,72 @@ function toEnterpriseEventEnvelope(event, metadata) {
1022
1453
  async function runOutboxRelayCycle(options) {
1023
1454
  const now = options.now ?? (() => /* @__PURE__ */ new Date());
1024
1455
  const maxAttempts = options.maxAttempts ?? 10;
1025
- const records = await options.store.claimOutbox({
1026
- workerId: options.workerId,
1027
- leaseDurationMs: options.leaseDurationMs ?? 3e4,
1028
- limit: options.batchSize ?? 100,
1029
- now: now()
1030
- });
1456
+ const leaseDurationMs = options.leaseDurationMs ?? 3e4;
1457
+ emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
1458
+ let records;
1459
+ try {
1460
+ records = await options.store.claimOutbox({
1461
+ workerId: options.workerId,
1462
+ leaseDurationMs,
1463
+ limit: options.batchSize ?? 100,
1464
+ now: now()
1465
+ });
1466
+ } catch (error) {
1467
+ emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
1468
+ throw error;
1469
+ }
1470
+ for (const record of records) {
1471
+ emitPlatformHostTelemetry(options.telemetry, {
1472
+ kind: "event",
1473
+ name: "outbox.lease_claimed",
1474
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
1475
+ occurredAt: now(),
1476
+ tenantId: record.tenantId,
1477
+ spaceId: record.spaceId,
1478
+ attributes: { attempt: record.attemptCount }
1479
+ });
1480
+ }
1031
1481
  const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
1032
1482
  for (const record of records) {
1483
+ const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
1033
1484
  try {
1034
- await options.publisher.publish(record.event);
1485
+ await publishWithTimeout(
1486
+ options.publisher,
1487
+ record.event,
1488
+ options.publishTimeoutMs ?? 1e4
1489
+ );
1490
+ await stopHeartbeat();
1035
1491
  try {
1036
- await options.store.markOutboxPublished(record.id, options.workerId, now());
1492
+ await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
1037
1493
  } catch {
1038
1494
  result.failed += 1;
1495
+ emitPlatformHostTelemetry(options.telemetry, {
1496
+ kind: "event",
1497
+ name: "outbox.failed",
1498
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
1499
+ occurredAt: now(),
1500
+ tenantId: record.tenantId,
1501
+ spaceId: record.spaceId
1502
+ });
1039
1503
  continue;
1040
1504
  }
1041
1505
  result.published += 1;
1506
+ emitPlatformHostTelemetry(options.telemetry, {
1507
+ kind: "event",
1508
+ name: "outbox.published",
1509
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
1510
+ occurredAt: now(),
1511
+ tenantId: record.tenantId,
1512
+ spaceId: record.spaceId
1513
+ });
1042
1514
  } catch {
1515
+ await stopHeartbeat();
1043
1516
  const deadLetter = record.attemptCount >= maxAttempts;
1044
1517
  try {
1045
1518
  await options.store.markOutboxFailed({
1046
1519
  id: record.id,
1047
1520
  workerId: options.workerId,
1521
+ leaseToken: record.leaseToken,
1048
1522
  error: "Event publisher failed",
1049
1523
  availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
1050
1524
  deadLetter
@@ -1053,16 +1527,84 @@ async function runOutboxRelayCycle(options) {
1053
1527
  }
1054
1528
  result.failed += 1;
1055
1529
  if (deadLetter) result.deadLettered += 1;
1530
+ emitPlatformHostTelemetry(options.telemetry, {
1531
+ kind: "event",
1532
+ name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
1533
+ metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
1534
+ occurredAt: now(),
1535
+ tenantId: record.tenantId,
1536
+ spaceId: record.spaceId,
1537
+ attributes: { attempt: record.attemptCount }
1538
+ });
1056
1539
  }
1057
1540
  }
1541
+ emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
1542
+ claimed: result.claimed,
1543
+ published: result.published,
1544
+ failed: result.failed,
1545
+ deadLettered: result.deadLettered
1546
+ });
1058
1547
  return result;
1059
1548
  }
1549
+ async function publishWithTimeout(publisher, event, timeoutMs) {
1550
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
1551
+ throw new Error("publishTimeoutMs must be a positive finite number");
1552
+ }
1553
+ const controller = new AbortController();
1554
+ let timer;
1555
+ const timeout = new Promise((_, reject) => {
1556
+ timer = setTimeout(() => {
1557
+ const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
1558
+ controller.abort(error);
1559
+ reject(error);
1560
+ }, timeoutMs);
1561
+ });
1562
+ try {
1563
+ await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
1564
+ } finally {
1565
+ if (timer) clearTimeout(timer);
1566
+ }
1567
+ }
1568
+ function startOutboxHeartbeat(options, record, leaseDurationMs) {
1569
+ if (!record.leaseToken) return async () => void 0;
1570
+ const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
1571
+ let stopped = false;
1572
+ let timer;
1573
+ let running;
1574
+ const schedule = () => {
1575
+ if (!stopped) timer = setTimeout(tick, intervalMs);
1576
+ };
1577
+ const tick = () => {
1578
+ running = options.store.renewOutboxLease({
1579
+ id: record.id,
1580
+ workerId: options.workerId,
1581
+ leaseToken: record.leaseToken,
1582
+ leaseDurationMs
1583
+ }).then(() => void 0).catch(() => void 0).finally(schedule);
1584
+ };
1585
+ schedule();
1586
+ return async () => {
1587
+ stopped = true;
1588
+ if (timer) clearTimeout(timer);
1589
+ await running;
1590
+ };
1591
+ }
1060
1592
  function cloneOutboxRecord(record) {
1061
1593
  return {
1062
1594
  ...record,
1063
1595
  event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
1064
1596
  };
1065
1597
  }
1598
+ function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
1599
+ const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
1600
+ emitPlatformHostTelemetry(telemetry, {
1601
+ kind: "event",
1602
+ name,
1603
+ metricName,
1604
+ occurredAt,
1605
+ attributes: { workerId, ...attributes }
1606
+ });
1607
+ }
1066
1608
 
1067
1609
  // src/memory-store.ts
1068
1610
  var MemoryPlatformHostStore = class {
@@ -1105,12 +1647,33 @@ var MemoryPlatformHostStore = class {
1105
1647
  };
1106
1648
  const result = await run({
1107
1649
  db: this.db,
1650
+ getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
1651
+ const record = await this.getActionInvocation(id, tenantId, spaceId);
1652
+ return record ? structuredClone(record) : void 0;
1653
+ },
1654
+ getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
1108
1655
  appendEvent: (event) => appendPending(event),
1109
1656
  appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
1110
1657
  nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
1111
1658
  listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
1659
+ getEntityState: async (tenantId, spaceId, entityType, entityId) => {
1660
+ let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
1661
+ for (const pending of pendingEvents) {
1662
+ const event = pending.event;
1663
+ if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
1664
+ const candidate = event.payload?.toState;
1665
+ if (typeof candidate === "string") state = candidate;
1666
+ }
1667
+ return state;
1668
+ },
1112
1669
  updateActionInvocation: async (id, tenantId, spaceId, patch) => {
1113
1670
  pendingUpdates.push({ id, tenantId, spaceId, patch });
1671
+ },
1672
+ updateLeasedActionInvocation: async (input) => {
1673
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1674
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
1675
+ pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
1676
+ return true;
1114
1677
  }
1115
1678
  });
1116
1679
  for (const update of pendingUpdates) {
@@ -1137,7 +1700,7 @@ var MemoryPlatformHostStore = class {
1137
1700
  if (existing) return existing;
1138
1701
  }
1139
1702
  const now = /* @__PURE__ */ new Date();
1140
- const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
1703
+ const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
1141
1704
  this.invocations.push(record);
1142
1705
  return record;
1143
1706
  }
@@ -1187,6 +1750,7 @@ var MemoryPlatformHostStore = class {
1187
1750
  record.error = void 0;
1188
1751
  record.leaseOwner = input.leaseOwner;
1189
1752
  record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
1753
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1190
1754
  record.attemptCount = Math.max(record.attemptCount, 1);
1191
1755
  return { applied: true, invocation: record };
1192
1756
  }
@@ -1264,31 +1828,41 @@ var MemoryPlatformHostStore = class {
1264
1828
  return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
1265
1829
  record.leaseOwner = input.workerId;
1266
1830
  record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1831
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1267
1832
  record.attemptCount += 1;
1268
1833
  return cloneOutboxRecord(record);
1269
1834
  });
1270
1835
  }
1271
- async markOutboxPublished(id, workerId, publishedAt) {
1272
- const record = this.requireLeasedOutbox(id, workerId);
1836
+ async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
1837
+ const record = this.requireLeasedOutbox(id, workerId, leaseToken);
1273
1838
  record.status = "published";
1274
1839
  record.publishedAt = publishedAt;
1275
1840
  delete record.leaseOwner;
1276
1841
  delete record.leaseExpiresAt;
1277
1842
  }
1278
1843
  async markOutboxFailed(input) {
1279
- const record = this.requireLeasedOutbox(input.id, input.workerId);
1844
+ const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
1280
1845
  record.status = input.deadLetter ? "dead_letter" : "pending";
1281
1846
  record.lastError = input.error;
1282
1847
  record.availableAt = input.availableAt;
1283
1848
  delete record.leaseOwner;
1284
1849
  delete record.leaseExpiresAt;
1285
1850
  }
1851
+ async renewOutboxLease(input) {
1852
+ const record = this.outbox.find((candidate) => candidate.id === input.id);
1853
+ const current = input.now ?? /* @__PURE__ */ new Date();
1854
+ if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
1855
+ record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1856
+ return true;
1857
+ }
1286
1858
  async listOutbox(input = {}) {
1287
1859
  return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
1288
1860
  }
1289
- requireLeasedOutbox(id, workerId) {
1861
+ requireLeasedOutbox(id, workerId, leaseToken) {
1290
1862
  const record = this.outbox.find((candidate) => candidate.id === id);
1291
- if (!record || record.leaseOwner !== workerId) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1863
+ if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
1864
+ throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1865
+ }
1292
1866
  return record;
1293
1867
  }
1294
1868
  async nextEventSequence(tenantId, spaceId) {
@@ -1310,24 +1884,128 @@ var MemoryPlatformHostStore = class {
1310
1884
  (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
1311
1885
  ).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
1312
1886
  }
1887
+ async getHealthCounts(input) {
1888
+ const invocations = this.invocations.filter(
1889
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
1890
+ );
1891
+ const outbox = this.outbox.filter(
1892
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
1893
+ );
1894
+ return {
1895
+ invocationBacklog: invocations.filter((item) => item.status === "pending").length,
1896
+ invocationRunning: invocations.filter((item) => item.status === "running").length,
1897
+ invocationExpiredLeases: invocations.filter(
1898
+ (item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
1899
+ ).length,
1900
+ invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
1901
+ invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
1902
+ outboxBacklog: outbox.filter((item) => item.status === "pending").length,
1903
+ outboxExpiredLeases: outbox.filter(
1904
+ (item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
1905
+ ).length,
1906
+ outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
1907
+ };
1908
+ }
1313
1909
  async claimActionInvocations(input) {
1314
1910
  const current = input.now ?? /* @__PURE__ */ new Date();
1315
1911
  const eligible = this.invocations.filter(
1316
- (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && (!record.leaseExpiresAt || record.leaseExpiresAt <= current))
1912
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
1317
1913
  ).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
1318
1914
  for (const record of eligible) {
1319
1915
  record.status = "running";
1320
1916
  record.leaseOwner = input.workerId;
1321
1917
  record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1918
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1322
1919
  record.attemptCount += 1;
1323
1920
  record.updatedAt = current;
1324
1921
  }
1325
- return eligible;
1922
+ return eligible.map((record) => structuredClone(record));
1923
+ }
1924
+ async updateLeasedActionInvocation(input) {
1925
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1926
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
1927
+ await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
1928
+ return true;
1929
+ }
1930
+ async renewActionInvocationLease(input) {
1931
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1932
+ const current = input.now ?? /* @__PURE__ */ new Date();
1933
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
1934
+ record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1935
+ record.updatedAt = current;
1936
+ return true;
1326
1937
  }
1327
1938
  async listEvents(tenantId, spaceId) {
1328
1939
  return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
1329
1940
  }
1330
1941
  };
1942
+ async function applyPostgresMigrations(client, migrations) {
1943
+ const ordered = validateMigrations(migrations);
1944
+ const statements = ordered.map((migration) => migrationStatement(migration));
1945
+ await client.query(`
1946
+ BEGIN;
1947
+ SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
1948
+ CREATE SCHEMA IF NOT EXISTS fabric_platform;
1949
+ CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
1950
+ version integer PRIMARY KEY,
1951
+ name text NOT NULL,
1952
+ checksum text NOT NULL,
1953
+ applied_at timestamptz NOT NULL DEFAULT now()
1954
+ );
1955
+ ${statements.join("\n")}
1956
+ COMMIT;
1957
+ `);
1958
+ }
1959
+ function validateMigrations(migrations) {
1960
+ const ordered = [...migrations].sort((left, right) => left.version - right.version);
1961
+ const versions = /* @__PURE__ */ new Set();
1962
+ for (const migration of ordered) {
1963
+ if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
1964
+ throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
1965
+ }
1966
+ if (versions.has(migration.version)) {
1967
+ throw new Error(`Duplicate migration version: ${migration.version}`);
1968
+ }
1969
+ if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
1970
+ throw new Error(`Invalid migration name: ${migration.name}`);
1971
+ }
1972
+ if (migration.sql.trim().length === 0) {
1973
+ throw new Error(`Migration ${migration.version} has no SQL`);
1974
+ }
1975
+ versions.add(migration.version);
1976
+ }
1977
+ return ordered;
1978
+ }
1979
+ function migrationStatement(migration) {
1980
+ const checksum = crypto.createHash("sha256").update(migration.sql).digest("hex");
1981
+ const delimiter = `$fabric_platform_migration_${migration.version}$`;
1982
+ if (migration.sql.includes(delimiter)) {
1983
+ throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
1984
+ }
1985
+ return `
1986
+ DO ${delimiter}
1987
+ DECLARE
1988
+ recorded_checksum text;
1989
+ BEGIN
1990
+ SELECT checksum INTO recorded_checksum
1991
+ FROM fabric_platform.schema_migrations
1992
+ WHERE version = ${migration.version};
1993
+
1994
+ IF FOUND THEN
1995
+ IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
1996
+ RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
1997
+ END IF;
1998
+ ELSE
1999
+ ${migration.sql}
2000
+ INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
2001
+ VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
2002
+ END IF;
2003
+ END
2004
+ ${delimiter};`;
2005
+ }
2006
+ function sqlLiteral(value) {
2007
+ return `'${value.replaceAll("'", "''")}'`;
2008
+ }
1331
2009
 
1332
2010
  // src/postgres-store.ts
1333
2011
  var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
@@ -1340,11 +2018,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1340
2018
  const scoped = new _PostgresPlatformHostStore(db2, sql2);
1341
2019
  return run({
1342
2020
  db: db2,
2021
+ getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
2022
+ const result = await sql2.query(
2023
+ `SELECT * FROM fabric_platform.action_invocations
2024
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2025
+ FOR UPDATE`,
2026
+ [id, tenantId, spaceId]
2027
+ );
2028
+ return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
2029
+ },
2030
+ getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
1343
2031
  appendEvent: (event) => scoped.appendEvent(event),
1344
2032
  appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
1345
2033
  nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
1346
2034
  listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
1347
- updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch)
2035
+ getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
2036
+ updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
2037
+ updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
1348
2038
  });
1349
2039
  });
1350
2040
  }
@@ -1354,7 +2044,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1354
2044
  transactionWithEvents;
1355
2045
  transactionalOutbox;
1356
2046
  async ensureSchema() {
1357
- await this.sql.query(`
2047
+ await applyPostgresMigrations(this.sql, [{
2048
+ version: 1,
2049
+ name: "host_ledger_baseline",
2050
+ sql: `
1358
2051
  CREATE SCHEMA IF NOT EXISTS fabric_platform;
1359
2052
  CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
1360
2053
  id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
@@ -1365,7 +2058,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1365
2058
  parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
1366
2059
  idempotency_authorization_binding_id text, invocation_provenance jsonb,
1367
2060
  authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
1368
- authorization_binding_id text, error text,
2061
+ adapter_reconciliation jsonb, authorization_binding_id text, error text,
1369
2062
  attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
1370
2063
  lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
1371
2064
  hitl_reason text, hitl_policy_version text, approval_decision jsonb,
@@ -1382,6 +2075,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1382
2075
  ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
1383
2076
  ADD COLUMN IF NOT EXISTS execution_reason text,
1384
2077
  ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
2078
+ ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
1385
2079
  ADD COLUMN IF NOT EXISTS authorization_binding_id text,
1386
2080
  ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
1387
2081
  ADD COLUMN IF NOT EXISTS lease_owner text,
@@ -1464,7 +2158,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1464
2158
  );
1465
2159
  CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
1466
2160
  ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
1467
- `);
2161
+ `
2162
+ }, {
2163
+ version: 2,
2164
+ name: "lease_fencing",
2165
+ sql: `
2166
+ ALTER TABLE fabric_platform.action_invocations
2167
+ ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
2168
+ ALTER TABLE fabric_platform.event_outbox
2169
+ ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
2170
+ `
2171
+ }]);
1468
2172
  }
1469
2173
  async transaction(run) {
1470
2174
  return run(this.db);
@@ -1525,6 +2229,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1525
2229
  status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
1526
2230
  error=CASE WHEN $6::boolean THEN $7 ELSE error END,
1527
2231
  authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
2232
+ adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
1528
2233
  lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
1529
2234
  THEN NULL ELSE lease_owner END,
1530
2235
  lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
@@ -1540,7 +2245,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1540
2245
  Object.hasOwn(patch, "error"),
1541
2246
  patch.error ?? null,
1542
2247
  Object.hasOwn(patch, "authorizationReconciliation"),
1543
- patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
2248
+ patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
2249
+ Object.hasOwn(patch, "adapterReconciliation"),
2250
+ patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
1544
2251
  ]
1545
2252
  );
1546
2253
  }
@@ -1574,6 +2281,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1574
2281
  error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
1575
2282
  lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
1576
2283
  lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
2284
+ lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
1577
2285
  attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
1578
2286
  updated_at=$9
1579
2287
  WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
@@ -1811,28 +2519,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1811
2519
  ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
1812
2520
  )
1813
2521
  UPDATE fabric_platform.event_outbox AS item
1814
- SET lease_owner=$5, lease_expires_at=$6, attempt_count=attempt_count+1
2522
+ SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
2523
+ attempt_count=attempt_count+1
1815
2524
  FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
1816
2525
  [current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
1817
2526
  );
1818
2527
  return result.rows.map(toOutboxRecord);
1819
2528
  }
1820
- async markOutboxPublished(id, workerId, publishedAt) {
2529
+ async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
1821
2530
  const result = await this.sql.query(
1822
2531
  `UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
1823
- lease_owner=NULL,lease_expires_at=NULL WHERE id=$1 AND lease_owner=$2 RETURNING id`,
1824
- [id, workerId, publishedAt]
2532
+ lease_owner=NULL,lease_expires_at=NULL
2533
+ WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
2534
+ [id, workerId, publishedAt, leaseToken ?? null]
1825
2535
  );
1826
2536
  if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1827
2537
  }
1828
2538
  async markOutboxFailed(input) {
1829
2539
  const result = await this.sql.query(
1830
2540
  `UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
1831
- lease_owner=NULL,lease_expires_at=NULL WHERE id=$1 AND lease_owner=$2 RETURNING id`,
1832
- [input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt]
2541
+ lease_owner=NULL,lease_expires_at=NULL
2542
+ WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
2543
+ [input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
1833
2544
  );
1834
2545
  if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
1835
2546
  }
2547
+ async renewOutboxLease(input) {
2548
+ const current = input.now ?? /* @__PURE__ */ new Date();
2549
+ const result = await this.sql.query(
2550
+ `UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
2551
+ WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
2552
+ AND lease_expires_at > $4
2553
+ RETURNING id`,
2554
+ [
2555
+ input.id,
2556
+ input.workerId,
2557
+ input.leaseToken,
2558
+ current,
2559
+ new Date(current.getTime() + input.leaseDurationMs)
2560
+ ]
2561
+ );
2562
+ return result.rows.length === 1;
2563
+ }
1836
2564
  async listOutbox(input = {}) {
1837
2565
  const result = await this.sql.query(
1838
2566
  `SELECT * FROM fabric_platform.event_outbox
@@ -1881,6 +2609,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1881
2609
  );
1882
2610
  return result.rows.map(toActionRecord);
1883
2611
  }
2612
+ async getHealthCounts(input) {
2613
+ const result = await this.sql.query(
2614
+ `SELECT
2615
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2616
+ WHERE status='pending'
2617
+ AND ($1::text IS NULL OR tenant_id=$1)
2618
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
2619
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2620
+ WHERE status='running'
2621
+ AND ($1::text IS NULL OR tenant_id=$1)
2622
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
2623
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2624
+ WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
2625
+ AND ($1::text IS NULL OR tenant_id=$1)
2626
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
2627
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2628
+ WHERE status='waiting_for_approval'
2629
+ AND ($1::text IS NULL OR tenant_id=$1)
2630
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
2631
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2632
+ WHERE status='reconciliation_required'
2633
+ AND ($1::text IS NULL OR tenant_id=$1)
2634
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
2635
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2636
+ WHERE status='pending'
2637
+ AND ($1::text IS NULL OR tenant_id=$1)
2638
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
2639
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2640
+ WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
2641
+ AND ($1::text IS NULL OR tenant_id=$1)
2642
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
2643
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2644
+ WHERE status='dead_letter'
2645
+ AND ($1::text IS NULL OR tenant_id=$1)
2646
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
2647
+ [input.tenantId ?? null, input.spaceId ?? null, input.now]
2648
+ );
2649
+ const row = result.rows[0] ?? {};
2650
+ return {
2651
+ invocationBacklog: Number(row.invocation_backlog ?? 0),
2652
+ invocationRunning: Number(row.invocation_running ?? 0),
2653
+ invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
2654
+ invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
2655
+ invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
2656
+ outboxBacklog: Number(row.outbox_backlog ?? 0),
2657
+ outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
2658
+ outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
2659
+ };
2660
+ }
1884
2661
  async claimActionInvocations(input) {
1885
2662
  const current = input.now ?? /* @__PURE__ */ new Date();
1886
2663
  const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
@@ -1889,7 +2666,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1889
2666
  `WITH claimable AS (
1890
2667
  SELECT id FROM fabric_platform.action_invocations
1891
2668
  WHERE (status='pending' OR
1892
- (status='running' AND (lease_expires_at IS NULL OR lease_expires_at <= $1)))
2669
+ (status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
1893
2670
  AND ($2::text IS NULL OR tenant_id=$2)
1894
2671
  AND ($3::text IS NULL OR space_id=$3)
1895
2672
  ORDER BY created_at
@@ -1898,7 +2675,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1898
2675
  )
1899
2676
  UPDATE fabric_platform.action_invocations AS invocation
1900
2677
  SET status='running', lease_owner=$5, lease_expires_at=$6,
1901
- attempt_count=attempt_count+1, updated_at=$1
2678
+ lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
1902
2679
  FROM claimable WHERE invocation.id=claimable.id
1903
2680
  RETURNING invocation.*`,
1904
2681
  [
@@ -1912,6 +2689,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1912
2689
  );
1913
2690
  return result.rows.map(toActionRecord);
1914
2691
  }
2692
+ async updateLeasedActionInvocation(input) {
2693
+ const patch = input.patch;
2694
+ const result = await this.sql.query(
2695
+ `UPDATE fabric_platform.action_invocations SET
2696
+ status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
2697
+ error=CASE WHEN $8::boolean THEN $9 ELSE error END,
2698
+ authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
2699
+ adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
2700
+ lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
2701
+ THEN NULL ELSE lease_owner END,
2702
+ lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
2703
+ THEN NULL ELSE lease_expires_at END,
2704
+ updated_at=now()
2705
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2706
+ AND status='running' AND lease_owner=$4 AND lease_token=$5
2707
+ RETURNING id`,
2708
+ [
2709
+ input.id,
2710
+ input.tenantId,
2711
+ input.spaceId,
2712
+ input.workerId,
2713
+ input.leaseToken,
2714
+ patch.status ?? null,
2715
+ patch.result === void 0 ? null : JSON.stringify(patch.result),
2716
+ Object.hasOwn(patch, "error"),
2717
+ patch.error ?? null,
2718
+ Object.hasOwn(patch, "authorizationReconciliation"),
2719
+ patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
2720
+ Object.hasOwn(patch, "adapterReconciliation"),
2721
+ patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
2722
+ ]
2723
+ );
2724
+ return result.rows.length === 1;
2725
+ }
2726
+ async renewActionInvocationLease(input) {
2727
+ const current = input.now ?? /* @__PURE__ */ new Date();
2728
+ const result = await this.sql.query(
2729
+ `UPDATE fabric_platform.action_invocations
2730
+ SET lease_expires_at=$6,updated_at=$5
2731
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2732
+ AND status='running' AND lease_owner=$4 AND lease_token=$7
2733
+ AND lease_expires_at > $5
2734
+ RETURNING id`,
2735
+ [
2736
+ input.id,
2737
+ input.tenantId,
2738
+ input.spaceId,
2739
+ input.workerId,
2740
+ current,
2741
+ new Date(current.getTime() + input.leaseDurationMs),
2742
+ input.leaseToken
2743
+ ]
2744
+ );
2745
+ return result.rows.length === 1;
2746
+ }
1915
2747
  async listEvents(tenantId, spaceId) {
1916
2748
  const result = await this.sql.query(
1917
2749
  `SELECT * FROM fabric_platform.asset_events
@@ -1920,6 +2752,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1920
2752
  );
1921
2753
  return result.rows.map(toEventRecord);
1922
2754
  }
2755
+ async getEvent(id, tenantId, spaceId) {
2756
+ const result = await this.sql.query(
2757
+ `SELECT * FROM fabric_platform.asset_events
2758
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
2759
+ [id, tenantId, spaceId]
2760
+ );
2761
+ return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
2762
+ }
1923
2763
  };
1924
2764
  function toAdapterRecord(row) {
1925
2765
  return {
@@ -1962,8 +2802,10 @@ function toActionRecord(row) {
1962
2802
  ...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
1963
2803
  ...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
1964
2804
  ...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
2805
+ ...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
1965
2806
  ...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
1966
2807
  attemptCount: Number(row.attempt_count ?? 0),
2808
+ leaseToken: Number(row.lease_token ?? 0),
1967
2809
  ...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
1968
2810
  ...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
1969
2811
  ...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
@@ -2059,6 +2901,7 @@ function toOutboxRecord(row) {
2059
2901
  correlationId: String(event.correlationId),
2060
2902
  ...event.causationId ? { causationId: String(event.causationId) } : {},
2061
2903
  ...event.provenance ? { provenance: event.provenance } : {},
2904
+ ...event.consistency ? { consistency: event.consistency } : {},
2062
2905
  occurredAt: new Date(event.occurredAt),
2063
2906
  recordedAt: new Date(event.recordedAt),
2064
2907
  producerModuleVersion: String(event.producerModuleVersion),
@@ -2071,6 +2914,7 @@ function toOutboxRecord(row) {
2071
2914
  availableAt: new Date(row.available_at),
2072
2915
  ...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
2073
2916
  ...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
2917
+ leaseToken: Number(row.lease_token ?? 0),
2074
2918
  ...row.last_error ? { lastError: String(row.last_error) } : {},
2075
2919
  createdAt: new Date(row.created_at),
2076
2920
  ...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
@@ -2087,38 +2931,93 @@ function createStoreBackedActionDispatcher() {
2087
2931
  };
2088
2932
  }
2089
2933
  async function runPlatformActionWorkerCycle(options) {
2090
- const claimed = await options.store.claimActionInvocations({
2091
- workerId: options.workerId,
2092
- limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
2093
- leaseDurationMs: options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS,
2094
- ...options.tenantId ? { tenantId: options.tenantId } : {},
2095
- ...options.spaceId ? { spaceId: options.spaceId } : {}
2096
- });
2934
+ const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
2935
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
2936
+ let claimed;
2937
+ try {
2938
+ claimed = await options.store.claimActionInvocations({
2939
+ workerId: options.workerId,
2940
+ limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
2941
+ leaseDurationMs,
2942
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
2943
+ ...options.spaceId ? { spaceId: options.spaceId } : {}
2944
+ });
2945
+ } catch (error) {
2946
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
2947
+ throw error;
2948
+ }
2949
+ for (const invocation of claimed) {
2950
+ emitPlatformHostTelemetry(options.telemetry, {
2951
+ kind: "event",
2952
+ name: "invocation.lease_claimed",
2953
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
2954
+ occurredAt: /* @__PURE__ */ new Date(),
2955
+ tenantId: invocation.tenantId,
2956
+ spaceId: invocation.spaceId,
2957
+ attributes: { attempt: invocation.attemptCount }
2958
+ });
2959
+ }
2097
2960
  let completed = 0;
2098
2961
  let failed = 0;
2099
2962
  let waitingForApproval = 0;
2100
2963
  for (const invocation of claimed) {
2964
+ const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
2101
2965
  try {
2102
- const result = await options.host.executeInvocation(
2966
+ const result2 = await options.host.executeInvocation(
2103
2967
  invocation.id,
2104
2968
  invocation.tenantId,
2105
2969
  invocation.spaceId,
2106
- { leaseOwner: options.workerId }
2970
+ { leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
2107
2971
  );
2108
- if (result.status === "completed") completed += 1;
2109
- else if (result.status === "waiting_for_approval") waitingForApproval += 1;
2972
+ if (result2.status === "completed") completed += 1;
2973
+ else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
2110
2974
  else failed += 1;
2111
2975
  } catch (error) {
2112
2976
  failed += 1;
2113
2977
  options.onError?.(error, invocation);
2978
+ } finally {
2979
+ await stopHeartbeat();
2114
2980
  }
2115
2981
  }
2116
- return {
2982
+ const result = {
2117
2983
  claimed: claimed.length,
2118
2984
  completed,
2119
2985
  failed,
2120
2986
  ...waitingForApproval > 0 ? { waitingForApproval } : {}
2121
2987
  };
2988
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
2989
+ return result;
2990
+ }
2991
+ function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
2992
+ if (!invocation?.leaseToken) return async () => void 0;
2993
+ const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
2994
+ let stopped = false;
2995
+ let timer;
2996
+ let running;
2997
+ const schedule = () => {
2998
+ if (!stopped) timer = setTimeout(tick, intervalMs);
2999
+ };
3000
+ const tick = () => {
3001
+ running = options.store.renewActionInvocationLease({
3002
+ id: invocation.id,
3003
+ tenantId: invocation.tenantId,
3004
+ spaceId: invocation.spaceId,
3005
+ workerId: options.workerId,
3006
+ leaseToken: invocation.leaseToken,
3007
+ leaseDurationMs
3008
+ }).then((renewed) => {
3009
+ if (!renewed) options.onError?.(
3010
+ new Error(`Invocation lease lost: ${invocation.id}`),
3011
+ invocation
3012
+ );
3013
+ }).catch((error) => options.onError?.(error, invocation)).finally(schedule);
3014
+ };
3015
+ schedule();
3016
+ return async () => {
3017
+ stopped = true;
3018
+ if (timer) clearTimeout(timer);
3019
+ await running;
3020
+ };
2122
3021
  }
2123
3022
  async function runPlatformActionWorker(options) {
2124
3023
  while (!options.signal?.aborted) {
@@ -2144,17 +3043,174 @@ async function abortableDelay(milliseconds, signal) {
2144
3043
  );
2145
3044
  });
2146
3045
  }
3046
+ function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
3047
+ const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
3048
+ emitPlatformHostTelemetry(telemetry, {
3049
+ kind: "event",
3050
+ name,
3051
+ metricName,
3052
+ occurredAt: /* @__PURE__ */ new Date(),
3053
+ attributes: { workerId, ...attributes }
3054
+ });
3055
+ }
3056
+
3057
+ // src/saga-parent-lifecycle.ts
3058
+ var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
3059
+ "completed",
3060
+ "failed",
3061
+ "blocked_by_policy",
3062
+ "reconciliation_required",
3063
+ "validation_failed"
3064
+ ]);
3065
+ function createDurableSagaParentLifecycle(options) {
3066
+ const store = options.store;
3067
+ const now = options.now ?? (() => /* @__PURE__ */ new Date());
3068
+ const transactionWithEvents = store.transactionWithEvents?.bind(store);
3069
+ if (!transactionWithEvents) {
3070
+ throw new Error(
3071
+ "Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
3072
+ );
3073
+ }
3074
+ const runInTransaction = transactionWithEvents;
3075
+ async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
3076
+ const eventId = `sagapar_${digestParameters({
3077
+ parentInvocationId: input.parentInvocationId,
3078
+ lifecycleId: lifecycleId2
3079
+ }).slice(0, 40)}`;
3080
+ await runInTransaction(async (transaction) => {
3081
+ if (!transaction.getActionInvocationForUpdate) {
3082
+ throw new Error(
3083
+ "Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
3084
+ );
3085
+ }
3086
+ if (!transaction.getEvent) {
3087
+ throw new Error(
3088
+ "Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
3089
+ );
3090
+ }
3091
+ const parent = await transaction.getActionInvocationForUpdate(
3092
+ input.parentInvocationId,
3093
+ input.tenantId,
3094
+ input.spaceId
3095
+ );
3096
+ if (!parent) {
3097
+ throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
3098
+ }
3099
+ const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
3100
+ if (prior) {
3101
+ if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
3102
+ throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
3103
+ }
3104
+ return;
3105
+ }
3106
+ if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
3107
+ throw new Error(
3108
+ `Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
3109
+ );
3110
+ }
3111
+ const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
3112
+ const timestamp = now();
3113
+ await transaction.appendEvent({
3114
+ id: eventId,
3115
+ tenantId: input.tenantId,
3116
+ spaceId: input.spaceId,
3117
+ eventType,
3118
+ eventSchemaVersion: 1,
3119
+ subjectType: "ActionInvocation",
3120
+ subjectId: input.parentInvocationId,
3121
+ actorId: "system",
3122
+ actorType: "system",
3123
+ actionInvocationId: input.parentInvocationId,
3124
+ payload,
3125
+ sequence,
3126
+ occurredAt: timestamp,
3127
+ recordedAt: timestamp,
3128
+ correlationId: input.parentInvocationId,
3129
+ causationId: lifecycleId2
3130
+ });
3131
+ if (patch) {
3132
+ await transaction.updateActionInvocation(
3133
+ input.parentInvocationId,
3134
+ input.tenantId,
3135
+ input.spaceId,
3136
+ patch
3137
+ );
3138
+ }
3139
+ });
3140
+ }
3141
+ return {
3142
+ async recordProgress(input) {
3143
+ await applyLifecycleTransition(
3144
+ input,
3145
+ input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
3146
+ "SagaParentProgress",
3147
+ {
3148
+ stepId: input.progress.stepId,
3149
+ index: input.progress.index,
3150
+ total: input.progress.total,
3151
+ completedAt: input.progress.completedAt
3152
+ }
3153
+ );
3154
+ },
3155
+ async parkApproval(input) {
3156
+ await applyLifecycleTransition(
3157
+ input,
3158
+ input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
3159
+ "SagaParentApprovalParked",
3160
+ {
3161
+ stepId: input.stepId,
3162
+ approvalId: input.approvalId,
3163
+ reason: input.reason
3164
+ },
3165
+ { status: "waiting_for_approval" }
3166
+ );
3167
+ },
3168
+ async cancel(input) {
3169
+ await applyLifecycleTransition(
3170
+ input,
3171
+ input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
3172
+ "SagaParentCancelled",
3173
+ { reason: input.reason, actorId: input.actorId },
3174
+ { status: "failed", error: `Saga cancelled: ${input.reason}` }
3175
+ );
3176
+ },
3177
+ async complete(input) {
3178
+ await applyLifecycleTransition(
3179
+ input,
3180
+ input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
3181
+ "SagaParentCompleted",
3182
+ { result: input.result },
3183
+ { status: "completed", result: input.result }
3184
+ );
3185
+ },
3186
+ async fail(input) {
3187
+ await applyLifecycleTransition(
3188
+ input,
3189
+ input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
3190
+ "SagaParentFailed",
3191
+ { error: input.error, compensation: input.compensation },
3192
+ { status: "failed", error: input.error }
3193
+ );
3194
+ }
3195
+ };
3196
+ }
2147
3197
 
2148
3198
  exports.IdempotencyConflictError = IdempotencyConflictError;
2149
3199
  exports.MemoryPlatformHostStore = MemoryPlatformHostStore;
2150
3200
  exports.PARAMETER_DIGEST_ALGORITHM = PARAMETER_DIGEST_ALGORITHM;
2151
3201
  exports.PLATFORM_HOST_CONTRACT_VERSION = PLATFORM_HOST_CONTRACT_VERSION;
3202
+ exports.PLATFORM_HOST_HEALTH_CONTRACT_VERSION = PLATFORM_HOST_HEALTH_CONTRACT_VERSION;
3203
+ exports.PLATFORM_HOST_METRIC_NAMES = PLATFORM_HOST_METRIC_NAMES;
2152
3204
  exports.PostgresPlatformHostStore = PostgresPlatformHostStore;
3205
+ exports.applyPostgresMigrations = applyPostgresMigrations;
2153
3206
  exports.canonicalJson = canonicalJson;
2154
3207
  exports.cloneOutboxRecord = cloneOutboxRecord;
3208
+ exports.createDurableSagaParentLifecycle = createDurableSagaParentLifecycle;
2155
3209
  exports.createGovernedActionHost = createGovernedActionHost;
2156
3210
  exports.createStoreBackedActionDispatcher = createStoreBackedActionDispatcher;
2157
3211
  exports.digestParameters = digestParameters;
3212
+ exports.emitPlatformHostTelemetry = emitPlatformHostTelemetry;
3213
+ exports.getPlatformHostHealthSnapshot = getPlatformHostHealthSnapshot;
2158
3214
  exports.runOutboxRelayCycle = runOutboxRelayCycle;
2159
3215
  exports.runPlatformActionWorker = runPlatformActionWorker;
2160
3216
  exports.runPlatformActionWorkerCycle = runPlatformActionWorkerCycle;