@fabricorg/platform-host 4.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,4 +1,5 @@
1
- import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, validateStateMachineTransition, executeWithAdapterRetry } from '@fabricorg/platform';
1
+ import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
2
+ import { assertAssemblyRuntimeCompatible } from '@fabricorg/assembly';
2
3
  import { createHash } from 'crypto';
3
4
 
4
5
  // src/host.ts
@@ -62,6 +63,213 @@ function normalizeProvenance(input, options) {
62
63
  };
63
64
  }
64
65
 
66
+ // src/observability.ts
67
+ var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
68
+ var PLATFORM_HOST_METRIC_NAMES = {
69
+ workerCyclesStarted: "fabric.platform.worker.cycles.started",
70
+ workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
71
+ workerCyclesFailed: "fabric.platform.worker.cycles.failed",
72
+ invocationSubmitted: "fabric.platform.invocation.submitted",
73
+ invocationExecutionStarted: "fabric.platform.invocation.execution_started",
74
+ invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
75
+ invocationCompleted: "fabric.platform.invocation.completed",
76
+ invocationFailed: "fabric.platform.invocation.failed",
77
+ invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
78
+ invocationValidationFailed: "fabric.platform.invocation.validation_failed",
79
+ invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
80
+ invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
81
+ outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
82
+ outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
83
+ outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
84
+ outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
85
+ outboxPublished: "fabric.platform.outbox.published",
86
+ outboxFailed: "fabric.platform.outbox.failed",
87
+ outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
88
+ invocationBacklog: "fabric.platform.invocation.backlog",
89
+ invocationRunning: "fabric.platform.invocation.running",
90
+ invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
91
+ invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
92
+ invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
93
+ outboxBacklog: "fabric.platform.outbox.backlog",
94
+ outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
95
+ outboxDeadLetters: "fabric.platform.outbox.dead_letters"
96
+ };
97
+ function emitPlatformHostTelemetry(telemetry, input) {
98
+ if (!telemetry) return;
99
+ const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
100
+ try {
101
+ const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
102
+ if (isPromiseLike(result)) void result.catch(() => void 0);
103
+ } catch {
104
+ }
105
+ }
106
+ async function getPlatformHostHealthSnapshot(options) {
107
+ const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
108
+ const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
109
+ const scope = {
110
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
111
+ ...options.spaceId ? { spaceId: options.spaceId } : {}
112
+ };
113
+ try {
114
+ const counts = await readHealthCounts(options.store, {
115
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
116
+ ...options.spaceId ? { spaceId: options.spaceId } : {},
117
+ now: generatedAt
118
+ });
119
+ const snapshot = createHealthSnapshot(
120
+ generatedAt,
121
+ scope,
122
+ counts,
123
+ workerSummary,
124
+ true
125
+ );
126
+ emitHealthMetrics(options.telemetry, snapshot);
127
+ return snapshot;
128
+ } catch {
129
+ const snapshot = createHealthSnapshot(
130
+ generatedAt,
131
+ scope,
132
+ zeroHealthCounts(),
133
+ workerSummary,
134
+ false
135
+ );
136
+ emitHealthMetrics(options.telemetry, snapshot);
137
+ return snapshot;
138
+ }
139
+ }
140
+ async function readHealthCounts(store, query) {
141
+ const candidate = store;
142
+ if (typeof candidate.getHealthCounts === "function") {
143
+ return assertHealthCounts(await candidate.getHealthCounts(query));
144
+ }
145
+ const recoverable = store;
146
+ const outbox = store;
147
+ if (typeof recoverable.listActionInvocations !== "function") {
148
+ throw new Error("Health counts are not supported by this store.");
149
+ }
150
+ const invocations = await recoverable.listActionInvocations({
151
+ ...query.tenantId ? { tenantId: query.tenantId } : {},
152
+ ...query.spaceId ? { spaceId: query.spaceId } : {},
153
+ limit: Number.MAX_SAFE_INTEGER
154
+ });
155
+ const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
156
+ ...query.tenantId ? { tenantId: query.tenantId } : {},
157
+ ...query.spaceId ? { spaceId: query.spaceId } : {}
158
+ }) : [];
159
+ return countsFromRecords(invocations, outboxRecords, query.now);
160
+ }
161
+ function countsFromRecords(invocations, outbox, now) {
162
+ return {
163
+ invocationBacklog: invocations.filter((item) => item.status === "pending").length,
164
+ invocationRunning: invocations.filter((item) => item.status === "running").length,
165
+ invocationExpiredLeases: invocations.filter(
166
+ (item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
167
+ ).length,
168
+ invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
169
+ invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
170
+ outboxBacklog: outbox.filter((item) => item.status === "pending").length,
171
+ outboxExpiredLeases: outbox.filter(
172
+ (item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
173
+ ).length,
174
+ outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
175
+ };
176
+ }
177
+ function assertHealthCounts(value) {
178
+ for (const [name, count] of Object.entries(value)) {
179
+ if (!Number.isSafeInteger(count) || count < 0) {
180
+ throw new Error(`Invalid health count: ${name}`);
181
+ }
182
+ }
183
+ return value;
184
+ }
185
+ function zeroHealthCounts() {
186
+ return {
187
+ invocationBacklog: 0,
188
+ invocationRunning: 0,
189
+ invocationExpiredLeases: 0,
190
+ invocationApprovalWaits: 0,
191
+ invocationReconciliationRequired: 0,
192
+ outboxBacklog: 0,
193
+ outboxExpiredLeases: 0,
194
+ outboxDeadLetters: 0
195
+ };
196
+ }
197
+ function summarizeWorkers(workers, now) {
198
+ let healthy = 0;
199
+ let stale = 0;
200
+ for (const worker of workers) {
201
+ if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
202
+ else stale += 1;
203
+ }
204
+ return { reported: workers.length, healthy, stale };
205
+ }
206
+ function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
207
+ const reasonCodes = [];
208
+ if (!dataAvailable) reasonCodes.push("health_source_unavailable");
209
+ if (workers.stale > 0) reasonCodes.push("workers_stale");
210
+ if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
211
+ if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
212
+ if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
213
+ if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
214
+ const unhealthy = !dataAvailable;
215
+ const degraded = unhealthy || reasonCodes.length > 0;
216
+ return {
217
+ contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
218
+ generatedAt,
219
+ scope,
220
+ dataAvailable,
221
+ status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
222
+ readiness: {
223
+ ready: dataAvailable && workers.stale === 0,
224
+ reasonCodes
225
+ },
226
+ workers,
227
+ invocations: {
228
+ backlog: counts.invocationBacklog,
229
+ running: counts.invocationRunning,
230
+ expiredLeases: counts.invocationExpiredLeases,
231
+ approvalWaits: counts.invocationApprovalWaits,
232
+ reconciliationRequired: counts.invocationReconciliationRequired
233
+ },
234
+ outbox: {
235
+ backlog: counts.outboxBacklog,
236
+ expiredLeases: counts.outboxExpiredLeases,
237
+ deadLetters: counts.outboxDeadLetters
238
+ }
239
+ };
240
+ }
241
+ function emitHealthMetrics(telemetry, snapshot) {
242
+ const scope = {
243
+ ...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
244
+ ...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
245
+ };
246
+ const metrics = [
247
+ [PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
248
+ [PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
249
+ [PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
250
+ [PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
251
+ [PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
252
+ [PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
253
+ [PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
254
+ [PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
255
+ ];
256
+ for (const [name, value] of metrics) {
257
+ emitPlatformHostTelemetry(telemetry, {
258
+ kind: "metric",
259
+ name,
260
+ metricType: "gauge",
261
+ value,
262
+ ...scope,
263
+ observedAt: snapshot.generatedAt
264
+ });
265
+ }
266
+ }
267
+ function isPromiseLike(value) {
268
+ return Boolean(
269
+ value && typeof value === "object" && "then" in value && typeof value.then === "function"
270
+ );
271
+ }
272
+
65
273
  // src/host.ts
66
274
  var DEFAULT_EXTRACT_EVENTS = (data) => {
67
275
  const value = data._events;
@@ -71,6 +279,19 @@ var RecoverableFinalizationError = class extends Error {
71
279
  name = "RecoverableFinalizationError";
72
280
  };
73
281
  function createGovernedActionHost(options) {
282
+ if (options.composition) {
283
+ if (!options.registry?.orderedModules) {
284
+ throw new Error("Composition-bound Platform Host requires an explicit module registry.");
285
+ }
286
+ assertAssemblyRuntimeCompatible(
287
+ options.composition.assembly,
288
+ options.registry.orderedModules.map((module) => ({
289
+ namespace: module.namespace,
290
+ version: module.version ?? "",
291
+ manifestDigest: module.manifestDigest ?? ""
292
+ }))
293
+ );
294
+ }
74
295
  if (options.outbox) {
75
296
  const outboxStore = asOutboxStore(options.store);
76
297
  if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
@@ -92,6 +313,12 @@ function createGovernedActionHost(options) {
92
313
  }
93
314
  const action = actionResolver(input.actionId);
94
315
  if (!action) throw new Error(`Unknown action: ${input.actionId}`);
316
+ if (action.execution?.connectivity === "online-required" && input.executionReason === "offline_replay") {
317
+ throw new Error(`Action ${input.actionId} is online-required and cannot be submitted as an offline replay.`);
318
+ }
319
+ if (action.execution?.sensitivity === "restricted" && Object.keys(input.provenance?.auditAttributes ?? {}).length > 0) {
320
+ throw new Error(`Action ${input.actionId} declares restricted sensitivity; provenance audit attributes must not be durably recorded.`);
321
+ }
95
322
  const authorizationInput = toAuthorizationInput(action, input);
96
323
  if (!await options.authorization.checkEntitlement(authorizationInput)) {
97
324
  throw new Error(`Module "${action.namespace}" is not enabled for tenant ${input.tenantId}`);
@@ -110,10 +337,13 @@ function createGovernedActionHost(options) {
110
337
  input.provenance,
111
338
  options.provenance
112
339
  );
340
+ const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
113
341
  const runtimeEvidence = {
114
342
  governanceContractVersion: FABRIC_GOVERNANCE_CONTRACT_VERSION,
115
343
  hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
116
- ...options.runtimeEvidence
344
+ ...options.runtimeEvidence,
345
+ ...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
346
+ ...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
117
347
  };
118
348
  assertGovernanceRuntimeEvidence(runtimeEvidence);
119
349
  const durableInvocation = await options.store.createActionInvocation({
@@ -142,6 +372,15 @@ function createGovernedActionHost(options) {
142
372
  ...input.executionReason ? { executionReason: input.executionReason } : {},
143
373
  ...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
144
374
  });
375
+ emitPlatformHostTelemetry(options.telemetry, {
376
+ kind: "event",
377
+ name: "invocation.submitted",
378
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
379
+ occurredAt: now(),
380
+ tenantId: input.tenantId,
381
+ spaceId: input.spaceId,
382
+ attributes: { actionVersion: action.version }
383
+ });
145
384
  if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
146
385
  const conflict = idempotencyConflict(durableInvocation, {
147
386
  actorId: input.actorId,
@@ -164,7 +403,24 @@ function createGovernedActionHost(options) {
164
403
  }
165
404
  const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
166
405
  if (durableInvocation.id !== actionInvocationId) {
167
- return {
406
+ if (options.dispatcher && durableInvocation.status === "pending") {
407
+ const dispatched = await options.dispatcher.dispatch({
408
+ actionInvocationId: durableInvocation.id,
409
+ actionId: durableInvocation.actionId,
410
+ tenantId: input.tenantId,
411
+ spaceId: input.spaceId,
412
+ workflowId: durableWorkflowId
413
+ });
414
+ return withConsistency({
415
+ actionInvocationId: durableInvocation.id,
416
+ status: durableInvocation.status,
417
+ workflowId: dispatched.workflowId,
418
+ ...dispatched.runId ? { runId: dispatched.runId } : {},
419
+ ...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
420
+ ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
421
+ }, input.actionId);
422
+ }
423
+ return withConsistency({
168
424
  actionInvocationId: durableInvocation.id,
169
425
  status: durableInvocation.status,
170
426
  workflowId: durableWorkflowId,
@@ -172,33 +428,28 @@ function createGovernedActionHost(options) {
172
428
  ...durableInvocation.error ? { error: durableInvocation.error } : {},
173
429
  ...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
174
430
  ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
175
- ...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
176
- };
431
+ ...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
432
+ ...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
433
+ }, input.actionId);
177
434
  }
178
435
  if (options.dispatcher) {
179
436
  try {
180
437
  const dispatched = await options.dispatcher.dispatch({
181
438
  actionInvocationId: durableInvocation.id,
439
+ actionId: durableInvocation.actionId,
182
440
  tenantId: input.tenantId,
183
441
  spaceId: input.spaceId,
184
442
  workflowId: durableWorkflowId
185
443
  });
186
- return {
444
+ return withConsistency({
187
445
  actionInvocationId: durableInvocation.id,
188
446
  status: durableInvocation.status,
189
447
  workflowId: dispatched.workflowId,
190
448
  ...dispatched.runId ? { runId: dispatched.runId } : {},
191
449
  ...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
192
450
  ...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
193
- };
451
+ }, input.actionId);
194
452
  } catch (error) {
195
- const message = errorMessage(error);
196
- await options.store.updateActionInvocation(
197
- durableInvocation.id,
198
- input.tenantId,
199
- input.spaceId,
200
- { status: "failed", error: message }
201
- );
202
453
  throw error;
203
454
  }
204
455
  }
@@ -207,7 +458,7 @@ function createGovernedActionHost(options) {
207
458
  input.tenantId,
208
459
  input.spaceId
209
460
  );
210
- return { ...executed, workflowId: durableWorkflowId };
461
+ return { ...withConsistency(executed, input.actionId), workflowId: durableWorkflowId };
211
462
  }
212
463
  async function executeInvocation(actionInvocationId, tenantId, spaceId, executionOptions = {}, executionReasonOverride) {
213
464
  const loadedInvocation = await options.store.getActionInvocation(
@@ -221,18 +472,32 @@ function createGovernedActionHost(options) {
221
472
  const resumingRunningInvocation = loadedInvocation.status === "running";
222
473
  let invocation = loadedInvocation;
223
474
  if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
224
- return actionResult(invocation);
475
+ return withConsistency(actionResult(invocation), invocation.actionId);
225
476
  }
226
- if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
477
+ if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
227
478
  return {
228
479
  ...actionResult(invocation),
229
- error: `Invocation is leased by ${invocation.leaseOwner}`
480
+ error: `Invocation lease is not owned by the supplied worker generation`
230
481
  };
231
482
  }
232
483
  const action = actionResolver(invocation.actionId);
233
484
  if (!action) {
234
485
  return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
235
486
  }
487
+ if (action.kind === "saga") {
488
+ return fail(
489
+ invocation,
490
+ "failed",
491
+ `Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
492
+ );
493
+ }
494
+ if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
495
+ return fail(
496
+ invocation,
497
+ "failed",
498
+ `Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
499
+ );
500
+ }
236
501
  if (invocation.attemptCount > 1 && !action.idempotent) {
237
502
  return fail(
238
503
  invocation,
@@ -245,6 +510,15 @@ function createGovernedActionHost(options) {
245
510
  status: "running"
246
511
  });
247
512
  }
513
+ emitPlatformHostTelemetry(options.telemetry, {
514
+ kind: "event",
515
+ name: "invocation.execution_started",
516
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
517
+ occurredAt: now(),
518
+ tenantId,
519
+ spaceId,
520
+ attributes: { actionVersion: invocation.actionVersion }
521
+ });
248
522
  try {
249
523
  const parsed = action.schema.safeParse(invocation.parameters);
250
524
  if (!parsed.success) {
@@ -310,7 +584,7 @@ function createGovernedActionHost(options) {
310
584
  );
311
585
  }
312
586
  if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
313
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, {
587
+ await persistInvocation(invocation, {
314
588
  status: "waiting_for_approval"
315
589
  });
316
590
  return {
@@ -337,8 +611,8 @@ function createGovernedActionHost(options) {
337
611
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
338
612
  message: `Action ${action.actionId} requires durable capture-time authorization evidence`
339
613
  };
340
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
341
- return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
614
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
615
+ return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
342
616
  }
343
617
  const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
344
618
  if (bindingExpired) {
@@ -350,8 +624,8 @@ function createGovernedActionHost(options) {
350
624
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
351
625
  message: `Authorization binding for action ${action.actionId} expired before execution`
352
626
  };
353
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
354
- return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
627
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
628
+ return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
355
629
  }
356
630
  const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
357
631
  ...authorizationInput,
@@ -372,8 +646,8 @@ function createGovernedActionHost(options) {
372
646
  ...invocation.provenance ? { provenance: invocation.provenance } : {},
373
647
  message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
374
648
  };
375
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
376
- return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
649
+ await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
650
+ return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
377
651
  }
378
652
  const definitions = options.resolvePolicies ? await options.resolvePolicies({
379
653
  ...authorizationInput,
@@ -434,30 +708,6 @@ function createGovernedActionHost(options) {
434
708
  })));
435
709
  }
436
710
  }
437
- const binding = action.stateMachine;
438
- if (binding) {
439
- const entityId = binding.getEntityId(parsed.data);
440
- const currentState = entityId ? await options.store.getEntityState(
441
- tenantId,
442
- spaceId,
443
- binding.entityType,
444
- entityId
445
- ) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
446
- const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
447
- if (targetState !== "") {
448
- const transition = validateStateMachineTransition(
449
- stateMachineResolver(binding.entityType),
450
- binding.entityType,
451
- currentState,
452
- targetState,
453
- action.actionId
454
- );
455
- const replayingAppliedTransition = action.idempotent && currentState === targetState;
456
- if (!transition.valid && !replayingAppliedTransition) {
457
- return fail(invocation, "failed", transition.error ?? "Invalid state transition");
458
- }
459
- }
460
- }
461
711
  let data;
462
712
  let domainEvents = [];
463
713
  try {
@@ -465,6 +715,42 @@ function createGovernedActionHost(options) {
465
715
  if (options.outbox && !transaction?.appendEventWithOutbox) {
466
716
  throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
467
717
  }
718
+ const binding = action.stateMachine;
719
+ if (binding) {
720
+ const machine = stateMachineResolver(binding.entityType);
721
+ const entityId = binding.getEntityId(parsed.data);
722
+ const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
723
+ const currentState = entityId ? await getEntityState(
724
+ tenantId,
725
+ spaceId,
726
+ binding.entityType,
727
+ entityId
728
+ ) ?? initialState(machine) : initialState(machine);
729
+ const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
730
+ if (targetState !== "") {
731
+ const transition = await evaluateStateMachineTransition(
732
+ machine,
733
+ binding.entityType,
734
+ currentState,
735
+ targetState,
736
+ action.actionId,
737
+ {
738
+ entity: {
739
+ ...entityId ? { id: entityId } : {},
740
+ state: currentState
741
+ },
742
+ actionInvocationId,
743
+ actorId: invocation.actorId,
744
+ parameters: parsed.data,
745
+ db
746
+ }
747
+ );
748
+ const replayingAppliedTransition = action.idempotent && currentState === targetState;
749
+ if (!transition.valid && !replayingAppliedTransition) {
750
+ throw new Error(transition.error ?? "Invalid state transition");
751
+ }
752
+ }
753
+ }
468
754
  const handlerResult = action.handler ? await action.handler(
469
755
  {
470
756
  actionInvocationId,
@@ -564,7 +850,21 @@ function createGovernedActionHost(options) {
564
850
  subjectId: adapterEventSubject.subjectId,
565
851
  payload: { adapterType: step.adapterType, operation: step.operation }
566
852
  }, `adapter:${stepIndex}:started`);
853
+ let adapterDeadlineTimer;
567
854
  try {
855
+ const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
856
+ const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
857
+ if (adapterController && deadlineMs) {
858
+ const remaining = deadlineMs - now().getTime();
859
+ if (remaining <= 0) {
860
+ throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
861
+ }
862
+ adapterDeadlineTimer = setTimeout(
863
+ () => adapterController.abort(new Error("Adapter deadline exceeded")),
864
+ remaining
865
+ );
866
+ adapterDeadlineTimer.unref?.();
867
+ }
568
868
  const result2 = await executeWithAdapterRetry({
569
869
  policy: step.retryPolicy ?? adapter.retryPolicy,
570
870
  defaultIdempotent: adapter.idempotent,
@@ -581,13 +881,77 @@ function createGovernedActionHost(options) {
581
881
  correlationId: invocation.correlationId,
582
882
  ...invocation.causationId ? { causationId: invocation.causationId } : {},
583
883
  attempt,
584
- maxAttempts
884
+ maxAttempts,
885
+ ...deadlineMs ? { deadlineMs } : {},
886
+ ...adapterController ? { signal: adapterController.signal } : {}
585
887
  });
586
888
  },
587
889
  isSuccessful: (result3) => result3.success,
588
- getError: (result3) => result3.error
890
+ getError: (result3) => result3.error,
891
+ classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
892
+ classifyThrownError: (error) => {
893
+ if (adapterController?.signal.aborted) return "timeout";
894
+ const message = error instanceof Error ? error.message : String(error);
895
+ if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
896
+ return "transient_failure";
897
+ }
589
898
  });
899
+ assertAdapterOutcomeConsistency(result2);
590
900
  if (!result2.success) {
901
+ const outcome = result2.outcome ?? "transient_failure";
902
+ if (outcome === "ambiguous") {
903
+ const evidence = result2.outcomeEvidence;
904
+ const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
905
+ const adapterReconciliation = {
906
+ kind: "adapter_outcome_ambiguous",
907
+ adapterType: step.adapterType,
908
+ operation: step.operation,
909
+ vendor: adapter.vendor,
910
+ adapterInvocationId,
911
+ ...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
912
+ ...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
913
+ message
914
+ };
915
+ await options.store.updateAdapterInvocation(adapterInvocationId, {
916
+ status: "ambiguous",
917
+ error: message,
918
+ updatedAt: now()
919
+ });
920
+ await appendEvent(invocation, {
921
+ eventType: "AdapterInvocationAmbiguous",
922
+ subjectType: adapterEventSubject.subjectType,
923
+ subjectId: adapterEventSubject.subjectId,
924
+ payload: {
925
+ adapterType: step.adapterType,
926
+ operation: step.operation,
927
+ ...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
928
+ }
929
+ }, `adapter:${stepIndex}:ambiguous`);
930
+ const governanceStore2 = asGovernanceStore(options.store);
931
+ if (governanceStore2 && evidence) {
932
+ await governanceStore2.appendExternalReconciliation({
933
+ id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
934
+ actionInvocationId,
935
+ tenantId,
936
+ spaceId,
937
+ status: "pending",
938
+ provider: adapter.vendor,
939
+ ...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
940
+ attempt: 1,
941
+ reason: message,
942
+ observedAt: now()
943
+ });
944
+ }
945
+ await persistInvocation(invocation, {
946
+ status: "reconciliation_required",
947
+ error: message,
948
+ adapterReconciliation
949
+ });
950
+ return withConsistency(
951
+ { actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
952
+ action.actionId
953
+ );
954
+ }
591
955
  await options.store.updateAdapterInvocation(adapterInvocationId, {
592
956
  status: "failed",
593
957
  error: result2.error ?? "Adapter failed",
@@ -647,6 +1011,8 @@ function createGovernedActionHost(options) {
647
1011
  updatedAt: now()
648
1012
  });
649
1013
  return fail(invocation, "failed", message);
1014
+ } finally {
1015
+ if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
650
1016
  }
651
1017
  }
652
1018
  const governanceStore = asGovernanceStore(options.store);
@@ -671,20 +1037,41 @@ function createGovernedActionHost(options) {
671
1037
  transaction
672
1038
  );
673
1039
  }
674
- await transaction.updateActionInvocation(
675
- actionInvocationId,
676
- tenantId,
677
- spaceId,
678
- { status: "completed", result }
679
- );
1040
+ if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
1041
+ if (!transaction.updateLeasedActionInvocation) {
1042
+ throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
1043
+ }
1044
+ const updated = await transaction.updateLeasedActionInvocation({
1045
+ id: actionInvocationId,
1046
+ tenantId,
1047
+ spaceId,
1048
+ workerId: invocation.leaseOwner,
1049
+ leaseToken: invocation.leaseToken,
1050
+ patch: { status: "completed", result }
1051
+ });
1052
+ if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
1053
+ } else {
1054
+ await transaction.updateActionInvocation(
1055
+ actionInvocationId,
1056
+ tenantId,
1057
+ spaceId,
1058
+ { status: "completed", result }
1059
+ );
1060
+ }
680
1061
  });
1062
+ emitInvocationStatusTelemetry(
1063
+ invocation,
1064
+ "completed",
1065
+ options.telemetry,
1066
+ now()
1067
+ );
681
1068
  } else {
682
1069
  if (action.eventPhase === "after_adapters") {
683
1070
  for (const [index, event] of domainEvents.entries()) {
684
1071
  await appendEvent(invocation, event, `domain:${index}`, action.version);
685
1072
  }
686
1073
  }
687
- await options.store.updateActionInvocation(actionInvocationId, tenantId, spaceId, {
1074
+ await persistInvocation(invocation, {
688
1075
  status: "completed",
689
1076
  result
690
1077
  });
@@ -715,7 +1102,7 @@ function createGovernedActionHost(options) {
715
1102
  spaceId
716
1103
  );
717
1104
  if (!invocation) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
718
- if (isTerminal(invocation.status)) return actionResult(invocation);
1105
+ if (isTerminal(invocation.status)) return withConsistency(actionResult(invocation), invocation.actionId);
719
1106
  if (invocation.status !== "waiting_for_approval") {
720
1107
  return {
721
1108
  ...actionResult(invocation),
@@ -765,14 +1152,23 @@ function createGovernedActionHost(options) {
765
1152
  const transitioned = transition.invocation ?? await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
766
1153
  if (!transition.applied || !transitioned) {
767
1154
  if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
768
- return actionResult(transitioned);
1155
+ return withConsistency(actionResult(transitioned), transitioned.actionId);
1156
+ }
1157
+ if (!decision.approved) {
1158
+ emitInvocationStatusTelemetry(
1159
+ transitioned,
1160
+ "failed",
1161
+ options.telemetry,
1162
+ now(),
1163
+ "waiting_for_approval"
1164
+ );
1165
+ return withConsistency(actionResult(transitioned), transitioned.actionId);
769
1166
  }
770
- if (!decision.approved) return actionResult(transitioned);
771
1167
  return executeInvocation(
772
1168
  actionInvocationId,
773
1169
  tenantId,
774
1170
  spaceId,
775
- { leaseOwner },
1171
+ { leaseOwner, leaseToken: transitioned.leaseToken },
776
1172
  "approval_resume"
777
1173
  );
778
1174
  }
@@ -808,6 +1204,13 @@ function createGovernedActionHost(options) {
808
1204
  spaceId
809
1205
  });
810
1206
  }
1207
+ function declaredConsistency(actionId) {
1208
+ return actionResolver(actionId)?.execution?.consistency;
1209
+ }
1210
+ function withConsistency(result, actionId) {
1211
+ const consistency = declaredConsistency(actionId);
1212
+ return consistency ? { ...result, consistency } : result;
1213
+ }
811
1214
  async function appendEvent(invocation, event, deduplicationKey, defaultEventSchemaVersion = 1, transaction) {
812
1215
  const timestamp = now();
813
1216
  const envelope = {
@@ -830,7 +1233,8 @@ function createGovernedActionHost(options) {
830
1233
  recordedAt: timestamp,
831
1234
  correlationId: invocation.correlationId,
832
1235
  ...invocation.causationId ? { causationId: invocation.causationId } : {},
833
- ...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {}
1236
+ ...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {},
1237
+ ...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
834
1238
  };
835
1239
  if (options.outbox) {
836
1240
  const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
@@ -860,12 +1264,7 @@ function createGovernedActionHost(options) {
860
1264
  }
861
1265
  }
862
1266
  async function fail(invocation, status, error) {
863
- await options.store.updateActionInvocation(
864
- invocation.id,
865
- invocation.tenantId,
866
- invocation.spaceId,
867
- { status, error }
868
- );
1267
+ await persistInvocation(invocation, { status, error });
869
1268
  return {
870
1269
  actionInvocationId: invocation.id,
871
1270
  status,
@@ -874,6 +1273,33 @@ function createGovernedActionHost(options) {
874
1273
  ...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
875
1274
  };
876
1275
  }
1276
+ async function persistInvocation(invocation, patch) {
1277
+ const previousStatus = invocation.status;
1278
+ if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
1279
+ const recoverable = options.store;
1280
+ if (!recoverable.updateLeasedActionInvocation) {
1281
+ throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
1282
+ }
1283
+ const updated = await recoverable.updateLeasedActionInvocation({
1284
+ id: invocation.id,
1285
+ tenantId: invocation.tenantId,
1286
+ spaceId: invocation.spaceId,
1287
+ workerId: invocation.leaseOwner,
1288
+ leaseToken: invocation.leaseToken,
1289
+ patch
1290
+ });
1291
+ if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
1292
+ emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
1293
+ return;
1294
+ }
1295
+ await options.store.updateActionInvocation(
1296
+ invocation.id,
1297
+ invocation.tenantId,
1298
+ invocation.spaceId,
1299
+ patch
1300
+ );
1301
+ emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
1302
+ }
877
1303
  return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
878
1304
  }
879
1305
  function actionResult(invocation) {
@@ -884,7 +1310,8 @@ function actionResult(invocation) {
884
1310
  ...invocation.error ? { error: invocation.error } : {},
885
1311
  ...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
886
1312
  ...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
887
- ...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
1313
+ ...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
1314
+ ...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
888
1315
  };
889
1316
  }
890
1317
  function asApprovalStore(store) {
@@ -938,6 +1365,24 @@ function initialState(machine) {
938
1365
  function isTerminal(status) {
939
1366
  return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
940
1367
  }
1368
+ function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
1369
+ if (!status) return;
1370
+ const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
1371
+ if (!event) return;
1372
+ emitPlatformHostTelemetry(telemetry, {
1373
+ kind: "event",
1374
+ name: event.name,
1375
+ metricName: event.metricName,
1376
+ occurredAt,
1377
+ tenantId: invocation.tenantId,
1378
+ spaceId: invocation.spaceId,
1379
+ attributes: {
1380
+ ...invocation.actionId ? { actionId: invocation.actionId } : {},
1381
+ ...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
1382
+ ...previousStatus ? { fromStatus: previousStatus } : {}
1383
+ }
1384
+ });
1385
+ }
941
1386
  function withoutPrivateHostFields(data, eventResultFields) {
942
1387
  return Object.fromEntries(
943
1388
  Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
@@ -998,35 +1443,80 @@ function toEnterpriseEventEnvelope(event, metadata) {
998
1443
  payloadClassification: metadata.payloadClassification,
999
1444
  ...metadata.traceContext ? { traceContext: { ...metadata.traceContext } } : {},
1000
1445
  ...metadata.includeProvenance && event.provenance ? { provenance: event.provenance } : {},
1446
+ // A provisional fact must stay marked as provisional once it leaves the platform.
1447
+ ...event.consistency ? { consistency: event.consistency } : {},
1001
1448
  payload: event.payload
1002
1449
  };
1003
1450
  }
1004
1451
  async function runOutboxRelayCycle(options) {
1005
1452
  const now = options.now ?? (() => /* @__PURE__ */ new Date());
1006
1453
  const maxAttempts = options.maxAttempts ?? 10;
1007
- const records = await options.store.claimOutbox({
1008
- workerId: options.workerId,
1009
- leaseDurationMs: options.leaseDurationMs ?? 3e4,
1010
- limit: options.batchSize ?? 100,
1011
- now: now()
1012
- });
1454
+ const leaseDurationMs = options.leaseDurationMs ?? 3e4;
1455
+ emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
1456
+ let records;
1457
+ try {
1458
+ records = await options.store.claimOutbox({
1459
+ workerId: options.workerId,
1460
+ leaseDurationMs,
1461
+ limit: options.batchSize ?? 100,
1462
+ now: now()
1463
+ });
1464
+ } catch (error) {
1465
+ emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
1466
+ throw error;
1467
+ }
1468
+ for (const record of records) {
1469
+ emitPlatformHostTelemetry(options.telemetry, {
1470
+ kind: "event",
1471
+ name: "outbox.lease_claimed",
1472
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
1473
+ occurredAt: now(),
1474
+ tenantId: record.tenantId,
1475
+ spaceId: record.spaceId,
1476
+ attributes: { attempt: record.attemptCount }
1477
+ });
1478
+ }
1013
1479
  const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
1014
1480
  for (const record of records) {
1481
+ const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
1015
1482
  try {
1016
- await options.publisher.publish(record.event);
1483
+ await publishWithTimeout(
1484
+ options.publisher,
1485
+ record.event,
1486
+ options.publishTimeoutMs ?? 1e4
1487
+ );
1488
+ await stopHeartbeat();
1017
1489
  try {
1018
- await options.store.markOutboxPublished(record.id, options.workerId, now());
1490
+ await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
1019
1491
  } catch {
1020
1492
  result.failed += 1;
1493
+ emitPlatformHostTelemetry(options.telemetry, {
1494
+ kind: "event",
1495
+ name: "outbox.failed",
1496
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
1497
+ occurredAt: now(),
1498
+ tenantId: record.tenantId,
1499
+ spaceId: record.spaceId
1500
+ });
1021
1501
  continue;
1022
1502
  }
1023
1503
  result.published += 1;
1504
+ emitPlatformHostTelemetry(options.telemetry, {
1505
+ kind: "event",
1506
+ name: "outbox.published",
1507
+ metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
1508
+ occurredAt: now(),
1509
+ tenantId: record.tenantId,
1510
+ spaceId: record.spaceId
1511
+ });
1024
1512
  } catch {
1513
+ await stopHeartbeat();
1025
1514
  const deadLetter = record.attemptCount >= maxAttempts;
1026
1515
  try {
1027
1516
  await options.store.markOutboxFailed({
1028
1517
  id: record.id,
1029
1518
  workerId: options.workerId,
1519
+ leaseToken: record.leaseToken,
1030
1520
  error: "Event publisher failed",
1031
1521
  availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
1032
1522
  deadLetter
@@ -1035,16 +1525,84 @@ async function runOutboxRelayCycle(options) {
1035
1525
  }
1036
1526
  result.failed += 1;
1037
1527
  if (deadLetter) result.deadLettered += 1;
1528
+ emitPlatformHostTelemetry(options.telemetry, {
1529
+ kind: "event",
1530
+ name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
1531
+ metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
1532
+ occurredAt: now(),
1533
+ tenantId: record.tenantId,
1534
+ spaceId: record.spaceId,
1535
+ attributes: { attempt: record.attemptCount }
1536
+ });
1038
1537
  }
1039
1538
  }
1539
+ emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
1540
+ claimed: result.claimed,
1541
+ published: result.published,
1542
+ failed: result.failed,
1543
+ deadLettered: result.deadLettered
1544
+ });
1040
1545
  return result;
1041
1546
  }
1547
+ async function publishWithTimeout(publisher, event, timeoutMs) {
1548
+ if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
1549
+ throw new Error("publishTimeoutMs must be a positive finite number");
1550
+ }
1551
+ const controller = new AbortController();
1552
+ let timer;
1553
+ const timeout = new Promise((_, reject) => {
1554
+ timer = setTimeout(() => {
1555
+ const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
1556
+ controller.abort(error);
1557
+ reject(error);
1558
+ }, timeoutMs);
1559
+ });
1560
+ try {
1561
+ await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
1562
+ } finally {
1563
+ if (timer) clearTimeout(timer);
1564
+ }
1565
+ }
1566
+ function startOutboxHeartbeat(options, record, leaseDurationMs) {
1567
+ if (!record.leaseToken) return async () => void 0;
1568
+ const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
1569
+ let stopped = false;
1570
+ let timer;
1571
+ let running;
1572
+ const schedule = () => {
1573
+ if (!stopped) timer = setTimeout(tick, intervalMs);
1574
+ };
1575
+ const tick = () => {
1576
+ running = options.store.renewOutboxLease({
1577
+ id: record.id,
1578
+ workerId: options.workerId,
1579
+ leaseToken: record.leaseToken,
1580
+ leaseDurationMs
1581
+ }).then(() => void 0).catch(() => void 0).finally(schedule);
1582
+ };
1583
+ schedule();
1584
+ return async () => {
1585
+ stopped = true;
1586
+ if (timer) clearTimeout(timer);
1587
+ await running;
1588
+ };
1589
+ }
1042
1590
  function cloneOutboxRecord(record) {
1043
1591
  return {
1044
1592
  ...record,
1045
1593
  event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
1046
1594
  };
1047
1595
  }
1596
+ function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
1597
+ const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
1598
+ emitPlatformHostTelemetry(telemetry, {
1599
+ kind: "event",
1600
+ name,
1601
+ metricName,
1602
+ occurredAt,
1603
+ attributes: { workerId, ...attributes }
1604
+ });
1605
+ }
1048
1606
 
1049
1607
  // src/memory-store.ts
1050
1608
  var MemoryPlatformHostStore = class {
@@ -1087,12 +1645,33 @@ var MemoryPlatformHostStore = class {
1087
1645
  };
1088
1646
  const result = await run({
1089
1647
  db: this.db,
1648
+ getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
1649
+ const record = await this.getActionInvocation(id, tenantId, spaceId);
1650
+ return record ? structuredClone(record) : void 0;
1651
+ },
1652
+ getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
1090
1653
  appendEvent: (event) => appendPending(event),
1091
1654
  appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
1092
1655
  nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
1093
1656
  listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
1657
+ getEntityState: async (tenantId, spaceId, entityType, entityId) => {
1658
+ let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
1659
+ for (const pending of pendingEvents) {
1660
+ const event = pending.event;
1661
+ if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
1662
+ const candidate = event.payload?.toState;
1663
+ if (typeof candidate === "string") state = candidate;
1664
+ }
1665
+ return state;
1666
+ },
1094
1667
  updateActionInvocation: async (id, tenantId, spaceId, patch) => {
1095
1668
  pendingUpdates.push({ id, tenantId, spaceId, patch });
1669
+ },
1670
+ updateLeasedActionInvocation: async (input) => {
1671
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1672
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
1673
+ pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
1674
+ return true;
1096
1675
  }
1097
1676
  });
1098
1677
  for (const update of pendingUpdates) {
@@ -1119,7 +1698,7 @@ var MemoryPlatformHostStore = class {
1119
1698
  if (existing) return existing;
1120
1699
  }
1121
1700
  const now = /* @__PURE__ */ new Date();
1122
- const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
1701
+ const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
1123
1702
  this.invocations.push(record);
1124
1703
  return record;
1125
1704
  }
@@ -1169,6 +1748,7 @@ var MemoryPlatformHostStore = class {
1169
1748
  record.error = void 0;
1170
1749
  record.leaseOwner = input.leaseOwner;
1171
1750
  record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
1751
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1172
1752
  record.attemptCount = Math.max(record.attemptCount, 1);
1173
1753
  return { applied: true, invocation: record };
1174
1754
  }
@@ -1246,31 +1826,41 @@ var MemoryPlatformHostStore = class {
1246
1826
  return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
1247
1827
  record.leaseOwner = input.workerId;
1248
1828
  record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1829
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1249
1830
  record.attemptCount += 1;
1250
1831
  return cloneOutboxRecord(record);
1251
1832
  });
1252
1833
  }
1253
- async markOutboxPublished(id, workerId, publishedAt) {
1254
- const record = this.requireLeasedOutbox(id, workerId);
1834
+ async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
1835
+ const record = this.requireLeasedOutbox(id, workerId, leaseToken);
1255
1836
  record.status = "published";
1256
1837
  record.publishedAt = publishedAt;
1257
1838
  delete record.leaseOwner;
1258
1839
  delete record.leaseExpiresAt;
1259
1840
  }
1260
1841
  async markOutboxFailed(input) {
1261
- const record = this.requireLeasedOutbox(input.id, input.workerId);
1842
+ const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
1262
1843
  record.status = input.deadLetter ? "dead_letter" : "pending";
1263
1844
  record.lastError = input.error;
1264
1845
  record.availableAt = input.availableAt;
1265
1846
  delete record.leaseOwner;
1266
1847
  delete record.leaseExpiresAt;
1267
1848
  }
1849
+ async renewOutboxLease(input) {
1850
+ const record = this.outbox.find((candidate) => candidate.id === input.id);
1851
+ const current = input.now ?? /* @__PURE__ */ new Date();
1852
+ if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
1853
+ record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1854
+ return true;
1855
+ }
1268
1856
  async listOutbox(input = {}) {
1269
1857
  return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
1270
1858
  }
1271
- requireLeasedOutbox(id, workerId) {
1859
+ requireLeasedOutbox(id, workerId, leaseToken) {
1272
1860
  const record = this.outbox.find((candidate) => candidate.id === id);
1273
- if (!record || record.leaseOwner !== workerId) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1861
+ if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
1862
+ throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1863
+ }
1274
1864
  return record;
1275
1865
  }
1276
1866
  async nextEventSequence(tenantId, spaceId) {
@@ -1292,24 +1882,128 @@ var MemoryPlatformHostStore = class {
1292
1882
  (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
1293
1883
  ).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
1294
1884
  }
1885
+ async getHealthCounts(input) {
1886
+ const invocations = this.invocations.filter(
1887
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
1888
+ );
1889
+ const outbox = this.outbox.filter(
1890
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
1891
+ );
1892
+ return {
1893
+ invocationBacklog: invocations.filter((item) => item.status === "pending").length,
1894
+ invocationRunning: invocations.filter((item) => item.status === "running").length,
1895
+ invocationExpiredLeases: invocations.filter(
1896
+ (item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
1897
+ ).length,
1898
+ invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
1899
+ invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
1900
+ outboxBacklog: outbox.filter((item) => item.status === "pending").length,
1901
+ outboxExpiredLeases: outbox.filter(
1902
+ (item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
1903
+ ).length,
1904
+ outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
1905
+ };
1906
+ }
1295
1907
  async claimActionInvocations(input) {
1296
1908
  const current = input.now ?? /* @__PURE__ */ new Date();
1297
1909
  const eligible = this.invocations.filter(
1298
- (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && (!record.leaseExpiresAt || record.leaseExpiresAt <= current))
1910
+ (record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
1299
1911
  ).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
1300
1912
  for (const record of eligible) {
1301
1913
  record.status = "running";
1302
1914
  record.leaseOwner = input.workerId;
1303
1915
  record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1916
+ record.leaseToken = (record.leaseToken ?? 0) + 1;
1304
1917
  record.attemptCount += 1;
1305
1918
  record.updatedAt = current;
1306
1919
  }
1307
- return eligible;
1920
+ return eligible.map((record) => structuredClone(record));
1921
+ }
1922
+ async updateLeasedActionInvocation(input) {
1923
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1924
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
1925
+ await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
1926
+ return true;
1927
+ }
1928
+ async renewActionInvocationLease(input) {
1929
+ const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
1930
+ const current = input.now ?? /* @__PURE__ */ new Date();
1931
+ if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
1932
+ record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
1933
+ record.updatedAt = current;
1934
+ return true;
1308
1935
  }
1309
1936
  async listEvents(tenantId, spaceId) {
1310
1937
  return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
1311
1938
  }
1312
1939
  };
1940
+ async function applyPostgresMigrations(client, migrations) {
1941
+ const ordered = validateMigrations(migrations);
1942
+ const statements = ordered.map((migration) => migrationStatement(migration));
1943
+ await client.query(`
1944
+ BEGIN;
1945
+ SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
1946
+ CREATE SCHEMA IF NOT EXISTS fabric_platform;
1947
+ CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
1948
+ version integer PRIMARY KEY,
1949
+ name text NOT NULL,
1950
+ checksum text NOT NULL,
1951
+ applied_at timestamptz NOT NULL DEFAULT now()
1952
+ );
1953
+ ${statements.join("\n")}
1954
+ COMMIT;
1955
+ `);
1956
+ }
1957
+ function validateMigrations(migrations) {
1958
+ const ordered = [...migrations].sort((left, right) => left.version - right.version);
1959
+ const versions = /* @__PURE__ */ new Set();
1960
+ for (const migration of ordered) {
1961
+ if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
1962
+ throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
1963
+ }
1964
+ if (versions.has(migration.version)) {
1965
+ throw new Error(`Duplicate migration version: ${migration.version}`);
1966
+ }
1967
+ if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
1968
+ throw new Error(`Invalid migration name: ${migration.name}`);
1969
+ }
1970
+ if (migration.sql.trim().length === 0) {
1971
+ throw new Error(`Migration ${migration.version} has no SQL`);
1972
+ }
1973
+ versions.add(migration.version);
1974
+ }
1975
+ return ordered;
1976
+ }
1977
+ function migrationStatement(migration) {
1978
+ const checksum = createHash("sha256").update(migration.sql).digest("hex");
1979
+ const delimiter = `$fabric_platform_migration_${migration.version}$`;
1980
+ if (migration.sql.includes(delimiter)) {
1981
+ throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
1982
+ }
1983
+ return `
1984
+ DO ${delimiter}
1985
+ DECLARE
1986
+ recorded_checksum text;
1987
+ BEGIN
1988
+ SELECT checksum INTO recorded_checksum
1989
+ FROM fabric_platform.schema_migrations
1990
+ WHERE version = ${migration.version};
1991
+
1992
+ IF FOUND THEN
1993
+ IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
1994
+ RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
1995
+ END IF;
1996
+ ELSE
1997
+ ${migration.sql}
1998
+ INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
1999
+ VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
2000
+ END IF;
2001
+ END
2002
+ ${delimiter};`;
2003
+ }
2004
+ function sqlLiteral(value) {
2005
+ return `'${value.replaceAll("'", "''")}'`;
2006
+ }
1313
2007
 
1314
2008
  // src/postgres-store.ts
1315
2009
  var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
@@ -1322,11 +2016,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1322
2016
  const scoped = new _PostgresPlatformHostStore(db2, sql2);
1323
2017
  return run({
1324
2018
  db: db2,
2019
+ getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
2020
+ const result = await sql2.query(
2021
+ `SELECT * FROM fabric_platform.action_invocations
2022
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2023
+ FOR UPDATE`,
2024
+ [id, tenantId, spaceId]
2025
+ );
2026
+ return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
2027
+ },
2028
+ getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
1325
2029
  appendEvent: (event) => scoped.appendEvent(event),
1326
2030
  appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
1327
2031
  nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
1328
2032
  listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
1329
- updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch)
2033
+ getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
2034
+ updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
2035
+ updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
1330
2036
  });
1331
2037
  });
1332
2038
  }
@@ -1336,7 +2042,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1336
2042
  transactionWithEvents;
1337
2043
  transactionalOutbox;
1338
2044
  async ensureSchema() {
1339
- await this.sql.query(`
2045
+ await applyPostgresMigrations(this.sql, [{
2046
+ version: 1,
2047
+ name: "host_ledger_baseline",
2048
+ sql: `
1340
2049
  CREATE SCHEMA IF NOT EXISTS fabric_platform;
1341
2050
  CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
1342
2051
  id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
@@ -1347,7 +2056,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1347
2056
  parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
1348
2057
  idempotency_authorization_binding_id text, invocation_provenance jsonb,
1349
2058
  authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
1350
- authorization_binding_id text, error text,
2059
+ adapter_reconciliation jsonb, authorization_binding_id text, error text,
1351
2060
  attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
1352
2061
  lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
1353
2062
  hitl_reason text, hitl_policy_version text, approval_decision jsonb,
@@ -1364,6 +2073,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1364
2073
  ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
1365
2074
  ADD COLUMN IF NOT EXISTS execution_reason text,
1366
2075
  ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
2076
+ ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
1367
2077
  ADD COLUMN IF NOT EXISTS authorization_binding_id text,
1368
2078
  ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
1369
2079
  ADD COLUMN IF NOT EXISTS lease_owner text,
@@ -1429,10 +2139,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1429
2139
  actor_id text NOT NULL, actor_type text NOT NULL, action_invocation_id text,
1430
2140
  payload jsonb NOT NULL, sequence bigint NOT NULL,
1431
2141
  occurred_at timestamptz NOT NULL, recorded_at timestamptz NOT NULL,
1432
- correlation_id text NOT NULL, causation_id text, provenance jsonb,
2142
+ correlation_id text NOT NULL, causation_id text, provenance jsonb, consistency text,
1433
2143
  UNIQUE (tenant_id, space_id, sequence)
1434
2144
  );
1435
2145
  ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS provenance jsonb;
2146
+ ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS consistency text;
1436
2147
  CREATE INDEX IF NOT EXISTS asset_events_subject_idx
1437
2148
  ON fabric_platform.asset_events
1438
2149
  (tenant_id, space_id, subject_type, subject_id, sequence);
@@ -1445,7 +2156,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1445
2156
  );
1446
2157
  CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
1447
2158
  ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
1448
- `);
2159
+ `
2160
+ }, {
2161
+ version: 2,
2162
+ name: "lease_fencing",
2163
+ sql: `
2164
+ ALTER TABLE fabric_platform.action_invocations
2165
+ ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
2166
+ ALTER TABLE fabric_platform.event_outbox
2167
+ ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
2168
+ `
2169
+ }]);
1449
2170
  }
1450
2171
  async transaction(run) {
1451
2172
  return run(this.db);
@@ -1506,6 +2227,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1506
2227
  status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
1507
2228
  error=CASE WHEN $6::boolean THEN $7 ELSE error END,
1508
2229
  authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
2230
+ adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
1509
2231
  lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
1510
2232
  THEN NULL ELSE lease_owner END,
1511
2233
  lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
@@ -1521,7 +2243,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1521
2243
  Object.hasOwn(patch, "error"),
1522
2244
  patch.error ?? null,
1523
2245
  Object.hasOwn(patch, "authorizationReconciliation"),
1524
- patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
2246
+ patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
2247
+ Object.hasOwn(patch, "adapterReconciliation"),
2248
+ patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
1525
2249
  ]
1526
2250
  );
1527
2251
  }
@@ -1555,6 +2279,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1555
2279
  error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
1556
2280
  lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
1557
2281
  lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
2282
+ lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
1558
2283
  attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
1559
2284
  updated_at=$9
1560
2285
  WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
@@ -1716,8 +2441,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1716
2441
  `INSERT INTO fabric_platform.asset_events
1717
2442
  (id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
1718
2443
  actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
1719
- correlation_id,causation_id,provenance)
1720
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
2444
+ correlation_id,causation_id,provenance,consistency)
2445
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$18)
1721
2446
  ON CONFLICT (id) DO NOTHING`,
1722
2447
  [
1723
2448
  event.id,
@@ -1736,7 +2461,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1736
2461
  event.recordedAt,
1737
2462
  event.correlationId,
1738
2463
  event.causationId ?? null,
1739
- event.provenance ? JSON.stringify(event.provenance) : null
2464
+ event.provenance ? JSON.stringify(event.provenance) : null,
2465
+ event.consistency ?? null
1740
2466
  ]
1741
2467
  );
1742
2468
  }
@@ -1747,8 +2473,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1747
2473
  INSERT INTO fabric_platform.asset_events
1748
2474
  (id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
1749
2475
  actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
1750
- correlation_id,causation_id,provenance)
1751
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
2476
+ correlation_id,causation_id,provenance,consistency)
2477
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$19)
1752
2478
  ON CONFLICT (id) DO NOTHING RETURNING id
1753
2479
  )
1754
2480
  INSERT INTO fabric_platform.event_outbox
@@ -1773,7 +2499,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1773
2499
  event.correlationId,
1774
2500
  event.causationId ?? null,
1775
2501
  event.provenance ? JSON.stringify(event.provenance) : null,
1776
- JSON.stringify(envelope)
2502
+ JSON.stringify(envelope),
2503
+ event.consistency ?? null
1777
2504
  ]
1778
2505
  );
1779
2506
  }
@@ -1790,28 +2517,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1790
2517
  ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
1791
2518
  )
1792
2519
  UPDATE fabric_platform.event_outbox AS item
1793
- SET lease_owner=$5, lease_expires_at=$6, attempt_count=attempt_count+1
2520
+ SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
2521
+ attempt_count=attempt_count+1
1794
2522
  FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
1795
2523
  [current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
1796
2524
  );
1797
2525
  return result.rows.map(toOutboxRecord);
1798
2526
  }
1799
- async markOutboxPublished(id, workerId, publishedAt) {
2527
+ async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
1800
2528
  const result = await this.sql.query(
1801
2529
  `UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
1802
- lease_owner=NULL,lease_expires_at=NULL WHERE id=$1 AND lease_owner=$2 RETURNING id`,
1803
- [id, workerId, publishedAt]
2530
+ lease_owner=NULL,lease_expires_at=NULL
2531
+ WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
2532
+ [id, workerId, publishedAt, leaseToken ?? null]
1804
2533
  );
1805
2534
  if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
1806
2535
  }
1807
2536
  async markOutboxFailed(input) {
1808
2537
  const result = await this.sql.query(
1809
2538
  `UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
1810
- lease_owner=NULL,lease_expires_at=NULL WHERE id=$1 AND lease_owner=$2 RETURNING id`,
1811
- [input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt]
2539
+ lease_owner=NULL,lease_expires_at=NULL
2540
+ WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
2541
+ [input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
1812
2542
  );
1813
2543
  if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
1814
2544
  }
2545
+ async renewOutboxLease(input) {
2546
+ const current = input.now ?? /* @__PURE__ */ new Date();
2547
+ const result = await this.sql.query(
2548
+ `UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
2549
+ WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
2550
+ AND lease_expires_at > $4
2551
+ RETURNING id`,
2552
+ [
2553
+ input.id,
2554
+ input.workerId,
2555
+ input.leaseToken,
2556
+ current,
2557
+ new Date(current.getTime() + input.leaseDurationMs)
2558
+ ]
2559
+ );
2560
+ return result.rows.length === 1;
2561
+ }
1815
2562
  async listOutbox(input = {}) {
1816
2563
  const result = await this.sql.query(
1817
2564
  `SELECT * FROM fabric_platform.event_outbox
@@ -1860,6 +2607,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1860
2607
  );
1861
2608
  return result.rows.map(toActionRecord);
1862
2609
  }
2610
+ async getHealthCounts(input) {
2611
+ const result = await this.sql.query(
2612
+ `SELECT
2613
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2614
+ WHERE status='pending'
2615
+ AND ($1::text IS NULL OR tenant_id=$1)
2616
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
2617
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2618
+ WHERE status='running'
2619
+ AND ($1::text IS NULL OR tenant_id=$1)
2620
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
2621
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2622
+ WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
2623
+ AND ($1::text IS NULL OR tenant_id=$1)
2624
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
2625
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2626
+ WHERE status='waiting_for_approval'
2627
+ AND ($1::text IS NULL OR tenant_id=$1)
2628
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
2629
+ (SELECT COUNT(*) FROM fabric_platform.action_invocations
2630
+ WHERE status='reconciliation_required'
2631
+ AND ($1::text IS NULL OR tenant_id=$1)
2632
+ AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
2633
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2634
+ WHERE status='pending'
2635
+ AND ($1::text IS NULL OR tenant_id=$1)
2636
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
2637
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2638
+ WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
2639
+ AND ($1::text IS NULL OR tenant_id=$1)
2640
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
2641
+ (SELECT COUNT(*) FROM fabric_platform.event_outbox
2642
+ WHERE status='dead_letter'
2643
+ AND ($1::text IS NULL OR tenant_id=$1)
2644
+ AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
2645
+ [input.tenantId ?? null, input.spaceId ?? null, input.now]
2646
+ );
2647
+ const row = result.rows[0] ?? {};
2648
+ return {
2649
+ invocationBacklog: Number(row.invocation_backlog ?? 0),
2650
+ invocationRunning: Number(row.invocation_running ?? 0),
2651
+ invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
2652
+ invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
2653
+ invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
2654
+ outboxBacklog: Number(row.outbox_backlog ?? 0),
2655
+ outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
2656
+ outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
2657
+ };
2658
+ }
1863
2659
  async claimActionInvocations(input) {
1864
2660
  const current = input.now ?? /* @__PURE__ */ new Date();
1865
2661
  const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
@@ -1868,7 +2664,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1868
2664
  `WITH claimable AS (
1869
2665
  SELECT id FROM fabric_platform.action_invocations
1870
2666
  WHERE (status='pending' OR
1871
- (status='running' AND (lease_expires_at IS NULL OR lease_expires_at <= $1)))
2667
+ (status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
1872
2668
  AND ($2::text IS NULL OR tenant_id=$2)
1873
2669
  AND ($3::text IS NULL OR space_id=$3)
1874
2670
  ORDER BY created_at
@@ -1877,7 +2673,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1877
2673
  )
1878
2674
  UPDATE fabric_platform.action_invocations AS invocation
1879
2675
  SET status='running', lease_owner=$5, lease_expires_at=$6,
1880
- attempt_count=attempt_count+1, updated_at=$1
2676
+ lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
1881
2677
  FROM claimable WHERE invocation.id=claimable.id
1882
2678
  RETURNING invocation.*`,
1883
2679
  [
@@ -1891,6 +2687,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1891
2687
  );
1892
2688
  return result.rows.map(toActionRecord);
1893
2689
  }
2690
+ async updateLeasedActionInvocation(input) {
2691
+ const patch = input.patch;
2692
+ const result = await this.sql.query(
2693
+ `UPDATE fabric_platform.action_invocations SET
2694
+ status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
2695
+ error=CASE WHEN $8::boolean THEN $9 ELSE error END,
2696
+ authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
2697
+ adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
2698
+ lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
2699
+ THEN NULL ELSE lease_owner END,
2700
+ lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
2701
+ THEN NULL ELSE lease_expires_at END,
2702
+ updated_at=now()
2703
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2704
+ AND status='running' AND lease_owner=$4 AND lease_token=$5
2705
+ RETURNING id`,
2706
+ [
2707
+ input.id,
2708
+ input.tenantId,
2709
+ input.spaceId,
2710
+ input.workerId,
2711
+ input.leaseToken,
2712
+ patch.status ?? null,
2713
+ patch.result === void 0 ? null : JSON.stringify(patch.result),
2714
+ Object.hasOwn(patch, "error"),
2715
+ patch.error ?? null,
2716
+ Object.hasOwn(patch, "authorizationReconciliation"),
2717
+ patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
2718
+ Object.hasOwn(patch, "adapterReconciliation"),
2719
+ patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
2720
+ ]
2721
+ );
2722
+ return result.rows.length === 1;
2723
+ }
2724
+ async renewActionInvocationLease(input) {
2725
+ const current = input.now ?? /* @__PURE__ */ new Date();
2726
+ const result = await this.sql.query(
2727
+ `UPDATE fabric_platform.action_invocations
2728
+ SET lease_expires_at=$6,updated_at=$5
2729
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3
2730
+ AND status='running' AND lease_owner=$4 AND lease_token=$7
2731
+ AND lease_expires_at > $5
2732
+ RETURNING id`,
2733
+ [
2734
+ input.id,
2735
+ input.tenantId,
2736
+ input.spaceId,
2737
+ input.workerId,
2738
+ current,
2739
+ new Date(current.getTime() + input.leaseDurationMs),
2740
+ input.leaseToken
2741
+ ]
2742
+ );
2743
+ return result.rows.length === 1;
2744
+ }
1894
2745
  async listEvents(tenantId, spaceId) {
1895
2746
  const result = await this.sql.query(
1896
2747
  `SELECT * FROM fabric_platform.asset_events
@@ -1899,6 +2750,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
1899
2750
  );
1900
2751
  return result.rows.map(toEventRecord);
1901
2752
  }
2753
+ async getEvent(id, tenantId, spaceId) {
2754
+ const result = await this.sql.query(
2755
+ `SELECT * FROM fabric_platform.asset_events
2756
+ WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
2757
+ [id, tenantId, spaceId]
2758
+ );
2759
+ return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
2760
+ }
1902
2761
  };
1903
2762
  function toAdapterRecord(row) {
1904
2763
  return {
@@ -1941,8 +2800,10 @@ function toActionRecord(row) {
1941
2800
  ...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
1942
2801
  ...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
1943
2802
  ...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
2803
+ ...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
1944
2804
  ...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
1945
2805
  attemptCount: Number(row.attempt_count ?? 0),
2806
+ leaseToken: Number(row.lease_token ?? 0),
1946
2807
  ...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
1947
2808
  ...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
1948
2809
  ...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
@@ -2015,7 +2876,8 @@ function toEventRecord(row) {
2015
2876
  recordedAt: new Date(row.recorded_at),
2016
2877
  correlationId: String(row.correlation_id),
2017
2878
  ...row.causation_id ? { causationId: String(row.causation_id) } : {},
2018
- ...row.provenance ? { provenance: row.provenance } : {}
2879
+ ...row.provenance ? { provenance: row.provenance } : {},
2880
+ ...row.consistency ? { consistency: row.consistency } : {}
2019
2881
  };
2020
2882
  }
2021
2883
  function toOutboxRecord(row) {
@@ -2037,6 +2899,7 @@ function toOutboxRecord(row) {
2037
2899
  correlationId: String(event.correlationId),
2038
2900
  ...event.causationId ? { causationId: String(event.causationId) } : {},
2039
2901
  ...event.provenance ? { provenance: event.provenance } : {},
2902
+ ...event.consistency ? { consistency: event.consistency } : {},
2040
2903
  occurredAt: new Date(event.occurredAt),
2041
2904
  recordedAt: new Date(event.recordedAt),
2042
2905
  producerModuleVersion: String(event.producerModuleVersion),
@@ -2049,6 +2912,7 @@ function toOutboxRecord(row) {
2049
2912
  availableAt: new Date(row.available_at),
2050
2913
  ...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
2051
2914
  ...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
2915
+ leaseToken: Number(row.lease_token ?? 0),
2052
2916
  ...row.last_error ? { lastError: String(row.last_error) } : {},
2053
2917
  createdAt: new Date(row.created_at),
2054
2918
  ...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
@@ -2065,38 +2929,93 @@ function createStoreBackedActionDispatcher() {
2065
2929
  };
2066
2930
  }
2067
2931
  async function runPlatformActionWorkerCycle(options) {
2068
- const claimed = await options.store.claimActionInvocations({
2069
- workerId: options.workerId,
2070
- limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
2071
- leaseDurationMs: options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS,
2072
- ...options.tenantId ? { tenantId: options.tenantId } : {},
2073
- ...options.spaceId ? { spaceId: options.spaceId } : {}
2074
- });
2932
+ const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
2933
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
2934
+ let claimed;
2935
+ try {
2936
+ claimed = await options.store.claimActionInvocations({
2937
+ workerId: options.workerId,
2938
+ limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
2939
+ leaseDurationMs,
2940
+ ...options.tenantId ? { tenantId: options.tenantId } : {},
2941
+ ...options.spaceId ? { spaceId: options.spaceId } : {}
2942
+ });
2943
+ } catch (error) {
2944
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
2945
+ throw error;
2946
+ }
2947
+ for (const invocation of claimed) {
2948
+ emitPlatformHostTelemetry(options.telemetry, {
2949
+ kind: "event",
2950
+ name: "invocation.lease_claimed",
2951
+ metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
2952
+ occurredAt: /* @__PURE__ */ new Date(),
2953
+ tenantId: invocation.tenantId,
2954
+ spaceId: invocation.spaceId,
2955
+ attributes: { attempt: invocation.attemptCount }
2956
+ });
2957
+ }
2075
2958
  let completed = 0;
2076
2959
  let failed = 0;
2077
2960
  let waitingForApproval = 0;
2078
2961
  for (const invocation of claimed) {
2962
+ const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
2079
2963
  try {
2080
- const result = await options.host.executeInvocation(
2964
+ const result2 = await options.host.executeInvocation(
2081
2965
  invocation.id,
2082
2966
  invocation.tenantId,
2083
2967
  invocation.spaceId,
2084
- { leaseOwner: options.workerId }
2968
+ { leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
2085
2969
  );
2086
- if (result.status === "completed") completed += 1;
2087
- else if (result.status === "waiting_for_approval") waitingForApproval += 1;
2970
+ if (result2.status === "completed") completed += 1;
2971
+ else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
2088
2972
  else failed += 1;
2089
2973
  } catch (error) {
2090
2974
  failed += 1;
2091
2975
  options.onError?.(error, invocation);
2976
+ } finally {
2977
+ await stopHeartbeat();
2092
2978
  }
2093
2979
  }
2094
- return {
2980
+ const result = {
2095
2981
  claimed: claimed.length,
2096
2982
  completed,
2097
2983
  failed,
2098
2984
  ...waitingForApproval > 0 ? { waitingForApproval } : {}
2099
2985
  };
2986
+ emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
2987
+ return result;
2988
+ }
2989
+ function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
2990
+ if (!invocation?.leaseToken) return async () => void 0;
2991
+ const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
2992
+ let stopped = false;
2993
+ let timer;
2994
+ let running;
2995
+ const schedule = () => {
2996
+ if (!stopped) timer = setTimeout(tick, intervalMs);
2997
+ };
2998
+ const tick = () => {
2999
+ running = options.store.renewActionInvocationLease({
3000
+ id: invocation.id,
3001
+ tenantId: invocation.tenantId,
3002
+ spaceId: invocation.spaceId,
3003
+ workerId: options.workerId,
3004
+ leaseToken: invocation.leaseToken,
3005
+ leaseDurationMs
3006
+ }).then((renewed) => {
3007
+ if (!renewed) options.onError?.(
3008
+ new Error(`Invocation lease lost: ${invocation.id}`),
3009
+ invocation
3010
+ );
3011
+ }).catch((error) => options.onError?.(error, invocation)).finally(schedule);
3012
+ };
3013
+ schedule();
3014
+ return async () => {
3015
+ stopped = true;
3016
+ if (timer) clearTimeout(timer);
3017
+ await running;
3018
+ };
2100
3019
  }
2101
3020
  async function runPlatformActionWorker(options) {
2102
3021
  while (!options.signal?.aborted) {
@@ -2122,7 +3041,158 @@ async function abortableDelay(milliseconds, signal) {
2122
3041
  );
2123
3042
  });
2124
3043
  }
3044
+ function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
3045
+ const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
3046
+ emitPlatformHostTelemetry(telemetry, {
3047
+ kind: "event",
3048
+ name,
3049
+ metricName,
3050
+ occurredAt: /* @__PURE__ */ new Date(),
3051
+ attributes: { workerId, ...attributes }
3052
+ });
3053
+ }
3054
+
3055
+ // src/saga-parent-lifecycle.ts
3056
+ var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
3057
+ "completed",
3058
+ "failed",
3059
+ "blocked_by_policy",
3060
+ "reconciliation_required",
3061
+ "validation_failed"
3062
+ ]);
3063
+ function createDurableSagaParentLifecycle(options) {
3064
+ const store = options.store;
3065
+ const now = options.now ?? (() => /* @__PURE__ */ new Date());
3066
+ const transactionWithEvents = store.transactionWithEvents?.bind(store);
3067
+ if (!transactionWithEvents) {
3068
+ throw new Error(
3069
+ "Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
3070
+ );
3071
+ }
3072
+ const runInTransaction = transactionWithEvents;
3073
+ async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
3074
+ const eventId = `sagapar_${digestParameters({
3075
+ parentInvocationId: input.parentInvocationId,
3076
+ lifecycleId: lifecycleId2
3077
+ }).slice(0, 40)}`;
3078
+ await runInTransaction(async (transaction) => {
3079
+ if (!transaction.getActionInvocationForUpdate) {
3080
+ throw new Error(
3081
+ "Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
3082
+ );
3083
+ }
3084
+ if (!transaction.getEvent) {
3085
+ throw new Error(
3086
+ "Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
3087
+ );
3088
+ }
3089
+ const parent = await transaction.getActionInvocationForUpdate(
3090
+ input.parentInvocationId,
3091
+ input.tenantId,
3092
+ input.spaceId
3093
+ );
3094
+ if (!parent) {
3095
+ throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
3096
+ }
3097
+ const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
3098
+ if (prior) {
3099
+ if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
3100
+ throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
3101
+ }
3102
+ return;
3103
+ }
3104
+ if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
3105
+ throw new Error(
3106
+ `Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
3107
+ );
3108
+ }
3109
+ const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
3110
+ const timestamp = now();
3111
+ await transaction.appendEvent({
3112
+ id: eventId,
3113
+ tenantId: input.tenantId,
3114
+ spaceId: input.spaceId,
3115
+ eventType,
3116
+ eventSchemaVersion: 1,
3117
+ subjectType: "ActionInvocation",
3118
+ subjectId: input.parentInvocationId,
3119
+ actorId: "system",
3120
+ actorType: "system",
3121
+ actionInvocationId: input.parentInvocationId,
3122
+ payload,
3123
+ sequence,
3124
+ occurredAt: timestamp,
3125
+ recordedAt: timestamp,
3126
+ correlationId: input.parentInvocationId,
3127
+ causationId: lifecycleId2
3128
+ });
3129
+ if (patch) {
3130
+ await transaction.updateActionInvocation(
3131
+ input.parentInvocationId,
3132
+ input.tenantId,
3133
+ input.spaceId,
3134
+ patch
3135
+ );
3136
+ }
3137
+ });
3138
+ }
3139
+ return {
3140
+ async recordProgress(input) {
3141
+ await applyLifecycleTransition(
3142
+ input,
3143
+ input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
3144
+ "SagaParentProgress",
3145
+ {
3146
+ stepId: input.progress.stepId,
3147
+ index: input.progress.index,
3148
+ total: input.progress.total,
3149
+ completedAt: input.progress.completedAt
3150
+ }
3151
+ );
3152
+ },
3153
+ async parkApproval(input) {
3154
+ await applyLifecycleTransition(
3155
+ input,
3156
+ input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
3157
+ "SagaParentApprovalParked",
3158
+ {
3159
+ stepId: input.stepId,
3160
+ approvalId: input.approvalId,
3161
+ reason: input.reason
3162
+ },
3163
+ { status: "waiting_for_approval" }
3164
+ );
3165
+ },
3166
+ async cancel(input) {
3167
+ await applyLifecycleTransition(
3168
+ input,
3169
+ input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
3170
+ "SagaParentCancelled",
3171
+ { reason: input.reason, actorId: input.actorId },
3172
+ { status: "failed", error: `Saga cancelled: ${input.reason}` }
3173
+ );
3174
+ },
3175
+ async complete(input) {
3176
+ await applyLifecycleTransition(
3177
+ input,
3178
+ input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
3179
+ "SagaParentCompleted",
3180
+ { result: input.result },
3181
+ { status: "completed", result: input.result }
3182
+ );
3183
+ },
3184
+ async fail(input) {
3185
+ await applyLifecycleTransition(
3186
+ input,
3187
+ input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
3188
+ "SagaParentFailed",
3189
+ { error: input.error, compensation: input.compensation },
3190
+ { status: "failed", error: input.error }
3191
+ );
3192
+ }
3193
+ };
3194
+ }
2125
3195
 
2126
- export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PostgresPlatformHostStore, canonicalJson, cloneOutboxRecord, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
3196
+ export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, getPlatformHostHealthSnapshot, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
2127
3197
  //# sourceMappingURL=index.js.map
2128
3198
  //# sourceMappingURL=index.js.map