@fabricorg/platform-host 4.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/MIGRATION-6-PRODUCTION.md +59 -0
- package/README.md +89 -11
- package/dist/index.cjs +1194 -118
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +383 -13
- package/dist/index.d.ts +383 -13
- package/dist/index.js +1190 -120
- package/dist/index.js.map +1 -1
- package/package.json +6 -3
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes,
|
|
1
|
+
import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
2
|
+
import { assertAssemblyRuntimeCompatible } from '@fabricorg/assembly';
|
|
2
3
|
import { createHash } from 'crypto';
|
|
3
4
|
|
|
4
5
|
// src/host.ts
|
|
@@ -62,6 +63,213 @@ function normalizeProvenance(input, options) {
|
|
|
62
63
|
};
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
// src/observability.ts
|
|
67
|
+
var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
|
|
68
|
+
var PLATFORM_HOST_METRIC_NAMES = {
|
|
69
|
+
workerCyclesStarted: "fabric.platform.worker.cycles.started",
|
|
70
|
+
workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
|
|
71
|
+
workerCyclesFailed: "fabric.platform.worker.cycles.failed",
|
|
72
|
+
invocationSubmitted: "fabric.platform.invocation.submitted",
|
|
73
|
+
invocationExecutionStarted: "fabric.platform.invocation.execution_started",
|
|
74
|
+
invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
|
|
75
|
+
invocationCompleted: "fabric.platform.invocation.completed",
|
|
76
|
+
invocationFailed: "fabric.platform.invocation.failed",
|
|
77
|
+
invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
|
|
78
|
+
invocationValidationFailed: "fabric.platform.invocation.validation_failed",
|
|
79
|
+
invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
|
|
80
|
+
invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
|
|
81
|
+
outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
|
|
82
|
+
outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
|
|
83
|
+
outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
|
|
84
|
+
outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
|
|
85
|
+
outboxPublished: "fabric.platform.outbox.published",
|
|
86
|
+
outboxFailed: "fabric.platform.outbox.failed",
|
|
87
|
+
outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
|
|
88
|
+
invocationBacklog: "fabric.platform.invocation.backlog",
|
|
89
|
+
invocationRunning: "fabric.platform.invocation.running",
|
|
90
|
+
invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
|
|
91
|
+
invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
|
|
92
|
+
invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
|
|
93
|
+
outboxBacklog: "fabric.platform.outbox.backlog",
|
|
94
|
+
outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
|
|
95
|
+
outboxDeadLetters: "fabric.platform.outbox.dead_letters"
|
|
96
|
+
};
|
|
97
|
+
function emitPlatformHostTelemetry(telemetry, input) {
|
|
98
|
+
if (!telemetry) return;
|
|
99
|
+
const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
|
|
100
|
+
try {
|
|
101
|
+
const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
|
|
102
|
+
if (isPromiseLike(result)) void result.catch(() => void 0);
|
|
103
|
+
} catch {
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function getPlatformHostHealthSnapshot(options) {
|
|
107
|
+
const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
|
|
108
|
+
const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
|
|
109
|
+
const scope = {
|
|
110
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
111
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
112
|
+
};
|
|
113
|
+
try {
|
|
114
|
+
const counts = await readHealthCounts(options.store, {
|
|
115
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
116
|
+
...options.spaceId ? { spaceId: options.spaceId } : {},
|
|
117
|
+
now: generatedAt
|
|
118
|
+
});
|
|
119
|
+
const snapshot = createHealthSnapshot(
|
|
120
|
+
generatedAt,
|
|
121
|
+
scope,
|
|
122
|
+
counts,
|
|
123
|
+
workerSummary,
|
|
124
|
+
true
|
|
125
|
+
);
|
|
126
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
127
|
+
return snapshot;
|
|
128
|
+
} catch {
|
|
129
|
+
const snapshot = createHealthSnapshot(
|
|
130
|
+
generatedAt,
|
|
131
|
+
scope,
|
|
132
|
+
zeroHealthCounts(),
|
|
133
|
+
workerSummary,
|
|
134
|
+
false
|
|
135
|
+
);
|
|
136
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
137
|
+
return snapshot;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function readHealthCounts(store, query) {
|
|
141
|
+
const candidate = store;
|
|
142
|
+
if (typeof candidate.getHealthCounts === "function") {
|
|
143
|
+
return assertHealthCounts(await candidate.getHealthCounts(query));
|
|
144
|
+
}
|
|
145
|
+
const recoverable = store;
|
|
146
|
+
const outbox = store;
|
|
147
|
+
if (typeof recoverable.listActionInvocations !== "function") {
|
|
148
|
+
throw new Error("Health counts are not supported by this store.");
|
|
149
|
+
}
|
|
150
|
+
const invocations = await recoverable.listActionInvocations({
|
|
151
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
152
|
+
...query.spaceId ? { spaceId: query.spaceId } : {},
|
|
153
|
+
limit: Number.MAX_SAFE_INTEGER
|
|
154
|
+
});
|
|
155
|
+
const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
|
|
156
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
157
|
+
...query.spaceId ? { spaceId: query.spaceId } : {}
|
|
158
|
+
}) : [];
|
|
159
|
+
return countsFromRecords(invocations, outboxRecords, query.now);
|
|
160
|
+
}
|
|
161
|
+
function countsFromRecords(invocations, outbox, now) {
|
|
162
|
+
return {
|
|
163
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
164
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
165
|
+
invocationExpiredLeases: invocations.filter(
|
|
166
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
167
|
+
).length,
|
|
168
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
169
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
170
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
171
|
+
outboxExpiredLeases: outbox.filter(
|
|
172
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
173
|
+
).length,
|
|
174
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function assertHealthCounts(value) {
|
|
178
|
+
for (const [name, count] of Object.entries(value)) {
|
|
179
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
180
|
+
throw new Error(`Invalid health count: ${name}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return value;
|
|
184
|
+
}
|
|
185
|
+
function zeroHealthCounts() {
|
|
186
|
+
return {
|
|
187
|
+
invocationBacklog: 0,
|
|
188
|
+
invocationRunning: 0,
|
|
189
|
+
invocationExpiredLeases: 0,
|
|
190
|
+
invocationApprovalWaits: 0,
|
|
191
|
+
invocationReconciliationRequired: 0,
|
|
192
|
+
outboxBacklog: 0,
|
|
193
|
+
outboxExpiredLeases: 0,
|
|
194
|
+
outboxDeadLetters: 0
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
function summarizeWorkers(workers, now) {
|
|
198
|
+
let healthy = 0;
|
|
199
|
+
let stale = 0;
|
|
200
|
+
for (const worker of workers) {
|
|
201
|
+
if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
|
|
202
|
+
else stale += 1;
|
|
203
|
+
}
|
|
204
|
+
return { reported: workers.length, healthy, stale };
|
|
205
|
+
}
|
|
206
|
+
function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
|
|
207
|
+
const reasonCodes = [];
|
|
208
|
+
if (!dataAvailable) reasonCodes.push("health_source_unavailable");
|
|
209
|
+
if (workers.stale > 0) reasonCodes.push("workers_stale");
|
|
210
|
+
if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
|
|
211
|
+
if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
|
|
212
|
+
if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
|
|
213
|
+
if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
|
|
214
|
+
const unhealthy = !dataAvailable;
|
|
215
|
+
const degraded = unhealthy || reasonCodes.length > 0;
|
|
216
|
+
return {
|
|
217
|
+
contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
|
|
218
|
+
generatedAt,
|
|
219
|
+
scope,
|
|
220
|
+
dataAvailable,
|
|
221
|
+
status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
|
|
222
|
+
readiness: {
|
|
223
|
+
ready: dataAvailable && workers.stale === 0,
|
|
224
|
+
reasonCodes
|
|
225
|
+
},
|
|
226
|
+
workers,
|
|
227
|
+
invocations: {
|
|
228
|
+
backlog: counts.invocationBacklog,
|
|
229
|
+
running: counts.invocationRunning,
|
|
230
|
+
expiredLeases: counts.invocationExpiredLeases,
|
|
231
|
+
approvalWaits: counts.invocationApprovalWaits,
|
|
232
|
+
reconciliationRequired: counts.invocationReconciliationRequired
|
|
233
|
+
},
|
|
234
|
+
outbox: {
|
|
235
|
+
backlog: counts.outboxBacklog,
|
|
236
|
+
expiredLeases: counts.outboxExpiredLeases,
|
|
237
|
+
deadLetters: counts.outboxDeadLetters
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function emitHealthMetrics(telemetry, snapshot) {
|
|
242
|
+
const scope = {
|
|
243
|
+
...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
|
|
244
|
+
...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
|
|
245
|
+
};
|
|
246
|
+
const metrics = [
|
|
247
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
|
|
248
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
|
|
249
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
|
|
250
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
|
|
251
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
|
|
252
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
|
|
253
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
|
|
254
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
|
|
255
|
+
];
|
|
256
|
+
for (const [name, value] of metrics) {
|
|
257
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
258
|
+
kind: "metric",
|
|
259
|
+
name,
|
|
260
|
+
metricType: "gauge",
|
|
261
|
+
value,
|
|
262
|
+
...scope,
|
|
263
|
+
observedAt: snapshot.generatedAt
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
function isPromiseLike(value) {
|
|
268
|
+
return Boolean(
|
|
269
|
+
value && typeof value === "object" && "then" in value && typeof value.then === "function"
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
|
|
65
273
|
// src/host.ts
|
|
66
274
|
var DEFAULT_EXTRACT_EVENTS = (data) => {
|
|
67
275
|
const value = data._events;
|
|
@@ -71,6 +279,19 @@ var RecoverableFinalizationError = class extends Error {
|
|
|
71
279
|
name = "RecoverableFinalizationError";
|
|
72
280
|
};
|
|
73
281
|
function createGovernedActionHost(options) {
|
|
282
|
+
if (options.composition) {
|
|
283
|
+
if (!options.registry?.orderedModules) {
|
|
284
|
+
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
285
|
+
}
|
|
286
|
+
assertAssemblyRuntimeCompatible(
|
|
287
|
+
options.composition.assembly,
|
|
288
|
+
options.registry.orderedModules.map((module) => ({
|
|
289
|
+
namespace: module.namespace,
|
|
290
|
+
version: module.version ?? "",
|
|
291
|
+
manifestDigest: module.manifestDigest ?? ""
|
|
292
|
+
}))
|
|
293
|
+
);
|
|
294
|
+
}
|
|
74
295
|
if (options.outbox) {
|
|
75
296
|
const outboxStore = asOutboxStore(options.store);
|
|
76
297
|
if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
|
|
@@ -92,6 +313,12 @@ function createGovernedActionHost(options) {
|
|
|
92
313
|
}
|
|
93
314
|
const action = actionResolver(input.actionId);
|
|
94
315
|
if (!action) throw new Error(`Unknown action: ${input.actionId}`);
|
|
316
|
+
if (action.execution?.connectivity === "online-required" && input.executionReason === "offline_replay") {
|
|
317
|
+
throw new Error(`Action ${input.actionId} is online-required and cannot be submitted as an offline replay.`);
|
|
318
|
+
}
|
|
319
|
+
if (action.execution?.sensitivity === "restricted" && Object.keys(input.provenance?.auditAttributes ?? {}).length > 0) {
|
|
320
|
+
throw new Error(`Action ${input.actionId} declares restricted sensitivity; provenance audit attributes must not be durably recorded.`);
|
|
321
|
+
}
|
|
95
322
|
const authorizationInput = toAuthorizationInput(action, input);
|
|
96
323
|
if (!await options.authorization.checkEntitlement(authorizationInput)) {
|
|
97
324
|
throw new Error(`Module "${action.namespace}" is not enabled for tenant ${input.tenantId}`);
|
|
@@ -110,10 +337,13 @@ function createGovernedActionHost(options) {
|
|
|
110
337
|
input.provenance,
|
|
111
338
|
options.provenance
|
|
112
339
|
);
|
|
340
|
+
const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
|
|
113
341
|
const runtimeEvidence = {
|
|
114
342
|
governanceContractVersion: FABRIC_GOVERNANCE_CONTRACT_VERSION,
|
|
115
343
|
hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
|
|
116
|
-
...options.runtimeEvidence
|
|
344
|
+
...options.runtimeEvidence,
|
|
345
|
+
...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
|
|
346
|
+
...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
|
|
117
347
|
};
|
|
118
348
|
assertGovernanceRuntimeEvidence(runtimeEvidence);
|
|
119
349
|
const durableInvocation = await options.store.createActionInvocation({
|
|
@@ -142,6 +372,15 @@ function createGovernedActionHost(options) {
|
|
|
142
372
|
...input.executionReason ? { executionReason: input.executionReason } : {},
|
|
143
373
|
...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
|
|
144
374
|
});
|
|
375
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
376
|
+
kind: "event",
|
|
377
|
+
name: "invocation.submitted",
|
|
378
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
|
|
379
|
+
occurredAt: now(),
|
|
380
|
+
tenantId: input.tenantId,
|
|
381
|
+
spaceId: input.spaceId,
|
|
382
|
+
attributes: { actionVersion: action.version }
|
|
383
|
+
});
|
|
145
384
|
if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
|
|
146
385
|
const conflict = idempotencyConflict(durableInvocation, {
|
|
147
386
|
actorId: input.actorId,
|
|
@@ -164,7 +403,24 @@ function createGovernedActionHost(options) {
|
|
|
164
403
|
}
|
|
165
404
|
const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
|
|
166
405
|
if (durableInvocation.id !== actionInvocationId) {
|
|
167
|
-
|
|
406
|
+
if (options.dispatcher && durableInvocation.status === "pending") {
|
|
407
|
+
const dispatched = await options.dispatcher.dispatch({
|
|
408
|
+
actionInvocationId: durableInvocation.id,
|
|
409
|
+
actionId: durableInvocation.actionId,
|
|
410
|
+
tenantId: input.tenantId,
|
|
411
|
+
spaceId: input.spaceId,
|
|
412
|
+
workflowId: durableWorkflowId
|
|
413
|
+
});
|
|
414
|
+
return withConsistency({
|
|
415
|
+
actionInvocationId: durableInvocation.id,
|
|
416
|
+
status: durableInvocation.status,
|
|
417
|
+
workflowId: dispatched.workflowId,
|
|
418
|
+
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
419
|
+
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
420
|
+
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
421
|
+
}, input.actionId);
|
|
422
|
+
}
|
|
423
|
+
return withConsistency({
|
|
168
424
|
actionInvocationId: durableInvocation.id,
|
|
169
425
|
status: durableInvocation.status,
|
|
170
426
|
workflowId: durableWorkflowId,
|
|
@@ -172,33 +428,28 @@ function createGovernedActionHost(options) {
|
|
|
172
428
|
...durableInvocation.error ? { error: durableInvocation.error } : {},
|
|
173
429
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
174
430
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
175
|
-
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
|
|
176
|
-
|
|
431
|
+
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
432
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
433
|
+
}, input.actionId);
|
|
177
434
|
}
|
|
178
435
|
if (options.dispatcher) {
|
|
179
436
|
try {
|
|
180
437
|
const dispatched = await options.dispatcher.dispatch({
|
|
181
438
|
actionInvocationId: durableInvocation.id,
|
|
439
|
+
actionId: durableInvocation.actionId,
|
|
182
440
|
tenantId: input.tenantId,
|
|
183
441
|
spaceId: input.spaceId,
|
|
184
442
|
workflowId: durableWorkflowId
|
|
185
443
|
});
|
|
186
|
-
return {
|
|
444
|
+
return withConsistency({
|
|
187
445
|
actionInvocationId: durableInvocation.id,
|
|
188
446
|
status: durableInvocation.status,
|
|
189
447
|
workflowId: dispatched.workflowId,
|
|
190
448
|
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
191
449
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
192
450
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
193
|
-
};
|
|
451
|
+
}, input.actionId);
|
|
194
452
|
} catch (error) {
|
|
195
|
-
const message = errorMessage(error);
|
|
196
|
-
await options.store.updateActionInvocation(
|
|
197
|
-
durableInvocation.id,
|
|
198
|
-
input.tenantId,
|
|
199
|
-
input.spaceId,
|
|
200
|
-
{ status: "failed", error: message }
|
|
201
|
-
);
|
|
202
453
|
throw error;
|
|
203
454
|
}
|
|
204
455
|
}
|
|
@@ -207,7 +458,7 @@ function createGovernedActionHost(options) {
|
|
|
207
458
|
input.tenantId,
|
|
208
459
|
input.spaceId
|
|
209
460
|
);
|
|
210
|
-
return { ...executed, workflowId: durableWorkflowId };
|
|
461
|
+
return { ...withConsistency(executed, input.actionId), workflowId: durableWorkflowId };
|
|
211
462
|
}
|
|
212
463
|
async function executeInvocation(actionInvocationId, tenantId, spaceId, executionOptions = {}, executionReasonOverride) {
|
|
213
464
|
const loadedInvocation = await options.store.getActionInvocation(
|
|
@@ -221,18 +472,32 @@ function createGovernedActionHost(options) {
|
|
|
221
472
|
const resumingRunningInvocation = loadedInvocation.status === "running";
|
|
222
473
|
let invocation = loadedInvocation;
|
|
223
474
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
224
|
-
return actionResult(invocation);
|
|
475
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
225
476
|
}
|
|
226
|
-
if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
|
|
477
|
+
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
227
478
|
return {
|
|
228
479
|
...actionResult(invocation),
|
|
229
|
-
error: `Invocation is
|
|
480
|
+
error: `Invocation lease is not owned by the supplied worker generation`
|
|
230
481
|
};
|
|
231
482
|
}
|
|
232
483
|
const action = actionResolver(invocation.actionId);
|
|
233
484
|
if (!action) {
|
|
234
485
|
return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
|
|
235
486
|
}
|
|
487
|
+
if (action.kind === "saga") {
|
|
488
|
+
return fail(
|
|
489
|
+
invocation,
|
|
490
|
+
"failed",
|
|
491
|
+
`Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
|
|
495
|
+
return fail(
|
|
496
|
+
invocation,
|
|
497
|
+
"failed",
|
|
498
|
+
`Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
|
|
499
|
+
);
|
|
500
|
+
}
|
|
236
501
|
if (invocation.attemptCount > 1 && !action.idempotent) {
|
|
237
502
|
return fail(
|
|
238
503
|
invocation,
|
|
@@ -245,6 +510,15 @@ function createGovernedActionHost(options) {
|
|
|
245
510
|
status: "running"
|
|
246
511
|
});
|
|
247
512
|
}
|
|
513
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
514
|
+
kind: "event",
|
|
515
|
+
name: "invocation.execution_started",
|
|
516
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
|
|
517
|
+
occurredAt: now(),
|
|
518
|
+
tenantId,
|
|
519
|
+
spaceId,
|
|
520
|
+
attributes: { actionVersion: invocation.actionVersion }
|
|
521
|
+
});
|
|
248
522
|
try {
|
|
249
523
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
250
524
|
if (!parsed.success) {
|
|
@@ -310,7 +584,7 @@ function createGovernedActionHost(options) {
|
|
|
310
584
|
);
|
|
311
585
|
}
|
|
312
586
|
if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
|
|
313
|
-
await
|
|
587
|
+
await persistInvocation(invocation, {
|
|
314
588
|
status: "waiting_for_approval"
|
|
315
589
|
});
|
|
316
590
|
return {
|
|
@@ -337,8 +611,8 @@ function createGovernedActionHost(options) {
|
|
|
337
611
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
338
612
|
message: `Action ${action.actionId} requires durable capture-time authorization evidence`
|
|
339
613
|
};
|
|
340
|
-
await
|
|
341
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
614
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
615
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
342
616
|
}
|
|
343
617
|
const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
|
|
344
618
|
if (bindingExpired) {
|
|
@@ -350,8 +624,8 @@ function createGovernedActionHost(options) {
|
|
|
350
624
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
351
625
|
message: `Authorization binding for action ${action.actionId} expired before execution`
|
|
352
626
|
};
|
|
353
|
-
await
|
|
354
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
627
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
628
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
355
629
|
}
|
|
356
630
|
const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
|
|
357
631
|
...authorizationInput,
|
|
@@ -372,8 +646,8 @@ function createGovernedActionHost(options) {
|
|
|
372
646
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
373
647
|
message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
|
|
374
648
|
};
|
|
375
|
-
await
|
|
376
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
649
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
650
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
377
651
|
}
|
|
378
652
|
const definitions = options.resolvePolicies ? await options.resolvePolicies({
|
|
379
653
|
...authorizationInput,
|
|
@@ -434,30 +708,6 @@ function createGovernedActionHost(options) {
|
|
|
434
708
|
})));
|
|
435
709
|
}
|
|
436
710
|
}
|
|
437
|
-
const binding = action.stateMachine;
|
|
438
|
-
if (binding) {
|
|
439
|
-
const entityId = binding.getEntityId(parsed.data);
|
|
440
|
-
const currentState = entityId ? await options.store.getEntityState(
|
|
441
|
-
tenantId,
|
|
442
|
-
spaceId,
|
|
443
|
-
binding.entityType,
|
|
444
|
-
entityId
|
|
445
|
-
) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
|
|
446
|
-
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
447
|
-
if (targetState !== "") {
|
|
448
|
-
const transition = validateStateMachineTransition(
|
|
449
|
-
stateMachineResolver(binding.entityType),
|
|
450
|
-
binding.entityType,
|
|
451
|
-
currentState,
|
|
452
|
-
targetState,
|
|
453
|
-
action.actionId
|
|
454
|
-
);
|
|
455
|
-
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
456
|
-
if (!transition.valid && !replayingAppliedTransition) {
|
|
457
|
-
return fail(invocation, "failed", transition.error ?? "Invalid state transition");
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
711
|
let data;
|
|
462
712
|
let domainEvents = [];
|
|
463
713
|
try {
|
|
@@ -465,6 +715,42 @@ function createGovernedActionHost(options) {
|
|
|
465
715
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
466
716
|
throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
467
717
|
}
|
|
718
|
+
const binding = action.stateMachine;
|
|
719
|
+
if (binding) {
|
|
720
|
+
const machine = stateMachineResolver(binding.entityType);
|
|
721
|
+
const entityId = binding.getEntityId(parsed.data);
|
|
722
|
+
const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
|
|
723
|
+
const currentState = entityId ? await getEntityState(
|
|
724
|
+
tenantId,
|
|
725
|
+
spaceId,
|
|
726
|
+
binding.entityType,
|
|
727
|
+
entityId
|
|
728
|
+
) ?? initialState(machine) : initialState(machine);
|
|
729
|
+
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
730
|
+
if (targetState !== "") {
|
|
731
|
+
const transition = await evaluateStateMachineTransition(
|
|
732
|
+
machine,
|
|
733
|
+
binding.entityType,
|
|
734
|
+
currentState,
|
|
735
|
+
targetState,
|
|
736
|
+
action.actionId,
|
|
737
|
+
{
|
|
738
|
+
entity: {
|
|
739
|
+
...entityId ? { id: entityId } : {},
|
|
740
|
+
state: currentState
|
|
741
|
+
},
|
|
742
|
+
actionInvocationId,
|
|
743
|
+
actorId: invocation.actorId,
|
|
744
|
+
parameters: parsed.data,
|
|
745
|
+
db
|
|
746
|
+
}
|
|
747
|
+
);
|
|
748
|
+
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
749
|
+
if (!transition.valid && !replayingAppliedTransition) {
|
|
750
|
+
throw new Error(transition.error ?? "Invalid state transition");
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
}
|
|
468
754
|
const handlerResult = action.handler ? await action.handler(
|
|
469
755
|
{
|
|
470
756
|
actionInvocationId,
|
|
@@ -564,7 +850,21 @@ function createGovernedActionHost(options) {
|
|
|
564
850
|
subjectId: adapterEventSubject.subjectId,
|
|
565
851
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
566
852
|
}, `adapter:${stepIndex}:started`);
|
|
853
|
+
let adapterDeadlineTimer;
|
|
567
854
|
try {
|
|
855
|
+
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
856
|
+
const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
|
|
857
|
+
if (adapterController && deadlineMs) {
|
|
858
|
+
const remaining = deadlineMs - now().getTime();
|
|
859
|
+
if (remaining <= 0) {
|
|
860
|
+
throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
861
|
+
}
|
|
862
|
+
adapterDeadlineTimer = setTimeout(
|
|
863
|
+
() => adapterController.abort(new Error("Adapter deadline exceeded")),
|
|
864
|
+
remaining
|
|
865
|
+
);
|
|
866
|
+
adapterDeadlineTimer.unref?.();
|
|
867
|
+
}
|
|
568
868
|
const result2 = await executeWithAdapterRetry({
|
|
569
869
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
570
870
|
defaultIdempotent: adapter.idempotent,
|
|
@@ -581,13 +881,77 @@ function createGovernedActionHost(options) {
|
|
|
581
881
|
correlationId: invocation.correlationId,
|
|
582
882
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
583
883
|
attempt,
|
|
584
|
-
maxAttempts
|
|
884
|
+
maxAttempts,
|
|
885
|
+
...deadlineMs ? { deadlineMs } : {},
|
|
886
|
+
...adapterController ? { signal: adapterController.signal } : {}
|
|
585
887
|
});
|
|
586
888
|
},
|
|
587
889
|
isSuccessful: (result3) => result3.success,
|
|
588
|
-
getError: (result3) => result3.error
|
|
890
|
+
getError: (result3) => result3.error,
|
|
891
|
+
classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
|
|
892
|
+
classifyThrownError: (error) => {
|
|
893
|
+
if (adapterController?.signal.aborted) return "timeout";
|
|
894
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
895
|
+
if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
|
|
896
|
+
return "transient_failure";
|
|
897
|
+
}
|
|
589
898
|
});
|
|
899
|
+
assertAdapterOutcomeConsistency(result2);
|
|
590
900
|
if (!result2.success) {
|
|
901
|
+
const outcome = result2.outcome ?? "transient_failure";
|
|
902
|
+
if (outcome === "ambiguous") {
|
|
903
|
+
const evidence = result2.outcomeEvidence;
|
|
904
|
+
const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
905
|
+
const adapterReconciliation = {
|
|
906
|
+
kind: "adapter_outcome_ambiguous",
|
|
907
|
+
adapterType: step.adapterType,
|
|
908
|
+
operation: step.operation,
|
|
909
|
+
vendor: adapter.vendor,
|
|
910
|
+
adapterInvocationId,
|
|
911
|
+
...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
|
|
912
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
913
|
+
message
|
|
914
|
+
};
|
|
915
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
916
|
+
status: "ambiguous",
|
|
917
|
+
error: message,
|
|
918
|
+
updatedAt: now()
|
|
919
|
+
});
|
|
920
|
+
await appendEvent(invocation, {
|
|
921
|
+
eventType: "AdapterInvocationAmbiguous",
|
|
922
|
+
subjectType: adapterEventSubject.subjectType,
|
|
923
|
+
subjectId: adapterEventSubject.subjectId,
|
|
924
|
+
payload: {
|
|
925
|
+
adapterType: step.adapterType,
|
|
926
|
+
operation: step.operation,
|
|
927
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
|
|
928
|
+
}
|
|
929
|
+
}, `adapter:${stepIndex}:ambiguous`);
|
|
930
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
931
|
+
if (governanceStore2 && evidence) {
|
|
932
|
+
await governanceStore2.appendExternalReconciliation({
|
|
933
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
934
|
+
actionInvocationId,
|
|
935
|
+
tenantId,
|
|
936
|
+
spaceId,
|
|
937
|
+
status: "pending",
|
|
938
|
+
provider: adapter.vendor,
|
|
939
|
+
...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
|
|
940
|
+
attempt: 1,
|
|
941
|
+
reason: message,
|
|
942
|
+
observedAt: now()
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
await persistInvocation(invocation, {
|
|
946
|
+
status: "reconciliation_required",
|
|
947
|
+
error: message,
|
|
948
|
+
adapterReconciliation
|
|
949
|
+
});
|
|
950
|
+
return withConsistency(
|
|
951
|
+
{ actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
|
|
952
|
+
action.actionId
|
|
953
|
+
);
|
|
954
|
+
}
|
|
591
955
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
592
956
|
status: "failed",
|
|
593
957
|
error: result2.error ?? "Adapter failed",
|
|
@@ -647,6 +1011,8 @@ function createGovernedActionHost(options) {
|
|
|
647
1011
|
updatedAt: now()
|
|
648
1012
|
});
|
|
649
1013
|
return fail(invocation, "failed", message);
|
|
1014
|
+
} finally {
|
|
1015
|
+
if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
|
|
650
1016
|
}
|
|
651
1017
|
}
|
|
652
1018
|
const governanceStore = asGovernanceStore(options.store);
|
|
@@ -671,20 +1037,41 @@ function createGovernedActionHost(options) {
|
|
|
671
1037
|
transaction
|
|
672
1038
|
);
|
|
673
1039
|
}
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
1040
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1041
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1042
|
+
throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1043
|
+
}
|
|
1044
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1045
|
+
id: actionInvocationId,
|
|
1046
|
+
tenantId,
|
|
1047
|
+
spaceId,
|
|
1048
|
+
workerId: invocation.leaseOwner,
|
|
1049
|
+
leaseToken: invocation.leaseToken,
|
|
1050
|
+
patch: { status: "completed", result }
|
|
1051
|
+
});
|
|
1052
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1053
|
+
} else {
|
|
1054
|
+
await transaction.updateActionInvocation(
|
|
1055
|
+
actionInvocationId,
|
|
1056
|
+
tenantId,
|
|
1057
|
+
spaceId,
|
|
1058
|
+
{ status: "completed", result }
|
|
1059
|
+
);
|
|
1060
|
+
}
|
|
680
1061
|
});
|
|
1062
|
+
emitInvocationStatusTelemetry(
|
|
1063
|
+
invocation,
|
|
1064
|
+
"completed",
|
|
1065
|
+
options.telemetry,
|
|
1066
|
+
now()
|
|
1067
|
+
);
|
|
681
1068
|
} else {
|
|
682
1069
|
if (action.eventPhase === "after_adapters") {
|
|
683
1070
|
for (const [index, event] of domainEvents.entries()) {
|
|
684
1071
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
685
1072
|
}
|
|
686
1073
|
}
|
|
687
|
-
await
|
|
1074
|
+
await persistInvocation(invocation, {
|
|
688
1075
|
status: "completed",
|
|
689
1076
|
result
|
|
690
1077
|
});
|
|
@@ -715,7 +1102,7 @@ function createGovernedActionHost(options) {
|
|
|
715
1102
|
spaceId
|
|
716
1103
|
);
|
|
717
1104
|
if (!invocation) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
718
|
-
if (isTerminal(invocation.status)) return actionResult(invocation);
|
|
1105
|
+
if (isTerminal(invocation.status)) return withConsistency(actionResult(invocation), invocation.actionId);
|
|
719
1106
|
if (invocation.status !== "waiting_for_approval") {
|
|
720
1107
|
return {
|
|
721
1108
|
...actionResult(invocation),
|
|
@@ -765,14 +1152,23 @@ function createGovernedActionHost(options) {
|
|
|
765
1152
|
const transitioned = transition.invocation ?? await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
766
1153
|
if (!transition.applied || !transitioned) {
|
|
767
1154
|
if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
768
|
-
return actionResult(transitioned);
|
|
1155
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
1156
|
+
}
|
|
1157
|
+
if (!decision.approved) {
|
|
1158
|
+
emitInvocationStatusTelemetry(
|
|
1159
|
+
transitioned,
|
|
1160
|
+
"failed",
|
|
1161
|
+
options.telemetry,
|
|
1162
|
+
now(),
|
|
1163
|
+
"waiting_for_approval"
|
|
1164
|
+
);
|
|
1165
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
769
1166
|
}
|
|
770
|
-
if (!decision.approved) return actionResult(transitioned);
|
|
771
1167
|
return executeInvocation(
|
|
772
1168
|
actionInvocationId,
|
|
773
1169
|
tenantId,
|
|
774
1170
|
spaceId,
|
|
775
|
-
{ leaseOwner },
|
|
1171
|
+
{ leaseOwner, leaseToken: transitioned.leaseToken },
|
|
776
1172
|
"approval_resume"
|
|
777
1173
|
);
|
|
778
1174
|
}
|
|
@@ -808,6 +1204,13 @@ function createGovernedActionHost(options) {
|
|
|
808
1204
|
spaceId
|
|
809
1205
|
});
|
|
810
1206
|
}
|
|
1207
|
+
function declaredConsistency(actionId) {
|
|
1208
|
+
return actionResolver(actionId)?.execution?.consistency;
|
|
1209
|
+
}
|
|
1210
|
+
function withConsistency(result, actionId) {
|
|
1211
|
+
const consistency = declaredConsistency(actionId);
|
|
1212
|
+
return consistency ? { ...result, consistency } : result;
|
|
1213
|
+
}
|
|
811
1214
|
async function appendEvent(invocation, event, deduplicationKey, defaultEventSchemaVersion = 1, transaction) {
|
|
812
1215
|
const timestamp = now();
|
|
813
1216
|
const envelope = {
|
|
@@ -830,7 +1233,8 @@ function createGovernedActionHost(options) {
|
|
|
830
1233
|
recordedAt: timestamp,
|
|
831
1234
|
correlationId: invocation.correlationId,
|
|
832
1235
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
833
|
-
...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {}
|
|
1236
|
+
...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {},
|
|
1237
|
+
...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
|
|
834
1238
|
};
|
|
835
1239
|
if (options.outbox) {
|
|
836
1240
|
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
|
|
@@ -860,12 +1264,7 @@ function createGovernedActionHost(options) {
|
|
|
860
1264
|
}
|
|
861
1265
|
}
|
|
862
1266
|
async function fail(invocation, status, error) {
|
|
863
|
-
await
|
|
864
|
-
invocation.id,
|
|
865
|
-
invocation.tenantId,
|
|
866
|
-
invocation.spaceId,
|
|
867
|
-
{ status, error }
|
|
868
|
-
);
|
|
1267
|
+
await persistInvocation(invocation, { status, error });
|
|
869
1268
|
return {
|
|
870
1269
|
actionInvocationId: invocation.id,
|
|
871
1270
|
status,
|
|
@@ -874,6 +1273,33 @@ function createGovernedActionHost(options) {
|
|
|
874
1273
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
875
1274
|
};
|
|
876
1275
|
}
|
|
1276
|
+
async function persistInvocation(invocation, patch) {
|
|
1277
|
+
const previousStatus = invocation.status;
|
|
1278
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1279
|
+
const recoverable = options.store;
|
|
1280
|
+
if (!recoverable.updateLeasedActionInvocation) {
|
|
1281
|
+
throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
1282
|
+
}
|
|
1283
|
+
const updated = await recoverable.updateLeasedActionInvocation({
|
|
1284
|
+
id: invocation.id,
|
|
1285
|
+
tenantId: invocation.tenantId,
|
|
1286
|
+
spaceId: invocation.spaceId,
|
|
1287
|
+
workerId: invocation.leaseOwner,
|
|
1288
|
+
leaseToken: invocation.leaseToken,
|
|
1289
|
+
patch
|
|
1290
|
+
});
|
|
1291
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
|
|
1292
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
await options.store.updateActionInvocation(
|
|
1296
|
+
invocation.id,
|
|
1297
|
+
invocation.tenantId,
|
|
1298
|
+
invocation.spaceId,
|
|
1299
|
+
patch
|
|
1300
|
+
);
|
|
1301
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1302
|
+
}
|
|
877
1303
|
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
878
1304
|
}
|
|
879
1305
|
function actionResult(invocation) {
|
|
@@ -884,7 +1310,8 @@ function actionResult(invocation) {
|
|
|
884
1310
|
...invocation.error ? { error: invocation.error } : {},
|
|
885
1311
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
886
1312
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
887
|
-
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
|
|
1313
|
+
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1314
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
888
1315
|
};
|
|
889
1316
|
}
|
|
890
1317
|
function asApprovalStore(store) {
|
|
@@ -938,6 +1365,24 @@ function initialState(machine) {
|
|
|
938
1365
|
function isTerminal(status) {
|
|
939
1366
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
940
1367
|
}
|
|
1368
|
+
function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
|
|
1369
|
+
if (!status) return;
|
|
1370
|
+
const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
|
|
1371
|
+
if (!event) return;
|
|
1372
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1373
|
+
kind: "event",
|
|
1374
|
+
name: event.name,
|
|
1375
|
+
metricName: event.metricName,
|
|
1376
|
+
occurredAt,
|
|
1377
|
+
tenantId: invocation.tenantId,
|
|
1378
|
+
spaceId: invocation.spaceId,
|
|
1379
|
+
attributes: {
|
|
1380
|
+
...invocation.actionId ? { actionId: invocation.actionId } : {},
|
|
1381
|
+
...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
|
|
1382
|
+
...previousStatus ? { fromStatus: previousStatus } : {}
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
941
1386
|
function withoutPrivateHostFields(data, eventResultFields) {
|
|
942
1387
|
return Object.fromEntries(
|
|
943
1388
|
Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
|
|
@@ -998,35 +1443,80 @@ function toEnterpriseEventEnvelope(event, metadata) {
|
|
|
998
1443
|
payloadClassification: metadata.payloadClassification,
|
|
999
1444
|
...metadata.traceContext ? { traceContext: { ...metadata.traceContext } } : {},
|
|
1000
1445
|
...metadata.includeProvenance && event.provenance ? { provenance: event.provenance } : {},
|
|
1446
|
+
// A provisional fact must stay marked as provisional once it leaves the platform.
|
|
1447
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
1001
1448
|
payload: event.payload
|
|
1002
1449
|
};
|
|
1003
1450
|
}
|
|
1004
1451
|
async function runOutboxRelayCycle(options) {
|
|
1005
1452
|
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
1006
1453
|
const maxAttempts = options.maxAttempts ?? 10;
|
|
1007
|
-
const
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1454
|
+
const leaseDurationMs = options.leaseDurationMs ?? 3e4;
|
|
1455
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
|
|
1456
|
+
let records;
|
|
1457
|
+
try {
|
|
1458
|
+
records = await options.store.claimOutbox({
|
|
1459
|
+
workerId: options.workerId,
|
|
1460
|
+
leaseDurationMs,
|
|
1461
|
+
limit: options.batchSize ?? 100,
|
|
1462
|
+
now: now()
|
|
1463
|
+
});
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
|
|
1466
|
+
throw error;
|
|
1467
|
+
}
|
|
1468
|
+
for (const record of records) {
|
|
1469
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1470
|
+
kind: "event",
|
|
1471
|
+
name: "outbox.lease_claimed",
|
|
1472
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
|
|
1473
|
+
occurredAt: now(),
|
|
1474
|
+
tenantId: record.tenantId,
|
|
1475
|
+
spaceId: record.spaceId,
|
|
1476
|
+
attributes: { attempt: record.attemptCount }
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1013
1479
|
const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
|
|
1014
1480
|
for (const record of records) {
|
|
1481
|
+
const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
|
|
1015
1482
|
try {
|
|
1016
|
-
await
|
|
1483
|
+
await publishWithTimeout(
|
|
1484
|
+
options.publisher,
|
|
1485
|
+
record.event,
|
|
1486
|
+
options.publishTimeoutMs ?? 1e4
|
|
1487
|
+
);
|
|
1488
|
+
await stopHeartbeat();
|
|
1017
1489
|
try {
|
|
1018
|
-
await options.store.markOutboxPublished(record.id, options.workerId, now());
|
|
1490
|
+
await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
|
|
1019
1491
|
} catch {
|
|
1020
1492
|
result.failed += 1;
|
|
1493
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1494
|
+
kind: "event",
|
|
1495
|
+
name: "outbox.failed",
|
|
1496
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1497
|
+
occurredAt: now(),
|
|
1498
|
+
tenantId: record.tenantId,
|
|
1499
|
+
spaceId: record.spaceId
|
|
1500
|
+
});
|
|
1021
1501
|
continue;
|
|
1022
1502
|
}
|
|
1023
1503
|
result.published += 1;
|
|
1504
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1505
|
+
kind: "event",
|
|
1506
|
+
name: "outbox.published",
|
|
1507
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
|
|
1508
|
+
occurredAt: now(),
|
|
1509
|
+
tenantId: record.tenantId,
|
|
1510
|
+
spaceId: record.spaceId
|
|
1511
|
+
});
|
|
1024
1512
|
} catch {
|
|
1513
|
+
await stopHeartbeat();
|
|
1025
1514
|
const deadLetter = record.attemptCount >= maxAttempts;
|
|
1026
1515
|
try {
|
|
1027
1516
|
await options.store.markOutboxFailed({
|
|
1028
1517
|
id: record.id,
|
|
1029
1518
|
workerId: options.workerId,
|
|
1519
|
+
leaseToken: record.leaseToken,
|
|
1030
1520
|
error: "Event publisher failed",
|
|
1031
1521
|
availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
|
|
1032
1522
|
deadLetter
|
|
@@ -1035,16 +1525,84 @@ async function runOutboxRelayCycle(options) {
|
|
|
1035
1525
|
}
|
|
1036
1526
|
result.failed += 1;
|
|
1037
1527
|
if (deadLetter) result.deadLettered += 1;
|
|
1528
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1529
|
+
kind: "event",
|
|
1530
|
+
name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
|
|
1531
|
+
metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1532
|
+
occurredAt: now(),
|
|
1533
|
+
tenantId: record.tenantId,
|
|
1534
|
+
spaceId: record.spaceId,
|
|
1535
|
+
attributes: { attempt: record.attemptCount }
|
|
1536
|
+
});
|
|
1038
1537
|
}
|
|
1039
1538
|
}
|
|
1539
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
|
|
1540
|
+
claimed: result.claimed,
|
|
1541
|
+
published: result.published,
|
|
1542
|
+
failed: result.failed,
|
|
1543
|
+
deadLettered: result.deadLettered
|
|
1544
|
+
});
|
|
1040
1545
|
return result;
|
|
1041
1546
|
}
|
|
1547
|
+
async function publishWithTimeout(publisher, event, timeoutMs) {
|
|
1548
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
1549
|
+
throw new Error("publishTimeoutMs must be a positive finite number");
|
|
1550
|
+
}
|
|
1551
|
+
const controller = new AbortController();
|
|
1552
|
+
let timer;
|
|
1553
|
+
const timeout = new Promise((_, reject) => {
|
|
1554
|
+
timer = setTimeout(() => {
|
|
1555
|
+
const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
|
|
1556
|
+
controller.abort(error);
|
|
1557
|
+
reject(error);
|
|
1558
|
+
}, timeoutMs);
|
|
1559
|
+
});
|
|
1560
|
+
try {
|
|
1561
|
+
await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
|
|
1562
|
+
} finally {
|
|
1563
|
+
if (timer) clearTimeout(timer);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
function startOutboxHeartbeat(options, record, leaseDurationMs) {
|
|
1567
|
+
if (!record.leaseToken) return async () => void 0;
|
|
1568
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
1569
|
+
let stopped = false;
|
|
1570
|
+
let timer;
|
|
1571
|
+
let running;
|
|
1572
|
+
const schedule = () => {
|
|
1573
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
1574
|
+
};
|
|
1575
|
+
const tick = () => {
|
|
1576
|
+
running = options.store.renewOutboxLease({
|
|
1577
|
+
id: record.id,
|
|
1578
|
+
workerId: options.workerId,
|
|
1579
|
+
leaseToken: record.leaseToken,
|
|
1580
|
+
leaseDurationMs
|
|
1581
|
+
}).then(() => void 0).catch(() => void 0).finally(schedule);
|
|
1582
|
+
};
|
|
1583
|
+
schedule();
|
|
1584
|
+
return async () => {
|
|
1585
|
+
stopped = true;
|
|
1586
|
+
if (timer) clearTimeout(timer);
|
|
1587
|
+
await running;
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1042
1590
|
function cloneOutboxRecord(record) {
|
|
1043
1591
|
return {
|
|
1044
1592
|
...record,
|
|
1045
1593
|
event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
|
|
1046
1594
|
};
|
|
1047
1595
|
}
|
|
1596
|
+
function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
|
|
1597
|
+
const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
|
|
1598
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1599
|
+
kind: "event",
|
|
1600
|
+
name,
|
|
1601
|
+
metricName,
|
|
1602
|
+
occurredAt,
|
|
1603
|
+
attributes: { workerId, ...attributes }
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1048
1606
|
|
|
1049
1607
|
// src/memory-store.ts
|
|
1050
1608
|
var MemoryPlatformHostStore = class {
|
|
@@ -1087,12 +1645,33 @@ var MemoryPlatformHostStore = class {
|
|
|
1087
1645
|
};
|
|
1088
1646
|
const result = await run({
|
|
1089
1647
|
db: this.db,
|
|
1648
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
1649
|
+
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1650
|
+
return record ? structuredClone(record) : void 0;
|
|
1651
|
+
},
|
|
1652
|
+
getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
|
|
1090
1653
|
appendEvent: (event) => appendPending(event),
|
|
1091
1654
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
1092
1655
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
1093
1656
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
1657
|
+
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
1658
|
+
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
1659
|
+
for (const pending of pendingEvents) {
|
|
1660
|
+
const event = pending.event;
|
|
1661
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
1662
|
+
const candidate = event.payload?.toState;
|
|
1663
|
+
if (typeof candidate === "string") state = candidate;
|
|
1664
|
+
}
|
|
1665
|
+
return state;
|
|
1666
|
+
},
|
|
1094
1667
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
1095
1668
|
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
1669
|
+
},
|
|
1670
|
+
updateLeasedActionInvocation: async (input) => {
|
|
1671
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1672
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1673
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
1674
|
+
return true;
|
|
1096
1675
|
}
|
|
1097
1676
|
});
|
|
1098
1677
|
for (const update of pendingUpdates) {
|
|
@@ -1119,7 +1698,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1119
1698
|
if (existing) return existing;
|
|
1120
1699
|
}
|
|
1121
1700
|
const now = /* @__PURE__ */ new Date();
|
|
1122
|
-
const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
|
|
1701
|
+
const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
|
|
1123
1702
|
this.invocations.push(record);
|
|
1124
1703
|
return record;
|
|
1125
1704
|
}
|
|
@@ -1169,6 +1748,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1169
1748
|
record.error = void 0;
|
|
1170
1749
|
record.leaseOwner = input.leaseOwner;
|
|
1171
1750
|
record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
|
|
1751
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1172
1752
|
record.attemptCount = Math.max(record.attemptCount, 1);
|
|
1173
1753
|
return { applied: true, invocation: record };
|
|
1174
1754
|
}
|
|
@@ -1246,31 +1826,41 @@ var MemoryPlatformHostStore = class {
|
|
|
1246
1826
|
return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
|
|
1247
1827
|
record.leaseOwner = input.workerId;
|
|
1248
1828
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1829
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1249
1830
|
record.attemptCount += 1;
|
|
1250
1831
|
return cloneOutboxRecord(record);
|
|
1251
1832
|
});
|
|
1252
1833
|
}
|
|
1253
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
1254
|
-
const record = this.requireLeasedOutbox(id, workerId);
|
|
1834
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1835
|
+
const record = this.requireLeasedOutbox(id, workerId, leaseToken);
|
|
1255
1836
|
record.status = "published";
|
|
1256
1837
|
record.publishedAt = publishedAt;
|
|
1257
1838
|
delete record.leaseOwner;
|
|
1258
1839
|
delete record.leaseExpiresAt;
|
|
1259
1840
|
}
|
|
1260
1841
|
async markOutboxFailed(input) {
|
|
1261
|
-
const record = this.requireLeasedOutbox(input.id, input.workerId);
|
|
1842
|
+
const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
|
|
1262
1843
|
record.status = input.deadLetter ? "dead_letter" : "pending";
|
|
1263
1844
|
record.lastError = input.error;
|
|
1264
1845
|
record.availableAt = input.availableAt;
|
|
1265
1846
|
delete record.leaseOwner;
|
|
1266
1847
|
delete record.leaseExpiresAt;
|
|
1267
1848
|
}
|
|
1849
|
+
async renewOutboxLease(input) {
|
|
1850
|
+
const record = this.outbox.find((candidate) => candidate.id === input.id);
|
|
1851
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1852
|
+
if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1853
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1854
|
+
return true;
|
|
1855
|
+
}
|
|
1268
1856
|
async listOutbox(input = {}) {
|
|
1269
1857
|
return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
|
|
1270
1858
|
}
|
|
1271
|
-
requireLeasedOutbox(id, workerId) {
|
|
1859
|
+
requireLeasedOutbox(id, workerId, leaseToken) {
|
|
1272
1860
|
const record = this.outbox.find((candidate) => candidate.id === id);
|
|
1273
|
-
if (!record || record.leaseOwner !== workerId
|
|
1861
|
+
if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
|
|
1862
|
+
throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1863
|
+
}
|
|
1274
1864
|
return record;
|
|
1275
1865
|
}
|
|
1276
1866
|
async nextEventSequence(tenantId, spaceId) {
|
|
@@ -1292,24 +1882,128 @@ var MemoryPlatformHostStore = class {
|
|
|
1292
1882
|
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
1293
1883
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1294
1884
|
}
|
|
1885
|
+
async getHealthCounts(input) {
|
|
1886
|
+
const invocations = this.invocations.filter(
|
|
1887
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1888
|
+
);
|
|
1889
|
+
const outbox = this.outbox.filter(
|
|
1890
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1891
|
+
);
|
|
1892
|
+
return {
|
|
1893
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
1894
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
1895
|
+
invocationExpiredLeases: invocations.filter(
|
|
1896
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1897
|
+
).length,
|
|
1898
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
1899
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
1900
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
1901
|
+
outboxExpiredLeases: outbox.filter(
|
|
1902
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1903
|
+
).length,
|
|
1904
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1295
1907
|
async claimActionInvocations(input) {
|
|
1296
1908
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1297
1909
|
const eligible = this.invocations.filter(
|
|
1298
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" &&
|
|
1910
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
|
|
1299
1911
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
|
|
1300
1912
|
for (const record of eligible) {
|
|
1301
1913
|
record.status = "running";
|
|
1302
1914
|
record.leaseOwner = input.workerId;
|
|
1303
1915
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1916
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1304
1917
|
record.attemptCount += 1;
|
|
1305
1918
|
record.updatedAt = current;
|
|
1306
1919
|
}
|
|
1307
|
-
return eligible;
|
|
1920
|
+
return eligible.map((record) => structuredClone(record));
|
|
1921
|
+
}
|
|
1922
|
+
async updateLeasedActionInvocation(input) {
|
|
1923
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1924
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1925
|
+
await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
|
|
1926
|
+
return true;
|
|
1927
|
+
}
|
|
1928
|
+
async renewActionInvocationLease(input) {
|
|
1929
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1930
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1931
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1932
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1933
|
+
record.updatedAt = current;
|
|
1934
|
+
return true;
|
|
1308
1935
|
}
|
|
1309
1936
|
async listEvents(tenantId, spaceId) {
|
|
1310
1937
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
|
|
1311
1938
|
}
|
|
1312
1939
|
};
|
|
1940
|
+
async function applyPostgresMigrations(client, migrations) {
|
|
1941
|
+
const ordered = validateMigrations(migrations);
|
|
1942
|
+
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
1943
|
+
await client.query(`
|
|
1944
|
+
BEGIN;
|
|
1945
|
+
SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
|
|
1946
|
+
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1947
|
+
CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
|
|
1948
|
+
version integer PRIMARY KEY,
|
|
1949
|
+
name text NOT NULL,
|
|
1950
|
+
checksum text NOT NULL,
|
|
1951
|
+
applied_at timestamptz NOT NULL DEFAULT now()
|
|
1952
|
+
);
|
|
1953
|
+
${statements.join("\n")}
|
|
1954
|
+
COMMIT;
|
|
1955
|
+
`);
|
|
1956
|
+
}
|
|
1957
|
+
function validateMigrations(migrations) {
|
|
1958
|
+
const ordered = [...migrations].sort((left, right) => left.version - right.version);
|
|
1959
|
+
const versions = /* @__PURE__ */ new Set();
|
|
1960
|
+
for (const migration of ordered) {
|
|
1961
|
+
if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
|
|
1962
|
+
throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
|
|
1963
|
+
}
|
|
1964
|
+
if (versions.has(migration.version)) {
|
|
1965
|
+
throw new Error(`Duplicate migration version: ${migration.version}`);
|
|
1966
|
+
}
|
|
1967
|
+
if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
|
|
1968
|
+
throw new Error(`Invalid migration name: ${migration.name}`);
|
|
1969
|
+
}
|
|
1970
|
+
if (migration.sql.trim().length === 0) {
|
|
1971
|
+
throw new Error(`Migration ${migration.version} has no SQL`);
|
|
1972
|
+
}
|
|
1973
|
+
versions.add(migration.version);
|
|
1974
|
+
}
|
|
1975
|
+
return ordered;
|
|
1976
|
+
}
|
|
1977
|
+
function migrationStatement(migration) {
|
|
1978
|
+
const checksum = createHash("sha256").update(migration.sql).digest("hex");
|
|
1979
|
+
const delimiter = `$fabric_platform_migration_${migration.version}$`;
|
|
1980
|
+
if (migration.sql.includes(delimiter)) {
|
|
1981
|
+
throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
|
|
1982
|
+
}
|
|
1983
|
+
return `
|
|
1984
|
+
DO ${delimiter}
|
|
1985
|
+
DECLARE
|
|
1986
|
+
recorded_checksum text;
|
|
1987
|
+
BEGIN
|
|
1988
|
+
SELECT checksum INTO recorded_checksum
|
|
1989
|
+
FROM fabric_platform.schema_migrations
|
|
1990
|
+
WHERE version = ${migration.version};
|
|
1991
|
+
|
|
1992
|
+
IF FOUND THEN
|
|
1993
|
+
IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
|
|
1994
|
+
RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
|
|
1995
|
+
END IF;
|
|
1996
|
+
ELSE
|
|
1997
|
+
${migration.sql}
|
|
1998
|
+
INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
|
|
1999
|
+
VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
|
|
2000
|
+
END IF;
|
|
2001
|
+
END
|
|
2002
|
+
${delimiter};`;
|
|
2003
|
+
}
|
|
2004
|
+
function sqlLiteral(value) {
|
|
2005
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
2006
|
+
}
|
|
1313
2007
|
|
|
1314
2008
|
// src/postgres-store.ts
|
|
1315
2009
|
var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
@@ -1322,11 +2016,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1322
2016
|
const scoped = new _PostgresPlatformHostStore(db2, sql2);
|
|
1323
2017
|
return run({
|
|
1324
2018
|
db: db2,
|
|
2019
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
2020
|
+
const result = await sql2.query(
|
|
2021
|
+
`SELECT * FROM fabric_platform.action_invocations
|
|
2022
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2023
|
+
FOR UPDATE`,
|
|
2024
|
+
[id, tenantId, spaceId]
|
|
2025
|
+
);
|
|
2026
|
+
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
2027
|
+
},
|
|
2028
|
+
getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
|
|
1325
2029
|
appendEvent: (event) => scoped.appendEvent(event),
|
|
1326
2030
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
1327
2031
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
1328
2032
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
1329
|
-
|
|
2033
|
+
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2034
|
+
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2035
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
1330
2036
|
});
|
|
1331
2037
|
});
|
|
1332
2038
|
}
|
|
@@ -1336,7 +2042,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1336
2042
|
transactionWithEvents;
|
|
1337
2043
|
transactionalOutbox;
|
|
1338
2044
|
async ensureSchema() {
|
|
1339
|
-
await this.sql
|
|
2045
|
+
await applyPostgresMigrations(this.sql, [{
|
|
2046
|
+
version: 1,
|
|
2047
|
+
name: "host_ledger_baseline",
|
|
2048
|
+
sql: `
|
|
1340
2049
|
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1341
2050
|
CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
|
|
1342
2051
|
id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
|
|
@@ -1347,7 +2056,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1347
2056
|
parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
|
|
1348
2057
|
idempotency_authorization_binding_id text, invocation_provenance jsonb,
|
|
1349
2058
|
authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
|
|
1350
|
-
authorization_binding_id text, error text,
|
|
2059
|
+
adapter_reconciliation jsonb, authorization_binding_id text, error text,
|
|
1351
2060
|
attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
|
|
1352
2061
|
lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
|
|
1353
2062
|
hitl_reason text, hitl_policy_version text, approval_decision jsonb,
|
|
@@ -1364,6 +2073,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1364
2073
|
ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
|
|
1365
2074
|
ADD COLUMN IF NOT EXISTS execution_reason text,
|
|
1366
2075
|
ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
|
|
2076
|
+
ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
|
|
1367
2077
|
ADD COLUMN IF NOT EXISTS authorization_binding_id text,
|
|
1368
2078
|
ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
|
|
1369
2079
|
ADD COLUMN IF NOT EXISTS lease_owner text,
|
|
@@ -1429,10 +2139,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1429
2139
|
actor_id text NOT NULL, actor_type text NOT NULL, action_invocation_id text,
|
|
1430
2140
|
payload jsonb NOT NULL, sequence bigint NOT NULL,
|
|
1431
2141
|
occurred_at timestamptz NOT NULL, recorded_at timestamptz NOT NULL,
|
|
1432
|
-
correlation_id text NOT NULL, causation_id text, provenance jsonb,
|
|
2142
|
+
correlation_id text NOT NULL, causation_id text, provenance jsonb, consistency text,
|
|
1433
2143
|
UNIQUE (tenant_id, space_id, sequence)
|
|
1434
2144
|
);
|
|
1435
2145
|
ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS provenance jsonb;
|
|
2146
|
+
ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS consistency text;
|
|
1436
2147
|
CREATE INDEX IF NOT EXISTS asset_events_subject_idx
|
|
1437
2148
|
ON fabric_platform.asset_events
|
|
1438
2149
|
(tenant_id, space_id, subject_type, subject_id, sequence);
|
|
@@ -1445,7 +2156,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1445
2156
|
);
|
|
1446
2157
|
CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
|
|
1447
2158
|
ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
|
|
1448
|
-
|
|
2159
|
+
`
|
|
2160
|
+
}, {
|
|
2161
|
+
version: 2,
|
|
2162
|
+
name: "lease_fencing",
|
|
2163
|
+
sql: `
|
|
2164
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2165
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2166
|
+
ALTER TABLE fabric_platform.event_outbox
|
|
2167
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2168
|
+
`
|
|
2169
|
+
}]);
|
|
1449
2170
|
}
|
|
1450
2171
|
async transaction(run) {
|
|
1451
2172
|
return run(this.db);
|
|
@@ -1506,6 +2227,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1506
2227
|
status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
|
|
1507
2228
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
1508
2229
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2230
|
+
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
1509
2231
|
lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
1510
2232
|
THEN NULL ELSE lease_owner END,
|
|
1511
2233
|
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -1521,7 +2243,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1521
2243
|
Object.hasOwn(patch, "error"),
|
|
1522
2244
|
patch.error ?? null,
|
|
1523
2245
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
1524
|
-
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
|
|
2246
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2247
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2248
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
1525
2249
|
]
|
|
1526
2250
|
);
|
|
1527
2251
|
}
|
|
@@ -1555,6 +2279,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1555
2279
|
error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
|
|
1556
2280
|
lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
|
|
1557
2281
|
lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
|
|
2282
|
+
lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
|
|
1558
2283
|
attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
|
|
1559
2284
|
updated_at=$9
|
|
1560
2285
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
|
|
@@ -1716,8 +2441,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1716
2441
|
`INSERT INTO fabric_platform.asset_events
|
|
1717
2442
|
(id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
|
|
1718
2443
|
actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
|
|
1719
|
-
correlation_id,causation_id,provenance)
|
|
1720
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
|
|
2444
|
+
correlation_id,causation_id,provenance,consistency)
|
|
2445
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$18)
|
|
1721
2446
|
ON CONFLICT (id) DO NOTHING`,
|
|
1722
2447
|
[
|
|
1723
2448
|
event.id,
|
|
@@ -1736,7 +2461,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1736
2461
|
event.recordedAt,
|
|
1737
2462
|
event.correlationId,
|
|
1738
2463
|
event.causationId ?? null,
|
|
1739
|
-
event.provenance ? JSON.stringify(event.provenance) : null
|
|
2464
|
+
event.provenance ? JSON.stringify(event.provenance) : null,
|
|
2465
|
+
event.consistency ?? null
|
|
1740
2466
|
]
|
|
1741
2467
|
);
|
|
1742
2468
|
}
|
|
@@ -1747,8 +2473,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1747
2473
|
INSERT INTO fabric_platform.asset_events
|
|
1748
2474
|
(id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
|
|
1749
2475
|
actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
|
|
1750
|
-
correlation_id,causation_id,provenance)
|
|
1751
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
|
|
2476
|
+
correlation_id,causation_id,provenance,consistency)
|
|
2477
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$19)
|
|
1752
2478
|
ON CONFLICT (id) DO NOTHING RETURNING id
|
|
1753
2479
|
)
|
|
1754
2480
|
INSERT INTO fabric_platform.event_outbox
|
|
@@ -1773,7 +2499,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1773
2499
|
event.correlationId,
|
|
1774
2500
|
event.causationId ?? null,
|
|
1775
2501
|
event.provenance ? JSON.stringify(event.provenance) : null,
|
|
1776
|
-
JSON.stringify(envelope)
|
|
2502
|
+
JSON.stringify(envelope),
|
|
2503
|
+
event.consistency ?? null
|
|
1777
2504
|
]
|
|
1778
2505
|
);
|
|
1779
2506
|
}
|
|
@@ -1790,28 +2517,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1790
2517
|
ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
|
|
1791
2518
|
)
|
|
1792
2519
|
UPDATE fabric_platform.event_outbox AS item
|
|
1793
|
-
SET lease_owner=$5, lease_expires_at=$6,
|
|
2520
|
+
SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
|
|
2521
|
+
attempt_count=attempt_count+1
|
|
1794
2522
|
FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
|
|
1795
2523
|
[current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
|
|
1796
2524
|
);
|
|
1797
2525
|
return result.rows.map(toOutboxRecord);
|
|
1798
2526
|
}
|
|
1799
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
2527
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1800
2528
|
const result = await this.sql.query(
|
|
1801
2529
|
`UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
|
|
1802
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1803
|
-
|
|
2530
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2531
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
|
|
2532
|
+
[id, workerId, publishedAt, leaseToken ?? null]
|
|
1804
2533
|
);
|
|
1805
2534
|
if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1806
2535
|
}
|
|
1807
2536
|
async markOutboxFailed(input) {
|
|
1808
2537
|
const result = await this.sql.query(
|
|
1809
2538
|
`UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
|
|
1810
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1811
|
-
|
|
2539
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2540
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
|
|
2541
|
+
[input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
|
|
1812
2542
|
);
|
|
1813
2543
|
if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
|
|
1814
2544
|
}
|
|
2545
|
+
async renewOutboxLease(input) {
|
|
2546
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2547
|
+
const result = await this.sql.query(
|
|
2548
|
+
`UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
|
|
2549
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
2550
|
+
AND lease_expires_at > $4
|
|
2551
|
+
RETURNING id`,
|
|
2552
|
+
[
|
|
2553
|
+
input.id,
|
|
2554
|
+
input.workerId,
|
|
2555
|
+
input.leaseToken,
|
|
2556
|
+
current,
|
|
2557
|
+
new Date(current.getTime() + input.leaseDurationMs)
|
|
2558
|
+
]
|
|
2559
|
+
);
|
|
2560
|
+
return result.rows.length === 1;
|
|
2561
|
+
}
|
|
1815
2562
|
async listOutbox(input = {}) {
|
|
1816
2563
|
const result = await this.sql.query(
|
|
1817
2564
|
`SELECT * FROM fabric_platform.event_outbox
|
|
@@ -1860,6 +2607,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1860
2607
|
);
|
|
1861
2608
|
return result.rows.map(toActionRecord);
|
|
1862
2609
|
}
|
|
2610
|
+
async getHealthCounts(input) {
|
|
2611
|
+
const result = await this.sql.query(
|
|
2612
|
+
`SELECT
|
|
2613
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2614
|
+
WHERE status='pending'
|
|
2615
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2616
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
|
|
2617
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2618
|
+
WHERE status='running'
|
|
2619
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2620
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
|
|
2621
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2622
|
+
WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2623
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2624
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
|
|
2625
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2626
|
+
WHERE status='waiting_for_approval'
|
|
2627
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2628
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
|
|
2629
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2630
|
+
WHERE status='reconciliation_required'
|
|
2631
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2632
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
|
|
2633
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2634
|
+
WHERE status='pending'
|
|
2635
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2636
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
|
|
2637
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2638
|
+
WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2639
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2640
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
|
|
2641
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2642
|
+
WHERE status='dead_letter'
|
|
2643
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2644
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
|
|
2645
|
+
[input.tenantId ?? null, input.spaceId ?? null, input.now]
|
|
2646
|
+
);
|
|
2647
|
+
const row = result.rows[0] ?? {};
|
|
2648
|
+
return {
|
|
2649
|
+
invocationBacklog: Number(row.invocation_backlog ?? 0),
|
|
2650
|
+
invocationRunning: Number(row.invocation_running ?? 0),
|
|
2651
|
+
invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
|
|
2652
|
+
invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
|
|
2653
|
+
invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
|
|
2654
|
+
outboxBacklog: Number(row.outbox_backlog ?? 0),
|
|
2655
|
+
outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
|
|
2656
|
+
outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
|
|
2657
|
+
};
|
|
2658
|
+
}
|
|
1863
2659
|
async claimActionInvocations(input) {
|
|
1864
2660
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1865
2661
|
const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
|
|
@@ -1868,7 +2664,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1868
2664
|
`WITH claimable AS (
|
|
1869
2665
|
SELECT id FROM fabric_platform.action_invocations
|
|
1870
2666
|
WHERE (status='pending' OR
|
|
1871
|
-
(status='running' AND
|
|
2667
|
+
(status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
|
|
1872
2668
|
AND ($2::text IS NULL OR tenant_id=$2)
|
|
1873
2669
|
AND ($3::text IS NULL OR space_id=$3)
|
|
1874
2670
|
ORDER BY created_at
|
|
@@ -1877,7 +2673,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1877
2673
|
)
|
|
1878
2674
|
UPDATE fabric_platform.action_invocations AS invocation
|
|
1879
2675
|
SET status='running', lease_owner=$5, lease_expires_at=$6,
|
|
1880
|
-
attempt_count=attempt_count+1, updated_at=$1
|
|
2676
|
+
lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
|
|
1881
2677
|
FROM claimable WHERE invocation.id=claimable.id
|
|
1882
2678
|
RETURNING invocation.*`,
|
|
1883
2679
|
[
|
|
@@ -1891,6 +2687,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1891
2687
|
);
|
|
1892
2688
|
return result.rows.map(toActionRecord);
|
|
1893
2689
|
}
|
|
2690
|
+
async updateLeasedActionInvocation(input) {
|
|
2691
|
+
const patch = input.patch;
|
|
2692
|
+
const result = await this.sql.query(
|
|
2693
|
+
`UPDATE fabric_platform.action_invocations SET
|
|
2694
|
+
status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
|
|
2695
|
+
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2696
|
+
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2697
|
+
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2698
|
+
lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2699
|
+
THEN NULL ELSE lease_owner END,
|
|
2700
|
+
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
|
+
THEN NULL ELSE lease_expires_at END,
|
|
2702
|
+
updated_at=now()
|
|
2703
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2704
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$5
|
|
2705
|
+
RETURNING id`,
|
|
2706
|
+
[
|
|
2707
|
+
input.id,
|
|
2708
|
+
input.tenantId,
|
|
2709
|
+
input.spaceId,
|
|
2710
|
+
input.workerId,
|
|
2711
|
+
input.leaseToken,
|
|
2712
|
+
patch.status ?? null,
|
|
2713
|
+
patch.result === void 0 ? null : JSON.stringify(patch.result),
|
|
2714
|
+
Object.hasOwn(patch, "error"),
|
|
2715
|
+
patch.error ?? null,
|
|
2716
|
+
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2717
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2718
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2719
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2720
|
+
]
|
|
2721
|
+
);
|
|
2722
|
+
return result.rows.length === 1;
|
|
2723
|
+
}
|
|
2724
|
+
async renewActionInvocationLease(input) {
|
|
2725
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2726
|
+
const result = await this.sql.query(
|
|
2727
|
+
`UPDATE fabric_platform.action_invocations
|
|
2728
|
+
SET lease_expires_at=$6,updated_at=$5
|
|
2729
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2730
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$7
|
|
2731
|
+
AND lease_expires_at > $5
|
|
2732
|
+
RETURNING id`,
|
|
2733
|
+
[
|
|
2734
|
+
input.id,
|
|
2735
|
+
input.tenantId,
|
|
2736
|
+
input.spaceId,
|
|
2737
|
+
input.workerId,
|
|
2738
|
+
current,
|
|
2739
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
2740
|
+
input.leaseToken
|
|
2741
|
+
]
|
|
2742
|
+
);
|
|
2743
|
+
return result.rows.length === 1;
|
|
2744
|
+
}
|
|
1894
2745
|
async listEvents(tenantId, spaceId) {
|
|
1895
2746
|
const result = await this.sql.query(
|
|
1896
2747
|
`SELECT * FROM fabric_platform.asset_events
|
|
@@ -1899,6 +2750,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1899
2750
|
);
|
|
1900
2751
|
return result.rows.map(toEventRecord);
|
|
1901
2752
|
}
|
|
2753
|
+
async getEvent(id, tenantId, spaceId) {
|
|
2754
|
+
const result = await this.sql.query(
|
|
2755
|
+
`SELECT * FROM fabric_platform.asset_events
|
|
2756
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
2757
|
+
[id, tenantId, spaceId]
|
|
2758
|
+
);
|
|
2759
|
+
return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
|
|
2760
|
+
}
|
|
1902
2761
|
};
|
|
1903
2762
|
function toAdapterRecord(row) {
|
|
1904
2763
|
return {
|
|
@@ -1941,8 +2800,10 @@ function toActionRecord(row) {
|
|
|
1941
2800
|
...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
|
|
1942
2801
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
1943
2802
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2803
|
+
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
1944
2804
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
1945
2805
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2806
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
1946
2807
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
1947
2808
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
1948
2809
|
...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
|
|
@@ -2015,7 +2876,8 @@ function toEventRecord(row) {
|
|
|
2015
2876
|
recordedAt: new Date(row.recorded_at),
|
|
2016
2877
|
correlationId: String(row.correlation_id),
|
|
2017
2878
|
...row.causation_id ? { causationId: String(row.causation_id) } : {},
|
|
2018
|
-
...row.provenance ? { provenance: row.provenance } : {}
|
|
2879
|
+
...row.provenance ? { provenance: row.provenance } : {},
|
|
2880
|
+
...row.consistency ? { consistency: row.consistency } : {}
|
|
2019
2881
|
};
|
|
2020
2882
|
}
|
|
2021
2883
|
function toOutboxRecord(row) {
|
|
@@ -2037,6 +2899,7 @@ function toOutboxRecord(row) {
|
|
|
2037
2899
|
correlationId: String(event.correlationId),
|
|
2038
2900
|
...event.causationId ? { causationId: String(event.causationId) } : {},
|
|
2039
2901
|
...event.provenance ? { provenance: event.provenance } : {},
|
|
2902
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
2040
2903
|
occurredAt: new Date(event.occurredAt),
|
|
2041
2904
|
recordedAt: new Date(event.recordedAt),
|
|
2042
2905
|
producerModuleVersion: String(event.producerModuleVersion),
|
|
@@ -2049,6 +2912,7 @@ function toOutboxRecord(row) {
|
|
|
2049
2912
|
availableAt: new Date(row.available_at),
|
|
2050
2913
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
2051
2914
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
2915
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
2052
2916
|
...row.last_error ? { lastError: String(row.last_error) } : {},
|
|
2053
2917
|
createdAt: new Date(row.created_at),
|
|
2054
2918
|
...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
|
|
@@ -2065,38 +2929,93 @@ function createStoreBackedActionDispatcher() {
|
|
|
2065
2929
|
};
|
|
2066
2930
|
}
|
|
2067
2931
|
async function runPlatformActionWorkerCycle(options) {
|
|
2068
|
-
const
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2932
|
+
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2933
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
2934
|
+
let claimed;
|
|
2935
|
+
try {
|
|
2936
|
+
claimed = await options.store.claimActionInvocations({
|
|
2937
|
+
workerId: options.workerId,
|
|
2938
|
+
limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
2939
|
+
leaseDurationMs,
|
|
2940
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
2941
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
2942
|
+
});
|
|
2943
|
+
} catch (error) {
|
|
2944
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
|
|
2945
|
+
throw error;
|
|
2946
|
+
}
|
|
2947
|
+
for (const invocation of claimed) {
|
|
2948
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
2949
|
+
kind: "event",
|
|
2950
|
+
name: "invocation.lease_claimed",
|
|
2951
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
|
|
2952
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
2953
|
+
tenantId: invocation.tenantId,
|
|
2954
|
+
spaceId: invocation.spaceId,
|
|
2955
|
+
attributes: { attempt: invocation.attemptCount }
|
|
2956
|
+
});
|
|
2957
|
+
}
|
|
2075
2958
|
let completed = 0;
|
|
2076
2959
|
let failed = 0;
|
|
2077
2960
|
let waitingForApproval = 0;
|
|
2078
2961
|
for (const invocation of claimed) {
|
|
2962
|
+
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
2079
2963
|
try {
|
|
2080
|
-
const
|
|
2964
|
+
const result2 = await options.host.executeInvocation(
|
|
2081
2965
|
invocation.id,
|
|
2082
2966
|
invocation.tenantId,
|
|
2083
2967
|
invocation.spaceId,
|
|
2084
|
-
{ leaseOwner: options.workerId }
|
|
2968
|
+
{ leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
|
|
2085
2969
|
);
|
|
2086
|
-
if (
|
|
2087
|
-
else if (
|
|
2970
|
+
if (result2.status === "completed") completed += 1;
|
|
2971
|
+
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
2088
2972
|
else failed += 1;
|
|
2089
2973
|
} catch (error) {
|
|
2090
2974
|
failed += 1;
|
|
2091
2975
|
options.onError?.(error, invocation);
|
|
2976
|
+
} finally {
|
|
2977
|
+
await stopHeartbeat();
|
|
2092
2978
|
}
|
|
2093
2979
|
}
|
|
2094
|
-
|
|
2980
|
+
const result = {
|
|
2095
2981
|
claimed: claimed.length,
|
|
2096
2982
|
completed,
|
|
2097
2983
|
failed,
|
|
2098
2984
|
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
2099
2985
|
};
|
|
2986
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
2987
|
+
return result;
|
|
2988
|
+
}
|
|
2989
|
+
function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
|
|
2990
|
+
if (!invocation?.leaseToken) return async () => void 0;
|
|
2991
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
2992
|
+
let stopped = false;
|
|
2993
|
+
let timer;
|
|
2994
|
+
let running;
|
|
2995
|
+
const schedule = () => {
|
|
2996
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
2997
|
+
};
|
|
2998
|
+
const tick = () => {
|
|
2999
|
+
running = options.store.renewActionInvocationLease({
|
|
3000
|
+
id: invocation.id,
|
|
3001
|
+
tenantId: invocation.tenantId,
|
|
3002
|
+
spaceId: invocation.spaceId,
|
|
3003
|
+
workerId: options.workerId,
|
|
3004
|
+
leaseToken: invocation.leaseToken,
|
|
3005
|
+
leaseDurationMs
|
|
3006
|
+
}).then((renewed) => {
|
|
3007
|
+
if (!renewed) options.onError?.(
|
|
3008
|
+
new Error(`Invocation lease lost: ${invocation.id}`),
|
|
3009
|
+
invocation
|
|
3010
|
+
);
|
|
3011
|
+
}).catch((error) => options.onError?.(error, invocation)).finally(schedule);
|
|
3012
|
+
};
|
|
3013
|
+
schedule();
|
|
3014
|
+
return async () => {
|
|
3015
|
+
stopped = true;
|
|
3016
|
+
if (timer) clearTimeout(timer);
|
|
3017
|
+
await running;
|
|
3018
|
+
};
|
|
2100
3019
|
}
|
|
2101
3020
|
async function runPlatformActionWorker(options) {
|
|
2102
3021
|
while (!options.signal?.aborted) {
|
|
@@ -2122,7 +3041,158 @@ async function abortableDelay(milliseconds, signal) {
|
|
|
2122
3041
|
);
|
|
2123
3042
|
});
|
|
2124
3043
|
}
|
|
3044
|
+
function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
|
|
3045
|
+
const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
|
|
3046
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
3047
|
+
kind: "event",
|
|
3048
|
+
name,
|
|
3049
|
+
metricName,
|
|
3050
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
3051
|
+
attributes: { workerId, ...attributes }
|
|
3052
|
+
});
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
// src/saga-parent-lifecycle.ts
|
|
3056
|
+
var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
|
|
3057
|
+
"completed",
|
|
3058
|
+
"failed",
|
|
3059
|
+
"blocked_by_policy",
|
|
3060
|
+
"reconciliation_required",
|
|
3061
|
+
"validation_failed"
|
|
3062
|
+
]);
|
|
3063
|
+
function createDurableSagaParentLifecycle(options) {
|
|
3064
|
+
const store = options.store;
|
|
3065
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3066
|
+
const transactionWithEvents = store.transactionWithEvents?.bind(store);
|
|
3067
|
+
if (!transactionWithEvents) {
|
|
3068
|
+
throw new Error(
|
|
3069
|
+
"Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
|
|
3070
|
+
);
|
|
3071
|
+
}
|
|
3072
|
+
const runInTransaction = transactionWithEvents;
|
|
3073
|
+
async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
|
|
3074
|
+
const eventId = `sagapar_${digestParameters({
|
|
3075
|
+
parentInvocationId: input.parentInvocationId,
|
|
3076
|
+
lifecycleId: lifecycleId2
|
|
3077
|
+
}).slice(0, 40)}`;
|
|
3078
|
+
await runInTransaction(async (transaction) => {
|
|
3079
|
+
if (!transaction.getActionInvocationForUpdate) {
|
|
3080
|
+
throw new Error(
|
|
3081
|
+
"Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
|
|
3082
|
+
);
|
|
3083
|
+
}
|
|
3084
|
+
if (!transaction.getEvent) {
|
|
3085
|
+
throw new Error(
|
|
3086
|
+
"Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
|
|
3087
|
+
);
|
|
3088
|
+
}
|
|
3089
|
+
const parent = await transaction.getActionInvocationForUpdate(
|
|
3090
|
+
input.parentInvocationId,
|
|
3091
|
+
input.tenantId,
|
|
3092
|
+
input.spaceId
|
|
3093
|
+
);
|
|
3094
|
+
if (!parent) {
|
|
3095
|
+
throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
|
|
3096
|
+
}
|
|
3097
|
+
const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
|
|
3098
|
+
if (prior) {
|
|
3099
|
+
if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
|
|
3100
|
+
throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
|
|
3101
|
+
}
|
|
3102
|
+
return;
|
|
3103
|
+
}
|
|
3104
|
+
if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
|
|
3105
|
+
throw new Error(
|
|
3106
|
+
`Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
|
|
3107
|
+
);
|
|
3108
|
+
}
|
|
3109
|
+
const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
|
|
3110
|
+
const timestamp = now();
|
|
3111
|
+
await transaction.appendEvent({
|
|
3112
|
+
id: eventId,
|
|
3113
|
+
tenantId: input.tenantId,
|
|
3114
|
+
spaceId: input.spaceId,
|
|
3115
|
+
eventType,
|
|
3116
|
+
eventSchemaVersion: 1,
|
|
3117
|
+
subjectType: "ActionInvocation",
|
|
3118
|
+
subjectId: input.parentInvocationId,
|
|
3119
|
+
actorId: "system",
|
|
3120
|
+
actorType: "system",
|
|
3121
|
+
actionInvocationId: input.parentInvocationId,
|
|
3122
|
+
payload,
|
|
3123
|
+
sequence,
|
|
3124
|
+
occurredAt: timestamp,
|
|
3125
|
+
recordedAt: timestamp,
|
|
3126
|
+
correlationId: input.parentInvocationId,
|
|
3127
|
+
causationId: lifecycleId2
|
|
3128
|
+
});
|
|
3129
|
+
if (patch) {
|
|
3130
|
+
await transaction.updateActionInvocation(
|
|
3131
|
+
input.parentInvocationId,
|
|
3132
|
+
input.tenantId,
|
|
3133
|
+
input.spaceId,
|
|
3134
|
+
patch
|
|
3135
|
+
);
|
|
3136
|
+
}
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
return {
|
|
3140
|
+
async recordProgress(input) {
|
|
3141
|
+
await applyLifecycleTransition(
|
|
3142
|
+
input,
|
|
3143
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
|
|
3144
|
+
"SagaParentProgress",
|
|
3145
|
+
{
|
|
3146
|
+
stepId: input.progress.stepId,
|
|
3147
|
+
index: input.progress.index,
|
|
3148
|
+
total: input.progress.total,
|
|
3149
|
+
completedAt: input.progress.completedAt
|
|
3150
|
+
}
|
|
3151
|
+
);
|
|
3152
|
+
},
|
|
3153
|
+
async parkApproval(input) {
|
|
3154
|
+
await applyLifecycleTransition(
|
|
3155
|
+
input,
|
|
3156
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
|
|
3157
|
+
"SagaParentApprovalParked",
|
|
3158
|
+
{
|
|
3159
|
+
stepId: input.stepId,
|
|
3160
|
+
approvalId: input.approvalId,
|
|
3161
|
+
reason: input.reason
|
|
3162
|
+
},
|
|
3163
|
+
{ status: "waiting_for_approval" }
|
|
3164
|
+
);
|
|
3165
|
+
},
|
|
3166
|
+
async cancel(input) {
|
|
3167
|
+
await applyLifecycleTransition(
|
|
3168
|
+
input,
|
|
3169
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
|
|
3170
|
+
"SagaParentCancelled",
|
|
3171
|
+
{ reason: input.reason, actorId: input.actorId },
|
|
3172
|
+
{ status: "failed", error: `Saga cancelled: ${input.reason}` }
|
|
3173
|
+
);
|
|
3174
|
+
},
|
|
3175
|
+
async complete(input) {
|
|
3176
|
+
await applyLifecycleTransition(
|
|
3177
|
+
input,
|
|
3178
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
|
|
3179
|
+
"SagaParentCompleted",
|
|
3180
|
+
{ result: input.result },
|
|
3181
|
+
{ status: "completed", result: input.result }
|
|
3182
|
+
);
|
|
3183
|
+
},
|
|
3184
|
+
async fail(input) {
|
|
3185
|
+
await applyLifecycleTransition(
|
|
3186
|
+
input,
|
|
3187
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
|
|
3188
|
+
"SagaParentFailed",
|
|
3189
|
+
{ error: input.error, compensation: input.compensation },
|
|
3190
|
+
{ status: "failed", error: input.error }
|
|
3191
|
+
);
|
|
3192
|
+
}
|
|
3193
|
+
};
|
|
3194
|
+
}
|
|
2125
3195
|
|
|
2126
|
-
export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PostgresPlatformHostStore, canonicalJson, cloneOutboxRecord, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
3196
|
+
export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, getPlatformHostHealthSnapshot, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
2127
3197
|
//# sourceMappingURL=index.js.map
|
|
2128
3198
|
//# sourceMappingURL=index.js.map
|