@fabricorg/platform-host 4.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/MIGRATION-6-PRODUCTION.md +59 -0
- package/README.md +89 -11
- package/dist/index.cjs +1194 -118
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +383 -13
- package/dist/index.d.ts +383 -13
- package/dist/index.js +1190 -120
- package/dist/index.js.map +1 -1
- package/package.json +6 -3
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var platform = require('@fabricorg/platform');
|
|
4
|
+
var assembly = require('@fabricorg/assembly');
|
|
4
5
|
var crypto = require('crypto');
|
|
5
6
|
|
|
6
7
|
// src/host.ts
|
|
@@ -64,6 +65,213 @@ function normalizeProvenance(input, options) {
|
|
|
64
65
|
};
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
// src/observability.ts
|
|
69
|
+
var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
|
|
70
|
+
var PLATFORM_HOST_METRIC_NAMES = {
|
|
71
|
+
workerCyclesStarted: "fabric.platform.worker.cycles.started",
|
|
72
|
+
workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
|
|
73
|
+
workerCyclesFailed: "fabric.platform.worker.cycles.failed",
|
|
74
|
+
invocationSubmitted: "fabric.platform.invocation.submitted",
|
|
75
|
+
invocationExecutionStarted: "fabric.platform.invocation.execution_started",
|
|
76
|
+
invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
|
|
77
|
+
invocationCompleted: "fabric.platform.invocation.completed",
|
|
78
|
+
invocationFailed: "fabric.platform.invocation.failed",
|
|
79
|
+
invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
|
|
80
|
+
invocationValidationFailed: "fabric.platform.invocation.validation_failed",
|
|
81
|
+
invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
|
|
82
|
+
invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
|
|
83
|
+
outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
|
|
84
|
+
outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
|
|
85
|
+
outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
|
|
86
|
+
outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
|
|
87
|
+
outboxPublished: "fabric.platform.outbox.published",
|
|
88
|
+
outboxFailed: "fabric.platform.outbox.failed",
|
|
89
|
+
outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
|
|
90
|
+
invocationBacklog: "fabric.platform.invocation.backlog",
|
|
91
|
+
invocationRunning: "fabric.platform.invocation.running",
|
|
92
|
+
invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
|
|
93
|
+
invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
|
|
94
|
+
invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
|
|
95
|
+
outboxBacklog: "fabric.platform.outbox.backlog",
|
|
96
|
+
outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
|
|
97
|
+
outboxDeadLetters: "fabric.platform.outbox.dead_letters"
|
|
98
|
+
};
|
|
99
|
+
function emitPlatformHostTelemetry(telemetry, input) {
|
|
100
|
+
if (!telemetry) return;
|
|
101
|
+
const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
|
|
102
|
+
try {
|
|
103
|
+
const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
|
|
104
|
+
if (isPromiseLike(result)) void result.catch(() => void 0);
|
|
105
|
+
} catch {
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async function getPlatformHostHealthSnapshot(options) {
|
|
109
|
+
const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
|
|
110
|
+
const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
|
|
111
|
+
const scope = {
|
|
112
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
113
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
114
|
+
};
|
|
115
|
+
try {
|
|
116
|
+
const counts = await readHealthCounts(options.store, {
|
|
117
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
118
|
+
...options.spaceId ? { spaceId: options.spaceId } : {},
|
|
119
|
+
now: generatedAt
|
|
120
|
+
});
|
|
121
|
+
const snapshot = createHealthSnapshot(
|
|
122
|
+
generatedAt,
|
|
123
|
+
scope,
|
|
124
|
+
counts,
|
|
125
|
+
workerSummary,
|
|
126
|
+
true
|
|
127
|
+
);
|
|
128
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
129
|
+
return snapshot;
|
|
130
|
+
} catch {
|
|
131
|
+
const snapshot = createHealthSnapshot(
|
|
132
|
+
generatedAt,
|
|
133
|
+
scope,
|
|
134
|
+
zeroHealthCounts(),
|
|
135
|
+
workerSummary,
|
|
136
|
+
false
|
|
137
|
+
);
|
|
138
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
139
|
+
return snapshot;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
async function readHealthCounts(store, query) {
|
|
143
|
+
const candidate = store;
|
|
144
|
+
if (typeof candidate.getHealthCounts === "function") {
|
|
145
|
+
return assertHealthCounts(await candidate.getHealthCounts(query));
|
|
146
|
+
}
|
|
147
|
+
const recoverable = store;
|
|
148
|
+
const outbox = store;
|
|
149
|
+
if (typeof recoverable.listActionInvocations !== "function") {
|
|
150
|
+
throw new Error("Health counts are not supported by this store.");
|
|
151
|
+
}
|
|
152
|
+
const invocations = await recoverable.listActionInvocations({
|
|
153
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
154
|
+
...query.spaceId ? { spaceId: query.spaceId } : {},
|
|
155
|
+
limit: Number.MAX_SAFE_INTEGER
|
|
156
|
+
});
|
|
157
|
+
const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
|
|
158
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
159
|
+
...query.spaceId ? { spaceId: query.spaceId } : {}
|
|
160
|
+
}) : [];
|
|
161
|
+
return countsFromRecords(invocations, outboxRecords, query.now);
|
|
162
|
+
}
|
|
163
|
+
function countsFromRecords(invocations, outbox, now) {
|
|
164
|
+
return {
|
|
165
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
166
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
167
|
+
invocationExpiredLeases: invocations.filter(
|
|
168
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
169
|
+
).length,
|
|
170
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
171
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
172
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
173
|
+
outboxExpiredLeases: outbox.filter(
|
|
174
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
175
|
+
).length,
|
|
176
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function assertHealthCounts(value) {
|
|
180
|
+
for (const [name, count] of Object.entries(value)) {
|
|
181
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
182
|
+
throw new Error(`Invalid health count: ${name}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return value;
|
|
186
|
+
}
|
|
187
|
+
function zeroHealthCounts() {
|
|
188
|
+
return {
|
|
189
|
+
invocationBacklog: 0,
|
|
190
|
+
invocationRunning: 0,
|
|
191
|
+
invocationExpiredLeases: 0,
|
|
192
|
+
invocationApprovalWaits: 0,
|
|
193
|
+
invocationReconciliationRequired: 0,
|
|
194
|
+
outboxBacklog: 0,
|
|
195
|
+
outboxExpiredLeases: 0,
|
|
196
|
+
outboxDeadLetters: 0
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function summarizeWorkers(workers, now) {
|
|
200
|
+
let healthy = 0;
|
|
201
|
+
let stale = 0;
|
|
202
|
+
for (const worker of workers) {
|
|
203
|
+
if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
|
|
204
|
+
else stale += 1;
|
|
205
|
+
}
|
|
206
|
+
return { reported: workers.length, healthy, stale };
|
|
207
|
+
}
|
|
208
|
+
function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
|
|
209
|
+
const reasonCodes = [];
|
|
210
|
+
if (!dataAvailable) reasonCodes.push("health_source_unavailable");
|
|
211
|
+
if (workers.stale > 0) reasonCodes.push("workers_stale");
|
|
212
|
+
if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
|
|
213
|
+
if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
|
|
214
|
+
if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
|
|
215
|
+
if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
|
|
216
|
+
const unhealthy = !dataAvailable;
|
|
217
|
+
const degraded = unhealthy || reasonCodes.length > 0;
|
|
218
|
+
return {
|
|
219
|
+
contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
|
|
220
|
+
generatedAt,
|
|
221
|
+
scope,
|
|
222
|
+
dataAvailable,
|
|
223
|
+
status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
|
|
224
|
+
readiness: {
|
|
225
|
+
ready: dataAvailable && workers.stale === 0,
|
|
226
|
+
reasonCodes
|
|
227
|
+
},
|
|
228
|
+
workers,
|
|
229
|
+
invocations: {
|
|
230
|
+
backlog: counts.invocationBacklog,
|
|
231
|
+
running: counts.invocationRunning,
|
|
232
|
+
expiredLeases: counts.invocationExpiredLeases,
|
|
233
|
+
approvalWaits: counts.invocationApprovalWaits,
|
|
234
|
+
reconciliationRequired: counts.invocationReconciliationRequired
|
|
235
|
+
},
|
|
236
|
+
outbox: {
|
|
237
|
+
backlog: counts.outboxBacklog,
|
|
238
|
+
expiredLeases: counts.outboxExpiredLeases,
|
|
239
|
+
deadLetters: counts.outboxDeadLetters
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
function emitHealthMetrics(telemetry, snapshot) {
|
|
244
|
+
const scope = {
|
|
245
|
+
...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
|
|
246
|
+
...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
|
|
247
|
+
};
|
|
248
|
+
const metrics = [
|
|
249
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
|
|
250
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
|
|
251
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
|
|
252
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
|
|
253
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
|
|
254
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
|
|
255
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
|
|
256
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
|
|
257
|
+
];
|
|
258
|
+
for (const [name, value] of metrics) {
|
|
259
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
260
|
+
kind: "metric",
|
|
261
|
+
name,
|
|
262
|
+
metricType: "gauge",
|
|
263
|
+
value,
|
|
264
|
+
...scope,
|
|
265
|
+
observedAt: snapshot.generatedAt
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
function isPromiseLike(value) {
|
|
270
|
+
return Boolean(
|
|
271
|
+
value && typeof value === "object" && "then" in value && typeof value.then === "function"
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
|
|
67
275
|
// src/host.ts
|
|
68
276
|
var DEFAULT_EXTRACT_EVENTS = (data) => {
|
|
69
277
|
const value = data._events;
|
|
@@ -73,6 +281,19 @@ var RecoverableFinalizationError = class extends Error {
|
|
|
73
281
|
name = "RecoverableFinalizationError";
|
|
74
282
|
};
|
|
75
283
|
function createGovernedActionHost(options) {
|
|
284
|
+
if (options.composition) {
|
|
285
|
+
if (!options.registry?.orderedModules) {
|
|
286
|
+
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
287
|
+
}
|
|
288
|
+
assembly.assertAssemblyRuntimeCompatible(
|
|
289
|
+
options.composition.assembly,
|
|
290
|
+
options.registry.orderedModules.map((module) => ({
|
|
291
|
+
namespace: module.namespace,
|
|
292
|
+
version: module.version ?? "",
|
|
293
|
+
manifestDigest: module.manifestDigest ?? ""
|
|
294
|
+
}))
|
|
295
|
+
);
|
|
296
|
+
}
|
|
76
297
|
if (options.outbox) {
|
|
77
298
|
const outboxStore = asOutboxStore(options.store);
|
|
78
299
|
if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
|
|
@@ -94,6 +315,12 @@ function createGovernedActionHost(options) {
|
|
|
94
315
|
}
|
|
95
316
|
const action = actionResolver(input.actionId);
|
|
96
317
|
if (!action) throw new Error(`Unknown action: ${input.actionId}`);
|
|
318
|
+
if (action.execution?.connectivity === "online-required" && input.executionReason === "offline_replay") {
|
|
319
|
+
throw new Error(`Action ${input.actionId} is online-required and cannot be submitted as an offline replay.`);
|
|
320
|
+
}
|
|
321
|
+
if (action.execution?.sensitivity === "restricted" && Object.keys(input.provenance?.auditAttributes ?? {}).length > 0) {
|
|
322
|
+
throw new Error(`Action ${input.actionId} declares restricted sensitivity; provenance audit attributes must not be durably recorded.`);
|
|
323
|
+
}
|
|
97
324
|
const authorizationInput = toAuthorizationInput(action, input);
|
|
98
325
|
if (!await options.authorization.checkEntitlement(authorizationInput)) {
|
|
99
326
|
throw new Error(`Module "${action.namespace}" is not enabled for tenant ${input.tenantId}`);
|
|
@@ -112,10 +339,13 @@ function createGovernedActionHost(options) {
|
|
|
112
339
|
input.provenance,
|
|
113
340
|
options.provenance
|
|
114
341
|
);
|
|
342
|
+
const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
|
|
115
343
|
const runtimeEvidence = {
|
|
116
344
|
governanceContractVersion: platform.FABRIC_GOVERNANCE_CONTRACT_VERSION,
|
|
117
345
|
hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
|
|
118
|
-
...options.runtimeEvidence
|
|
346
|
+
...options.runtimeEvidence,
|
|
347
|
+
...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
|
|
348
|
+
...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
|
|
119
349
|
};
|
|
120
350
|
platform.assertGovernanceRuntimeEvidence(runtimeEvidence);
|
|
121
351
|
const durableInvocation = await options.store.createActionInvocation({
|
|
@@ -144,6 +374,15 @@ function createGovernedActionHost(options) {
|
|
|
144
374
|
...input.executionReason ? { executionReason: input.executionReason } : {},
|
|
145
375
|
...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
|
|
146
376
|
});
|
|
377
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
378
|
+
kind: "event",
|
|
379
|
+
name: "invocation.submitted",
|
|
380
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
|
|
381
|
+
occurredAt: now(),
|
|
382
|
+
tenantId: input.tenantId,
|
|
383
|
+
spaceId: input.spaceId,
|
|
384
|
+
attributes: { actionVersion: action.version }
|
|
385
|
+
});
|
|
147
386
|
if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
|
|
148
387
|
const conflict = idempotencyConflict(durableInvocation, {
|
|
149
388
|
actorId: input.actorId,
|
|
@@ -166,7 +405,24 @@ function createGovernedActionHost(options) {
|
|
|
166
405
|
}
|
|
167
406
|
const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
|
|
168
407
|
if (durableInvocation.id !== actionInvocationId) {
|
|
169
|
-
|
|
408
|
+
if (options.dispatcher && durableInvocation.status === "pending") {
|
|
409
|
+
const dispatched = await options.dispatcher.dispatch({
|
|
410
|
+
actionInvocationId: durableInvocation.id,
|
|
411
|
+
actionId: durableInvocation.actionId,
|
|
412
|
+
tenantId: input.tenantId,
|
|
413
|
+
spaceId: input.spaceId,
|
|
414
|
+
workflowId: durableWorkflowId
|
|
415
|
+
});
|
|
416
|
+
return withConsistency({
|
|
417
|
+
actionInvocationId: durableInvocation.id,
|
|
418
|
+
status: durableInvocation.status,
|
|
419
|
+
workflowId: dispatched.workflowId,
|
|
420
|
+
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
421
|
+
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
422
|
+
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
423
|
+
}, input.actionId);
|
|
424
|
+
}
|
|
425
|
+
return withConsistency({
|
|
170
426
|
actionInvocationId: durableInvocation.id,
|
|
171
427
|
status: durableInvocation.status,
|
|
172
428
|
workflowId: durableWorkflowId,
|
|
@@ -174,33 +430,28 @@ function createGovernedActionHost(options) {
|
|
|
174
430
|
...durableInvocation.error ? { error: durableInvocation.error } : {},
|
|
175
431
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
176
432
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
177
|
-
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
|
|
178
|
-
|
|
433
|
+
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
434
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
435
|
+
}, input.actionId);
|
|
179
436
|
}
|
|
180
437
|
if (options.dispatcher) {
|
|
181
438
|
try {
|
|
182
439
|
const dispatched = await options.dispatcher.dispatch({
|
|
183
440
|
actionInvocationId: durableInvocation.id,
|
|
441
|
+
actionId: durableInvocation.actionId,
|
|
184
442
|
tenantId: input.tenantId,
|
|
185
443
|
spaceId: input.spaceId,
|
|
186
444
|
workflowId: durableWorkflowId
|
|
187
445
|
});
|
|
188
|
-
return {
|
|
446
|
+
return withConsistency({
|
|
189
447
|
actionInvocationId: durableInvocation.id,
|
|
190
448
|
status: durableInvocation.status,
|
|
191
449
|
workflowId: dispatched.workflowId,
|
|
192
450
|
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
193
451
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
194
452
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
195
|
-
};
|
|
453
|
+
}, input.actionId);
|
|
196
454
|
} catch (error) {
|
|
197
|
-
const message = errorMessage(error);
|
|
198
|
-
await options.store.updateActionInvocation(
|
|
199
|
-
durableInvocation.id,
|
|
200
|
-
input.tenantId,
|
|
201
|
-
input.spaceId,
|
|
202
|
-
{ status: "failed", error: message }
|
|
203
|
-
);
|
|
204
455
|
throw error;
|
|
205
456
|
}
|
|
206
457
|
}
|
|
@@ -209,7 +460,7 @@ function createGovernedActionHost(options) {
|
|
|
209
460
|
input.tenantId,
|
|
210
461
|
input.spaceId
|
|
211
462
|
);
|
|
212
|
-
return { ...executed, workflowId: durableWorkflowId };
|
|
463
|
+
return { ...withConsistency(executed, input.actionId), workflowId: durableWorkflowId };
|
|
213
464
|
}
|
|
214
465
|
async function executeInvocation(actionInvocationId, tenantId, spaceId, executionOptions = {}, executionReasonOverride) {
|
|
215
466
|
const loadedInvocation = await options.store.getActionInvocation(
|
|
@@ -223,18 +474,32 @@ function createGovernedActionHost(options) {
|
|
|
223
474
|
const resumingRunningInvocation = loadedInvocation.status === "running";
|
|
224
475
|
let invocation = loadedInvocation;
|
|
225
476
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
226
|
-
return actionResult(invocation);
|
|
477
|
+
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
227
478
|
}
|
|
228
|
-
if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
|
|
479
|
+
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
229
480
|
return {
|
|
230
481
|
...actionResult(invocation),
|
|
231
|
-
error: `Invocation is
|
|
482
|
+
error: `Invocation lease is not owned by the supplied worker generation`
|
|
232
483
|
};
|
|
233
484
|
}
|
|
234
485
|
const action = actionResolver(invocation.actionId);
|
|
235
486
|
if (!action) {
|
|
236
487
|
return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
|
|
237
488
|
}
|
|
489
|
+
if (action.kind === "saga") {
|
|
490
|
+
return fail(
|
|
491
|
+
invocation,
|
|
492
|
+
"failed",
|
|
493
|
+
`Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
|
|
497
|
+
return fail(
|
|
498
|
+
invocation,
|
|
499
|
+
"failed",
|
|
500
|
+
`Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
|
|
501
|
+
);
|
|
502
|
+
}
|
|
238
503
|
if (invocation.attemptCount > 1 && !action.idempotent) {
|
|
239
504
|
return fail(
|
|
240
505
|
invocation,
|
|
@@ -247,6 +512,15 @@ function createGovernedActionHost(options) {
|
|
|
247
512
|
status: "running"
|
|
248
513
|
});
|
|
249
514
|
}
|
|
515
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
516
|
+
kind: "event",
|
|
517
|
+
name: "invocation.execution_started",
|
|
518
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
|
|
519
|
+
occurredAt: now(),
|
|
520
|
+
tenantId,
|
|
521
|
+
spaceId,
|
|
522
|
+
attributes: { actionVersion: invocation.actionVersion }
|
|
523
|
+
});
|
|
250
524
|
try {
|
|
251
525
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
252
526
|
if (!parsed.success) {
|
|
@@ -312,7 +586,7 @@ function createGovernedActionHost(options) {
|
|
|
312
586
|
);
|
|
313
587
|
}
|
|
314
588
|
if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
|
|
315
|
-
await
|
|
589
|
+
await persistInvocation(invocation, {
|
|
316
590
|
status: "waiting_for_approval"
|
|
317
591
|
});
|
|
318
592
|
return {
|
|
@@ -339,8 +613,8 @@ function createGovernedActionHost(options) {
|
|
|
339
613
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
340
614
|
message: `Action ${action.actionId} requires durable capture-time authorization evidence`
|
|
341
615
|
};
|
|
342
|
-
await
|
|
343
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
616
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
617
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
344
618
|
}
|
|
345
619
|
const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
|
|
346
620
|
if (bindingExpired) {
|
|
@@ -352,8 +626,8 @@ function createGovernedActionHost(options) {
|
|
|
352
626
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
353
627
|
message: `Authorization binding for action ${action.actionId} expired before execution`
|
|
354
628
|
};
|
|
355
|
-
await
|
|
356
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
629
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
630
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
357
631
|
}
|
|
358
632
|
const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
|
|
359
633
|
...authorizationInput,
|
|
@@ -374,8 +648,8 @@ function createGovernedActionHost(options) {
|
|
|
374
648
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
375
649
|
message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
|
|
376
650
|
};
|
|
377
|
-
await
|
|
378
|
-
return { actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation };
|
|
651
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
652
|
+
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
379
653
|
}
|
|
380
654
|
const definitions = options.resolvePolicies ? await options.resolvePolicies({
|
|
381
655
|
...authorizationInput,
|
|
@@ -436,30 +710,6 @@ function createGovernedActionHost(options) {
|
|
|
436
710
|
})));
|
|
437
711
|
}
|
|
438
712
|
}
|
|
439
|
-
const binding = action.stateMachine;
|
|
440
|
-
if (binding) {
|
|
441
|
-
const entityId = binding.getEntityId(parsed.data);
|
|
442
|
-
const currentState = entityId ? await options.store.getEntityState(
|
|
443
|
-
tenantId,
|
|
444
|
-
spaceId,
|
|
445
|
-
binding.entityType,
|
|
446
|
-
entityId
|
|
447
|
-
) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
|
|
448
|
-
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
449
|
-
if (targetState !== "") {
|
|
450
|
-
const transition = platform.validateStateMachineTransition(
|
|
451
|
-
stateMachineResolver(binding.entityType),
|
|
452
|
-
binding.entityType,
|
|
453
|
-
currentState,
|
|
454
|
-
targetState,
|
|
455
|
-
action.actionId
|
|
456
|
-
);
|
|
457
|
-
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
458
|
-
if (!transition.valid && !replayingAppliedTransition) {
|
|
459
|
-
return fail(invocation, "failed", transition.error ?? "Invalid state transition");
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
713
|
let data;
|
|
464
714
|
let domainEvents = [];
|
|
465
715
|
try {
|
|
@@ -467,6 +717,42 @@ function createGovernedActionHost(options) {
|
|
|
467
717
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
468
718
|
throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
469
719
|
}
|
|
720
|
+
const binding = action.stateMachine;
|
|
721
|
+
if (binding) {
|
|
722
|
+
const machine = stateMachineResolver(binding.entityType);
|
|
723
|
+
const entityId = binding.getEntityId(parsed.data);
|
|
724
|
+
const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
|
|
725
|
+
const currentState = entityId ? await getEntityState(
|
|
726
|
+
tenantId,
|
|
727
|
+
spaceId,
|
|
728
|
+
binding.entityType,
|
|
729
|
+
entityId
|
|
730
|
+
) ?? initialState(machine) : initialState(machine);
|
|
731
|
+
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
732
|
+
if (targetState !== "") {
|
|
733
|
+
const transition = await platform.evaluateStateMachineTransition(
|
|
734
|
+
machine,
|
|
735
|
+
binding.entityType,
|
|
736
|
+
currentState,
|
|
737
|
+
targetState,
|
|
738
|
+
action.actionId,
|
|
739
|
+
{
|
|
740
|
+
entity: {
|
|
741
|
+
...entityId ? { id: entityId } : {},
|
|
742
|
+
state: currentState
|
|
743
|
+
},
|
|
744
|
+
actionInvocationId,
|
|
745
|
+
actorId: invocation.actorId,
|
|
746
|
+
parameters: parsed.data,
|
|
747
|
+
db
|
|
748
|
+
}
|
|
749
|
+
);
|
|
750
|
+
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
751
|
+
if (!transition.valid && !replayingAppliedTransition) {
|
|
752
|
+
throw new Error(transition.error ?? "Invalid state transition");
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
470
756
|
const handlerResult = action.handler ? await action.handler(
|
|
471
757
|
{
|
|
472
758
|
actionInvocationId,
|
|
@@ -566,7 +852,21 @@ function createGovernedActionHost(options) {
|
|
|
566
852
|
subjectId: adapterEventSubject.subjectId,
|
|
567
853
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
568
854
|
}, `adapter:${stepIndex}:started`);
|
|
855
|
+
let adapterDeadlineTimer;
|
|
569
856
|
try {
|
|
857
|
+
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
858
|
+
const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
|
|
859
|
+
if (adapterController && deadlineMs) {
|
|
860
|
+
const remaining = deadlineMs - now().getTime();
|
|
861
|
+
if (remaining <= 0) {
|
|
862
|
+
throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
863
|
+
}
|
|
864
|
+
adapterDeadlineTimer = setTimeout(
|
|
865
|
+
() => adapterController.abort(new Error("Adapter deadline exceeded")),
|
|
866
|
+
remaining
|
|
867
|
+
);
|
|
868
|
+
adapterDeadlineTimer.unref?.();
|
|
869
|
+
}
|
|
570
870
|
const result2 = await platform.executeWithAdapterRetry({
|
|
571
871
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
572
872
|
defaultIdempotent: adapter.idempotent,
|
|
@@ -583,13 +883,77 @@ function createGovernedActionHost(options) {
|
|
|
583
883
|
correlationId: invocation.correlationId,
|
|
584
884
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
585
885
|
attempt,
|
|
586
|
-
maxAttempts
|
|
886
|
+
maxAttempts,
|
|
887
|
+
...deadlineMs ? { deadlineMs } : {},
|
|
888
|
+
...adapterController ? { signal: adapterController.signal } : {}
|
|
587
889
|
});
|
|
588
890
|
},
|
|
589
891
|
isSuccessful: (result3) => result3.success,
|
|
590
|
-
getError: (result3) => result3.error
|
|
892
|
+
getError: (result3) => result3.error,
|
|
893
|
+
classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
|
|
894
|
+
classifyThrownError: (error) => {
|
|
895
|
+
if (adapterController?.signal.aborted) return "timeout";
|
|
896
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
897
|
+
if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
|
|
898
|
+
return "transient_failure";
|
|
899
|
+
}
|
|
591
900
|
});
|
|
901
|
+
platform.assertAdapterOutcomeConsistency(result2);
|
|
592
902
|
if (!result2.success) {
|
|
903
|
+
const outcome = result2.outcome ?? "transient_failure";
|
|
904
|
+
if (outcome === "ambiguous") {
|
|
905
|
+
const evidence = result2.outcomeEvidence;
|
|
906
|
+
const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
907
|
+
const adapterReconciliation = {
|
|
908
|
+
kind: "adapter_outcome_ambiguous",
|
|
909
|
+
adapterType: step.adapterType,
|
|
910
|
+
operation: step.operation,
|
|
911
|
+
vendor: adapter.vendor,
|
|
912
|
+
adapterInvocationId,
|
|
913
|
+
...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
|
|
914
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
915
|
+
message
|
|
916
|
+
};
|
|
917
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
918
|
+
status: "ambiguous",
|
|
919
|
+
error: message,
|
|
920
|
+
updatedAt: now()
|
|
921
|
+
});
|
|
922
|
+
await appendEvent(invocation, {
|
|
923
|
+
eventType: "AdapterInvocationAmbiguous",
|
|
924
|
+
subjectType: adapterEventSubject.subjectType,
|
|
925
|
+
subjectId: adapterEventSubject.subjectId,
|
|
926
|
+
payload: {
|
|
927
|
+
adapterType: step.adapterType,
|
|
928
|
+
operation: step.operation,
|
|
929
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
|
|
930
|
+
}
|
|
931
|
+
}, `adapter:${stepIndex}:ambiguous`);
|
|
932
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
933
|
+
if (governanceStore2 && evidence) {
|
|
934
|
+
await governanceStore2.appendExternalReconciliation({
|
|
935
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
936
|
+
actionInvocationId,
|
|
937
|
+
tenantId,
|
|
938
|
+
spaceId,
|
|
939
|
+
status: "pending",
|
|
940
|
+
provider: adapter.vendor,
|
|
941
|
+
...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
|
|
942
|
+
attempt: 1,
|
|
943
|
+
reason: message,
|
|
944
|
+
observedAt: now()
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
await persistInvocation(invocation, {
|
|
948
|
+
status: "reconciliation_required",
|
|
949
|
+
error: message,
|
|
950
|
+
adapterReconciliation
|
|
951
|
+
});
|
|
952
|
+
return withConsistency(
|
|
953
|
+
{ actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
|
|
954
|
+
action.actionId
|
|
955
|
+
);
|
|
956
|
+
}
|
|
593
957
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
594
958
|
status: "failed",
|
|
595
959
|
error: result2.error ?? "Adapter failed",
|
|
@@ -649,6 +1013,8 @@ function createGovernedActionHost(options) {
|
|
|
649
1013
|
updatedAt: now()
|
|
650
1014
|
});
|
|
651
1015
|
return fail(invocation, "failed", message);
|
|
1016
|
+
} finally {
|
|
1017
|
+
if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
|
|
652
1018
|
}
|
|
653
1019
|
}
|
|
654
1020
|
const governanceStore = asGovernanceStore(options.store);
|
|
@@ -673,20 +1039,41 @@ function createGovernedActionHost(options) {
|
|
|
673
1039
|
transaction
|
|
674
1040
|
);
|
|
675
1041
|
}
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
1042
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1043
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1044
|
+
throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1045
|
+
}
|
|
1046
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1047
|
+
id: actionInvocationId,
|
|
1048
|
+
tenantId,
|
|
1049
|
+
spaceId,
|
|
1050
|
+
workerId: invocation.leaseOwner,
|
|
1051
|
+
leaseToken: invocation.leaseToken,
|
|
1052
|
+
patch: { status: "completed", result }
|
|
1053
|
+
});
|
|
1054
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1055
|
+
} else {
|
|
1056
|
+
await transaction.updateActionInvocation(
|
|
1057
|
+
actionInvocationId,
|
|
1058
|
+
tenantId,
|
|
1059
|
+
spaceId,
|
|
1060
|
+
{ status: "completed", result }
|
|
1061
|
+
);
|
|
1062
|
+
}
|
|
682
1063
|
});
|
|
1064
|
+
emitInvocationStatusTelemetry(
|
|
1065
|
+
invocation,
|
|
1066
|
+
"completed",
|
|
1067
|
+
options.telemetry,
|
|
1068
|
+
now()
|
|
1069
|
+
);
|
|
683
1070
|
} else {
|
|
684
1071
|
if (action.eventPhase === "after_adapters") {
|
|
685
1072
|
for (const [index, event] of domainEvents.entries()) {
|
|
686
1073
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
687
1074
|
}
|
|
688
1075
|
}
|
|
689
|
-
await
|
|
1076
|
+
await persistInvocation(invocation, {
|
|
690
1077
|
status: "completed",
|
|
691
1078
|
result
|
|
692
1079
|
});
|
|
@@ -717,7 +1104,7 @@ function createGovernedActionHost(options) {
|
|
|
717
1104
|
spaceId
|
|
718
1105
|
);
|
|
719
1106
|
if (!invocation) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
720
|
-
if (isTerminal(invocation.status)) return actionResult(invocation);
|
|
1107
|
+
if (isTerminal(invocation.status)) return withConsistency(actionResult(invocation), invocation.actionId);
|
|
721
1108
|
if (invocation.status !== "waiting_for_approval") {
|
|
722
1109
|
return {
|
|
723
1110
|
...actionResult(invocation),
|
|
@@ -767,14 +1154,23 @@ function createGovernedActionHost(options) {
|
|
|
767
1154
|
const transitioned = transition.invocation ?? await options.store.getActionInvocation(actionInvocationId, tenantId, spaceId);
|
|
768
1155
|
if (!transition.applied || !transitioned) {
|
|
769
1156
|
if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
770
|
-
return actionResult(transitioned);
|
|
1157
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
1158
|
+
}
|
|
1159
|
+
if (!decision.approved) {
|
|
1160
|
+
emitInvocationStatusTelemetry(
|
|
1161
|
+
transitioned,
|
|
1162
|
+
"failed",
|
|
1163
|
+
options.telemetry,
|
|
1164
|
+
now(),
|
|
1165
|
+
"waiting_for_approval"
|
|
1166
|
+
);
|
|
1167
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
771
1168
|
}
|
|
772
|
-
if (!decision.approved) return actionResult(transitioned);
|
|
773
1169
|
return executeInvocation(
|
|
774
1170
|
actionInvocationId,
|
|
775
1171
|
tenantId,
|
|
776
1172
|
spaceId,
|
|
777
|
-
{ leaseOwner },
|
|
1173
|
+
{ leaseOwner, leaseToken: transitioned.leaseToken },
|
|
778
1174
|
"approval_resume"
|
|
779
1175
|
);
|
|
780
1176
|
}
|
|
@@ -810,6 +1206,13 @@ function createGovernedActionHost(options) {
|
|
|
810
1206
|
spaceId
|
|
811
1207
|
});
|
|
812
1208
|
}
|
|
1209
|
+
function declaredConsistency(actionId) {
|
|
1210
|
+
return actionResolver(actionId)?.execution?.consistency;
|
|
1211
|
+
}
|
|
1212
|
+
function withConsistency(result, actionId) {
|
|
1213
|
+
const consistency = declaredConsistency(actionId);
|
|
1214
|
+
return consistency ? { ...result, consistency } : result;
|
|
1215
|
+
}
|
|
813
1216
|
async function appendEvent(invocation, event, deduplicationKey, defaultEventSchemaVersion = 1, transaction) {
|
|
814
1217
|
const timestamp = now();
|
|
815
1218
|
const envelope = {
|
|
@@ -832,7 +1235,8 @@ function createGovernedActionHost(options) {
|
|
|
832
1235
|
recordedAt: timestamp,
|
|
833
1236
|
correlationId: invocation.correlationId,
|
|
834
1237
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
835
|
-
...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {}
|
|
1238
|
+
...invocation.provenance ? { provenance: { source: invocation.provenance.source, ...invocation.provenance.auditAttributes ? { auditAttributes: invocation.provenance.auditAttributes } : {} } } : {},
|
|
1239
|
+
...declaredConsistency(invocation.actionId) === "provisional-until-reconciled" ? { consistency: "provisional-until-reconciled" } : {}
|
|
836
1240
|
};
|
|
837
1241
|
if (options.outbox) {
|
|
838
1242
|
const hostLifecycleEvent = (/* @__PURE__ */ new Set(["AdapterInvocationStarted", "AdapterInvocationSucceeded", "AdapterInvocationFailed", "ComplianceBlocked"])).has(envelope.eventType);
|
|
@@ -862,12 +1266,7 @@ function createGovernedActionHost(options) {
|
|
|
862
1266
|
}
|
|
863
1267
|
}
|
|
864
1268
|
async function fail(invocation, status, error) {
|
|
865
|
-
await
|
|
866
|
-
invocation.id,
|
|
867
|
-
invocation.tenantId,
|
|
868
|
-
invocation.spaceId,
|
|
869
|
-
{ status, error }
|
|
870
|
-
);
|
|
1269
|
+
await persistInvocation(invocation, { status, error });
|
|
871
1270
|
return {
|
|
872
1271
|
actionInvocationId: invocation.id,
|
|
873
1272
|
status,
|
|
@@ -876,6 +1275,33 @@ function createGovernedActionHost(options) {
|
|
|
876
1275
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
877
1276
|
};
|
|
878
1277
|
}
|
|
1278
|
+
async function persistInvocation(invocation, patch) {
|
|
1279
|
+
const previousStatus = invocation.status;
|
|
1280
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1281
|
+
const recoverable = options.store;
|
|
1282
|
+
if (!recoverable.updateLeasedActionInvocation) {
|
|
1283
|
+
throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
1284
|
+
}
|
|
1285
|
+
const updated = await recoverable.updateLeasedActionInvocation({
|
|
1286
|
+
id: invocation.id,
|
|
1287
|
+
tenantId: invocation.tenantId,
|
|
1288
|
+
spaceId: invocation.spaceId,
|
|
1289
|
+
workerId: invocation.leaseOwner,
|
|
1290
|
+
leaseToken: invocation.leaseToken,
|
|
1291
|
+
patch
|
|
1292
|
+
});
|
|
1293
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
|
|
1294
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
await options.store.updateActionInvocation(
|
|
1298
|
+
invocation.id,
|
|
1299
|
+
invocation.tenantId,
|
|
1300
|
+
invocation.spaceId,
|
|
1301
|
+
patch
|
|
1302
|
+
);
|
|
1303
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1304
|
+
}
|
|
879
1305
|
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
880
1306
|
}
|
|
881
1307
|
function actionResult(invocation) {
|
|
@@ -886,7 +1312,8 @@ function actionResult(invocation) {
|
|
|
886
1312
|
...invocation.error ? { error: invocation.error } : {},
|
|
887
1313
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
888
1314
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
889
|
-
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
|
|
1315
|
+
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1316
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
890
1317
|
};
|
|
891
1318
|
}
|
|
892
1319
|
function asApprovalStore(store) {
|
|
@@ -940,6 +1367,24 @@ function initialState(machine) {
|
|
|
940
1367
|
function isTerminal(status) {
|
|
941
1368
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
942
1369
|
}
|
|
1370
|
+
function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
|
|
1371
|
+
if (!status) return;
|
|
1372
|
+
const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
|
|
1373
|
+
if (!event) return;
|
|
1374
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1375
|
+
kind: "event",
|
|
1376
|
+
name: event.name,
|
|
1377
|
+
metricName: event.metricName,
|
|
1378
|
+
occurredAt,
|
|
1379
|
+
tenantId: invocation.tenantId,
|
|
1380
|
+
spaceId: invocation.spaceId,
|
|
1381
|
+
attributes: {
|
|
1382
|
+
...invocation.actionId ? { actionId: invocation.actionId } : {},
|
|
1383
|
+
...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
|
|
1384
|
+
...previousStatus ? { fromStatus: previousStatus } : {}
|
|
1385
|
+
}
|
|
1386
|
+
});
|
|
1387
|
+
}
|
|
943
1388
|
function withoutPrivateHostFields(data, eventResultFields) {
|
|
944
1389
|
return Object.fromEntries(
|
|
945
1390
|
Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
|
|
@@ -1000,35 +1445,80 @@ function toEnterpriseEventEnvelope(event, metadata) {
|
|
|
1000
1445
|
payloadClassification: metadata.payloadClassification,
|
|
1001
1446
|
...metadata.traceContext ? { traceContext: { ...metadata.traceContext } } : {},
|
|
1002
1447
|
...metadata.includeProvenance && event.provenance ? { provenance: event.provenance } : {},
|
|
1448
|
+
// A provisional fact must stay marked as provisional once it leaves the platform.
|
|
1449
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
1003
1450
|
payload: event.payload
|
|
1004
1451
|
};
|
|
1005
1452
|
}
|
|
1006
1453
|
async function runOutboxRelayCycle(options) {
|
|
1007
1454
|
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
1008
1455
|
const maxAttempts = options.maxAttempts ?? 10;
|
|
1009
|
-
const
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1456
|
+
const leaseDurationMs = options.leaseDurationMs ?? 3e4;
|
|
1457
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
|
|
1458
|
+
let records;
|
|
1459
|
+
try {
|
|
1460
|
+
records = await options.store.claimOutbox({
|
|
1461
|
+
workerId: options.workerId,
|
|
1462
|
+
leaseDurationMs,
|
|
1463
|
+
limit: options.batchSize ?? 100,
|
|
1464
|
+
now: now()
|
|
1465
|
+
});
|
|
1466
|
+
} catch (error) {
|
|
1467
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
|
|
1468
|
+
throw error;
|
|
1469
|
+
}
|
|
1470
|
+
for (const record of records) {
|
|
1471
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1472
|
+
kind: "event",
|
|
1473
|
+
name: "outbox.lease_claimed",
|
|
1474
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
|
|
1475
|
+
occurredAt: now(),
|
|
1476
|
+
tenantId: record.tenantId,
|
|
1477
|
+
spaceId: record.spaceId,
|
|
1478
|
+
attributes: { attempt: record.attemptCount }
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1015
1481
|
const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
|
|
1016
1482
|
for (const record of records) {
|
|
1483
|
+
const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
|
|
1017
1484
|
try {
|
|
1018
|
-
await
|
|
1485
|
+
await publishWithTimeout(
|
|
1486
|
+
options.publisher,
|
|
1487
|
+
record.event,
|
|
1488
|
+
options.publishTimeoutMs ?? 1e4
|
|
1489
|
+
);
|
|
1490
|
+
await stopHeartbeat();
|
|
1019
1491
|
try {
|
|
1020
|
-
await options.store.markOutboxPublished(record.id, options.workerId, now());
|
|
1492
|
+
await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
|
|
1021
1493
|
} catch {
|
|
1022
1494
|
result.failed += 1;
|
|
1495
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1496
|
+
kind: "event",
|
|
1497
|
+
name: "outbox.failed",
|
|
1498
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1499
|
+
occurredAt: now(),
|
|
1500
|
+
tenantId: record.tenantId,
|
|
1501
|
+
spaceId: record.spaceId
|
|
1502
|
+
});
|
|
1023
1503
|
continue;
|
|
1024
1504
|
}
|
|
1025
1505
|
result.published += 1;
|
|
1506
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1507
|
+
kind: "event",
|
|
1508
|
+
name: "outbox.published",
|
|
1509
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
|
|
1510
|
+
occurredAt: now(),
|
|
1511
|
+
tenantId: record.tenantId,
|
|
1512
|
+
spaceId: record.spaceId
|
|
1513
|
+
});
|
|
1026
1514
|
} catch {
|
|
1515
|
+
await stopHeartbeat();
|
|
1027
1516
|
const deadLetter = record.attemptCount >= maxAttempts;
|
|
1028
1517
|
try {
|
|
1029
1518
|
await options.store.markOutboxFailed({
|
|
1030
1519
|
id: record.id,
|
|
1031
1520
|
workerId: options.workerId,
|
|
1521
|
+
leaseToken: record.leaseToken,
|
|
1032
1522
|
error: "Event publisher failed",
|
|
1033
1523
|
availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
|
|
1034
1524
|
deadLetter
|
|
@@ -1037,16 +1527,84 @@ async function runOutboxRelayCycle(options) {
|
|
|
1037
1527
|
}
|
|
1038
1528
|
result.failed += 1;
|
|
1039
1529
|
if (deadLetter) result.deadLettered += 1;
|
|
1530
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1531
|
+
kind: "event",
|
|
1532
|
+
name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
|
|
1533
|
+
metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1534
|
+
occurredAt: now(),
|
|
1535
|
+
tenantId: record.tenantId,
|
|
1536
|
+
spaceId: record.spaceId,
|
|
1537
|
+
attributes: { attempt: record.attemptCount }
|
|
1538
|
+
});
|
|
1040
1539
|
}
|
|
1041
1540
|
}
|
|
1541
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
|
|
1542
|
+
claimed: result.claimed,
|
|
1543
|
+
published: result.published,
|
|
1544
|
+
failed: result.failed,
|
|
1545
|
+
deadLettered: result.deadLettered
|
|
1546
|
+
});
|
|
1042
1547
|
return result;
|
|
1043
1548
|
}
|
|
1549
|
+
async function publishWithTimeout(publisher, event, timeoutMs) {
|
|
1550
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
1551
|
+
throw new Error("publishTimeoutMs must be a positive finite number");
|
|
1552
|
+
}
|
|
1553
|
+
const controller = new AbortController();
|
|
1554
|
+
let timer;
|
|
1555
|
+
const timeout = new Promise((_, reject) => {
|
|
1556
|
+
timer = setTimeout(() => {
|
|
1557
|
+
const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
|
|
1558
|
+
controller.abort(error);
|
|
1559
|
+
reject(error);
|
|
1560
|
+
}, timeoutMs);
|
|
1561
|
+
});
|
|
1562
|
+
try {
|
|
1563
|
+
await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
|
|
1564
|
+
} finally {
|
|
1565
|
+
if (timer) clearTimeout(timer);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
function startOutboxHeartbeat(options, record, leaseDurationMs) {
|
|
1569
|
+
if (!record.leaseToken) return async () => void 0;
|
|
1570
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
1571
|
+
let stopped = false;
|
|
1572
|
+
let timer;
|
|
1573
|
+
let running;
|
|
1574
|
+
const schedule = () => {
|
|
1575
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
1576
|
+
};
|
|
1577
|
+
const tick = () => {
|
|
1578
|
+
running = options.store.renewOutboxLease({
|
|
1579
|
+
id: record.id,
|
|
1580
|
+
workerId: options.workerId,
|
|
1581
|
+
leaseToken: record.leaseToken,
|
|
1582
|
+
leaseDurationMs
|
|
1583
|
+
}).then(() => void 0).catch(() => void 0).finally(schedule);
|
|
1584
|
+
};
|
|
1585
|
+
schedule();
|
|
1586
|
+
return async () => {
|
|
1587
|
+
stopped = true;
|
|
1588
|
+
if (timer) clearTimeout(timer);
|
|
1589
|
+
await running;
|
|
1590
|
+
};
|
|
1591
|
+
}
|
|
1044
1592
|
function cloneOutboxRecord(record) {
|
|
1045
1593
|
return {
|
|
1046
1594
|
...record,
|
|
1047
1595
|
event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
|
|
1048
1596
|
};
|
|
1049
1597
|
}
|
|
1598
|
+
function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
|
|
1599
|
+
const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
|
|
1600
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1601
|
+
kind: "event",
|
|
1602
|
+
name,
|
|
1603
|
+
metricName,
|
|
1604
|
+
occurredAt,
|
|
1605
|
+
attributes: { workerId, ...attributes }
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1050
1608
|
|
|
1051
1609
|
// src/memory-store.ts
|
|
1052
1610
|
var MemoryPlatformHostStore = class {
|
|
@@ -1089,12 +1647,33 @@ var MemoryPlatformHostStore = class {
|
|
|
1089
1647
|
};
|
|
1090
1648
|
const result = await run({
|
|
1091
1649
|
db: this.db,
|
|
1650
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
1651
|
+
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1652
|
+
return record ? structuredClone(record) : void 0;
|
|
1653
|
+
},
|
|
1654
|
+
getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
|
|
1092
1655
|
appendEvent: (event) => appendPending(event),
|
|
1093
1656
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
1094
1657
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
1095
1658
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
1659
|
+
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
1660
|
+
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
1661
|
+
for (const pending of pendingEvents) {
|
|
1662
|
+
const event = pending.event;
|
|
1663
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
1664
|
+
const candidate = event.payload?.toState;
|
|
1665
|
+
if (typeof candidate === "string") state = candidate;
|
|
1666
|
+
}
|
|
1667
|
+
return state;
|
|
1668
|
+
},
|
|
1096
1669
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
1097
1670
|
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
1671
|
+
},
|
|
1672
|
+
updateLeasedActionInvocation: async (input) => {
|
|
1673
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1674
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1675
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
1676
|
+
return true;
|
|
1098
1677
|
}
|
|
1099
1678
|
});
|
|
1100
1679
|
for (const update of pendingUpdates) {
|
|
@@ -1121,7 +1700,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1121
1700
|
if (existing) return existing;
|
|
1122
1701
|
}
|
|
1123
1702
|
const now = /* @__PURE__ */ new Date();
|
|
1124
|
-
const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
|
|
1703
|
+
const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
|
|
1125
1704
|
this.invocations.push(record);
|
|
1126
1705
|
return record;
|
|
1127
1706
|
}
|
|
@@ -1171,6 +1750,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1171
1750
|
record.error = void 0;
|
|
1172
1751
|
record.leaseOwner = input.leaseOwner;
|
|
1173
1752
|
record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
|
|
1753
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1174
1754
|
record.attemptCount = Math.max(record.attemptCount, 1);
|
|
1175
1755
|
return { applied: true, invocation: record };
|
|
1176
1756
|
}
|
|
@@ -1248,31 +1828,41 @@ var MemoryPlatformHostStore = class {
|
|
|
1248
1828
|
return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
|
|
1249
1829
|
record.leaseOwner = input.workerId;
|
|
1250
1830
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1831
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1251
1832
|
record.attemptCount += 1;
|
|
1252
1833
|
return cloneOutboxRecord(record);
|
|
1253
1834
|
});
|
|
1254
1835
|
}
|
|
1255
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
1256
|
-
const record = this.requireLeasedOutbox(id, workerId);
|
|
1836
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1837
|
+
const record = this.requireLeasedOutbox(id, workerId, leaseToken);
|
|
1257
1838
|
record.status = "published";
|
|
1258
1839
|
record.publishedAt = publishedAt;
|
|
1259
1840
|
delete record.leaseOwner;
|
|
1260
1841
|
delete record.leaseExpiresAt;
|
|
1261
1842
|
}
|
|
1262
1843
|
async markOutboxFailed(input) {
|
|
1263
|
-
const record = this.requireLeasedOutbox(input.id, input.workerId);
|
|
1844
|
+
const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
|
|
1264
1845
|
record.status = input.deadLetter ? "dead_letter" : "pending";
|
|
1265
1846
|
record.lastError = input.error;
|
|
1266
1847
|
record.availableAt = input.availableAt;
|
|
1267
1848
|
delete record.leaseOwner;
|
|
1268
1849
|
delete record.leaseExpiresAt;
|
|
1269
1850
|
}
|
|
1851
|
+
async renewOutboxLease(input) {
|
|
1852
|
+
const record = this.outbox.find((candidate) => candidate.id === input.id);
|
|
1853
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1854
|
+
if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1855
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1856
|
+
return true;
|
|
1857
|
+
}
|
|
1270
1858
|
async listOutbox(input = {}) {
|
|
1271
1859
|
return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
|
|
1272
1860
|
}
|
|
1273
|
-
requireLeasedOutbox(id, workerId) {
|
|
1861
|
+
requireLeasedOutbox(id, workerId, leaseToken) {
|
|
1274
1862
|
const record = this.outbox.find((candidate) => candidate.id === id);
|
|
1275
|
-
if (!record || record.leaseOwner !== workerId
|
|
1863
|
+
if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
|
|
1864
|
+
throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1865
|
+
}
|
|
1276
1866
|
return record;
|
|
1277
1867
|
}
|
|
1278
1868
|
async nextEventSequence(tenantId, spaceId) {
|
|
@@ -1294,24 +1884,128 @@ var MemoryPlatformHostStore = class {
|
|
|
1294
1884
|
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
1295
1885
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1296
1886
|
}
|
|
1887
|
+
async getHealthCounts(input) {
|
|
1888
|
+
const invocations = this.invocations.filter(
|
|
1889
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1890
|
+
);
|
|
1891
|
+
const outbox = this.outbox.filter(
|
|
1892
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1893
|
+
);
|
|
1894
|
+
return {
|
|
1895
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
1896
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
1897
|
+
invocationExpiredLeases: invocations.filter(
|
|
1898
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1899
|
+
).length,
|
|
1900
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
1901
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
1902
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
1903
|
+
outboxExpiredLeases: outbox.filter(
|
|
1904
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1905
|
+
).length,
|
|
1906
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
1907
|
+
};
|
|
1908
|
+
}
|
|
1297
1909
|
async claimActionInvocations(input) {
|
|
1298
1910
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1299
1911
|
const eligible = this.invocations.filter(
|
|
1300
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" &&
|
|
1912
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
|
|
1301
1913
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
|
|
1302
1914
|
for (const record of eligible) {
|
|
1303
1915
|
record.status = "running";
|
|
1304
1916
|
record.leaseOwner = input.workerId;
|
|
1305
1917
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1918
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1306
1919
|
record.attemptCount += 1;
|
|
1307
1920
|
record.updatedAt = current;
|
|
1308
1921
|
}
|
|
1309
|
-
return eligible;
|
|
1922
|
+
return eligible.map((record) => structuredClone(record));
|
|
1923
|
+
}
|
|
1924
|
+
async updateLeasedActionInvocation(input) {
|
|
1925
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1926
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1927
|
+
await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
|
|
1928
|
+
return true;
|
|
1929
|
+
}
|
|
1930
|
+
async renewActionInvocationLease(input) {
|
|
1931
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1932
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1933
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1934
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1935
|
+
record.updatedAt = current;
|
|
1936
|
+
return true;
|
|
1310
1937
|
}
|
|
1311
1938
|
async listEvents(tenantId, spaceId) {
|
|
1312
1939
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
|
|
1313
1940
|
}
|
|
1314
1941
|
};
|
|
1942
|
+
async function applyPostgresMigrations(client, migrations) {
|
|
1943
|
+
const ordered = validateMigrations(migrations);
|
|
1944
|
+
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
1945
|
+
await client.query(`
|
|
1946
|
+
BEGIN;
|
|
1947
|
+
SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
|
|
1948
|
+
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1949
|
+
CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
|
|
1950
|
+
version integer PRIMARY KEY,
|
|
1951
|
+
name text NOT NULL,
|
|
1952
|
+
checksum text NOT NULL,
|
|
1953
|
+
applied_at timestamptz NOT NULL DEFAULT now()
|
|
1954
|
+
);
|
|
1955
|
+
${statements.join("\n")}
|
|
1956
|
+
COMMIT;
|
|
1957
|
+
`);
|
|
1958
|
+
}
|
|
1959
|
+
function validateMigrations(migrations) {
|
|
1960
|
+
const ordered = [...migrations].sort((left, right) => left.version - right.version);
|
|
1961
|
+
const versions = /* @__PURE__ */ new Set();
|
|
1962
|
+
for (const migration of ordered) {
|
|
1963
|
+
if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
|
|
1964
|
+
throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
|
|
1965
|
+
}
|
|
1966
|
+
if (versions.has(migration.version)) {
|
|
1967
|
+
throw new Error(`Duplicate migration version: ${migration.version}`);
|
|
1968
|
+
}
|
|
1969
|
+
if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
|
|
1970
|
+
throw new Error(`Invalid migration name: ${migration.name}`);
|
|
1971
|
+
}
|
|
1972
|
+
if (migration.sql.trim().length === 0) {
|
|
1973
|
+
throw new Error(`Migration ${migration.version} has no SQL`);
|
|
1974
|
+
}
|
|
1975
|
+
versions.add(migration.version);
|
|
1976
|
+
}
|
|
1977
|
+
return ordered;
|
|
1978
|
+
}
|
|
1979
|
+
function migrationStatement(migration) {
|
|
1980
|
+
const checksum = crypto.createHash("sha256").update(migration.sql).digest("hex");
|
|
1981
|
+
const delimiter = `$fabric_platform_migration_${migration.version}$`;
|
|
1982
|
+
if (migration.sql.includes(delimiter)) {
|
|
1983
|
+
throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
|
|
1984
|
+
}
|
|
1985
|
+
return `
|
|
1986
|
+
DO ${delimiter}
|
|
1987
|
+
DECLARE
|
|
1988
|
+
recorded_checksum text;
|
|
1989
|
+
BEGIN
|
|
1990
|
+
SELECT checksum INTO recorded_checksum
|
|
1991
|
+
FROM fabric_platform.schema_migrations
|
|
1992
|
+
WHERE version = ${migration.version};
|
|
1993
|
+
|
|
1994
|
+
IF FOUND THEN
|
|
1995
|
+
IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
|
|
1996
|
+
RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
|
|
1997
|
+
END IF;
|
|
1998
|
+
ELSE
|
|
1999
|
+
${migration.sql}
|
|
2000
|
+
INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
|
|
2001
|
+
VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
|
|
2002
|
+
END IF;
|
|
2003
|
+
END
|
|
2004
|
+
${delimiter};`;
|
|
2005
|
+
}
|
|
2006
|
+
function sqlLiteral(value) {
|
|
2007
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
2008
|
+
}
|
|
1315
2009
|
|
|
1316
2010
|
// src/postgres-store.ts
|
|
1317
2011
|
var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
@@ -1324,11 +2018,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1324
2018
|
const scoped = new _PostgresPlatformHostStore(db2, sql2);
|
|
1325
2019
|
return run({
|
|
1326
2020
|
db: db2,
|
|
2021
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
2022
|
+
const result = await sql2.query(
|
|
2023
|
+
`SELECT * FROM fabric_platform.action_invocations
|
|
2024
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2025
|
+
FOR UPDATE`,
|
|
2026
|
+
[id, tenantId, spaceId]
|
|
2027
|
+
);
|
|
2028
|
+
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
2029
|
+
},
|
|
2030
|
+
getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
|
|
1327
2031
|
appendEvent: (event) => scoped.appendEvent(event),
|
|
1328
2032
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
1329
2033
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
1330
2034
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
1331
|
-
|
|
2035
|
+
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2036
|
+
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2037
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
1332
2038
|
});
|
|
1333
2039
|
});
|
|
1334
2040
|
}
|
|
@@ -1338,7 +2044,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1338
2044
|
transactionWithEvents;
|
|
1339
2045
|
transactionalOutbox;
|
|
1340
2046
|
async ensureSchema() {
|
|
1341
|
-
await this.sql
|
|
2047
|
+
await applyPostgresMigrations(this.sql, [{
|
|
2048
|
+
version: 1,
|
|
2049
|
+
name: "host_ledger_baseline",
|
|
2050
|
+
sql: `
|
|
1342
2051
|
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1343
2052
|
CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
|
|
1344
2053
|
id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
|
|
@@ -1349,7 +2058,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1349
2058
|
parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
|
|
1350
2059
|
idempotency_authorization_binding_id text, invocation_provenance jsonb,
|
|
1351
2060
|
authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
|
|
1352
|
-
authorization_binding_id text, error text,
|
|
2061
|
+
adapter_reconciliation jsonb, authorization_binding_id text, error text,
|
|
1353
2062
|
attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
|
|
1354
2063
|
lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
|
|
1355
2064
|
hitl_reason text, hitl_policy_version text, approval_decision jsonb,
|
|
@@ -1366,6 +2075,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1366
2075
|
ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
|
|
1367
2076
|
ADD COLUMN IF NOT EXISTS execution_reason text,
|
|
1368
2077
|
ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
|
|
2078
|
+
ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
|
|
1369
2079
|
ADD COLUMN IF NOT EXISTS authorization_binding_id text,
|
|
1370
2080
|
ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
|
|
1371
2081
|
ADD COLUMN IF NOT EXISTS lease_owner text,
|
|
@@ -1431,10 +2141,11 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1431
2141
|
actor_id text NOT NULL, actor_type text NOT NULL, action_invocation_id text,
|
|
1432
2142
|
payload jsonb NOT NULL, sequence bigint NOT NULL,
|
|
1433
2143
|
occurred_at timestamptz NOT NULL, recorded_at timestamptz NOT NULL,
|
|
1434
|
-
correlation_id text NOT NULL, causation_id text, provenance jsonb,
|
|
2144
|
+
correlation_id text NOT NULL, causation_id text, provenance jsonb, consistency text,
|
|
1435
2145
|
UNIQUE (tenant_id, space_id, sequence)
|
|
1436
2146
|
);
|
|
1437
2147
|
ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS provenance jsonb;
|
|
2148
|
+
ALTER TABLE fabric_platform.asset_events ADD COLUMN IF NOT EXISTS consistency text;
|
|
1438
2149
|
CREATE INDEX IF NOT EXISTS asset_events_subject_idx
|
|
1439
2150
|
ON fabric_platform.asset_events
|
|
1440
2151
|
(tenant_id, space_id, subject_type, subject_id, sequence);
|
|
@@ -1447,7 +2158,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1447
2158
|
);
|
|
1448
2159
|
CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
|
|
1449
2160
|
ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
|
|
1450
|
-
|
|
2161
|
+
`
|
|
2162
|
+
}, {
|
|
2163
|
+
version: 2,
|
|
2164
|
+
name: "lease_fencing",
|
|
2165
|
+
sql: `
|
|
2166
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2167
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2168
|
+
ALTER TABLE fabric_platform.event_outbox
|
|
2169
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2170
|
+
`
|
|
2171
|
+
}]);
|
|
1451
2172
|
}
|
|
1452
2173
|
async transaction(run) {
|
|
1453
2174
|
return run(this.db);
|
|
@@ -1508,6 +2229,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1508
2229
|
status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
|
|
1509
2230
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
1510
2231
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2232
|
+
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
1511
2233
|
lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
1512
2234
|
THEN NULL ELSE lease_owner END,
|
|
1513
2235
|
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -1523,7 +2245,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1523
2245
|
Object.hasOwn(patch, "error"),
|
|
1524
2246
|
patch.error ?? null,
|
|
1525
2247
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
1526
|
-
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
|
|
2248
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2249
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2250
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
1527
2251
|
]
|
|
1528
2252
|
);
|
|
1529
2253
|
}
|
|
@@ -1557,6 +2281,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1557
2281
|
error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
|
|
1558
2282
|
lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
|
|
1559
2283
|
lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
|
|
2284
|
+
lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
|
|
1560
2285
|
attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
|
|
1561
2286
|
updated_at=$9
|
|
1562
2287
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
|
|
@@ -1718,8 +2443,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1718
2443
|
`INSERT INTO fabric_platform.asset_events
|
|
1719
2444
|
(id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
|
|
1720
2445
|
actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
|
|
1721
|
-
correlation_id,causation_id,provenance)
|
|
1722
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
|
|
2446
|
+
correlation_id,causation_id,provenance,consistency)
|
|
2447
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$18)
|
|
1723
2448
|
ON CONFLICT (id) DO NOTHING`,
|
|
1724
2449
|
[
|
|
1725
2450
|
event.id,
|
|
@@ -1738,7 +2463,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1738
2463
|
event.recordedAt,
|
|
1739
2464
|
event.correlationId,
|
|
1740
2465
|
event.causationId ?? null,
|
|
1741
|
-
event.provenance ? JSON.stringify(event.provenance) : null
|
|
2466
|
+
event.provenance ? JSON.stringify(event.provenance) : null,
|
|
2467
|
+
event.consistency ?? null
|
|
1742
2468
|
]
|
|
1743
2469
|
);
|
|
1744
2470
|
}
|
|
@@ -1749,8 +2475,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1749
2475
|
INSERT INTO fabric_platform.asset_events
|
|
1750
2476
|
(id,tenant_id,space_id,event_type,event_schema_version,subject_type,subject_id,
|
|
1751
2477
|
actor_id,actor_type,action_invocation_id,payload,sequence,occurred_at,recorded_at,
|
|
1752
|
-
correlation_id,causation_id,provenance)
|
|
1753
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb)
|
|
2478
|
+
correlation_id,causation_id,provenance,consistency)
|
|
2479
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16,$17::jsonb,$19)
|
|
1754
2480
|
ON CONFLICT (id) DO NOTHING RETURNING id
|
|
1755
2481
|
)
|
|
1756
2482
|
INSERT INTO fabric_platform.event_outbox
|
|
@@ -1775,7 +2501,8 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1775
2501
|
event.correlationId,
|
|
1776
2502
|
event.causationId ?? null,
|
|
1777
2503
|
event.provenance ? JSON.stringify(event.provenance) : null,
|
|
1778
|
-
JSON.stringify(envelope)
|
|
2504
|
+
JSON.stringify(envelope),
|
|
2505
|
+
event.consistency ?? null
|
|
1779
2506
|
]
|
|
1780
2507
|
);
|
|
1781
2508
|
}
|
|
@@ -1792,28 +2519,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1792
2519
|
ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
|
|
1793
2520
|
)
|
|
1794
2521
|
UPDATE fabric_platform.event_outbox AS item
|
|
1795
|
-
SET lease_owner=$5, lease_expires_at=$6,
|
|
2522
|
+
SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
|
|
2523
|
+
attempt_count=attempt_count+1
|
|
1796
2524
|
FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
|
|
1797
2525
|
[current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
|
|
1798
2526
|
);
|
|
1799
2527
|
return result.rows.map(toOutboxRecord);
|
|
1800
2528
|
}
|
|
1801
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
2529
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1802
2530
|
const result = await this.sql.query(
|
|
1803
2531
|
`UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
|
|
1804
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1805
|
-
|
|
2532
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2533
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
|
|
2534
|
+
[id, workerId, publishedAt, leaseToken ?? null]
|
|
1806
2535
|
);
|
|
1807
2536
|
if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1808
2537
|
}
|
|
1809
2538
|
async markOutboxFailed(input) {
|
|
1810
2539
|
const result = await this.sql.query(
|
|
1811
2540
|
`UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
|
|
1812
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1813
|
-
|
|
2541
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2542
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
|
|
2543
|
+
[input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
|
|
1814
2544
|
);
|
|
1815
2545
|
if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
|
|
1816
2546
|
}
|
|
2547
|
+
async renewOutboxLease(input) {
|
|
2548
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2549
|
+
const result = await this.sql.query(
|
|
2550
|
+
`UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
|
|
2551
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
2552
|
+
AND lease_expires_at > $4
|
|
2553
|
+
RETURNING id`,
|
|
2554
|
+
[
|
|
2555
|
+
input.id,
|
|
2556
|
+
input.workerId,
|
|
2557
|
+
input.leaseToken,
|
|
2558
|
+
current,
|
|
2559
|
+
new Date(current.getTime() + input.leaseDurationMs)
|
|
2560
|
+
]
|
|
2561
|
+
);
|
|
2562
|
+
return result.rows.length === 1;
|
|
2563
|
+
}
|
|
1817
2564
|
async listOutbox(input = {}) {
|
|
1818
2565
|
const result = await this.sql.query(
|
|
1819
2566
|
`SELECT * FROM fabric_platform.event_outbox
|
|
@@ -1862,6 +2609,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1862
2609
|
);
|
|
1863
2610
|
return result.rows.map(toActionRecord);
|
|
1864
2611
|
}
|
|
2612
|
+
async getHealthCounts(input) {
|
|
2613
|
+
const result = await this.sql.query(
|
|
2614
|
+
`SELECT
|
|
2615
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2616
|
+
WHERE status='pending'
|
|
2617
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2618
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
|
|
2619
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2620
|
+
WHERE status='running'
|
|
2621
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2622
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
|
|
2623
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2624
|
+
WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2625
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2626
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
|
|
2627
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2628
|
+
WHERE status='waiting_for_approval'
|
|
2629
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2630
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
|
|
2631
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2632
|
+
WHERE status='reconciliation_required'
|
|
2633
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2634
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
|
|
2635
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2636
|
+
WHERE status='pending'
|
|
2637
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2638
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
|
|
2639
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2640
|
+
WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2641
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2642
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
|
|
2643
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2644
|
+
WHERE status='dead_letter'
|
|
2645
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2646
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
|
|
2647
|
+
[input.tenantId ?? null, input.spaceId ?? null, input.now]
|
|
2648
|
+
);
|
|
2649
|
+
const row = result.rows[0] ?? {};
|
|
2650
|
+
return {
|
|
2651
|
+
invocationBacklog: Number(row.invocation_backlog ?? 0),
|
|
2652
|
+
invocationRunning: Number(row.invocation_running ?? 0),
|
|
2653
|
+
invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
|
|
2654
|
+
invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
|
|
2655
|
+
invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
|
|
2656
|
+
outboxBacklog: Number(row.outbox_backlog ?? 0),
|
|
2657
|
+
outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
|
|
2658
|
+
outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
|
|
2659
|
+
};
|
|
2660
|
+
}
|
|
1865
2661
|
async claimActionInvocations(input) {
|
|
1866
2662
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1867
2663
|
const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
|
|
@@ -1870,7 +2666,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1870
2666
|
`WITH claimable AS (
|
|
1871
2667
|
SELECT id FROM fabric_platform.action_invocations
|
|
1872
2668
|
WHERE (status='pending' OR
|
|
1873
|
-
(status='running' AND
|
|
2669
|
+
(status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
|
|
1874
2670
|
AND ($2::text IS NULL OR tenant_id=$2)
|
|
1875
2671
|
AND ($3::text IS NULL OR space_id=$3)
|
|
1876
2672
|
ORDER BY created_at
|
|
@@ -1879,7 +2675,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1879
2675
|
)
|
|
1880
2676
|
UPDATE fabric_platform.action_invocations AS invocation
|
|
1881
2677
|
SET status='running', lease_owner=$5, lease_expires_at=$6,
|
|
1882
|
-
attempt_count=attempt_count+1, updated_at=$1
|
|
2678
|
+
lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
|
|
1883
2679
|
FROM claimable WHERE invocation.id=claimable.id
|
|
1884
2680
|
RETURNING invocation.*`,
|
|
1885
2681
|
[
|
|
@@ -1893,6 +2689,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1893
2689
|
);
|
|
1894
2690
|
return result.rows.map(toActionRecord);
|
|
1895
2691
|
}
|
|
2692
|
+
async updateLeasedActionInvocation(input) {
|
|
2693
|
+
const patch = input.patch;
|
|
2694
|
+
const result = await this.sql.query(
|
|
2695
|
+
`UPDATE fabric_platform.action_invocations SET
|
|
2696
|
+
status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
|
|
2697
|
+
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2698
|
+
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2699
|
+
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2700
|
+
lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
|
+
THEN NULL ELSE lease_owner END,
|
|
2702
|
+
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2703
|
+
THEN NULL ELSE lease_expires_at END,
|
|
2704
|
+
updated_at=now()
|
|
2705
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2706
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$5
|
|
2707
|
+
RETURNING id`,
|
|
2708
|
+
[
|
|
2709
|
+
input.id,
|
|
2710
|
+
input.tenantId,
|
|
2711
|
+
input.spaceId,
|
|
2712
|
+
input.workerId,
|
|
2713
|
+
input.leaseToken,
|
|
2714
|
+
patch.status ?? null,
|
|
2715
|
+
patch.result === void 0 ? null : JSON.stringify(patch.result),
|
|
2716
|
+
Object.hasOwn(patch, "error"),
|
|
2717
|
+
patch.error ?? null,
|
|
2718
|
+
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2719
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2720
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2721
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2722
|
+
]
|
|
2723
|
+
);
|
|
2724
|
+
return result.rows.length === 1;
|
|
2725
|
+
}
|
|
2726
|
+
async renewActionInvocationLease(input) {
|
|
2727
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2728
|
+
const result = await this.sql.query(
|
|
2729
|
+
`UPDATE fabric_platform.action_invocations
|
|
2730
|
+
SET lease_expires_at=$6,updated_at=$5
|
|
2731
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2732
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$7
|
|
2733
|
+
AND lease_expires_at > $5
|
|
2734
|
+
RETURNING id`,
|
|
2735
|
+
[
|
|
2736
|
+
input.id,
|
|
2737
|
+
input.tenantId,
|
|
2738
|
+
input.spaceId,
|
|
2739
|
+
input.workerId,
|
|
2740
|
+
current,
|
|
2741
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
2742
|
+
input.leaseToken
|
|
2743
|
+
]
|
|
2744
|
+
);
|
|
2745
|
+
return result.rows.length === 1;
|
|
2746
|
+
}
|
|
1896
2747
|
async listEvents(tenantId, spaceId) {
|
|
1897
2748
|
const result = await this.sql.query(
|
|
1898
2749
|
`SELECT * FROM fabric_platform.asset_events
|
|
@@ -1901,6 +2752,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1901
2752
|
);
|
|
1902
2753
|
return result.rows.map(toEventRecord);
|
|
1903
2754
|
}
|
|
2755
|
+
async getEvent(id, tenantId, spaceId) {
|
|
2756
|
+
const result = await this.sql.query(
|
|
2757
|
+
`SELECT * FROM fabric_platform.asset_events
|
|
2758
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
2759
|
+
[id, tenantId, spaceId]
|
|
2760
|
+
);
|
|
2761
|
+
return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
|
|
2762
|
+
}
|
|
1904
2763
|
};
|
|
1905
2764
|
function toAdapterRecord(row) {
|
|
1906
2765
|
return {
|
|
@@ -1943,8 +2802,10 @@ function toActionRecord(row) {
|
|
|
1943
2802
|
...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
|
|
1944
2803
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
1945
2804
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2805
|
+
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
1946
2806
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
1947
2807
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2808
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
1948
2809
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
1949
2810
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
1950
2811
|
...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
|
|
@@ -2017,7 +2878,8 @@ function toEventRecord(row) {
|
|
|
2017
2878
|
recordedAt: new Date(row.recorded_at),
|
|
2018
2879
|
correlationId: String(row.correlation_id),
|
|
2019
2880
|
...row.causation_id ? { causationId: String(row.causation_id) } : {},
|
|
2020
|
-
...row.provenance ? { provenance: row.provenance } : {}
|
|
2881
|
+
...row.provenance ? { provenance: row.provenance } : {},
|
|
2882
|
+
...row.consistency ? { consistency: row.consistency } : {}
|
|
2021
2883
|
};
|
|
2022
2884
|
}
|
|
2023
2885
|
function toOutboxRecord(row) {
|
|
@@ -2039,6 +2901,7 @@ function toOutboxRecord(row) {
|
|
|
2039
2901
|
correlationId: String(event.correlationId),
|
|
2040
2902
|
...event.causationId ? { causationId: String(event.causationId) } : {},
|
|
2041
2903
|
...event.provenance ? { provenance: event.provenance } : {},
|
|
2904
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
2042
2905
|
occurredAt: new Date(event.occurredAt),
|
|
2043
2906
|
recordedAt: new Date(event.recordedAt),
|
|
2044
2907
|
producerModuleVersion: String(event.producerModuleVersion),
|
|
@@ -2051,6 +2914,7 @@ function toOutboxRecord(row) {
|
|
|
2051
2914
|
availableAt: new Date(row.available_at),
|
|
2052
2915
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
2053
2916
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
2917
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
2054
2918
|
...row.last_error ? { lastError: String(row.last_error) } : {},
|
|
2055
2919
|
createdAt: new Date(row.created_at),
|
|
2056
2920
|
...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
|
|
@@ -2067,38 +2931,93 @@ function createStoreBackedActionDispatcher() {
|
|
|
2067
2931
|
};
|
|
2068
2932
|
}
|
|
2069
2933
|
async function runPlatformActionWorkerCycle(options) {
|
|
2070
|
-
const
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2934
|
+
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2935
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
2936
|
+
let claimed;
|
|
2937
|
+
try {
|
|
2938
|
+
claimed = await options.store.claimActionInvocations({
|
|
2939
|
+
workerId: options.workerId,
|
|
2940
|
+
limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
2941
|
+
leaseDurationMs,
|
|
2942
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
2943
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
2944
|
+
});
|
|
2945
|
+
} catch (error) {
|
|
2946
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
|
|
2947
|
+
throw error;
|
|
2948
|
+
}
|
|
2949
|
+
for (const invocation of claimed) {
|
|
2950
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
2951
|
+
kind: "event",
|
|
2952
|
+
name: "invocation.lease_claimed",
|
|
2953
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
|
|
2954
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
2955
|
+
tenantId: invocation.tenantId,
|
|
2956
|
+
spaceId: invocation.spaceId,
|
|
2957
|
+
attributes: { attempt: invocation.attemptCount }
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2077
2960
|
let completed = 0;
|
|
2078
2961
|
let failed = 0;
|
|
2079
2962
|
let waitingForApproval = 0;
|
|
2080
2963
|
for (const invocation of claimed) {
|
|
2964
|
+
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
2081
2965
|
try {
|
|
2082
|
-
const
|
|
2966
|
+
const result2 = await options.host.executeInvocation(
|
|
2083
2967
|
invocation.id,
|
|
2084
2968
|
invocation.tenantId,
|
|
2085
2969
|
invocation.spaceId,
|
|
2086
|
-
{ leaseOwner: options.workerId }
|
|
2970
|
+
{ leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
|
|
2087
2971
|
);
|
|
2088
|
-
if (
|
|
2089
|
-
else if (
|
|
2972
|
+
if (result2.status === "completed") completed += 1;
|
|
2973
|
+
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
2090
2974
|
else failed += 1;
|
|
2091
2975
|
} catch (error) {
|
|
2092
2976
|
failed += 1;
|
|
2093
2977
|
options.onError?.(error, invocation);
|
|
2978
|
+
} finally {
|
|
2979
|
+
await stopHeartbeat();
|
|
2094
2980
|
}
|
|
2095
2981
|
}
|
|
2096
|
-
|
|
2982
|
+
const result = {
|
|
2097
2983
|
claimed: claimed.length,
|
|
2098
2984
|
completed,
|
|
2099
2985
|
failed,
|
|
2100
2986
|
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
2101
2987
|
};
|
|
2988
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
2989
|
+
return result;
|
|
2990
|
+
}
|
|
2991
|
+
function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
|
|
2992
|
+
if (!invocation?.leaseToken) return async () => void 0;
|
|
2993
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
2994
|
+
let stopped = false;
|
|
2995
|
+
let timer;
|
|
2996
|
+
let running;
|
|
2997
|
+
const schedule = () => {
|
|
2998
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
2999
|
+
};
|
|
3000
|
+
const tick = () => {
|
|
3001
|
+
running = options.store.renewActionInvocationLease({
|
|
3002
|
+
id: invocation.id,
|
|
3003
|
+
tenantId: invocation.tenantId,
|
|
3004
|
+
spaceId: invocation.spaceId,
|
|
3005
|
+
workerId: options.workerId,
|
|
3006
|
+
leaseToken: invocation.leaseToken,
|
|
3007
|
+
leaseDurationMs
|
|
3008
|
+
}).then((renewed) => {
|
|
3009
|
+
if (!renewed) options.onError?.(
|
|
3010
|
+
new Error(`Invocation lease lost: ${invocation.id}`),
|
|
3011
|
+
invocation
|
|
3012
|
+
);
|
|
3013
|
+
}).catch((error) => options.onError?.(error, invocation)).finally(schedule);
|
|
3014
|
+
};
|
|
3015
|
+
schedule();
|
|
3016
|
+
return async () => {
|
|
3017
|
+
stopped = true;
|
|
3018
|
+
if (timer) clearTimeout(timer);
|
|
3019
|
+
await running;
|
|
3020
|
+
};
|
|
2102
3021
|
}
|
|
2103
3022
|
async function runPlatformActionWorker(options) {
|
|
2104
3023
|
while (!options.signal?.aborted) {
|
|
@@ -2124,17 +3043,174 @@ async function abortableDelay(milliseconds, signal) {
|
|
|
2124
3043
|
);
|
|
2125
3044
|
});
|
|
2126
3045
|
}
|
|
3046
|
+
function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
|
|
3047
|
+
const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
|
|
3048
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
3049
|
+
kind: "event",
|
|
3050
|
+
name,
|
|
3051
|
+
metricName,
|
|
3052
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
3053
|
+
attributes: { workerId, ...attributes }
|
|
3054
|
+
});
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
// src/saga-parent-lifecycle.ts
|
|
3058
|
+
var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
|
|
3059
|
+
"completed",
|
|
3060
|
+
"failed",
|
|
3061
|
+
"blocked_by_policy",
|
|
3062
|
+
"reconciliation_required",
|
|
3063
|
+
"validation_failed"
|
|
3064
|
+
]);
|
|
3065
|
+
function createDurableSagaParentLifecycle(options) {
|
|
3066
|
+
const store = options.store;
|
|
3067
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3068
|
+
const transactionWithEvents = store.transactionWithEvents?.bind(store);
|
|
3069
|
+
if (!transactionWithEvents) {
|
|
3070
|
+
throw new Error(
|
|
3071
|
+
"Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
|
|
3072
|
+
);
|
|
3073
|
+
}
|
|
3074
|
+
const runInTransaction = transactionWithEvents;
|
|
3075
|
+
async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
|
|
3076
|
+
const eventId = `sagapar_${digestParameters({
|
|
3077
|
+
parentInvocationId: input.parentInvocationId,
|
|
3078
|
+
lifecycleId: lifecycleId2
|
|
3079
|
+
}).slice(0, 40)}`;
|
|
3080
|
+
await runInTransaction(async (transaction) => {
|
|
3081
|
+
if (!transaction.getActionInvocationForUpdate) {
|
|
3082
|
+
throw new Error(
|
|
3083
|
+
"Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
|
|
3084
|
+
);
|
|
3085
|
+
}
|
|
3086
|
+
if (!transaction.getEvent) {
|
|
3087
|
+
throw new Error(
|
|
3088
|
+
"Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
|
|
3089
|
+
);
|
|
3090
|
+
}
|
|
3091
|
+
const parent = await transaction.getActionInvocationForUpdate(
|
|
3092
|
+
input.parentInvocationId,
|
|
3093
|
+
input.tenantId,
|
|
3094
|
+
input.spaceId
|
|
3095
|
+
);
|
|
3096
|
+
if (!parent) {
|
|
3097
|
+
throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
|
|
3098
|
+
}
|
|
3099
|
+
const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
|
|
3100
|
+
if (prior) {
|
|
3101
|
+
if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
|
|
3102
|
+
throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
|
|
3103
|
+
}
|
|
3104
|
+
return;
|
|
3105
|
+
}
|
|
3106
|
+
if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
|
|
3107
|
+
throw new Error(
|
|
3108
|
+
`Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
|
|
3109
|
+
);
|
|
3110
|
+
}
|
|
3111
|
+
const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
|
|
3112
|
+
const timestamp = now();
|
|
3113
|
+
await transaction.appendEvent({
|
|
3114
|
+
id: eventId,
|
|
3115
|
+
tenantId: input.tenantId,
|
|
3116
|
+
spaceId: input.spaceId,
|
|
3117
|
+
eventType,
|
|
3118
|
+
eventSchemaVersion: 1,
|
|
3119
|
+
subjectType: "ActionInvocation",
|
|
3120
|
+
subjectId: input.parentInvocationId,
|
|
3121
|
+
actorId: "system",
|
|
3122
|
+
actorType: "system",
|
|
3123
|
+
actionInvocationId: input.parentInvocationId,
|
|
3124
|
+
payload,
|
|
3125
|
+
sequence,
|
|
3126
|
+
occurredAt: timestamp,
|
|
3127
|
+
recordedAt: timestamp,
|
|
3128
|
+
correlationId: input.parentInvocationId,
|
|
3129
|
+
causationId: lifecycleId2
|
|
3130
|
+
});
|
|
3131
|
+
if (patch) {
|
|
3132
|
+
await transaction.updateActionInvocation(
|
|
3133
|
+
input.parentInvocationId,
|
|
3134
|
+
input.tenantId,
|
|
3135
|
+
input.spaceId,
|
|
3136
|
+
patch
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
3139
|
+
});
|
|
3140
|
+
}
|
|
3141
|
+
return {
|
|
3142
|
+
async recordProgress(input) {
|
|
3143
|
+
await applyLifecycleTransition(
|
|
3144
|
+
input,
|
|
3145
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
|
|
3146
|
+
"SagaParentProgress",
|
|
3147
|
+
{
|
|
3148
|
+
stepId: input.progress.stepId,
|
|
3149
|
+
index: input.progress.index,
|
|
3150
|
+
total: input.progress.total,
|
|
3151
|
+
completedAt: input.progress.completedAt
|
|
3152
|
+
}
|
|
3153
|
+
);
|
|
3154
|
+
},
|
|
3155
|
+
async parkApproval(input) {
|
|
3156
|
+
await applyLifecycleTransition(
|
|
3157
|
+
input,
|
|
3158
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
|
|
3159
|
+
"SagaParentApprovalParked",
|
|
3160
|
+
{
|
|
3161
|
+
stepId: input.stepId,
|
|
3162
|
+
approvalId: input.approvalId,
|
|
3163
|
+
reason: input.reason
|
|
3164
|
+
},
|
|
3165
|
+
{ status: "waiting_for_approval" }
|
|
3166
|
+
);
|
|
3167
|
+
},
|
|
3168
|
+
async cancel(input) {
|
|
3169
|
+
await applyLifecycleTransition(
|
|
3170
|
+
input,
|
|
3171
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
|
|
3172
|
+
"SagaParentCancelled",
|
|
3173
|
+
{ reason: input.reason, actorId: input.actorId },
|
|
3174
|
+
{ status: "failed", error: `Saga cancelled: ${input.reason}` }
|
|
3175
|
+
);
|
|
3176
|
+
},
|
|
3177
|
+
async complete(input) {
|
|
3178
|
+
await applyLifecycleTransition(
|
|
3179
|
+
input,
|
|
3180
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
|
|
3181
|
+
"SagaParentCompleted",
|
|
3182
|
+
{ result: input.result },
|
|
3183
|
+
{ status: "completed", result: input.result }
|
|
3184
|
+
);
|
|
3185
|
+
},
|
|
3186
|
+
async fail(input) {
|
|
3187
|
+
await applyLifecycleTransition(
|
|
3188
|
+
input,
|
|
3189
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
|
|
3190
|
+
"SagaParentFailed",
|
|
3191
|
+
{ error: input.error, compensation: input.compensation },
|
|
3192
|
+
{ status: "failed", error: input.error }
|
|
3193
|
+
);
|
|
3194
|
+
}
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
2127
3197
|
|
|
2128
3198
|
exports.IdempotencyConflictError = IdempotencyConflictError;
|
|
2129
3199
|
exports.MemoryPlatformHostStore = MemoryPlatformHostStore;
|
|
2130
3200
|
exports.PARAMETER_DIGEST_ALGORITHM = PARAMETER_DIGEST_ALGORITHM;
|
|
2131
3201
|
exports.PLATFORM_HOST_CONTRACT_VERSION = PLATFORM_HOST_CONTRACT_VERSION;
|
|
3202
|
+
exports.PLATFORM_HOST_HEALTH_CONTRACT_VERSION = PLATFORM_HOST_HEALTH_CONTRACT_VERSION;
|
|
3203
|
+
exports.PLATFORM_HOST_METRIC_NAMES = PLATFORM_HOST_METRIC_NAMES;
|
|
2132
3204
|
exports.PostgresPlatformHostStore = PostgresPlatformHostStore;
|
|
3205
|
+
exports.applyPostgresMigrations = applyPostgresMigrations;
|
|
2133
3206
|
exports.canonicalJson = canonicalJson;
|
|
2134
3207
|
exports.cloneOutboxRecord = cloneOutboxRecord;
|
|
3208
|
+
exports.createDurableSagaParentLifecycle = createDurableSagaParentLifecycle;
|
|
2135
3209
|
exports.createGovernedActionHost = createGovernedActionHost;
|
|
2136
3210
|
exports.createStoreBackedActionDispatcher = createStoreBackedActionDispatcher;
|
|
2137
3211
|
exports.digestParameters = digestParameters;
|
|
3212
|
+
exports.emitPlatformHostTelemetry = emitPlatformHostTelemetry;
|
|
3213
|
+
exports.getPlatformHostHealthSnapshot = getPlatformHostHealthSnapshot;
|
|
2138
3214
|
exports.runOutboxRelayCycle = runOutboxRelayCycle;
|
|
2139
3215
|
exports.runPlatformActionWorker = runPlatformActionWorker;
|
|
2140
3216
|
exports.runPlatformActionWorkerCycle = runPlatformActionWorkerCycle;
|