@fabricorg/platform-host 5.0.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/MIGRATION-6-PRODUCTION.md +59 -0
- package/README.md +89 -11
- package/dist/index.cjs +1154 -98
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +377 -13
- package/dist/index.d.ts +377 -13
- package/dist/index.js +1150 -100
- package/dist/index.js.map +1 -1
- package/package.json +6 -3
package/dist/index.cjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
var platform = require('@fabricorg/platform');
|
|
4
|
+
var assembly = require('@fabricorg/assembly');
|
|
4
5
|
var crypto = require('crypto');
|
|
5
6
|
|
|
6
7
|
// src/host.ts
|
|
@@ -64,6 +65,213 @@ function normalizeProvenance(input, options) {
|
|
|
64
65
|
};
|
|
65
66
|
}
|
|
66
67
|
|
|
68
|
+
// src/observability.ts
|
|
69
|
+
var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
|
|
70
|
+
var PLATFORM_HOST_METRIC_NAMES = {
|
|
71
|
+
workerCyclesStarted: "fabric.platform.worker.cycles.started",
|
|
72
|
+
workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
|
|
73
|
+
workerCyclesFailed: "fabric.platform.worker.cycles.failed",
|
|
74
|
+
invocationSubmitted: "fabric.platform.invocation.submitted",
|
|
75
|
+
invocationExecutionStarted: "fabric.platform.invocation.execution_started",
|
|
76
|
+
invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
|
|
77
|
+
invocationCompleted: "fabric.platform.invocation.completed",
|
|
78
|
+
invocationFailed: "fabric.platform.invocation.failed",
|
|
79
|
+
invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
|
|
80
|
+
invocationValidationFailed: "fabric.platform.invocation.validation_failed",
|
|
81
|
+
invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
|
|
82
|
+
invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
|
|
83
|
+
outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
|
|
84
|
+
outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
|
|
85
|
+
outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
|
|
86
|
+
outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
|
|
87
|
+
outboxPublished: "fabric.platform.outbox.published",
|
|
88
|
+
outboxFailed: "fabric.platform.outbox.failed",
|
|
89
|
+
outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
|
|
90
|
+
invocationBacklog: "fabric.platform.invocation.backlog",
|
|
91
|
+
invocationRunning: "fabric.platform.invocation.running",
|
|
92
|
+
invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
|
|
93
|
+
invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
|
|
94
|
+
invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
|
|
95
|
+
outboxBacklog: "fabric.platform.outbox.backlog",
|
|
96
|
+
outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
|
|
97
|
+
outboxDeadLetters: "fabric.platform.outbox.dead_letters"
|
|
98
|
+
};
|
|
99
|
+
function emitPlatformHostTelemetry(telemetry, input) {
|
|
100
|
+
if (!telemetry) return;
|
|
101
|
+
const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
|
|
102
|
+
try {
|
|
103
|
+
const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
|
|
104
|
+
if (isPromiseLike(result)) void result.catch(() => void 0);
|
|
105
|
+
} catch {
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async function getPlatformHostHealthSnapshot(options) {
|
|
109
|
+
const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
|
|
110
|
+
const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
|
|
111
|
+
const scope = {
|
|
112
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
113
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
114
|
+
};
|
|
115
|
+
try {
|
|
116
|
+
const counts = await readHealthCounts(options.store, {
|
|
117
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
118
|
+
...options.spaceId ? { spaceId: options.spaceId } : {},
|
|
119
|
+
now: generatedAt
|
|
120
|
+
});
|
|
121
|
+
const snapshot = createHealthSnapshot(
|
|
122
|
+
generatedAt,
|
|
123
|
+
scope,
|
|
124
|
+
counts,
|
|
125
|
+
workerSummary,
|
|
126
|
+
true
|
|
127
|
+
);
|
|
128
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
129
|
+
return snapshot;
|
|
130
|
+
} catch {
|
|
131
|
+
const snapshot = createHealthSnapshot(
|
|
132
|
+
generatedAt,
|
|
133
|
+
scope,
|
|
134
|
+
zeroHealthCounts(),
|
|
135
|
+
workerSummary,
|
|
136
|
+
false
|
|
137
|
+
);
|
|
138
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
139
|
+
return snapshot;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
async function readHealthCounts(store, query) {
|
|
143
|
+
const candidate = store;
|
|
144
|
+
if (typeof candidate.getHealthCounts === "function") {
|
|
145
|
+
return assertHealthCounts(await candidate.getHealthCounts(query));
|
|
146
|
+
}
|
|
147
|
+
const recoverable = store;
|
|
148
|
+
const outbox = store;
|
|
149
|
+
if (typeof recoverable.listActionInvocations !== "function") {
|
|
150
|
+
throw new Error("Health counts are not supported by this store.");
|
|
151
|
+
}
|
|
152
|
+
const invocations = await recoverable.listActionInvocations({
|
|
153
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
154
|
+
...query.spaceId ? { spaceId: query.spaceId } : {},
|
|
155
|
+
limit: Number.MAX_SAFE_INTEGER
|
|
156
|
+
});
|
|
157
|
+
const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
|
|
158
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
159
|
+
...query.spaceId ? { spaceId: query.spaceId } : {}
|
|
160
|
+
}) : [];
|
|
161
|
+
return countsFromRecords(invocations, outboxRecords, query.now);
|
|
162
|
+
}
|
|
163
|
+
function countsFromRecords(invocations, outbox, now) {
|
|
164
|
+
return {
|
|
165
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
166
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
167
|
+
invocationExpiredLeases: invocations.filter(
|
|
168
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
169
|
+
).length,
|
|
170
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
171
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
172
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
173
|
+
outboxExpiredLeases: outbox.filter(
|
|
174
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
175
|
+
).length,
|
|
176
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
function assertHealthCounts(value) {
|
|
180
|
+
for (const [name, count] of Object.entries(value)) {
|
|
181
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
182
|
+
throw new Error(`Invalid health count: ${name}`);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return value;
|
|
186
|
+
}
|
|
187
|
+
function zeroHealthCounts() {
|
|
188
|
+
return {
|
|
189
|
+
invocationBacklog: 0,
|
|
190
|
+
invocationRunning: 0,
|
|
191
|
+
invocationExpiredLeases: 0,
|
|
192
|
+
invocationApprovalWaits: 0,
|
|
193
|
+
invocationReconciliationRequired: 0,
|
|
194
|
+
outboxBacklog: 0,
|
|
195
|
+
outboxExpiredLeases: 0,
|
|
196
|
+
outboxDeadLetters: 0
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
function summarizeWorkers(workers, now) {
|
|
200
|
+
let healthy = 0;
|
|
201
|
+
let stale = 0;
|
|
202
|
+
for (const worker of workers) {
|
|
203
|
+
if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
|
|
204
|
+
else stale += 1;
|
|
205
|
+
}
|
|
206
|
+
return { reported: workers.length, healthy, stale };
|
|
207
|
+
}
|
|
208
|
+
function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
|
|
209
|
+
const reasonCodes = [];
|
|
210
|
+
if (!dataAvailable) reasonCodes.push("health_source_unavailable");
|
|
211
|
+
if (workers.stale > 0) reasonCodes.push("workers_stale");
|
|
212
|
+
if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
|
|
213
|
+
if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
|
|
214
|
+
if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
|
|
215
|
+
if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
|
|
216
|
+
const unhealthy = !dataAvailable;
|
|
217
|
+
const degraded = unhealthy || reasonCodes.length > 0;
|
|
218
|
+
return {
|
|
219
|
+
contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
|
|
220
|
+
generatedAt,
|
|
221
|
+
scope,
|
|
222
|
+
dataAvailable,
|
|
223
|
+
status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
|
|
224
|
+
readiness: {
|
|
225
|
+
ready: dataAvailable && workers.stale === 0,
|
|
226
|
+
reasonCodes
|
|
227
|
+
},
|
|
228
|
+
workers,
|
|
229
|
+
invocations: {
|
|
230
|
+
backlog: counts.invocationBacklog,
|
|
231
|
+
running: counts.invocationRunning,
|
|
232
|
+
expiredLeases: counts.invocationExpiredLeases,
|
|
233
|
+
approvalWaits: counts.invocationApprovalWaits,
|
|
234
|
+
reconciliationRequired: counts.invocationReconciliationRequired
|
|
235
|
+
},
|
|
236
|
+
outbox: {
|
|
237
|
+
backlog: counts.outboxBacklog,
|
|
238
|
+
expiredLeases: counts.outboxExpiredLeases,
|
|
239
|
+
deadLetters: counts.outboxDeadLetters
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
function emitHealthMetrics(telemetry, snapshot) {
|
|
244
|
+
const scope = {
|
|
245
|
+
...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
|
|
246
|
+
...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
|
|
247
|
+
};
|
|
248
|
+
const metrics = [
|
|
249
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
|
|
250
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
|
|
251
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
|
|
252
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
|
|
253
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
|
|
254
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
|
|
255
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
|
|
256
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
|
|
257
|
+
];
|
|
258
|
+
for (const [name, value] of metrics) {
|
|
259
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
260
|
+
kind: "metric",
|
|
261
|
+
name,
|
|
262
|
+
metricType: "gauge",
|
|
263
|
+
value,
|
|
264
|
+
...scope,
|
|
265
|
+
observedAt: snapshot.generatedAt
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
function isPromiseLike(value) {
|
|
270
|
+
return Boolean(
|
|
271
|
+
value && typeof value === "object" && "then" in value && typeof value.then === "function"
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
|
|
67
275
|
// src/host.ts
|
|
68
276
|
var DEFAULT_EXTRACT_EVENTS = (data) => {
|
|
69
277
|
const value = data._events;
|
|
@@ -73,6 +281,19 @@ var RecoverableFinalizationError = class extends Error {
|
|
|
73
281
|
name = "RecoverableFinalizationError";
|
|
74
282
|
};
|
|
75
283
|
function createGovernedActionHost(options) {
|
|
284
|
+
if (options.composition) {
|
|
285
|
+
if (!options.registry?.orderedModules) {
|
|
286
|
+
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
287
|
+
}
|
|
288
|
+
assembly.assertAssemblyRuntimeCompatible(
|
|
289
|
+
options.composition.assembly,
|
|
290
|
+
options.registry.orderedModules.map((module) => ({
|
|
291
|
+
namespace: module.namespace,
|
|
292
|
+
version: module.version ?? "",
|
|
293
|
+
manifestDigest: module.manifestDigest ?? ""
|
|
294
|
+
}))
|
|
295
|
+
);
|
|
296
|
+
}
|
|
76
297
|
if (options.outbox) {
|
|
77
298
|
const outboxStore = asOutboxStore(options.store);
|
|
78
299
|
if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
|
|
@@ -118,10 +339,13 @@ function createGovernedActionHost(options) {
|
|
|
118
339
|
input.provenance,
|
|
119
340
|
options.provenance
|
|
120
341
|
);
|
|
342
|
+
const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
|
|
121
343
|
const runtimeEvidence = {
|
|
122
344
|
governanceContractVersion: platform.FABRIC_GOVERNANCE_CONTRACT_VERSION,
|
|
123
345
|
hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
|
|
124
|
-
...options.runtimeEvidence
|
|
346
|
+
...options.runtimeEvidence,
|
|
347
|
+
...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
|
|
348
|
+
...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
|
|
125
349
|
};
|
|
126
350
|
platform.assertGovernanceRuntimeEvidence(runtimeEvidence);
|
|
127
351
|
const durableInvocation = await options.store.createActionInvocation({
|
|
@@ -150,6 +374,15 @@ function createGovernedActionHost(options) {
|
|
|
150
374
|
...input.executionReason ? { executionReason: input.executionReason } : {},
|
|
151
375
|
...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
|
|
152
376
|
});
|
|
377
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
378
|
+
kind: "event",
|
|
379
|
+
name: "invocation.submitted",
|
|
380
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
|
|
381
|
+
occurredAt: now(),
|
|
382
|
+
tenantId: input.tenantId,
|
|
383
|
+
spaceId: input.spaceId,
|
|
384
|
+
attributes: { actionVersion: action.version }
|
|
385
|
+
});
|
|
153
386
|
if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
|
|
154
387
|
const conflict = idempotencyConflict(durableInvocation, {
|
|
155
388
|
actorId: input.actorId,
|
|
@@ -172,6 +405,23 @@ function createGovernedActionHost(options) {
|
|
|
172
405
|
}
|
|
173
406
|
const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
|
|
174
407
|
if (durableInvocation.id !== actionInvocationId) {
|
|
408
|
+
if (options.dispatcher && durableInvocation.status === "pending") {
|
|
409
|
+
const dispatched = await options.dispatcher.dispatch({
|
|
410
|
+
actionInvocationId: durableInvocation.id,
|
|
411
|
+
actionId: durableInvocation.actionId,
|
|
412
|
+
tenantId: input.tenantId,
|
|
413
|
+
spaceId: input.spaceId,
|
|
414
|
+
workflowId: durableWorkflowId
|
|
415
|
+
});
|
|
416
|
+
return withConsistency({
|
|
417
|
+
actionInvocationId: durableInvocation.id,
|
|
418
|
+
status: durableInvocation.status,
|
|
419
|
+
workflowId: dispatched.workflowId,
|
|
420
|
+
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
421
|
+
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
422
|
+
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
423
|
+
}, input.actionId);
|
|
424
|
+
}
|
|
175
425
|
return withConsistency({
|
|
176
426
|
actionInvocationId: durableInvocation.id,
|
|
177
427
|
status: durableInvocation.status,
|
|
@@ -180,13 +430,15 @@ function createGovernedActionHost(options) {
|
|
|
180
430
|
...durableInvocation.error ? { error: durableInvocation.error } : {},
|
|
181
431
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
182
432
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
183
|
-
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
|
|
433
|
+
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
434
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
184
435
|
}, input.actionId);
|
|
185
436
|
}
|
|
186
437
|
if (options.dispatcher) {
|
|
187
438
|
try {
|
|
188
439
|
const dispatched = await options.dispatcher.dispatch({
|
|
189
440
|
actionInvocationId: durableInvocation.id,
|
|
441
|
+
actionId: durableInvocation.actionId,
|
|
190
442
|
tenantId: input.tenantId,
|
|
191
443
|
spaceId: input.spaceId,
|
|
192
444
|
workflowId: durableWorkflowId
|
|
@@ -200,13 +452,6 @@ function createGovernedActionHost(options) {
|
|
|
200
452
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
201
453
|
}, input.actionId);
|
|
202
454
|
} catch (error) {
|
|
203
|
-
const message = errorMessage(error);
|
|
204
|
-
await options.store.updateActionInvocation(
|
|
205
|
-
durableInvocation.id,
|
|
206
|
-
input.tenantId,
|
|
207
|
-
input.spaceId,
|
|
208
|
-
{ status: "failed", error: message }
|
|
209
|
-
);
|
|
210
455
|
throw error;
|
|
211
456
|
}
|
|
212
457
|
}
|
|
@@ -231,16 +476,30 @@ function createGovernedActionHost(options) {
|
|
|
231
476
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
232
477
|
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
233
478
|
}
|
|
234
|
-
if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
|
|
479
|
+
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
235
480
|
return {
|
|
236
481
|
...actionResult(invocation),
|
|
237
|
-
error: `Invocation is
|
|
482
|
+
error: `Invocation lease is not owned by the supplied worker generation`
|
|
238
483
|
};
|
|
239
484
|
}
|
|
240
485
|
const action = actionResolver(invocation.actionId);
|
|
241
486
|
if (!action) {
|
|
242
487
|
return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
|
|
243
488
|
}
|
|
489
|
+
if (action.kind === "saga") {
|
|
490
|
+
return fail(
|
|
491
|
+
invocation,
|
|
492
|
+
"failed",
|
|
493
|
+
`Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
|
|
497
|
+
return fail(
|
|
498
|
+
invocation,
|
|
499
|
+
"failed",
|
|
500
|
+
`Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
|
|
501
|
+
);
|
|
502
|
+
}
|
|
244
503
|
if (invocation.attemptCount > 1 && !action.idempotent) {
|
|
245
504
|
return fail(
|
|
246
505
|
invocation,
|
|
@@ -253,6 +512,15 @@ function createGovernedActionHost(options) {
|
|
|
253
512
|
status: "running"
|
|
254
513
|
});
|
|
255
514
|
}
|
|
515
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
516
|
+
kind: "event",
|
|
517
|
+
name: "invocation.execution_started",
|
|
518
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
|
|
519
|
+
occurredAt: now(),
|
|
520
|
+
tenantId,
|
|
521
|
+
spaceId,
|
|
522
|
+
attributes: { actionVersion: invocation.actionVersion }
|
|
523
|
+
});
|
|
256
524
|
try {
|
|
257
525
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
258
526
|
if (!parsed.success) {
|
|
@@ -318,7 +586,7 @@ function createGovernedActionHost(options) {
|
|
|
318
586
|
);
|
|
319
587
|
}
|
|
320
588
|
if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
|
|
321
|
-
await
|
|
589
|
+
await persistInvocation(invocation, {
|
|
322
590
|
status: "waiting_for_approval"
|
|
323
591
|
});
|
|
324
592
|
return {
|
|
@@ -345,7 +613,7 @@ function createGovernedActionHost(options) {
|
|
|
345
613
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
346
614
|
message: `Action ${action.actionId} requires durable capture-time authorization evidence`
|
|
347
615
|
};
|
|
348
|
-
await
|
|
616
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
349
617
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
350
618
|
}
|
|
351
619
|
const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
|
|
@@ -358,7 +626,7 @@ function createGovernedActionHost(options) {
|
|
|
358
626
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
359
627
|
message: `Authorization binding for action ${action.actionId} expired before execution`
|
|
360
628
|
};
|
|
361
|
-
await
|
|
629
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
362
630
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
363
631
|
}
|
|
364
632
|
const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
|
|
@@ -380,7 +648,7 @@ function createGovernedActionHost(options) {
|
|
|
380
648
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
381
649
|
message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
|
|
382
650
|
};
|
|
383
|
-
await
|
|
651
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
384
652
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
385
653
|
}
|
|
386
654
|
const definitions = options.resolvePolicies ? await options.resolvePolicies({
|
|
@@ -442,30 +710,6 @@ function createGovernedActionHost(options) {
|
|
|
442
710
|
})));
|
|
443
711
|
}
|
|
444
712
|
}
|
|
445
|
-
const binding = action.stateMachine;
|
|
446
|
-
if (binding) {
|
|
447
|
-
const entityId = binding.getEntityId(parsed.data);
|
|
448
|
-
const currentState = entityId ? await options.store.getEntityState(
|
|
449
|
-
tenantId,
|
|
450
|
-
spaceId,
|
|
451
|
-
binding.entityType,
|
|
452
|
-
entityId
|
|
453
|
-
) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
|
|
454
|
-
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
455
|
-
if (targetState !== "") {
|
|
456
|
-
const transition = platform.validateStateMachineTransition(
|
|
457
|
-
stateMachineResolver(binding.entityType),
|
|
458
|
-
binding.entityType,
|
|
459
|
-
currentState,
|
|
460
|
-
targetState,
|
|
461
|
-
action.actionId
|
|
462
|
-
);
|
|
463
|
-
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
464
|
-
if (!transition.valid && !replayingAppliedTransition) {
|
|
465
|
-
return fail(invocation, "failed", transition.error ?? "Invalid state transition");
|
|
466
|
-
}
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
713
|
let data;
|
|
470
714
|
let domainEvents = [];
|
|
471
715
|
try {
|
|
@@ -473,6 +717,42 @@ function createGovernedActionHost(options) {
|
|
|
473
717
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
474
718
|
throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
475
719
|
}
|
|
720
|
+
const binding = action.stateMachine;
|
|
721
|
+
if (binding) {
|
|
722
|
+
const machine = stateMachineResolver(binding.entityType);
|
|
723
|
+
const entityId = binding.getEntityId(parsed.data);
|
|
724
|
+
const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
|
|
725
|
+
const currentState = entityId ? await getEntityState(
|
|
726
|
+
tenantId,
|
|
727
|
+
spaceId,
|
|
728
|
+
binding.entityType,
|
|
729
|
+
entityId
|
|
730
|
+
) ?? initialState(machine) : initialState(machine);
|
|
731
|
+
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
732
|
+
if (targetState !== "") {
|
|
733
|
+
const transition = await platform.evaluateStateMachineTransition(
|
|
734
|
+
machine,
|
|
735
|
+
binding.entityType,
|
|
736
|
+
currentState,
|
|
737
|
+
targetState,
|
|
738
|
+
action.actionId,
|
|
739
|
+
{
|
|
740
|
+
entity: {
|
|
741
|
+
...entityId ? { id: entityId } : {},
|
|
742
|
+
state: currentState
|
|
743
|
+
},
|
|
744
|
+
actionInvocationId,
|
|
745
|
+
actorId: invocation.actorId,
|
|
746
|
+
parameters: parsed.data,
|
|
747
|
+
db
|
|
748
|
+
}
|
|
749
|
+
);
|
|
750
|
+
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
751
|
+
if (!transition.valid && !replayingAppliedTransition) {
|
|
752
|
+
throw new Error(transition.error ?? "Invalid state transition");
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
476
756
|
const handlerResult = action.handler ? await action.handler(
|
|
477
757
|
{
|
|
478
758
|
actionInvocationId,
|
|
@@ -572,7 +852,21 @@ function createGovernedActionHost(options) {
|
|
|
572
852
|
subjectId: adapterEventSubject.subjectId,
|
|
573
853
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
574
854
|
}, `adapter:${stepIndex}:started`);
|
|
855
|
+
let adapterDeadlineTimer;
|
|
575
856
|
try {
|
|
857
|
+
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
858
|
+
const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
|
|
859
|
+
if (adapterController && deadlineMs) {
|
|
860
|
+
const remaining = deadlineMs - now().getTime();
|
|
861
|
+
if (remaining <= 0) {
|
|
862
|
+
throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
863
|
+
}
|
|
864
|
+
adapterDeadlineTimer = setTimeout(
|
|
865
|
+
() => adapterController.abort(new Error("Adapter deadline exceeded")),
|
|
866
|
+
remaining
|
|
867
|
+
);
|
|
868
|
+
adapterDeadlineTimer.unref?.();
|
|
869
|
+
}
|
|
576
870
|
const result2 = await platform.executeWithAdapterRetry({
|
|
577
871
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
578
872
|
defaultIdempotent: adapter.idempotent,
|
|
@@ -589,13 +883,77 @@ function createGovernedActionHost(options) {
|
|
|
589
883
|
correlationId: invocation.correlationId,
|
|
590
884
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
591
885
|
attempt,
|
|
592
|
-
maxAttempts
|
|
886
|
+
maxAttempts,
|
|
887
|
+
...deadlineMs ? { deadlineMs } : {},
|
|
888
|
+
...adapterController ? { signal: adapterController.signal } : {}
|
|
593
889
|
});
|
|
594
890
|
},
|
|
595
891
|
isSuccessful: (result3) => result3.success,
|
|
596
|
-
getError: (result3) => result3.error
|
|
892
|
+
getError: (result3) => result3.error,
|
|
893
|
+
classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
|
|
894
|
+
classifyThrownError: (error) => {
|
|
895
|
+
if (adapterController?.signal.aborted) return "timeout";
|
|
896
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
897
|
+
if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
|
|
898
|
+
return "transient_failure";
|
|
899
|
+
}
|
|
597
900
|
});
|
|
901
|
+
platform.assertAdapterOutcomeConsistency(result2);
|
|
598
902
|
if (!result2.success) {
|
|
903
|
+
const outcome = result2.outcome ?? "transient_failure";
|
|
904
|
+
if (outcome === "ambiguous") {
|
|
905
|
+
const evidence = result2.outcomeEvidence;
|
|
906
|
+
const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
907
|
+
const adapterReconciliation = {
|
|
908
|
+
kind: "adapter_outcome_ambiguous",
|
|
909
|
+
adapterType: step.adapterType,
|
|
910
|
+
operation: step.operation,
|
|
911
|
+
vendor: adapter.vendor,
|
|
912
|
+
adapterInvocationId,
|
|
913
|
+
...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
|
|
914
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
915
|
+
message
|
|
916
|
+
};
|
|
917
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
918
|
+
status: "ambiguous",
|
|
919
|
+
error: message,
|
|
920
|
+
updatedAt: now()
|
|
921
|
+
});
|
|
922
|
+
await appendEvent(invocation, {
|
|
923
|
+
eventType: "AdapterInvocationAmbiguous",
|
|
924
|
+
subjectType: adapterEventSubject.subjectType,
|
|
925
|
+
subjectId: adapterEventSubject.subjectId,
|
|
926
|
+
payload: {
|
|
927
|
+
adapterType: step.adapterType,
|
|
928
|
+
operation: step.operation,
|
|
929
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
|
|
930
|
+
}
|
|
931
|
+
}, `adapter:${stepIndex}:ambiguous`);
|
|
932
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
933
|
+
if (governanceStore2 && evidence) {
|
|
934
|
+
await governanceStore2.appendExternalReconciliation({
|
|
935
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
936
|
+
actionInvocationId,
|
|
937
|
+
tenantId,
|
|
938
|
+
spaceId,
|
|
939
|
+
status: "pending",
|
|
940
|
+
provider: adapter.vendor,
|
|
941
|
+
...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
|
|
942
|
+
attempt: 1,
|
|
943
|
+
reason: message,
|
|
944
|
+
observedAt: now()
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
await persistInvocation(invocation, {
|
|
948
|
+
status: "reconciliation_required",
|
|
949
|
+
error: message,
|
|
950
|
+
adapterReconciliation
|
|
951
|
+
});
|
|
952
|
+
return withConsistency(
|
|
953
|
+
{ actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
|
|
954
|
+
action.actionId
|
|
955
|
+
);
|
|
956
|
+
}
|
|
599
957
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
600
958
|
status: "failed",
|
|
601
959
|
error: result2.error ?? "Adapter failed",
|
|
@@ -655,6 +1013,8 @@ function createGovernedActionHost(options) {
|
|
|
655
1013
|
updatedAt: now()
|
|
656
1014
|
});
|
|
657
1015
|
return fail(invocation, "failed", message);
|
|
1016
|
+
} finally {
|
|
1017
|
+
if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
|
|
658
1018
|
}
|
|
659
1019
|
}
|
|
660
1020
|
const governanceStore = asGovernanceStore(options.store);
|
|
@@ -679,20 +1039,41 @@ function createGovernedActionHost(options) {
|
|
|
679
1039
|
transaction
|
|
680
1040
|
);
|
|
681
1041
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
1042
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1043
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1044
|
+
throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1045
|
+
}
|
|
1046
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1047
|
+
id: actionInvocationId,
|
|
1048
|
+
tenantId,
|
|
1049
|
+
spaceId,
|
|
1050
|
+
workerId: invocation.leaseOwner,
|
|
1051
|
+
leaseToken: invocation.leaseToken,
|
|
1052
|
+
patch: { status: "completed", result }
|
|
1053
|
+
});
|
|
1054
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1055
|
+
} else {
|
|
1056
|
+
await transaction.updateActionInvocation(
|
|
1057
|
+
actionInvocationId,
|
|
1058
|
+
tenantId,
|
|
1059
|
+
spaceId,
|
|
1060
|
+
{ status: "completed", result }
|
|
1061
|
+
);
|
|
1062
|
+
}
|
|
688
1063
|
});
|
|
1064
|
+
emitInvocationStatusTelemetry(
|
|
1065
|
+
invocation,
|
|
1066
|
+
"completed",
|
|
1067
|
+
options.telemetry,
|
|
1068
|
+
now()
|
|
1069
|
+
);
|
|
689
1070
|
} else {
|
|
690
1071
|
if (action.eventPhase === "after_adapters") {
|
|
691
1072
|
for (const [index, event] of domainEvents.entries()) {
|
|
692
1073
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
693
1074
|
}
|
|
694
1075
|
}
|
|
695
|
-
await
|
|
1076
|
+
await persistInvocation(invocation, {
|
|
696
1077
|
status: "completed",
|
|
697
1078
|
result
|
|
698
1079
|
});
|
|
@@ -775,12 +1156,21 @@ function createGovernedActionHost(options) {
|
|
|
775
1156
|
if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
776
1157
|
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
777
1158
|
}
|
|
778
|
-
if (!decision.approved)
|
|
1159
|
+
if (!decision.approved) {
|
|
1160
|
+
emitInvocationStatusTelemetry(
|
|
1161
|
+
transitioned,
|
|
1162
|
+
"failed",
|
|
1163
|
+
options.telemetry,
|
|
1164
|
+
now(),
|
|
1165
|
+
"waiting_for_approval"
|
|
1166
|
+
);
|
|
1167
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
1168
|
+
}
|
|
779
1169
|
return executeInvocation(
|
|
780
1170
|
actionInvocationId,
|
|
781
1171
|
tenantId,
|
|
782
1172
|
spaceId,
|
|
783
|
-
{ leaseOwner },
|
|
1173
|
+
{ leaseOwner, leaseToken: transitioned.leaseToken },
|
|
784
1174
|
"approval_resume"
|
|
785
1175
|
);
|
|
786
1176
|
}
|
|
@@ -876,12 +1266,7 @@ function createGovernedActionHost(options) {
|
|
|
876
1266
|
}
|
|
877
1267
|
}
|
|
878
1268
|
async function fail(invocation, status, error) {
|
|
879
|
-
await
|
|
880
|
-
invocation.id,
|
|
881
|
-
invocation.tenantId,
|
|
882
|
-
invocation.spaceId,
|
|
883
|
-
{ status, error }
|
|
884
|
-
);
|
|
1269
|
+
await persistInvocation(invocation, { status, error });
|
|
885
1270
|
return {
|
|
886
1271
|
actionInvocationId: invocation.id,
|
|
887
1272
|
status,
|
|
@@ -890,6 +1275,33 @@ function createGovernedActionHost(options) {
|
|
|
890
1275
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
891
1276
|
};
|
|
892
1277
|
}
|
|
1278
|
+
async function persistInvocation(invocation, patch) {
|
|
1279
|
+
const previousStatus = invocation.status;
|
|
1280
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1281
|
+
const recoverable = options.store;
|
|
1282
|
+
if (!recoverable.updateLeasedActionInvocation) {
|
|
1283
|
+
throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
1284
|
+
}
|
|
1285
|
+
const updated = await recoverable.updateLeasedActionInvocation({
|
|
1286
|
+
id: invocation.id,
|
|
1287
|
+
tenantId: invocation.tenantId,
|
|
1288
|
+
spaceId: invocation.spaceId,
|
|
1289
|
+
workerId: invocation.leaseOwner,
|
|
1290
|
+
leaseToken: invocation.leaseToken,
|
|
1291
|
+
patch
|
|
1292
|
+
});
|
|
1293
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
|
|
1294
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
await options.store.updateActionInvocation(
|
|
1298
|
+
invocation.id,
|
|
1299
|
+
invocation.tenantId,
|
|
1300
|
+
invocation.spaceId,
|
|
1301
|
+
patch
|
|
1302
|
+
);
|
|
1303
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1304
|
+
}
|
|
893
1305
|
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
894
1306
|
}
|
|
895
1307
|
function actionResult(invocation) {
|
|
@@ -900,7 +1312,8 @@ function actionResult(invocation) {
|
|
|
900
1312
|
...invocation.error ? { error: invocation.error } : {},
|
|
901
1313
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
902
1314
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
903
|
-
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
|
|
1315
|
+
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1316
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
904
1317
|
};
|
|
905
1318
|
}
|
|
906
1319
|
function asApprovalStore(store) {
|
|
@@ -954,6 +1367,24 @@ function initialState(machine) {
|
|
|
954
1367
|
function isTerminal(status) {
|
|
955
1368
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
956
1369
|
}
|
|
1370
|
+
function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
|
|
1371
|
+
if (!status) return;
|
|
1372
|
+
const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
|
|
1373
|
+
if (!event) return;
|
|
1374
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1375
|
+
kind: "event",
|
|
1376
|
+
name: event.name,
|
|
1377
|
+
metricName: event.metricName,
|
|
1378
|
+
occurredAt,
|
|
1379
|
+
tenantId: invocation.tenantId,
|
|
1380
|
+
spaceId: invocation.spaceId,
|
|
1381
|
+
attributes: {
|
|
1382
|
+
...invocation.actionId ? { actionId: invocation.actionId } : {},
|
|
1383
|
+
...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
|
|
1384
|
+
...previousStatus ? { fromStatus: previousStatus } : {}
|
|
1385
|
+
}
|
|
1386
|
+
});
|
|
1387
|
+
}
|
|
957
1388
|
function withoutPrivateHostFields(data, eventResultFields) {
|
|
958
1389
|
return Object.fromEntries(
|
|
959
1390
|
Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
|
|
@@ -1022,29 +1453,72 @@ function toEnterpriseEventEnvelope(event, metadata) {
|
|
|
1022
1453
|
async function runOutboxRelayCycle(options) {
|
|
1023
1454
|
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
1024
1455
|
const maxAttempts = options.maxAttempts ?? 10;
|
|
1025
|
-
const
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1456
|
+
const leaseDurationMs = options.leaseDurationMs ?? 3e4;
|
|
1457
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
|
|
1458
|
+
let records;
|
|
1459
|
+
try {
|
|
1460
|
+
records = await options.store.claimOutbox({
|
|
1461
|
+
workerId: options.workerId,
|
|
1462
|
+
leaseDurationMs,
|
|
1463
|
+
limit: options.batchSize ?? 100,
|
|
1464
|
+
now: now()
|
|
1465
|
+
});
|
|
1466
|
+
} catch (error) {
|
|
1467
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
|
|
1468
|
+
throw error;
|
|
1469
|
+
}
|
|
1470
|
+
for (const record of records) {
|
|
1471
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1472
|
+
kind: "event",
|
|
1473
|
+
name: "outbox.lease_claimed",
|
|
1474
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
|
|
1475
|
+
occurredAt: now(),
|
|
1476
|
+
tenantId: record.tenantId,
|
|
1477
|
+
spaceId: record.spaceId,
|
|
1478
|
+
attributes: { attempt: record.attemptCount }
|
|
1479
|
+
});
|
|
1480
|
+
}
|
|
1031
1481
|
const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
|
|
1032
1482
|
for (const record of records) {
|
|
1483
|
+
const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
|
|
1033
1484
|
try {
|
|
1034
|
-
await
|
|
1485
|
+
await publishWithTimeout(
|
|
1486
|
+
options.publisher,
|
|
1487
|
+
record.event,
|
|
1488
|
+
options.publishTimeoutMs ?? 1e4
|
|
1489
|
+
);
|
|
1490
|
+
await stopHeartbeat();
|
|
1035
1491
|
try {
|
|
1036
|
-
await options.store.markOutboxPublished(record.id, options.workerId, now());
|
|
1492
|
+
await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
|
|
1037
1493
|
} catch {
|
|
1038
1494
|
result.failed += 1;
|
|
1495
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1496
|
+
kind: "event",
|
|
1497
|
+
name: "outbox.failed",
|
|
1498
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1499
|
+
occurredAt: now(),
|
|
1500
|
+
tenantId: record.tenantId,
|
|
1501
|
+
spaceId: record.spaceId
|
|
1502
|
+
});
|
|
1039
1503
|
continue;
|
|
1040
1504
|
}
|
|
1041
1505
|
result.published += 1;
|
|
1506
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1507
|
+
kind: "event",
|
|
1508
|
+
name: "outbox.published",
|
|
1509
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
|
|
1510
|
+
occurredAt: now(),
|
|
1511
|
+
tenantId: record.tenantId,
|
|
1512
|
+
spaceId: record.spaceId
|
|
1513
|
+
});
|
|
1042
1514
|
} catch {
|
|
1515
|
+
await stopHeartbeat();
|
|
1043
1516
|
const deadLetter = record.attemptCount >= maxAttempts;
|
|
1044
1517
|
try {
|
|
1045
1518
|
await options.store.markOutboxFailed({
|
|
1046
1519
|
id: record.id,
|
|
1047
1520
|
workerId: options.workerId,
|
|
1521
|
+
leaseToken: record.leaseToken,
|
|
1048
1522
|
error: "Event publisher failed",
|
|
1049
1523
|
availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
|
|
1050
1524
|
deadLetter
|
|
@@ -1053,16 +1527,84 @@ async function runOutboxRelayCycle(options) {
|
|
|
1053
1527
|
}
|
|
1054
1528
|
result.failed += 1;
|
|
1055
1529
|
if (deadLetter) result.deadLettered += 1;
|
|
1530
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1531
|
+
kind: "event",
|
|
1532
|
+
name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
|
|
1533
|
+
metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1534
|
+
occurredAt: now(),
|
|
1535
|
+
tenantId: record.tenantId,
|
|
1536
|
+
spaceId: record.spaceId,
|
|
1537
|
+
attributes: { attempt: record.attemptCount }
|
|
1538
|
+
});
|
|
1056
1539
|
}
|
|
1057
1540
|
}
|
|
1541
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
|
|
1542
|
+
claimed: result.claimed,
|
|
1543
|
+
published: result.published,
|
|
1544
|
+
failed: result.failed,
|
|
1545
|
+
deadLettered: result.deadLettered
|
|
1546
|
+
});
|
|
1058
1547
|
return result;
|
|
1059
1548
|
}
|
|
1549
|
+
async function publishWithTimeout(publisher, event, timeoutMs) {
|
|
1550
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
1551
|
+
throw new Error("publishTimeoutMs must be a positive finite number");
|
|
1552
|
+
}
|
|
1553
|
+
const controller = new AbortController();
|
|
1554
|
+
let timer;
|
|
1555
|
+
const timeout = new Promise((_, reject) => {
|
|
1556
|
+
timer = setTimeout(() => {
|
|
1557
|
+
const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
|
|
1558
|
+
controller.abort(error);
|
|
1559
|
+
reject(error);
|
|
1560
|
+
}, timeoutMs);
|
|
1561
|
+
});
|
|
1562
|
+
try {
|
|
1563
|
+
await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
|
|
1564
|
+
} finally {
|
|
1565
|
+
if (timer) clearTimeout(timer);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
function startOutboxHeartbeat(options, record, leaseDurationMs) {
|
|
1569
|
+
if (!record.leaseToken) return async () => void 0;
|
|
1570
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
1571
|
+
let stopped = false;
|
|
1572
|
+
let timer;
|
|
1573
|
+
let running;
|
|
1574
|
+
const schedule = () => {
|
|
1575
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
1576
|
+
};
|
|
1577
|
+
const tick = () => {
|
|
1578
|
+
running = options.store.renewOutboxLease({
|
|
1579
|
+
id: record.id,
|
|
1580
|
+
workerId: options.workerId,
|
|
1581
|
+
leaseToken: record.leaseToken,
|
|
1582
|
+
leaseDurationMs
|
|
1583
|
+
}).then(() => void 0).catch(() => void 0).finally(schedule);
|
|
1584
|
+
};
|
|
1585
|
+
schedule();
|
|
1586
|
+
return async () => {
|
|
1587
|
+
stopped = true;
|
|
1588
|
+
if (timer) clearTimeout(timer);
|
|
1589
|
+
await running;
|
|
1590
|
+
};
|
|
1591
|
+
}
|
|
1060
1592
|
function cloneOutboxRecord(record) {
|
|
1061
1593
|
return {
|
|
1062
1594
|
...record,
|
|
1063
1595
|
event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
|
|
1064
1596
|
};
|
|
1065
1597
|
}
|
|
1598
|
+
function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
|
|
1599
|
+
const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
|
|
1600
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1601
|
+
kind: "event",
|
|
1602
|
+
name,
|
|
1603
|
+
metricName,
|
|
1604
|
+
occurredAt,
|
|
1605
|
+
attributes: { workerId, ...attributes }
|
|
1606
|
+
});
|
|
1607
|
+
}
|
|
1066
1608
|
|
|
1067
1609
|
// src/memory-store.ts
|
|
1068
1610
|
var MemoryPlatformHostStore = class {
|
|
@@ -1105,12 +1647,33 @@ var MemoryPlatformHostStore = class {
|
|
|
1105
1647
|
};
|
|
1106
1648
|
const result = await run({
|
|
1107
1649
|
db: this.db,
|
|
1650
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
1651
|
+
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1652
|
+
return record ? structuredClone(record) : void 0;
|
|
1653
|
+
},
|
|
1654
|
+
getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
|
|
1108
1655
|
appendEvent: (event) => appendPending(event),
|
|
1109
1656
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
1110
1657
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
1111
1658
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
1659
|
+
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
1660
|
+
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
1661
|
+
for (const pending of pendingEvents) {
|
|
1662
|
+
const event = pending.event;
|
|
1663
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
1664
|
+
const candidate = event.payload?.toState;
|
|
1665
|
+
if (typeof candidate === "string") state = candidate;
|
|
1666
|
+
}
|
|
1667
|
+
return state;
|
|
1668
|
+
},
|
|
1112
1669
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
1113
1670
|
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
1671
|
+
},
|
|
1672
|
+
updateLeasedActionInvocation: async (input) => {
|
|
1673
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1674
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1675
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
1676
|
+
return true;
|
|
1114
1677
|
}
|
|
1115
1678
|
});
|
|
1116
1679
|
for (const update of pendingUpdates) {
|
|
@@ -1137,7 +1700,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1137
1700
|
if (existing) return existing;
|
|
1138
1701
|
}
|
|
1139
1702
|
const now = /* @__PURE__ */ new Date();
|
|
1140
|
-
const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
|
|
1703
|
+
const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
|
|
1141
1704
|
this.invocations.push(record);
|
|
1142
1705
|
return record;
|
|
1143
1706
|
}
|
|
@@ -1187,6 +1750,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1187
1750
|
record.error = void 0;
|
|
1188
1751
|
record.leaseOwner = input.leaseOwner;
|
|
1189
1752
|
record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
|
|
1753
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1190
1754
|
record.attemptCount = Math.max(record.attemptCount, 1);
|
|
1191
1755
|
return { applied: true, invocation: record };
|
|
1192
1756
|
}
|
|
@@ -1264,31 +1828,41 @@ var MemoryPlatformHostStore = class {
|
|
|
1264
1828
|
return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
|
|
1265
1829
|
record.leaseOwner = input.workerId;
|
|
1266
1830
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1831
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1267
1832
|
record.attemptCount += 1;
|
|
1268
1833
|
return cloneOutboxRecord(record);
|
|
1269
1834
|
});
|
|
1270
1835
|
}
|
|
1271
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
1272
|
-
const record = this.requireLeasedOutbox(id, workerId);
|
|
1836
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1837
|
+
const record = this.requireLeasedOutbox(id, workerId, leaseToken);
|
|
1273
1838
|
record.status = "published";
|
|
1274
1839
|
record.publishedAt = publishedAt;
|
|
1275
1840
|
delete record.leaseOwner;
|
|
1276
1841
|
delete record.leaseExpiresAt;
|
|
1277
1842
|
}
|
|
1278
1843
|
async markOutboxFailed(input) {
|
|
1279
|
-
const record = this.requireLeasedOutbox(input.id, input.workerId);
|
|
1844
|
+
const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
|
|
1280
1845
|
record.status = input.deadLetter ? "dead_letter" : "pending";
|
|
1281
1846
|
record.lastError = input.error;
|
|
1282
1847
|
record.availableAt = input.availableAt;
|
|
1283
1848
|
delete record.leaseOwner;
|
|
1284
1849
|
delete record.leaseExpiresAt;
|
|
1285
1850
|
}
|
|
1851
|
+
async renewOutboxLease(input) {
|
|
1852
|
+
const record = this.outbox.find((candidate) => candidate.id === input.id);
|
|
1853
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1854
|
+
if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1855
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1856
|
+
return true;
|
|
1857
|
+
}
|
|
1286
1858
|
async listOutbox(input = {}) {
|
|
1287
1859
|
return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
|
|
1288
1860
|
}
|
|
1289
|
-
requireLeasedOutbox(id, workerId) {
|
|
1861
|
+
requireLeasedOutbox(id, workerId, leaseToken) {
|
|
1290
1862
|
const record = this.outbox.find((candidate) => candidate.id === id);
|
|
1291
|
-
if (!record || record.leaseOwner !== workerId
|
|
1863
|
+
if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
|
|
1864
|
+
throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1865
|
+
}
|
|
1292
1866
|
return record;
|
|
1293
1867
|
}
|
|
1294
1868
|
async nextEventSequence(tenantId, spaceId) {
|
|
@@ -1310,24 +1884,128 @@ var MemoryPlatformHostStore = class {
|
|
|
1310
1884
|
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
1311
1885
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1312
1886
|
}
|
|
1887
|
+
async getHealthCounts(input) {
|
|
1888
|
+
const invocations = this.invocations.filter(
|
|
1889
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1890
|
+
);
|
|
1891
|
+
const outbox = this.outbox.filter(
|
|
1892
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1893
|
+
);
|
|
1894
|
+
return {
|
|
1895
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
1896
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
1897
|
+
invocationExpiredLeases: invocations.filter(
|
|
1898
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1899
|
+
).length,
|
|
1900
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
1901
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
1902
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
1903
|
+
outboxExpiredLeases: outbox.filter(
|
|
1904
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1905
|
+
).length,
|
|
1906
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
1907
|
+
};
|
|
1908
|
+
}
|
|
1313
1909
|
async claimActionInvocations(input) {
|
|
1314
1910
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1315
1911
|
const eligible = this.invocations.filter(
|
|
1316
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" &&
|
|
1912
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
|
|
1317
1913
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
|
|
1318
1914
|
for (const record of eligible) {
|
|
1319
1915
|
record.status = "running";
|
|
1320
1916
|
record.leaseOwner = input.workerId;
|
|
1321
1917
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1918
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1322
1919
|
record.attemptCount += 1;
|
|
1323
1920
|
record.updatedAt = current;
|
|
1324
1921
|
}
|
|
1325
|
-
return eligible;
|
|
1922
|
+
return eligible.map((record) => structuredClone(record));
|
|
1923
|
+
}
|
|
1924
|
+
async updateLeasedActionInvocation(input) {
|
|
1925
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1926
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1927
|
+
await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
|
|
1928
|
+
return true;
|
|
1929
|
+
}
|
|
1930
|
+
async renewActionInvocationLease(input) {
|
|
1931
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1932
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1933
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1934
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1935
|
+
record.updatedAt = current;
|
|
1936
|
+
return true;
|
|
1326
1937
|
}
|
|
1327
1938
|
async listEvents(tenantId, spaceId) {
|
|
1328
1939
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
|
|
1329
1940
|
}
|
|
1330
1941
|
};
|
|
1942
|
+
async function applyPostgresMigrations(client, migrations) {
|
|
1943
|
+
const ordered = validateMigrations(migrations);
|
|
1944
|
+
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
1945
|
+
await client.query(`
|
|
1946
|
+
BEGIN;
|
|
1947
|
+
SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
|
|
1948
|
+
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1949
|
+
CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
|
|
1950
|
+
version integer PRIMARY KEY,
|
|
1951
|
+
name text NOT NULL,
|
|
1952
|
+
checksum text NOT NULL,
|
|
1953
|
+
applied_at timestamptz NOT NULL DEFAULT now()
|
|
1954
|
+
);
|
|
1955
|
+
${statements.join("\n")}
|
|
1956
|
+
COMMIT;
|
|
1957
|
+
`);
|
|
1958
|
+
}
|
|
1959
|
+
function validateMigrations(migrations) {
|
|
1960
|
+
const ordered = [...migrations].sort((left, right) => left.version - right.version);
|
|
1961
|
+
const versions = /* @__PURE__ */ new Set();
|
|
1962
|
+
for (const migration of ordered) {
|
|
1963
|
+
if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
|
|
1964
|
+
throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
|
|
1965
|
+
}
|
|
1966
|
+
if (versions.has(migration.version)) {
|
|
1967
|
+
throw new Error(`Duplicate migration version: ${migration.version}`);
|
|
1968
|
+
}
|
|
1969
|
+
if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
|
|
1970
|
+
throw new Error(`Invalid migration name: ${migration.name}`);
|
|
1971
|
+
}
|
|
1972
|
+
if (migration.sql.trim().length === 0) {
|
|
1973
|
+
throw new Error(`Migration ${migration.version} has no SQL`);
|
|
1974
|
+
}
|
|
1975
|
+
versions.add(migration.version);
|
|
1976
|
+
}
|
|
1977
|
+
return ordered;
|
|
1978
|
+
}
|
|
1979
|
+
function migrationStatement(migration) {
|
|
1980
|
+
const checksum = crypto.createHash("sha256").update(migration.sql).digest("hex");
|
|
1981
|
+
const delimiter = `$fabric_platform_migration_${migration.version}$`;
|
|
1982
|
+
if (migration.sql.includes(delimiter)) {
|
|
1983
|
+
throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
|
|
1984
|
+
}
|
|
1985
|
+
return `
|
|
1986
|
+
DO ${delimiter}
|
|
1987
|
+
DECLARE
|
|
1988
|
+
recorded_checksum text;
|
|
1989
|
+
BEGIN
|
|
1990
|
+
SELECT checksum INTO recorded_checksum
|
|
1991
|
+
FROM fabric_platform.schema_migrations
|
|
1992
|
+
WHERE version = ${migration.version};
|
|
1993
|
+
|
|
1994
|
+
IF FOUND THEN
|
|
1995
|
+
IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
|
|
1996
|
+
RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
|
|
1997
|
+
END IF;
|
|
1998
|
+
ELSE
|
|
1999
|
+
${migration.sql}
|
|
2000
|
+
INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
|
|
2001
|
+
VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
|
|
2002
|
+
END IF;
|
|
2003
|
+
END
|
|
2004
|
+
${delimiter};`;
|
|
2005
|
+
}
|
|
2006
|
+
function sqlLiteral(value) {
|
|
2007
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
2008
|
+
}
|
|
1331
2009
|
|
|
1332
2010
|
// src/postgres-store.ts
|
|
1333
2011
|
var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
@@ -1340,11 +2018,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1340
2018
|
const scoped = new _PostgresPlatformHostStore(db2, sql2);
|
|
1341
2019
|
return run({
|
|
1342
2020
|
db: db2,
|
|
2021
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
2022
|
+
const result = await sql2.query(
|
|
2023
|
+
`SELECT * FROM fabric_platform.action_invocations
|
|
2024
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2025
|
+
FOR UPDATE`,
|
|
2026
|
+
[id, tenantId, spaceId]
|
|
2027
|
+
);
|
|
2028
|
+
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
2029
|
+
},
|
|
2030
|
+
getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
|
|
1343
2031
|
appendEvent: (event) => scoped.appendEvent(event),
|
|
1344
2032
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
1345
2033
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
1346
2034
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
1347
|
-
|
|
2035
|
+
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2036
|
+
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2037
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
1348
2038
|
});
|
|
1349
2039
|
});
|
|
1350
2040
|
}
|
|
@@ -1354,7 +2044,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1354
2044
|
transactionWithEvents;
|
|
1355
2045
|
transactionalOutbox;
|
|
1356
2046
|
async ensureSchema() {
|
|
1357
|
-
await this.sql
|
|
2047
|
+
await applyPostgresMigrations(this.sql, [{
|
|
2048
|
+
version: 1,
|
|
2049
|
+
name: "host_ledger_baseline",
|
|
2050
|
+
sql: `
|
|
1358
2051
|
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1359
2052
|
CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
|
|
1360
2053
|
id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
|
|
@@ -1365,7 +2058,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1365
2058
|
parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
|
|
1366
2059
|
idempotency_authorization_binding_id text, invocation_provenance jsonb,
|
|
1367
2060
|
authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
|
|
1368
|
-
authorization_binding_id text, error text,
|
|
2061
|
+
adapter_reconciliation jsonb, authorization_binding_id text, error text,
|
|
1369
2062
|
attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
|
|
1370
2063
|
lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
|
|
1371
2064
|
hitl_reason text, hitl_policy_version text, approval_decision jsonb,
|
|
@@ -1382,6 +2075,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1382
2075
|
ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
|
|
1383
2076
|
ADD COLUMN IF NOT EXISTS execution_reason text,
|
|
1384
2077
|
ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
|
|
2078
|
+
ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
|
|
1385
2079
|
ADD COLUMN IF NOT EXISTS authorization_binding_id text,
|
|
1386
2080
|
ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
|
|
1387
2081
|
ADD COLUMN IF NOT EXISTS lease_owner text,
|
|
@@ -1464,7 +2158,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1464
2158
|
);
|
|
1465
2159
|
CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
|
|
1466
2160
|
ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
|
|
1467
|
-
|
|
2161
|
+
`
|
|
2162
|
+
}, {
|
|
2163
|
+
version: 2,
|
|
2164
|
+
name: "lease_fencing",
|
|
2165
|
+
sql: `
|
|
2166
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2167
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2168
|
+
ALTER TABLE fabric_platform.event_outbox
|
|
2169
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2170
|
+
`
|
|
2171
|
+
}]);
|
|
1468
2172
|
}
|
|
1469
2173
|
async transaction(run) {
|
|
1470
2174
|
return run(this.db);
|
|
@@ -1525,6 +2229,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1525
2229
|
status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
|
|
1526
2230
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
1527
2231
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2232
|
+
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
1528
2233
|
lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
1529
2234
|
THEN NULL ELSE lease_owner END,
|
|
1530
2235
|
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -1540,7 +2245,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1540
2245
|
Object.hasOwn(patch, "error"),
|
|
1541
2246
|
patch.error ?? null,
|
|
1542
2247
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
1543
|
-
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
|
|
2248
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2249
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2250
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
1544
2251
|
]
|
|
1545
2252
|
);
|
|
1546
2253
|
}
|
|
@@ -1574,6 +2281,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1574
2281
|
error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
|
|
1575
2282
|
lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
|
|
1576
2283
|
lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
|
|
2284
|
+
lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
|
|
1577
2285
|
attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
|
|
1578
2286
|
updated_at=$9
|
|
1579
2287
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
|
|
@@ -1811,28 +2519,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1811
2519
|
ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
|
|
1812
2520
|
)
|
|
1813
2521
|
UPDATE fabric_platform.event_outbox AS item
|
|
1814
|
-
SET lease_owner=$5, lease_expires_at=$6,
|
|
2522
|
+
SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
|
|
2523
|
+
attempt_count=attempt_count+1
|
|
1815
2524
|
FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
|
|
1816
2525
|
[current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
|
|
1817
2526
|
);
|
|
1818
2527
|
return result.rows.map(toOutboxRecord);
|
|
1819
2528
|
}
|
|
1820
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
2529
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1821
2530
|
const result = await this.sql.query(
|
|
1822
2531
|
`UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
|
|
1823
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1824
|
-
|
|
2532
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2533
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
|
|
2534
|
+
[id, workerId, publishedAt, leaseToken ?? null]
|
|
1825
2535
|
);
|
|
1826
2536
|
if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1827
2537
|
}
|
|
1828
2538
|
async markOutboxFailed(input) {
|
|
1829
2539
|
const result = await this.sql.query(
|
|
1830
2540
|
`UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
|
|
1831
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1832
|
-
|
|
2541
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2542
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
|
|
2543
|
+
[input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
|
|
1833
2544
|
);
|
|
1834
2545
|
if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
|
|
1835
2546
|
}
|
|
2547
|
+
async renewOutboxLease(input) {
|
|
2548
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2549
|
+
const result = await this.sql.query(
|
|
2550
|
+
`UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
|
|
2551
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
2552
|
+
AND lease_expires_at > $4
|
|
2553
|
+
RETURNING id`,
|
|
2554
|
+
[
|
|
2555
|
+
input.id,
|
|
2556
|
+
input.workerId,
|
|
2557
|
+
input.leaseToken,
|
|
2558
|
+
current,
|
|
2559
|
+
new Date(current.getTime() + input.leaseDurationMs)
|
|
2560
|
+
]
|
|
2561
|
+
);
|
|
2562
|
+
return result.rows.length === 1;
|
|
2563
|
+
}
|
|
1836
2564
|
async listOutbox(input = {}) {
|
|
1837
2565
|
const result = await this.sql.query(
|
|
1838
2566
|
`SELECT * FROM fabric_platform.event_outbox
|
|
@@ -1881,6 +2609,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1881
2609
|
);
|
|
1882
2610
|
return result.rows.map(toActionRecord);
|
|
1883
2611
|
}
|
|
2612
|
+
async getHealthCounts(input) {
|
|
2613
|
+
const result = await this.sql.query(
|
|
2614
|
+
`SELECT
|
|
2615
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2616
|
+
WHERE status='pending'
|
|
2617
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2618
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
|
|
2619
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2620
|
+
WHERE status='running'
|
|
2621
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2622
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
|
|
2623
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2624
|
+
WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2625
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2626
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
|
|
2627
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2628
|
+
WHERE status='waiting_for_approval'
|
|
2629
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2630
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
|
|
2631
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2632
|
+
WHERE status='reconciliation_required'
|
|
2633
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2634
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
|
|
2635
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2636
|
+
WHERE status='pending'
|
|
2637
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2638
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
|
|
2639
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2640
|
+
WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2641
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2642
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
|
|
2643
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2644
|
+
WHERE status='dead_letter'
|
|
2645
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2646
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
|
|
2647
|
+
[input.tenantId ?? null, input.spaceId ?? null, input.now]
|
|
2648
|
+
);
|
|
2649
|
+
const row = result.rows[0] ?? {};
|
|
2650
|
+
return {
|
|
2651
|
+
invocationBacklog: Number(row.invocation_backlog ?? 0),
|
|
2652
|
+
invocationRunning: Number(row.invocation_running ?? 0),
|
|
2653
|
+
invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
|
|
2654
|
+
invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
|
|
2655
|
+
invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
|
|
2656
|
+
outboxBacklog: Number(row.outbox_backlog ?? 0),
|
|
2657
|
+
outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
|
|
2658
|
+
outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
|
|
2659
|
+
};
|
|
2660
|
+
}
|
|
1884
2661
|
async claimActionInvocations(input) {
|
|
1885
2662
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1886
2663
|
const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
|
|
@@ -1889,7 +2666,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1889
2666
|
`WITH claimable AS (
|
|
1890
2667
|
SELECT id FROM fabric_platform.action_invocations
|
|
1891
2668
|
WHERE (status='pending' OR
|
|
1892
|
-
(status='running' AND
|
|
2669
|
+
(status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
|
|
1893
2670
|
AND ($2::text IS NULL OR tenant_id=$2)
|
|
1894
2671
|
AND ($3::text IS NULL OR space_id=$3)
|
|
1895
2672
|
ORDER BY created_at
|
|
@@ -1898,7 +2675,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1898
2675
|
)
|
|
1899
2676
|
UPDATE fabric_platform.action_invocations AS invocation
|
|
1900
2677
|
SET status='running', lease_owner=$5, lease_expires_at=$6,
|
|
1901
|
-
attempt_count=attempt_count+1, updated_at=$1
|
|
2678
|
+
lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
|
|
1902
2679
|
FROM claimable WHERE invocation.id=claimable.id
|
|
1903
2680
|
RETURNING invocation.*`,
|
|
1904
2681
|
[
|
|
@@ -1912,6 +2689,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1912
2689
|
);
|
|
1913
2690
|
return result.rows.map(toActionRecord);
|
|
1914
2691
|
}
|
|
2692
|
+
async updateLeasedActionInvocation(input) {
|
|
2693
|
+
const patch = input.patch;
|
|
2694
|
+
const result = await this.sql.query(
|
|
2695
|
+
`UPDATE fabric_platform.action_invocations SET
|
|
2696
|
+
status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
|
|
2697
|
+
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2698
|
+
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2699
|
+
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2700
|
+
lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
|
+
THEN NULL ELSE lease_owner END,
|
|
2702
|
+
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2703
|
+
THEN NULL ELSE lease_expires_at END,
|
|
2704
|
+
updated_at=now()
|
|
2705
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2706
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$5
|
|
2707
|
+
RETURNING id`,
|
|
2708
|
+
[
|
|
2709
|
+
input.id,
|
|
2710
|
+
input.tenantId,
|
|
2711
|
+
input.spaceId,
|
|
2712
|
+
input.workerId,
|
|
2713
|
+
input.leaseToken,
|
|
2714
|
+
patch.status ?? null,
|
|
2715
|
+
patch.result === void 0 ? null : JSON.stringify(patch.result),
|
|
2716
|
+
Object.hasOwn(patch, "error"),
|
|
2717
|
+
patch.error ?? null,
|
|
2718
|
+
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2719
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2720
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2721
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2722
|
+
]
|
|
2723
|
+
);
|
|
2724
|
+
return result.rows.length === 1;
|
|
2725
|
+
}
|
|
2726
|
+
async renewActionInvocationLease(input) {
|
|
2727
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2728
|
+
const result = await this.sql.query(
|
|
2729
|
+
`UPDATE fabric_platform.action_invocations
|
|
2730
|
+
SET lease_expires_at=$6,updated_at=$5
|
|
2731
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2732
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$7
|
|
2733
|
+
AND lease_expires_at > $5
|
|
2734
|
+
RETURNING id`,
|
|
2735
|
+
[
|
|
2736
|
+
input.id,
|
|
2737
|
+
input.tenantId,
|
|
2738
|
+
input.spaceId,
|
|
2739
|
+
input.workerId,
|
|
2740
|
+
current,
|
|
2741
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
2742
|
+
input.leaseToken
|
|
2743
|
+
]
|
|
2744
|
+
);
|
|
2745
|
+
return result.rows.length === 1;
|
|
2746
|
+
}
|
|
1915
2747
|
async listEvents(tenantId, spaceId) {
|
|
1916
2748
|
const result = await this.sql.query(
|
|
1917
2749
|
`SELECT * FROM fabric_platform.asset_events
|
|
@@ -1920,6 +2752,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1920
2752
|
);
|
|
1921
2753
|
return result.rows.map(toEventRecord);
|
|
1922
2754
|
}
|
|
2755
|
+
async getEvent(id, tenantId, spaceId) {
|
|
2756
|
+
const result = await this.sql.query(
|
|
2757
|
+
`SELECT * FROM fabric_platform.asset_events
|
|
2758
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
2759
|
+
[id, tenantId, spaceId]
|
|
2760
|
+
);
|
|
2761
|
+
return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
|
|
2762
|
+
}
|
|
1923
2763
|
};
|
|
1924
2764
|
function toAdapterRecord(row) {
|
|
1925
2765
|
return {
|
|
@@ -1962,8 +2802,10 @@ function toActionRecord(row) {
|
|
|
1962
2802
|
...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
|
|
1963
2803
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
1964
2804
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2805
|
+
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
1965
2806
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
1966
2807
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2808
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
1967
2809
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
1968
2810
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
1969
2811
|
...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
|
|
@@ -2059,6 +2901,7 @@ function toOutboxRecord(row) {
|
|
|
2059
2901
|
correlationId: String(event.correlationId),
|
|
2060
2902
|
...event.causationId ? { causationId: String(event.causationId) } : {},
|
|
2061
2903
|
...event.provenance ? { provenance: event.provenance } : {},
|
|
2904
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
2062
2905
|
occurredAt: new Date(event.occurredAt),
|
|
2063
2906
|
recordedAt: new Date(event.recordedAt),
|
|
2064
2907
|
producerModuleVersion: String(event.producerModuleVersion),
|
|
@@ -2071,6 +2914,7 @@ function toOutboxRecord(row) {
|
|
|
2071
2914
|
availableAt: new Date(row.available_at),
|
|
2072
2915
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
2073
2916
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
2917
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
2074
2918
|
...row.last_error ? { lastError: String(row.last_error) } : {},
|
|
2075
2919
|
createdAt: new Date(row.created_at),
|
|
2076
2920
|
...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
|
|
@@ -2087,38 +2931,93 @@ function createStoreBackedActionDispatcher() {
|
|
|
2087
2931
|
};
|
|
2088
2932
|
}
|
|
2089
2933
|
async function runPlatformActionWorkerCycle(options) {
|
|
2090
|
-
const
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2934
|
+
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2935
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
2936
|
+
let claimed;
|
|
2937
|
+
try {
|
|
2938
|
+
claimed = await options.store.claimActionInvocations({
|
|
2939
|
+
workerId: options.workerId,
|
|
2940
|
+
limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
2941
|
+
leaseDurationMs,
|
|
2942
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
2943
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
2944
|
+
});
|
|
2945
|
+
} catch (error) {
|
|
2946
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
|
|
2947
|
+
throw error;
|
|
2948
|
+
}
|
|
2949
|
+
for (const invocation of claimed) {
|
|
2950
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
2951
|
+
kind: "event",
|
|
2952
|
+
name: "invocation.lease_claimed",
|
|
2953
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
|
|
2954
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
2955
|
+
tenantId: invocation.tenantId,
|
|
2956
|
+
spaceId: invocation.spaceId,
|
|
2957
|
+
attributes: { attempt: invocation.attemptCount }
|
|
2958
|
+
});
|
|
2959
|
+
}
|
|
2097
2960
|
let completed = 0;
|
|
2098
2961
|
let failed = 0;
|
|
2099
2962
|
let waitingForApproval = 0;
|
|
2100
2963
|
for (const invocation of claimed) {
|
|
2964
|
+
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
2101
2965
|
try {
|
|
2102
|
-
const
|
|
2966
|
+
const result2 = await options.host.executeInvocation(
|
|
2103
2967
|
invocation.id,
|
|
2104
2968
|
invocation.tenantId,
|
|
2105
2969
|
invocation.spaceId,
|
|
2106
|
-
{ leaseOwner: options.workerId }
|
|
2970
|
+
{ leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
|
|
2107
2971
|
);
|
|
2108
|
-
if (
|
|
2109
|
-
else if (
|
|
2972
|
+
if (result2.status === "completed") completed += 1;
|
|
2973
|
+
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
2110
2974
|
else failed += 1;
|
|
2111
2975
|
} catch (error) {
|
|
2112
2976
|
failed += 1;
|
|
2113
2977
|
options.onError?.(error, invocation);
|
|
2978
|
+
} finally {
|
|
2979
|
+
await stopHeartbeat();
|
|
2114
2980
|
}
|
|
2115
2981
|
}
|
|
2116
|
-
|
|
2982
|
+
const result = {
|
|
2117
2983
|
claimed: claimed.length,
|
|
2118
2984
|
completed,
|
|
2119
2985
|
failed,
|
|
2120
2986
|
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
2121
2987
|
};
|
|
2988
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
2989
|
+
return result;
|
|
2990
|
+
}
|
|
2991
|
+
function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
|
|
2992
|
+
if (!invocation?.leaseToken) return async () => void 0;
|
|
2993
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
2994
|
+
let stopped = false;
|
|
2995
|
+
let timer;
|
|
2996
|
+
let running;
|
|
2997
|
+
const schedule = () => {
|
|
2998
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
2999
|
+
};
|
|
3000
|
+
const tick = () => {
|
|
3001
|
+
running = options.store.renewActionInvocationLease({
|
|
3002
|
+
id: invocation.id,
|
|
3003
|
+
tenantId: invocation.tenantId,
|
|
3004
|
+
spaceId: invocation.spaceId,
|
|
3005
|
+
workerId: options.workerId,
|
|
3006
|
+
leaseToken: invocation.leaseToken,
|
|
3007
|
+
leaseDurationMs
|
|
3008
|
+
}).then((renewed) => {
|
|
3009
|
+
if (!renewed) options.onError?.(
|
|
3010
|
+
new Error(`Invocation lease lost: ${invocation.id}`),
|
|
3011
|
+
invocation
|
|
3012
|
+
);
|
|
3013
|
+
}).catch((error) => options.onError?.(error, invocation)).finally(schedule);
|
|
3014
|
+
};
|
|
3015
|
+
schedule();
|
|
3016
|
+
return async () => {
|
|
3017
|
+
stopped = true;
|
|
3018
|
+
if (timer) clearTimeout(timer);
|
|
3019
|
+
await running;
|
|
3020
|
+
};
|
|
2122
3021
|
}
|
|
2123
3022
|
async function runPlatformActionWorker(options) {
|
|
2124
3023
|
while (!options.signal?.aborted) {
|
|
@@ -2144,17 +3043,174 @@ async function abortableDelay(milliseconds, signal) {
|
|
|
2144
3043
|
);
|
|
2145
3044
|
});
|
|
2146
3045
|
}
|
|
3046
|
+
function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
|
|
3047
|
+
const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
|
|
3048
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
3049
|
+
kind: "event",
|
|
3050
|
+
name,
|
|
3051
|
+
metricName,
|
|
3052
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
3053
|
+
attributes: { workerId, ...attributes }
|
|
3054
|
+
});
|
|
3055
|
+
}
|
|
3056
|
+
|
|
3057
|
+
// src/saga-parent-lifecycle.ts
|
|
3058
|
+
var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
|
|
3059
|
+
"completed",
|
|
3060
|
+
"failed",
|
|
3061
|
+
"blocked_by_policy",
|
|
3062
|
+
"reconciliation_required",
|
|
3063
|
+
"validation_failed"
|
|
3064
|
+
]);
|
|
3065
|
+
function createDurableSagaParentLifecycle(options) {
|
|
3066
|
+
const store = options.store;
|
|
3067
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3068
|
+
const transactionWithEvents = store.transactionWithEvents?.bind(store);
|
|
3069
|
+
if (!transactionWithEvents) {
|
|
3070
|
+
throw new Error(
|
|
3071
|
+
"Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
|
|
3072
|
+
);
|
|
3073
|
+
}
|
|
3074
|
+
const runInTransaction = transactionWithEvents;
|
|
3075
|
+
async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
|
|
3076
|
+
const eventId = `sagapar_${digestParameters({
|
|
3077
|
+
parentInvocationId: input.parentInvocationId,
|
|
3078
|
+
lifecycleId: lifecycleId2
|
|
3079
|
+
}).slice(0, 40)}`;
|
|
3080
|
+
await runInTransaction(async (transaction) => {
|
|
3081
|
+
if (!transaction.getActionInvocationForUpdate) {
|
|
3082
|
+
throw new Error(
|
|
3083
|
+
"Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
|
|
3084
|
+
);
|
|
3085
|
+
}
|
|
3086
|
+
if (!transaction.getEvent) {
|
|
3087
|
+
throw new Error(
|
|
3088
|
+
"Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
|
|
3089
|
+
);
|
|
3090
|
+
}
|
|
3091
|
+
const parent = await transaction.getActionInvocationForUpdate(
|
|
3092
|
+
input.parentInvocationId,
|
|
3093
|
+
input.tenantId,
|
|
3094
|
+
input.spaceId
|
|
3095
|
+
);
|
|
3096
|
+
if (!parent) {
|
|
3097
|
+
throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
|
|
3098
|
+
}
|
|
3099
|
+
const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
|
|
3100
|
+
if (prior) {
|
|
3101
|
+
if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
|
|
3102
|
+
throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
|
|
3103
|
+
}
|
|
3104
|
+
return;
|
|
3105
|
+
}
|
|
3106
|
+
if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
|
|
3107
|
+
throw new Error(
|
|
3108
|
+
`Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
|
|
3109
|
+
);
|
|
3110
|
+
}
|
|
3111
|
+
const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
|
|
3112
|
+
const timestamp = now();
|
|
3113
|
+
await transaction.appendEvent({
|
|
3114
|
+
id: eventId,
|
|
3115
|
+
tenantId: input.tenantId,
|
|
3116
|
+
spaceId: input.spaceId,
|
|
3117
|
+
eventType,
|
|
3118
|
+
eventSchemaVersion: 1,
|
|
3119
|
+
subjectType: "ActionInvocation",
|
|
3120
|
+
subjectId: input.parentInvocationId,
|
|
3121
|
+
actorId: "system",
|
|
3122
|
+
actorType: "system",
|
|
3123
|
+
actionInvocationId: input.parentInvocationId,
|
|
3124
|
+
payload,
|
|
3125
|
+
sequence,
|
|
3126
|
+
occurredAt: timestamp,
|
|
3127
|
+
recordedAt: timestamp,
|
|
3128
|
+
correlationId: input.parentInvocationId,
|
|
3129
|
+
causationId: lifecycleId2
|
|
3130
|
+
});
|
|
3131
|
+
if (patch) {
|
|
3132
|
+
await transaction.updateActionInvocation(
|
|
3133
|
+
input.parentInvocationId,
|
|
3134
|
+
input.tenantId,
|
|
3135
|
+
input.spaceId,
|
|
3136
|
+
patch
|
|
3137
|
+
);
|
|
3138
|
+
}
|
|
3139
|
+
});
|
|
3140
|
+
}
|
|
3141
|
+
return {
|
|
3142
|
+
async recordProgress(input) {
|
|
3143
|
+
await applyLifecycleTransition(
|
|
3144
|
+
input,
|
|
3145
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
|
|
3146
|
+
"SagaParentProgress",
|
|
3147
|
+
{
|
|
3148
|
+
stepId: input.progress.stepId,
|
|
3149
|
+
index: input.progress.index,
|
|
3150
|
+
total: input.progress.total,
|
|
3151
|
+
completedAt: input.progress.completedAt
|
|
3152
|
+
}
|
|
3153
|
+
);
|
|
3154
|
+
},
|
|
3155
|
+
async parkApproval(input) {
|
|
3156
|
+
await applyLifecycleTransition(
|
|
3157
|
+
input,
|
|
3158
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
|
|
3159
|
+
"SagaParentApprovalParked",
|
|
3160
|
+
{
|
|
3161
|
+
stepId: input.stepId,
|
|
3162
|
+
approvalId: input.approvalId,
|
|
3163
|
+
reason: input.reason
|
|
3164
|
+
},
|
|
3165
|
+
{ status: "waiting_for_approval" }
|
|
3166
|
+
);
|
|
3167
|
+
},
|
|
3168
|
+
async cancel(input) {
|
|
3169
|
+
await applyLifecycleTransition(
|
|
3170
|
+
input,
|
|
3171
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
|
|
3172
|
+
"SagaParentCancelled",
|
|
3173
|
+
{ reason: input.reason, actorId: input.actorId },
|
|
3174
|
+
{ status: "failed", error: `Saga cancelled: ${input.reason}` }
|
|
3175
|
+
);
|
|
3176
|
+
},
|
|
3177
|
+
async complete(input) {
|
|
3178
|
+
await applyLifecycleTransition(
|
|
3179
|
+
input,
|
|
3180
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
|
|
3181
|
+
"SagaParentCompleted",
|
|
3182
|
+
{ result: input.result },
|
|
3183
|
+
{ status: "completed", result: input.result }
|
|
3184
|
+
);
|
|
3185
|
+
},
|
|
3186
|
+
async fail(input) {
|
|
3187
|
+
await applyLifecycleTransition(
|
|
3188
|
+
input,
|
|
3189
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
|
|
3190
|
+
"SagaParentFailed",
|
|
3191
|
+
{ error: input.error, compensation: input.compensation },
|
|
3192
|
+
{ status: "failed", error: input.error }
|
|
3193
|
+
);
|
|
3194
|
+
}
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
2147
3197
|
|
|
2148
3198
|
exports.IdempotencyConflictError = IdempotencyConflictError;
|
|
2149
3199
|
exports.MemoryPlatformHostStore = MemoryPlatformHostStore;
|
|
2150
3200
|
exports.PARAMETER_DIGEST_ALGORITHM = PARAMETER_DIGEST_ALGORITHM;
|
|
2151
3201
|
exports.PLATFORM_HOST_CONTRACT_VERSION = PLATFORM_HOST_CONTRACT_VERSION;
|
|
3202
|
+
exports.PLATFORM_HOST_HEALTH_CONTRACT_VERSION = PLATFORM_HOST_HEALTH_CONTRACT_VERSION;
|
|
3203
|
+
exports.PLATFORM_HOST_METRIC_NAMES = PLATFORM_HOST_METRIC_NAMES;
|
|
2152
3204
|
exports.PostgresPlatformHostStore = PostgresPlatformHostStore;
|
|
3205
|
+
exports.applyPostgresMigrations = applyPostgresMigrations;
|
|
2153
3206
|
exports.canonicalJson = canonicalJson;
|
|
2154
3207
|
exports.cloneOutboxRecord = cloneOutboxRecord;
|
|
3208
|
+
exports.createDurableSagaParentLifecycle = createDurableSagaParentLifecycle;
|
|
2155
3209
|
exports.createGovernedActionHost = createGovernedActionHost;
|
|
2156
3210
|
exports.createStoreBackedActionDispatcher = createStoreBackedActionDispatcher;
|
|
2157
3211
|
exports.digestParameters = digestParameters;
|
|
3212
|
+
exports.emitPlatformHostTelemetry = emitPlatformHostTelemetry;
|
|
3213
|
+
exports.getPlatformHostHealthSnapshot = getPlatformHostHealthSnapshot;
|
|
2158
3214
|
exports.runOutboxRelayCycle = runOutboxRelayCycle;
|
|
2159
3215
|
exports.runPlatformActionWorker = runPlatformActionWorker;
|
|
2160
3216
|
exports.runPlatformActionWorkerCycle = runPlatformActionWorkerCycle;
|