@fabricorg/platform-host 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/MIGRATION-6-PRODUCTION.md +59 -0
- package/README.md +89 -11
- package/dist/index.cjs +1154 -98
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +377 -13
- package/dist/index.d.ts +377 -13
- package/dist/index.js +1150 -100
- package/dist/index.js.map +1 -1
- package/package.json +6 -3
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes,
|
|
1
|
+
import { AdapterRegistry, createFabricId, assertGovernanceRuntimeEvidence, FABRIC_GOVERNANCE_CONTRACT_VERSION, assertMutationGovernanceContext, evaluatePolicyDefinitions, aggregatePolicyOutcomes, evaluateStateMachineTransition, executeWithAdapterRetry, assertAdapterOutcomeConsistency } from '@fabricorg/platform';
|
|
2
|
+
import { assertAssemblyRuntimeCompatible } from '@fabricorg/assembly';
|
|
2
3
|
import { createHash } from 'crypto';
|
|
3
4
|
|
|
4
5
|
// src/host.ts
|
|
@@ -62,6 +63,213 @@ function normalizeProvenance(input, options) {
|
|
|
62
63
|
};
|
|
63
64
|
}
|
|
64
65
|
|
|
66
|
+
// src/observability.ts
|
|
67
|
+
var PLATFORM_HOST_HEALTH_CONTRACT_VERSION = 1;
|
|
68
|
+
var PLATFORM_HOST_METRIC_NAMES = {
|
|
69
|
+
workerCyclesStarted: "fabric.platform.worker.cycles.started",
|
|
70
|
+
workerCyclesCompleted: "fabric.platform.worker.cycles.completed",
|
|
71
|
+
workerCyclesFailed: "fabric.platform.worker.cycles.failed",
|
|
72
|
+
invocationSubmitted: "fabric.platform.invocation.submitted",
|
|
73
|
+
invocationExecutionStarted: "fabric.platform.invocation.execution_started",
|
|
74
|
+
invocationLeaseClaimed: "fabric.platform.invocation.lease_claimed",
|
|
75
|
+
invocationCompleted: "fabric.platform.invocation.completed",
|
|
76
|
+
invocationFailed: "fabric.platform.invocation.failed",
|
|
77
|
+
invocationPolicyBlocked: "fabric.platform.invocation.policy_blocked",
|
|
78
|
+
invocationValidationFailed: "fabric.platform.invocation.validation_failed",
|
|
79
|
+
invocationApprovalWaited: "fabric.platform.invocation.approval_waited",
|
|
80
|
+
invocationReconciliationRequired: "fabric.platform.invocation.reconciliation_required",
|
|
81
|
+
outboxRelayCyclesStarted: "fabric.platform.outbox.relay_cycles.started",
|
|
82
|
+
outboxRelayCyclesCompleted: "fabric.platform.outbox.relay_cycles.completed",
|
|
83
|
+
outboxRelayCyclesFailed: "fabric.platform.outbox.relay_cycles.failed",
|
|
84
|
+
outboxLeaseClaimed: "fabric.platform.outbox.lease_claimed",
|
|
85
|
+
outboxPublished: "fabric.platform.outbox.published",
|
|
86
|
+
outboxFailed: "fabric.platform.outbox.failed",
|
|
87
|
+
outboxDeadLettered: "fabric.platform.outbox.dead_lettered",
|
|
88
|
+
invocationBacklog: "fabric.platform.invocation.backlog",
|
|
89
|
+
invocationRunning: "fabric.platform.invocation.running",
|
|
90
|
+
invocationExpiredLeases: "fabric.platform.invocation.expired_leases",
|
|
91
|
+
invocationApprovalWaits: "fabric.platform.invocation.approval_waits",
|
|
92
|
+
invocationReconciliationRequiredGauge: "fabric.platform.invocation.reconciliation_required.count",
|
|
93
|
+
outboxBacklog: "fabric.platform.outbox.backlog",
|
|
94
|
+
outboxExpiredLeases: "fabric.platform.outbox.expired_leases",
|
|
95
|
+
outboxDeadLetters: "fabric.platform.outbox.dead_letters"
|
|
96
|
+
};
|
|
97
|
+
function emitPlatformHostTelemetry(telemetry, input) {
|
|
98
|
+
if (!telemetry) return;
|
|
99
|
+
const record = input.kind === "event" ? { ...input, metricType: "counter", occurredAt: input.occurredAt ?? /* @__PURE__ */ new Date() } : { ...input, observedAt: input.observedAt ?? /* @__PURE__ */ new Date() };
|
|
100
|
+
try {
|
|
101
|
+
const result = typeof telemetry === "function" ? telemetry(record) : telemetry.record(record);
|
|
102
|
+
if (isPromiseLike(result)) void result.catch(() => void 0);
|
|
103
|
+
} catch {
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function getPlatformHostHealthSnapshot(options) {
|
|
107
|
+
const generatedAt = options.now?.() ?? /* @__PURE__ */ new Date();
|
|
108
|
+
const workerSummary = summarizeWorkers(options.workers ?? [], generatedAt);
|
|
109
|
+
const scope = {
|
|
110
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
111
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
112
|
+
};
|
|
113
|
+
try {
|
|
114
|
+
const counts = await readHealthCounts(options.store, {
|
|
115
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
116
|
+
...options.spaceId ? { spaceId: options.spaceId } : {},
|
|
117
|
+
now: generatedAt
|
|
118
|
+
});
|
|
119
|
+
const snapshot = createHealthSnapshot(
|
|
120
|
+
generatedAt,
|
|
121
|
+
scope,
|
|
122
|
+
counts,
|
|
123
|
+
workerSummary,
|
|
124
|
+
true
|
|
125
|
+
);
|
|
126
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
127
|
+
return snapshot;
|
|
128
|
+
} catch {
|
|
129
|
+
const snapshot = createHealthSnapshot(
|
|
130
|
+
generatedAt,
|
|
131
|
+
scope,
|
|
132
|
+
zeroHealthCounts(),
|
|
133
|
+
workerSummary,
|
|
134
|
+
false
|
|
135
|
+
);
|
|
136
|
+
emitHealthMetrics(options.telemetry, snapshot);
|
|
137
|
+
return snapshot;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function readHealthCounts(store, query) {
|
|
141
|
+
const candidate = store;
|
|
142
|
+
if (typeof candidate.getHealthCounts === "function") {
|
|
143
|
+
return assertHealthCounts(await candidate.getHealthCounts(query));
|
|
144
|
+
}
|
|
145
|
+
const recoverable = store;
|
|
146
|
+
const outbox = store;
|
|
147
|
+
if (typeof recoverable.listActionInvocations !== "function") {
|
|
148
|
+
throw new Error("Health counts are not supported by this store.");
|
|
149
|
+
}
|
|
150
|
+
const invocations = await recoverable.listActionInvocations({
|
|
151
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
152
|
+
...query.spaceId ? { spaceId: query.spaceId } : {},
|
|
153
|
+
limit: Number.MAX_SAFE_INTEGER
|
|
154
|
+
});
|
|
155
|
+
const outboxRecords = typeof outbox.listOutbox === "function" ? await outbox.listOutbox({
|
|
156
|
+
...query.tenantId ? { tenantId: query.tenantId } : {},
|
|
157
|
+
...query.spaceId ? { spaceId: query.spaceId } : {}
|
|
158
|
+
}) : [];
|
|
159
|
+
return countsFromRecords(invocations, outboxRecords, query.now);
|
|
160
|
+
}
|
|
161
|
+
function countsFromRecords(invocations, outbox, now) {
|
|
162
|
+
return {
|
|
163
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
164
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
165
|
+
invocationExpiredLeases: invocations.filter(
|
|
166
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
167
|
+
).length,
|
|
168
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
169
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
170
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
171
|
+
outboxExpiredLeases: outbox.filter(
|
|
172
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= now.getTime()
|
|
173
|
+
).length,
|
|
174
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function assertHealthCounts(value) {
|
|
178
|
+
for (const [name, count] of Object.entries(value)) {
|
|
179
|
+
if (!Number.isSafeInteger(count) || count < 0) {
|
|
180
|
+
throw new Error(`Invalid health count: ${name}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
return value;
|
|
184
|
+
}
|
|
185
|
+
function zeroHealthCounts() {
|
|
186
|
+
return {
|
|
187
|
+
invocationBacklog: 0,
|
|
188
|
+
invocationRunning: 0,
|
|
189
|
+
invocationExpiredLeases: 0,
|
|
190
|
+
invocationApprovalWaits: 0,
|
|
191
|
+
invocationReconciliationRequired: 0,
|
|
192
|
+
outboxBacklog: 0,
|
|
193
|
+
outboxExpiredLeases: 0,
|
|
194
|
+
outboxDeadLetters: 0
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
function summarizeWorkers(workers, now) {
|
|
198
|
+
let healthy = 0;
|
|
199
|
+
let stale = 0;
|
|
200
|
+
for (const worker of workers) {
|
|
201
|
+
if (Number.isFinite(worker.staleAfterMs) && worker.staleAfterMs >= 0 && worker.lastHeartbeatAt.getTime() + worker.staleAfterMs >= now.getTime()) healthy += 1;
|
|
202
|
+
else stale += 1;
|
|
203
|
+
}
|
|
204
|
+
return { reported: workers.length, healthy, stale };
|
|
205
|
+
}
|
|
206
|
+
function createHealthSnapshot(generatedAt, scope, counts, workers, dataAvailable) {
|
|
207
|
+
const reasonCodes = [];
|
|
208
|
+
if (!dataAvailable) reasonCodes.push("health_source_unavailable");
|
|
209
|
+
if (workers.stale > 0) reasonCodes.push("workers_stale");
|
|
210
|
+
if (counts.invocationExpiredLeases > 0) reasonCodes.push("invocation_expired_leases");
|
|
211
|
+
if (counts.outboxExpiredLeases > 0) reasonCodes.push("outbox_expired_leases");
|
|
212
|
+
if (counts.outboxDeadLetters > 0) reasonCodes.push("outbox_dead_letters");
|
|
213
|
+
if (counts.invocationReconciliationRequired > 0) reasonCodes.push("reconciliation_required");
|
|
214
|
+
const unhealthy = !dataAvailable;
|
|
215
|
+
const degraded = unhealthy || reasonCodes.length > 0;
|
|
216
|
+
return {
|
|
217
|
+
contractVersion: PLATFORM_HOST_HEALTH_CONTRACT_VERSION,
|
|
218
|
+
generatedAt,
|
|
219
|
+
scope,
|
|
220
|
+
dataAvailable,
|
|
221
|
+
status: unhealthy ? "unhealthy" : degraded ? "degraded" : "healthy",
|
|
222
|
+
readiness: {
|
|
223
|
+
ready: dataAvailable && workers.stale === 0,
|
|
224
|
+
reasonCodes
|
|
225
|
+
},
|
|
226
|
+
workers,
|
|
227
|
+
invocations: {
|
|
228
|
+
backlog: counts.invocationBacklog,
|
|
229
|
+
running: counts.invocationRunning,
|
|
230
|
+
expiredLeases: counts.invocationExpiredLeases,
|
|
231
|
+
approvalWaits: counts.invocationApprovalWaits,
|
|
232
|
+
reconciliationRequired: counts.invocationReconciliationRequired
|
|
233
|
+
},
|
|
234
|
+
outbox: {
|
|
235
|
+
backlog: counts.outboxBacklog,
|
|
236
|
+
expiredLeases: counts.outboxExpiredLeases,
|
|
237
|
+
deadLetters: counts.outboxDeadLetters
|
|
238
|
+
}
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
function emitHealthMetrics(telemetry, snapshot) {
|
|
242
|
+
const scope = {
|
|
243
|
+
...snapshot.scope.tenantId ? { tenantId: snapshot.scope.tenantId } : {},
|
|
244
|
+
...snapshot.scope.spaceId ? { spaceId: snapshot.scope.spaceId } : {}
|
|
245
|
+
};
|
|
246
|
+
const metrics = [
|
|
247
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationBacklog, snapshot.invocations.backlog],
|
|
248
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationRunning, snapshot.invocations.running],
|
|
249
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationExpiredLeases, snapshot.invocations.expiredLeases],
|
|
250
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaits, snapshot.invocations.approvalWaits],
|
|
251
|
+
[PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequiredGauge, snapshot.invocations.reconciliationRequired],
|
|
252
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxBacklog, snapshot.outbox.backlog],
|
|
253
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxExpiredLeases, snapshot.outbox.expiredLeases],
|
|
254
|
+
[PLATFORM_HOST_METRIC_NAMES.outboxDeadLetters, snapshot.outbox.deadLetters]
|
|
255
|
+
];
|
|
256
|
+
for (const [name, value] of metrics) {
|
|
257
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
258
|
+
kind: "metric",
|
|
259
|
+
name,
|
|
260
|
+
metricType: "gauge",
|
|
261
|
+
value,
|
|
262
|
+
...scope,
|
|
263
|
+
observedAt: snapshot.generatedAt
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
function isPromiseLike(value) {
|
|
268
|
+
return Boolean(
|
|
269
|
+
value && typeof value === "object" && "then" in value && typeof value.then === "function"
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
|
|
65
273
|
// src/host.ts
|
|
66
274
|
var DEFAULT_EXTRACT_EVENTS = (data) => {
|
|
67
275
|
const value = data._events;
|
|
@@ -71,6 +279,19 @@ var RecoverableFinalizationError = class extends Error {
|
|
|
71
279
|
name = "RecoverableFinalizationError";
|
|
72
280
|
};
|
|
73
281
|
function createGovernedActionHost(options) {
|
|
282
|
+
if (options.composition) {
|
|
283
|
+
if (!options.registry?.orderedModules) {
|
|
284
|
+
throw new Error("Composition-bound Platform Host requires an explicit module registry.");
|
|
285
|
+
}
|
|
286
|
+
assertAssemblyRuntimeCompatible(
|
|
287
|
+
options.composition.assembly,
|
|
288
|
+
options.registry.orderedModules.map((module) => ({
|
|
289
|
+
namespace: module.namespace,
|
|
290
|
+
version: module.version ?? "",
|
|
291
|
+
manifestDigest: module.manifestDigest ?? ""
|
|
292
|
+
}))
|
|
293
|
+
);
|
|
294
|
+
}
|
|
74
295
|
if (options.outbox) {
|
|
75
296
|
const outboxStore = asOutboxStore(options.store);
|
|
76
297
|
if (!outboxStore || !asAtomicMutationStore(options.store) || outboxStore.transactionalOutbox !== true) {
|
|
@@ -116,10 +337,13 @@ function createGovernedActionHost(options) {
|
|
|
116
337
|
input.provenance,
|
|
117
338
|
options.provenance
|
|
118
339
|
);
|
|
340
|
+
const initiatingReleaseDigest = await options.composition?.resolveInitiatingReleaseDigest?.(input);
|
|
119
341
|
const runtimeEvidence = {
|
|
120
342
|
governanceContractVersion: FABRIC_GOVERNANCE_CONTRACT_VERSION,
|
|
121
343
|
hostContractVersion: PLATFORM_HOST_CONTRACT_VERSION,
|
|
122
|
-
...options.runtimeEvidence
|
|
344
|
+
...options.runtimeEvidence,
|
|
345
|
+
...options.composition ? { assemblyDigest: options.composition.assembly.assemblyDigest } : {},
|
|
346
|
+
...initiatingReleaseDigest ? { initiatingReleaseDigest } : {}
|
|
123
347
|
};
|
|
124
348
|
assertGovernanceRuntimeEvidence(runtimeEvidence);
|
|
125
349
|
const durableInvocation = await options.store.createActionInvocation({
|
|
@@ -148,6 +372,15 @@ function createGovernedActionHost(options) {
|
|
|
148
372
|
...input.executionReason ? { executionReason: input.executionReason } : {},
|
|
149
373
|
...input.authorizationBindingId ? { authorizationBindingId: input.authorizationBindingId } : {}
|
|
150
374
|
});
|
|
375
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
376
|
+
kind: "event",
|
|
377
|
+
name: "invocation.submitted",
|
|
378
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationSubmitted,
|
|
379
|
+
occurredAt: now(),
|
|
380
|
+
tenantId: input.tenantId,
|
|
381
|
+
spaceId: input.spaceId,
|
|
382
|
+
attributes: { actionVersion: action.version }
|
|
383
|
+
});
|
|
151
384
|
if (durableInvocation.id !== actionInvocationId && input.idempotencyKey) {
|
|
152
385
|
const conflict = idempotencyConflict(durableInvocation, {
|
|
153
386
|
actorId: input.actorId,
|
|
@@ -170,6 +403,23 @@ function createGovernedActionHost(options) {
|
|
|
170
403
|
}
|
|
171
404
|
const durableWorkflowId = `action-invocation-${durableInvocation.id}`;
|
|
172
405
|
if (durableInvocation.id !== actionInvocationId) {
|
|
406
|
+
if (options.dispatcher && durableInvocation.status === "pending") {
|
|
407
|
+
const dispatched = await options.dispatcher.dispatch({
|
|
408
|
+
actionInvocationId: durableInvocation.id,
|
|
409
|
+
actionId: durableInvocation.actionId,
|
|
410
|
+
tenantId: input.tenantId,
|
|
411
|
+
spaceId: input.spaceId,
|
|
412
|
+
workflowId: durableWorkflowId
|
|
413
|
+
});
|
|
414
|
+
return withConsistency({
|
|
415
|
+
actionInvocationId: durableInvocation.id,
|
|
416
|
+
status: durableInvocation.status,
|
|
417
|
+
workflowId: dispatched.workflowId,
|
|
418
|
+
...dispatched.runId ? { runId: dispatched.runId } : {},
|
|
419
|
+
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
420
|
+
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
421
|
+
}, input.actionId);
|
|
422
|
+
}
|
|
173
423
|
return withConsistency({
|
|
174
424
|
actionInvocationId: durableInvocation.id,
|
|
175
425
|
status: durableInvocation.status,
|
|
@@ -178,13 +428,15 @@ function createGovernedActionHost(options) {
|
|
|
178
428
|
...durableInvocation.error ? { error: durableInvocation.error } : {},
|
|
179
429
|
...durableInvocation.hitlRoute ? { hitlRoute: durableInvocation.hitlRoute } : {},
|
|
180
430
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {},
|
|
181
|
-
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {}
|
|
431
|
+
...durableInvocation.authorizationReconciliation ? { reconciliation: durableInvocation.authorizationReconciliation } : {},
|
|
432
|
+
...durableInvocation.adapterReconciliation ? { adapterReconciliation: durableInvocation.adapterReconciliation } : {}
|
|
182
433
|
}, input.actionId);
|
|
183
434
|
}
|
|
184
435
|
if (options.dispatcher) {
|
|
185
436
|
try {
|
|
186
437
|
const dispatched = await options.dispatcher.dispatch({
|
|
187
438
|
actionInvocationId: durableInvocation.id,
|
|
439
|
+
actionId: durableInvocation.actionId,
|
|
188
440
|
tenantId: input.tenantId,
|
|
189
441
|
spaceId: input.spaceId,
|
|
190
442
|
workflowId: durableWorkflowId
|
|
@@ -198,13 +450,6 @@ function createGovernedActionHost(options) {
|
|
|
198
450
|
...durableInvocation.hitlRiskTier ? { hitlRiskTier: durableInvocation.hitlRiskTier } : {}
|
|
199
451
|
}, input.actionId);
|
|
200
452
|
} catch (error) {
|
|
201
|
-
const message = errorMessage(error);
|
|
202
|
-
await options.store.updateActionInvocation(
|
|
203
|
-
durableInvocation.id,
|
|
204
|
-
input.tenantId,
|
|
205
|
-
input.spaceId,
|
|
206
|
-
{ status: "failed", error: message }
|
|
207
|
-
);
|
|
208
453
|
throw error;
|
|
209
454
|
}
|
|
210
455
|
}
|
|
@@ -229,16 +474,30 @@ function createGovernedActionHost(options) {
|
|
|
229
474
|
if (isTerminal(invocation.status) || invocation.status === "waiting_for_approval") {
|
|
230
475
|
return withConsistency(actionResult(invocation), invocation.actionId);
|
|
231
476
|
}
|
|
232
|
-
if (invocation.status === "running" && invocation.leaseOwner && executionOptions.leaseOwner !== invocation.leaseOwner) {
|
|
477
|
+
if (invocation.status === "running" && invocation.leaseOwner && (executionOptions.leaseOwner !== invocation.leaseOwner || (invocation.leaseToken ?? 0) > 0 && executionOptions.leaseToken !== invocation.leaseToken)) {
|
|
233
478
|
return {
|
|
234
479
|
...actionResult(invocation),
|
|
235
|
-
error: `Invocation is
|
|
480
|
+
error: `Invocation lease is not owned by the supplied worker generation`
|
|
236
481
|
};
|
|
237
482
|
}
|
|
238
483
|
const action = actionResolver(invocation.actionId);
|
|
239
484
|
if (!action) {
|
|
240
485
|
return fail(invocation, "failed", `Unknown action: ${invocation.actionId}`);
|
|
241
486
|
}
|
|
487
|
+
if (action.kind === "saga") {
|
|
488
|
+
return fail(
|
|
489
|
+
invocation,
|
|
490
|
+
"failed",
|
|
491
|
+
`Saga action ${invocation.actionId} requires a durable PlatformActionDispatcher and cannot execute as an atomic Host action.`
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
if (resumingRunningInvocation && invocation.attemptCount === 0 && !invocation.leaseOwner && !action.idempotent) {
|
|
495
|
+
return fail(
|
|
496
|
+
invocation,
|
|
497
|
+
"failed",
|
|
498
|
+
`Interrupted inline action ${invocation.actionId} is not declared idempotent; manual reconciliation is required.`
|
|
499
|
+
);
|
|
500
|
+
}
|
|
242
501
|
if (invocation.attemptCount > 1 && !action.idempotent) {
|
|
243
502
|
return fail(
|
|
244
503
|
invocation,
|
|
@@ -251,6 +510,15 @@ function createGovernedActionHost(options) {
|
|
|
251
510
|
status: "running"
|
|
252
511
|
});
|
|
253
512
|
}
|
|
513
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
514
|
+
kind: "event",
|
|
515
|
+
name: "invocation.execution_started",
|
|
516
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationExecutionStarted,
|
|
517
|
+
occurredAt: now(),
|
|
518
|
+
tenantId,
|
|
519
|
+
spaceId,
|
|
520
|
+
attributes: { actionVersion: invocation.actionVersion }
|
|
521
|
+
});
|
|
254
522
|
try {
|
|
255
523
|
const parsed = action.schema.safeParse(invocation.parameters);
|
|
256
524
|
if (!parsed.success) {
|
|
@@ -316,7 +584,7 @@ function createGovernedActionHost(options) {
|
|
|
316
584
|
);
|
|
317
585
|
}
|
|
318
586
|
if ((invocation.hitlRoute === "needs-approval" || invocation.hitlRoute === "escalate") && !invocation.approvalDecision?.approved) {
|
|
319
|
-
await
|
|
587
|
+
await persistInvocation(invocation, {
|
|
320
588
|
status: "waiting_for_approval"
|
|
321
589
|
});
|
|
322
590
|
return {
|
|
@@ -343,7 +611,7 @@ function createGovernedActionHost(options) {
|
|
|
343
611
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
344
612
|
message: `Action ${action.actionId} requires durable capture-time authorization evidence`
|
|
345
613
|
};
|
|
346
|
-
await
|
|
614
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
347
615
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
348
616
|
}
|
|
349
617
|
const bindingExpired = authorityMoment !== "capture" && invocation.authorizationBinding?.expiresAt !== void 0 && Date.parse(invocation.authorizationBinding.expiresAt) <= now().getTime();
|
|
@@ -356,7 +624,7 @@ function createGovernedActionHost(options) {
|
|
|
356
624
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
357
625
|
message: `Authorization binding for action ${action.actionId} expired before execution`
|
|
358
626
|
};
|
|
359
|
-
await
|
|
627
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
360
628
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
361
629
|
}
|
|
362
630
|
const executionAuthorized = authorityMoment === "capture" ? invocation.authorizationBinding !== void 0 : options.authorization.authorizeExecution ? await options.authorization.authorizeExecution({
|
|
@@ -378,7 +646,7 @@ function createGovernedActionHost(options) {
|
|
|
378
646
|
...invocation.provenance ? { provenance: invocation.provenance } : {},
|
|
379
647
|
message: `Actor ${invocation.actorId} is not authorized to execute action ${action.actionId}`
|
|
380
648
|
};
|
|
381
|
-
await
|
|
649
|
+
await persistInvocation(invocation, { status: "reconciliation_required", error: reconciliation.message, authorizationReconciliation: reconciliation });
|
|
382
650
|
return withConsistency({ actionInvocationId, status: "reconciliation_required", error: reconciliation.message, reconciliation }, action.actionId);
|
|
383
651
|
}
|
|
384
652
|
const definitions = options.resolvePolicies ? await options.resolvePolicies({
|
|
@@ -440,30 +708,6 @@ function createGovernedActionHost(options) {
|
|
|
440
708
|
})));
|
|
441
709
|
}
|
|
442
710
|
}
|
|
443
|
-
const binding = action.stateMachine;
|
|
444
|
-
if (binding) {
|
|
445
|
-
const entityId = binding.getEntityId(parsed.data);
|
|
446
|
-
const currentState = entityId ? await options.store.getEntityState(
|
|
447
|
-
tenantId,
|
|
448
|
-
spaceId,
|
|
449
|
-
binding.entityType,
|
|
450
|
-
entityId
|
|
451
|
-
) ?? initialState(stateMachineResolver(binding.entityType)) : initialState(stateMachineResolver(binding.entityType));
|
|
452
|
-
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
453
|
-
if (targetState !== "") {
|
|
454
|
-
const transition = validateStateMachineTransition(
|
|
455
|
-
stateMachineResolver(binding.entityType),
|
|
456
|
-
binding.entityType,
|
|
457
|
-
currentState,
|
|
458
|
-
targetState,
|
|
459
|
-
action.actionId
|
|
460
|
-
);
|
|
461
|
-
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
462
|
-
if (!transition.valid && !replayingAppliedTransition) {
|
|
463
|
-
return fail(invocation, "failed", transition.error ?? "Invalid state transition");
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
711
|
let data;
|
|
468
712
|
let domainEvents = [];
|
|
469
713
|
try {
|
|
@@ -471,6 +715,42 @@ function createGovernedActionHost(options) {
|
|
|
471
715
|
if (options.outbox && !transaction?.appendEventWithOutbox) {
|
|
472
716
|
throw new Error("Outbox egress requires transactionWithEvents to provide appendEventWithOutbox before handler execution.");
|
|
473
717
|
}
|
|
718
|
+
const binding = action.stateMachine;
|
|
719
|
+
if (binding) {
|
|
720
|
+
const machine = stateMachineResolver(binding.entityType);
|
|
721
|
+
const entityId = binding.getEntityId(parsed.data);
|
|
722
|
+
const getEntityState = transaction?.getEntityState ? transaction.getEntityState.bind(transaction) : options.store.getEntityState.bind(options.store);
|
|
723
|
+
const currentState = entityId ? await getEntityState(
|
|
724
|
+
tenantId,
|
|
725
|
+
spaceId,
|
|
726
|
+
binding.entityType,
|
|
727
|
+
entityId
|
|
728
|
+
) ?? initialState(machine) : initialState(machine);
|
|
729
|
+
const targetState = typeof binding.targetState === "function" ? binding.targetState(parsed.data) : binding.targetState;
|
|
730
|
+
if (targetState !== "") {
|
|
731
|
+
const transition = await evaluateStateMachineTransition(
|
|
732
|
+
machine,
|
|
733
|
+
binding.entityType,
|
|
734
|
+
currentState,
|
|
735
|
+
targetState,
|
|
736
|
+
action.actionId,
|
|
737
|
+
{
|
|
738
|
+
entity: {
|
|
739
|
+
...entityId ? { id: entityId } : {},
|
|
740
|
+
state: currentState
|
|
741
|
+
},
|
|
742
|
+
actionInvocationId,
|
|
743
|
+
actorId: invocation.actorId,
|
|
744
|
+
parameters: parsed.data,
|
|
745
|
+
db
|
|
746
|
+
}
|
|
747
|
+
);
|
|
748
|
+
const replayingAppliedTransition = action.idempotent && currentState === targetState;
|
|
749
|
+
if (!transition.valid && !replayingAppliedTransition) {
|
|
750
|
+
throw new Error(transition.error ?? "Invalid state transition");
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
}
|
|
474
754
|
const handlerResult = action.handler ? await action.handler(
|
|
475
755
|
{
|
|
476
756
|
actionInvocationId,
|
|
@@ -570,7 +850,21 @@ function createGovernedActionHost(options) {
|
|
|
570
850
|
subjectId: adapterEventSubject.subjectId,
|
|
571
851
|
payload: { adapterType: step.adapterType, operation: step.operation }
|
|
572
852
|
}, `adapter:${stepIndex}:started`);
|
|
853
|
+
let adapterDeadlineTimer;
|
|
573
854
|
try {
|
|
855
|
+
const adapterController = options.adapterDeadlineMs ? new AbortController() : void 0;
|
|
856
|
+
const deadlineMs = options.adapterDeadlineMs ? now().getTime() + options.adapterDeadlineMs : void 0;
|
|
857
|
+
if (adapterController && deadlineMs) {
|
|
858
|
+
const remaining = deadlineMs - now().getTime();
|
|
859
|
+
if (remaining <= 0) {
|
|
860
|
+
throw new Error(`Adapter deadline already expired before execution: ${step.adapterType}:${step.operation}`);
|
|
861
|
+
}
|
|
862
|
+
adapterDeadlineTimer = setTimeout(
|
|
863
|
+
() => adapterController.abort(new Error("Adapter deadline exceeded")),
|
|
864
|
+
remaining
|
|
865
|
+
);
|
|
866
|
+
adapterDeadlineTimer.unref?.();
|
|
867
|
+
}
|
|
574
868
|
const result2 = await executeWithAdapterRetry({
|
|
575
869
|
policy: step.retryPolicy ?? adapter.retryPolicy,
|
|
576
870
|
defaultIdempotent: adapter.idempotent,
|
|
@@ -587,13 +881,77 @@ function createGovernedActionHost(options) {
|
|
|
587
881
|
correlationId: invocation.correlationId,
|
|
588
882
|
...invocation.causationId ? { causationId: invocation.causationId } : {},
|
|
589
883
|
attempt,
|
|
590
|
-
maxAttempts
|
|
884
|
+
maxAttempts,
|
|
885
|
+
...deadlineMs ? { deadlineMs } : {},
|
|
886
|
+
...adapterController ? { signal: adapterController.signal } : {}
|
|
591
887
|
});
|
|
592
888
|
},
|
|
593
889
|
isSuccessful: (result3) => result3.success,
|
|
594
|
-
getError: (result3) => result3.error
|
|
890
|
+
getError: (result3) => result3.error,
|
|
891
|
+
classifyOutcome: (result3) => result3.outcome ?? "transient_failure",
|
|
892
|
+
classifyThrownError: (error) => {
|
|
893
|
+
if (adapterController?.signal.aborted) return "timeout";
|
|
894
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
895
|
+
if (/timeout|deadline|timed?\s*out/i.test(message)) return "timeout";
|
|
896
|
+
return "transient_failure";
|
|
897
|
+
}
|
|
595
898
|
});
|
|
899
|
+
assertAdapterOutcomeConsistency(result2);
|
|
596
900
|
if (!result2.success) {
|
|
901
|
+
const outcome = result2.outcome ?? "transient_failure";
|
|
902
|
+
if (outcome === "ambiguous") {
|
|
903
|
+
const evidence = result2.outcomeEvidence;
|
|
904
|
+
const message = result2.error ?? "Adapter outcome is ambiguous; external effect may or may not have been applied";
|
|
905
|
+
const adapterReconciliation = {
|
|
906
|
+
kind: "adapter_outcome_ambiguous",
|
|
907
|
+
adapterType: step.adapterType,
|
|
908
|
+
operation: step.operation,
|
|
909
|
+
vendor: adapter.vendor,
|
|
910
|
+
adapterInvocationId,
|
|
911
|
+
...evidence?.reason ?? result2.error ? { reason: evidence?.reason ?? result2.error } : {},
|
|
912
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {},
|
|
913
|
+
message
|
|
914
|
+
};
|
|
915
|
+
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
916
|
+
status: "ambiguous",
|
|
917
|
+
error: message,
|
|
918
|
+
updatedAt: now()
|
|
919
|
+
});
|
|
920
|
+
await appendEvent(invocation, {
|
|
921
|
+
eventType: "AdapterInvocationAmbiguous",
|
|
922
|
+
subjectType: adapterEventSubject.subjectType,
|
|
923
|
+
subjectId: adapterEventSubject.subjectId,
|
|
924
|
+
payload: {
|
|
925
|
+
adapterType: step.adapterType,
|
|
926
|
+
operation: step.operation,
|
|
927
|
+
...evidence?.externalReference ? { externalReference: evidence.externalReference } : {}
|
|
928
|
+
}
|
|
929
|
+
}, `adapter:${stepIndex}:ambiguous`);
|
|
930
|
+
const governanceStore2 = asGovernanceStore(options.store);
|
|
931
|
+
if (governanceStore2 && evidence) {
|
|
932
|
+
await governanceStore2.appendExternalReconciliation({
|
|
933
|
+
id: lifecycleId("rec", actionInvocationId, `${adapter.vendor}:${adapterInvocationId}`),
|
|
934
|
+
actionInvocationId,
|
|
935
|
+
tenantId,
|
|
936
|
+
spaceId,
|
|
937
|
+
status: "pending",
|
|
938
|
+
provider: adapter.vendor,
|
|
939
|
+
...evidence.externalReference ? { externalOperationId: evidence.externalReference } : {},
|
|
940
|
+
attempt: 1,
|
|
941
|
+
reason: message,
|
|
942
|
+
observedAt: now()
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
await persistInvocation(invocation, {
|
|
946
|
+
status: "reconciliation_required",
|
|
947
|
+
error: message,
|
|
948
|
+
adapterReconciliation
|
|
949
|
+
});
|
|
950
|
+
return withConsistency(
|
|
951
|
+
{ actionInvocationId, status: "reconciliation_required", error: message, adapterReconciliation },
|
|
952
|
+
action.actionId
|
|
953
|
+
);
|
|
954
|
+
}
|
|
597
955
|
await options.store.updateAdapterInvocation(adapterInvocationId, {
|
|
598
956
|
status: "failed",
|
|
599
957
|
error: result2.error ?? "Adapter failed",
|
|
@@ -653,6 +1011,8 @@ function createGovernedActionHost(options) {
|
|
|
653
1011
|
updatedAt: now()
|
|
654
1012
|
});
|
|
655
1013
|
return fail(invocation, "failed", message);
|
|
1014
|
+
} finally {
|
|
1015
|
+
if (adapterDeadlineTimer) clearTimeout(adapterDeadlineTimer);
|
|
656
1016
|
}
|
|
657
1017
|
}
|
|
658
1018
|
const governanceStore = asGovernanceStore(options.store);
|
|
@@ -677,20 +1037,41 @@ function createGovernedActionHost(options) {
|
|
|
677
1037
|
transaction
|
|
678
1038
|
);
|
|
679
1039
|
}
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
1040
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1041
|
+
if (!transaction.updateLeasedActionInvocation) {
|
|
1042
|
+
throw new Error("Leased atomic finalization requires transaction-scoped fencing support.");
|
|
1043
|
+
}
|
|
1044
|
+
const updated = await transaction.updateLeasedActionInvocation({
|
|
1045
|
+
id: actionInvocationId,
|
|
1046
|
+
tenantId,
|
|
1047
|
+
spaceId,
|
|
1048
|
+
workerId: invocation.leaseOwner,
|
|
1049
|
+
leaseToken: invocation.leaseToken,
|
|
1050
|
+
patch: { status: "completed", result }
|
|
1051
|
+
});
|
|
1052
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${actionInvocationId}`);
|
|
1053
|
+
} else {
|
|
1054
|
+
await transaction.updateActionInvocation(
|
|
1055
|
+
actionInvocationId,
|
|
1056
|
+
tenantId,
|
|
1057
|
+
spaceId,
|
|
1058
|
+
{ status: "completed", result }
|
|
1059
|
+
);
|
|
1060
|
+
}
|
|
686
1061
|
});
|
|
1062
|
+
emitInvocationStatusTelemetry(
|
|
1063
|
+
invocation,
|
|
1064
|
+
"completed",
|
|
1065
|
+
options.telemetry,
|
|
1066
|
+
now()
|
|
1067
|
+
);
|
|
687
1068
|
} else {
|
|
688
1069
|
if (action.eventPhase === "after_adapters") {
|
|
689
1070
|
for (const [index, event] of domainEvents.entries()) {
|
|
690
1071
|
await appendEvent(invocation, event, `domain:${index}`, action.version);
|
|
691
1072
|
}
|
|
692
1073
|
}
|
|
693
|
-
await
|
|
1074
|
+
await persistInvocation(invocation, {
|
|
694
1075
|
status: "completed",
|
|
695
1076
|
result
|
|
696
1077
|
});
|
|
@@ -773,12 +1154,21 @@ function createGovernedActionHost(options) {
|
|
|
773
1154
|
if (!transitioned) throw new Error(`ActionInvocation not found: ${actionInvocationId}`);
|
|
774
1155
|
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
775
1156
|
}
|
|
776
|
-
if (!decision.approved)
|
|
1157
|
+
if (!decision.approved) {
|
|
1158
|
+
emitInvocationStatusTelemetry(
|
|
1159
|
+
transitioned,
|
|
1160
|
+
"failed",
|
|
1161
|
+
options.telemetry,
|
|
1162
|
+
now(),
|
|
1163
|
+
"waiting_for_approval"
|
|
1164
|
+
);
|
|
1165
|
+
return withConsistency(actionResult(transitioned), transitioned.actionId);
|
|
1166
|
+
}
|
|
777
1167
|
return executeInvocation(
|
|
778
1168
|
actionInvocationId,
|
|
779
1169
|
tenantId,
|
|
780
1170
|
spaceId,
|
|
781
|
-
{ leaseOwner },
|
|
1171
|
+
{ leaseOwner, leaseToken: transitioned.leaseToken },
|
|
782
1172
|
"approval_resume"
|
|
783
1173
|
);
|
|
784
1174
|
}
|
|
@@ -874,12 +1264,7 @@ function createGovernedActionHost(options) {
|
|
|
874
1264
|
}
|
|
875
1265
|
}
|
|
876
1266
|
async function fail(invocation, status, error) {
|
|
877
|
-
await
|
|
878
|
-
invocation.id,
|
|
879
|
-
invocation.tenantId,
|
|
880
|
-
invocation.spaceId,
|
|
881
|
-
{ status, error }
|
|
882
|
-
);
|
|
1267
|
+
await persistInvocation(invocation, { status, error });
|
|
883
1268
|
return {
|
|
884
1269
|
actionInvocationId: invocation.id,
|
|
885
1270
|
status,
|
|
@@ -888,6 +1273,33 @@ function createGovernedActionHost(options) {
|
|
|
888
1273
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {}
|
|
889
1274
|
};
|
|
890
1275
|
}
|
|
1276
|
+
async function persistInvocation(invocation, patch) {
|
|
1277
|
+
const previousStatus = invocation.status;
|
|
1278
|
+
if (invocation.leaseOwner && invocation.leaseToken !== void 0) {
|
|
1279
|
+
const recoverable = options.store;
|
|
1280
|
+
if (!recoverable.updateLeasedActionInvocation) {
|
|
1281
|
+
throw new Error("Leased execution requires a fencing-capable PlatformHostStore.");
|
|
1282
|
+
}
|
|
1283
|
+
const updated = await recoverable.updateLeasedActionInvocation({
|
|
1284
|
+
id: invocation.id,
|
|
1285
|
+
tenantId: invocation.tenantId,
|
|
1286
|
+
spaceId: invocation.spaceId,
|
|
1287
|
+
workerId: invocation.leaseOwner,
|
|
1288
|
+
leaseToken: invocation.leaseToken,
|
|
1289
|
+
patch
|
|
1290
|
+
});
|
|
1291
|
+
if (!updated) throw new RecoverableFinalizationError(`Invocation lease lost: ${invocation.id}`);
|
|
1292
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1293
|
+
return;
|
|
1294
|
+
}
|
|
1295
|
+
await options.store.updateActionInvocation(
|
|
1296
|
+
invocation.id,
|
|
1297
|
+
invocation.tenantId,
|
|
1298
|
+
invocation.spaceId,
|
|
1299
|
+
patch
|
|
1300
|
+
);
|
|
1301
|
+
emitInvocationStatusTelemetry(invocation, patch.status, options.telemetry, now(), previousStatus);
|
|
1302
|
+
}
|
|
891
1303
|
return { submitAction, executeInvocation, resumeApprovedInvocation, recordExecutionAttestation, recordExternalReconciliation };
|
|
892
1304
|
}
|
|
893
1305
|
function actionResult(invocation) {
|
|
@@ -898,7 +1310,8 @@ function actionResult(invocation) {
|
|
|
898
1310
|
...invocation.error ? { error: invocation.error } : {},
|
|
899
1311
|
...invocation.hitlRoute ? { hitlRoute: invocation.hitlRoute } : {},
|
|
900
1312
|
...invocation.hitlRiskTier ? { hitlRiskTier: invocation.hitlRiskTier } : {},
|
|
901
|
-
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {}
|
|
1313
|
+
...invocation.authorizationReconciliation ? { reconciliation: invocation.authorizationReconciliation } : {},
|
|
1314
|
+
...invocation.adapterReconciliation ? { adapterReconciliation: invocation.adapterReconciliation } : {}
|
|
902
1315
|
};
|
|
903
1316
|
}
|
|
904
1317
|
function asApprovalStore(store) {
|
|
@@ -952,6 +1365,24 @@ function initialState(machine) {
|
|
|
952
1365
|
function isTerminal(status) {
|
|
953
1366
|
return ["completed", "failed", "blocked_by_policy", "reconciliation_required", "validation_failed"].includes(status);
|
|
954
1367
|
}
|
|
1368
|
+
function emitInvocationStatusTelemetry(invocation, status, telemetry, occurredAt, previousStatus) {
|
|
1369
|
+
if (!status) return;
|
|
1370
|
+
const event = status === "completed" ? { name: "invocation.completed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationCompleted } : status === "blocked_by_policy" ? { name: "invocation.policy_blocked", metricName: PLATFORM_HOST_METRIC_NAMES.invocationPolicyBlocked } : status === "validation_failed" ? { name: "invocation.validation_failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationValidationFailed } : status === "waiting_for_approval" ? { name: "invocation.approval_waited", metricName: PLATFORM_HOST_METRIC_NAMES.invocationApprovalWaited } : status === "reconciliation_required" ? { name: "invocation.reconciliation_required", metricName: PLATFORM_HOST_METRIC_NAMES.invocationReconciliationRequired } : status === "failed" ? { name: "invocation.failed", metricName: PLATFORM_HOST_METRIC_NAMES.invocationFailed } : void 0;
|
|
1371
|
+
if (!event) return;
|
|
1372
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1373
|
+
kind: "event",
|
|
1374
|
+
name: event.name,
|
|
1375
|
+
metricName: event.metricName,
|
|
1376
|
+
occurredAt,
|
|
1377
|
+
tenantId: invocation.tenantId,
|
|
1378
|
+
spaceId: invocation.spaceId,
|
|
1379
|
+
attributes: {
|
|
1380
|
+
...invocation.actionId ? { actionId: invocation.actionId } : {},
|
|
1381
|
+
...invocation.actionVersion !== void 0 ? { actionVersion: invocation.actionVersion } : {},
|
|
1382
|
+
...previousStatus ? { fromStatus: previousStatus } : {}
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
955
1386
|
function withoutPrivateHostFields(data, eventResultFields) {
|
|
956
1387
|
return Object.fromEntries(
|
|
957
1388
|
Object.entries(data).filter(([key]) => !eventResultFields.includes(key))
|
|
@@ -1020,29 +1451,72 @@ function toEnterpriseEventEnvelope(event, metadata) {
|
|
|
1020
1451
|
async function runOutboxRelayCycle(options) {
|
|
1021
1452
|
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
1022
1453
|
const maxAttempts = options.maxAttempts ?? 10;
|
|
1023
|
-
const
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1454
|
+
const leaseDurationMs = options.leaseDurationMs ?? 3e4;
|
|
1455
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.started", now(), options.workerId);
|
|
1456
|
+
let records;
|
|
1457
|
+
try {
|
|
1458
|
+
records = await options.store.claimOutbox({
|
|
1459
|
+
workerId: options.workerId,
|
|
1460
|
+
leaseDurationMs,
|
|
1461
|
+
limit: options.batchSize ?? 100,
|
|
1462
|
+
now: now()
|
|
1463
|
+
});
|
|
1464
|
+
} catch (error) {
|
|
1465
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.failed", now(), options.workerId);
|
|
1466
|
+
throw error;
|
|
1467
|
+
}
|
|
1468
|
+
for (const record of records) {
|
|
1469
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1470
|
+
kind: "event",
|
|
1471
|
+
name: "outbox.lease_claimed",
|
|
1472
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxLeaseClaimed,
|
|
1473
|
+
occurredAt: now(),
|
|
1474
|
+
tenantId: record.tenantId,
|
|
1475
|
+
spaceId: record.spaceId,
|
|
1476
|
+
attributes: { attempt: record.attemptCount }
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1029
1479
|
const result = { claimed: records.length, published: 0, failed: 0, deadLettered: 0 };
|
|
1030
1480
|
for (const record of records) {
|
|
1481
|
+
const stopHeartbeat = startOutboxHeartbeat(options, record, leaseDurationMs);
|
|
1031
1482
|
try {
|
|
1032
|
-
await
|
|
1483
|
+
await publishWithTimeout(
|
|
1484
|
+
options.publisher,
|
|
1485
|
+
record.event,
|
|
1486
|
+
options.publishTimeoutMs ?? 1e4
|
|
1487
|
+
);
|
|
1488
|
+
await stopHeartbeat();
|
|
1033
1489
|
try {
|
|
1034
|
-
await options.store.markOutboxPublished(record.id, options.workerId, now());
|
|
1490
|
+
await options.store.markOutboxPublished(record.id, options.workerId, now(), record.leaseToken);
|
|
1035
1491
|
} catch {
|
|
1036
1492
|
result.failed += 1;
|
|
1493
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1494
|
+
kind: "event",
|
|
1495
|
+
name: "outbox.failed",
|
|
1496
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1497
|
+
occurredAt: now(),
|
|
1498
|
+
tenantId: record.tenantId,
|
|
1499
|
+
spaceId: record.spaceId
|
|
1500
|
+
});
|
|
1037
1501
|
continue;
|
|
1038
1502
|
}
|
|
1039
1503
|
result.published += 1;
|
|
1504
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1505
|
+
kind: "event",
|
|
1506
|
+
name: "outbox.published",
|
|
1507
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.outboxPublished,
|
|
1508
|
+
occurredAt: now(),
|
|
1509
|
+
tenantId: record.tenantId,
|
|
1510
|
+
spaceId: record.spaceId
|
|
1511
|
+
});
|
|
1040
1512
|
} catch {
|
|
1513
|
+
await stopHeartbeat();
|
|
1041
1514
|
const deadLetter = record.attemptCount >= maxAttempts;
|
|
1042
1515
|
try {
|
|
1043
1516
|
await options.store.markOutboxFailed({
|
|
1044
1517
|
id: record.id,
|
|
1045
1518
|
workerId: options.workerId,
|
|
1519
|
+
leaseToken: record.leaseToken,
|
|
1046
1520
|
error: "Event publisher failed",
|
|
1047
1521
|
availableAt: new Date(now().getTime() + (options.retryDelayMs?.(record.attemptCount) ?? 1e3)),
|
|
1048
1522
|
deadLetter
|
|
@@ -1051,16 +1525,84 @@ async function runOutboxRelayCycle(options) {
|
|
|
1051
1525
|
}
|
|
1052
1526
|
result.failed += 1;
|
|
1053
1527
|
if (deadLetter) result.deadLettered += 1;
|
|
1528
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
1529
|
+
kind: "event",
|
|
1530
|
+
name: deadLetter ? "outbox.dead_lettered" : "outbox.failed",
|
|
1531
|
+
metricName: deadLetter ? PLATFORM_HOST_METRIC_NAMES.outboxDeadLettered : PLATFORM_HOST_METRIC_NAMES.outboxFailed,
|
|
1532
|
+
occurredAt: now(),
|
|
1533
|
+
tenantId: record.tenantId,
|
|
1534
|
+
spaceId: record.spaceId,
|
|
1535
|
+
attributes: { attempt: record.attemptCount }
|
|
1536
|
+
});
|
|
1054
1537
|
}
|
|
1055
1538
|
}
|
|
1539
|
+
emitRelayTelemetry(options.telemetry, "outbox.relay.completed", now(), options.workerId, {
|
|
1540
|
+
claimed: result.claimed,
|
|
1541
|
+
published: result.published,
|
|
1542
|
+
failed: result.failed,
|
|
1543
|
+
deadLettered: result.deadLettered
|
|
1544
|
+
});
|
|
1056
1545
|
return result;
|
|
1057
1546
|
}
|
|
1547
|
+
async function publishWithTimeout(publisher, event, timeoutMs) {
|
|
1548
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
1549
|
+
throw new Error("publishTimeoutMs must be a positive finite number");
|
|
1550
|
+
}
|
|
1551
|
+
const controller = new AbortController();
|
|
1552
|
+
let timer;
|
|
1553
|
+
const timeout = new Promise((_, reject) => {
|
|
1554
|
+
timer = setTimeout(() => {
|
|
1555
|
+
const error = new Error(`Event publisher timed out after ${timeoutMs}ms`);
|
|
1556
|
+
controller.abort(error);
|
|
1557
|
+
reject(error);
|
|
1558
|
+
}, timeoutMs);
|
|
1559
|
+
});
|
|
1560
|
+
try {
|
|
1561
|
+
await Promise.race([publisher.publish(event, { signal: controller.signal }), timeout]);
|
|
1562
|
+
} finally {
|
|
1563
|
+
if (timer) clearTimeout(timer);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
function startOutboxHeartbeat(options, record, leaseDurationMs) {
|
|
1567
|
+
if (!record.leaseToken) return async () => void 0;
|
|
1568
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
1569
|
+
let stopped = false;
|
|
1570
|
+
let timer;
|
|
1571
|
+
let running;
|
|
1572
|
+
const schedule = () => {
|
|
1573
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
1574
|
+
};
|
|
1575
|
+
const tick = () => {
|
|
1576
|
+
running = options.store.renewOutboxLease({
|
|
1577
|
+
id: record.id,
|
|
1578
|
+
workerId: options.workerId,
|
|
1579
|
+
leaseToken: record.leaseToken,
|
|
1580
|
+
leaseDurationMs
|
|
1581
|
+
}).then(() => void 0).catch(() => void 0).finally(schedule);
|
|
1582
|
+
};
|
|
1583
|
+
schedule();
|
|
1584
|
+
return async () => {
|
|
1585
|
+
stopped = true;
|
|
1586
|
+
if (timer) clearTimeout(timer);
|
|
1587
|
+
await running;
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1058
1590
|
function cloneOutboxRecord(record) {
|
|
1059
1591
|
return {
|
|
1060
1592
|
...record,
|
|
1061
1593
|
event: { ...record.event, ...record.event.traceContext ? { traceContext: { ...record.event.traceContext } } : {} }
|
|
1062
1594
|
};
|
|
1063
1595
|
}
|
|
1596
|
+
function emitRelayTelemetry(telemetry, name, occurredAt, workerId, attributes) {
|
|
1597
|
+
const metricName = name === "outbox.relay.started" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesStarted : name === "outbox.relay.completed" ? PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.outboxRelayCyclesFailed;
|
|
1598
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
1599
|
+
kind: "event",
|
|
1600
|
+
name,
|
|
1601
|
+
metricName,
|
|
1602
|
+
occurredAt,
|
|
1603
|
+
attributes: { workerId, ...attributes }
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1064
1606
|
|
|
1065
1607
|
// src/memory-store.ts
|
|
1066
1608
|
var MemoryPlatformHostStore = class {
|
|
@@ -1103,12 +1645,33 @@ var MemoryPlatformHostStore = class {
|
|
|
1103
1645
|
};
|
|
1104
1646
|
const result = await run({
|
|
1105
1647
|
db: this.db,
|
|
1648
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
1649
|
+
const record = await this.getActionInvocation(id, tenantId, spaceId);
|
|
1650
|
+
return record ? structuredClone(record) : void 0;
|
|
1651
|
+
},
|
|
1652
|
+
getEvent: async (id, tenantId, spaceId) => pendingEvents.find((candidate) => candidate.event.id === id && candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId)?.event ?? this.events.find((event) => event.id === id && event.tenantId === tenantId && event.spaceId === spaceId),
|
|
1106
1653
|
appendEvent: (event) => appendPending(event),
|
|
1107
1654
|
appendEventWithOutbox: (event, metadata) => appendPending(event, metadata),
|
|
1108
1655
|
nextEventSequence: async (tenantId, spaceId) => (await this.listEvents(tenantId, spaceId)).length + pendingEvents.filter((candidate) => candidate.event.tenantId === tenantId && candidate.event.spaceId === spaceId).length + 1,
|
|
1109
1656
|
listEvents: async (tenantId, spaceId) => [...await this.listEvents(tenantId, spaceId), ...pendingEvents.map((candidate) => candidate.event).filter((event) => event.tenantId === tenantId && event.spaceId === spaceId)],
|
|
1657
|
+
getEntityState: async (tenantId, spaceId, entityType, entityId) => {
|
|
1658
|
+
let state = await this.getEntityState(tenantId, spaceId, entityType, entityId);
|
|
1659
|
+
for (const pending of pendingEvents) {
|
|
1660
|
+
const event = pending.event;
|
|
1661
|
+
if (event.tenantId !== tenantId || event.spaceId !== spaceId || event.subjectType !== entityType || event.subjectId !== entityId) continue;
|
|
1662
|
+
const candidate = event.payload?.toState;
|
|
1663
|
+
if (typeof candidate === "string") state = candidate;
|
|
1664
|
+
}
|
|
1665
|
+
return state;
|
|
1666
|
+
},
|
|
1110
1667
|
updateActionInvocation: async (id, tenantId, spaceId, patch) => {
|
|
1111
1668
|
pendingUpdates.push({ id, tenantId, spaceId, patch });
|
|
1669
|
+
},
|
|
1670
|
+
updateLeasedActionInvocation: async (input) => {
|
|
1671
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1672
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1673
|
+
pendingUpdates.push({ id: input.id, tenantId: input.tenantId, spaceId: input.spaceId, patch: input.patch });
|
|
1674
|
+
return true;
|
|
1112
1675
|
}
|
|
1113
1676
|
});
|
|
1114
1677
|
for (const update of pendingUpdates) {
|
|
@@ -1135,7 +1698,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1135
1698
|
if (existing) return existing;
|
|
1136
1699
|
}
|
|
1137
1700
|
const now = /* @__PURE__ */ new Date();
|
|
1138
|
-
const record = { ...input, attemptCount: 0, createdAt: now, updatedAt: now };
|
|
1701
|
+
const record = { ...input, attemptCount: 0, leaseToken: 0, createdAt: now, updatedAt: now };
|
|
1139
1702
|
this.invocations.push(record);
|
|
1140
1703
|
return record;
|
|
1141
1704
|
}
|
|
@@ -1185,6 +1748,7 @@ var MemoryPlatformHostStore = class {
|
|
|
1185
1748
|
record.error = void 0;
|
|
1186
1749
|
record.leaseOwner = input.leaseOwner;
|
|
1187
1750
|
record.leaseExpiresAt = new Date(input.now.getTime() + input.leaseDurationMs);
|
|
1751
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1188
1752
|
record.attemptCount = Math.max(record.attemptCount, 1);
|
|
1189
1753
|
return { applied: true, invocation: record };
|
|
1190
1754
|
}
|
|
@@ -1262,31 +1826,41 @@ var MemoryPlatformHostStore = class {
|
|
|
1262
1826
|
return this.outbox.filter((record) => record.status === "pending" && record.availableAt <= current && (!record.leaseExpiresAt || record.leaseExpiresAt <= current) && (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime() || left.id.localeCompare(right.id)).slice(0, Math.max(1, Math.min(input.limit ?? 100, 1e3))).map((record) => {
|
|
1263
1827
|
record.leaseOwner = input.workerId;
|
|
1264
1828
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1829
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1265
1830
|
record.attemptCount += 1;
|
|
1266
1831
|
return cloneOutboxRecord(record);
|
|
1267
1832
|
});
|
|
1268
1833
|
}
|
|
1269
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
1270
|
-
const record = this.requireLeasedOutbox(id, workerId);
|
|
1834
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1835
|
+
const record = this.requireLeasedOutbox(id, workerId, leaseToken);
|
|
1271
1836
|
record.status = "published";
|
|
1272
1837
|
record.publishedAt = publishedAt;
|
|
1273
1838
|
delete record.leaseOwner;
|
|
1274
1839
|
delete record.leaseExpiresAt;
|
|
1275
1840
|
}
|
|
1276
1841
|
async markOutboxFailed(input) {
|
|
1277
|
-
const record = this.requireLeasedOutbox(input.id, input.workerId);
|
|
1842
|
+
const record = this.requireLeasedOutbox(input.id, input.workerId, input.leaseToken);
|
|
1278
1843
|
record.status = input.deadLetter ? "dead_letter" : "pending";
|
|
1279
1844
|
record.lastError = input.error;
|
|
1280
1845
|
record.availableAt = input.availableAt;
|
|
1281
1846
|
delete record.leaseOwner;
|
|
1282
1847
|
delete record.leaseExpiresAt;
|
|
1283
1848
|
}
|
|
1849
|
+
async renewOutboxLease(input) {
|
|
1850
|
+
const record = this.outbox.find((candidate) => candidate.id === input.id);
|
|
1851
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1852
|
+
if (!record || record.status !== "pending" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1853
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1854
|
+
return true;
|
|
1855
|
+
}
|
|
1284
1856
|
async listOutbox(input = {}) {
|
|
1285
1857
|
return this.outbox.filter((record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status))).map(cloneOutboxRecord);
|
|
1286
1858
|
}
|
|
1287
|
-
requireLeasedOutbox(id, workerId) {
|
|
1859
|
+
requireLeasedOutbox(id, workerId, leaseToken) {
|
|
1288
1860
|
const record = this.outbox.find((candidate) => candidate.id === id);
|
|
1289
|
-
if (!record || record.leaseOwner !== workerId
|
|
1861
|
+
if (!record || record.leaseOwner !== workerId || (record.leaseToken ?? 0) !== (leaseToken ?? 0)) {
|
|
1862
|
+
throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1863
|
+
}
|
|
1290
1864
|
return record;
|
|
1291
1865
|
}
|
|
1292
1866
|
async nextEventSequence(tenantId, spaceId) {
|
|
@@ -1308,24 +1882,128 @@ var MemoryPlatformHostStore = class {
|
|
|
1308
1882
|
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (!input.statuses?.length || input.statuses.includes(record.status)) && (!input.updatedBefore || record.updatedAt < input.updatedBefore)
|
|
1309
1883
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 100);
|
|
1310
1884
|
}
|
|
1885
|
+
async getHealthCounts(input) {
|
|
1886
|
+
const invocations = this.invocations.filter(
|
|
1887
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1888
|
+
);
|
|
1889
|
+
const outbox = this.outbox.filter(
|
|
1890
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId)
|
|
1891
|
+
);
|
|
1892
|
+
return {
|
|
1893
|
+
invocationBacklog: invocations.filter((item) => item.status === "pending").length,
|
|
1894
|
+
invocationRunning: invocations.filter((item) => item.status === "running").length,
|
|
1895
|
+
invocationExpiredLeases: invocations.filter(
|
|
1896
|
+
(item) => item.status === "running" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1897
|
+
).length,
|
|
1898
|
+
invocationApprovalWaits: invocations.filter((item) => item.status === "waiting_for_approval").length,
|
|
1899
|
+
invocationReconciliationRequired: invocations.filter((item) => item.status === "reconciliation_required").length,
|
|
1900
|
+
outboxBacklog: outbox.filter((item) => item.status === "pending").length,
|
|
1901
|
+
outboxExpiredLeases: outbox.filter(
|
|
1902
|
+
(item) => item.status === "pending" && item.leaseExpiresAt !== void 0 && item.leaseExpiresAt.getTime() <= input.now.getTime()
|
|
1903
|
+
).length,
|
|
1904
|
+
outboxDeadLetters: outbox.filter((item) => item.status === "dead_letter").length
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1311
1907
|
async claimActionInvocations(input) {
|
|
1312
1908
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1313
1909
|
const eligible = this.invocations.filter(
|
|
1314
|
-
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" &&
|
|
1910
|
+
(record) => (!input.tenantId || record.tenantId === input.tenantId) && (!input.spaceId || record.spaceId === input.spaceId) && (record.status === "pending" || record.status === "running" && record.leaseExpiresAt !== void 0 && record.leaseExpiresAt <= current)
|
|
1315
1911
|
).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()).slice(0, input.limit ?? 10);
|
|
1316
1912
|
for (const record of eligible) {
|
|
1317
1913
|
record.status = "running";
|
|
1318
1914
|
record.leaseOwner = input.workerId;
|
|
1319
1915
|
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1916
|
+
record.leaseToken = (record.leaseToken ?? 0) + 1;
|
|
1320
1917
|
record.attemptCount += 1;
|
|
1321
1918
|
record.updatedAt = current;
|
|
1322
1919
|
}
|
|
1323
|
-
return eligible;
|
|
1920
|
+
return eligible.map((record) => structuredClone(record));
|
|
1921
|
+
}
|
|
1922
|
+
async updateLeasedActionInvocation(input) {
|
|
1923
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1924
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken) return false;
|
|
1925
|
+
await this.updateActionInvocation(input.id, input.tenantId, input.spaceId, input.patch);
|
|
1926
|
+
return true;
|
|
1927
|
+
}
|
|
1928
|
+
async renewActionInvocationLease(input) {
|
|
1929
|
+
const record = await this.getActionInvocation(input.id, input.tenantId, input.spaceId);
|
|
1930
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1931
|
+
if (!record || record.status !== "running" || record.leaseOwner !== input.workerId || record.leaseToken !== input.leaseToken || !record.leaseExpiresAt || record.leaseExpiresAt <= current) return false;
|
|
1932
|
+
record.leaseExpiresAt = new Date(current.getTime() + input.leaseDurationMs);
|
|
1933
|
+
record.updatedAt = current;
|
|
1934
|
+
return true;
|
|
1324
1935
|
}
|
|
1325
1936
|
async listEvents(tenantId, spaceId) {
|
|
1326
1937
|
return this.events.filter((event) => event.tenantId === tenantId && event.spaceId === spaceId).sort((left, right) => left.sequence - right.sequence);
|
|
1327
1938
|
}
|
|
1328
1939
|
};
|
|
1940
|
+
async function applyPostgresMigrations(client, migrations) {
|
|
1941
|
+
const ordered = validateMigrations(migrations);
|
|
1942
|
+
const statements = ordered.map((migration) => migrationStatement(migration));
|
|
1943
|
+
await client.query(`
|
|
1944
|
+
BEGIN;
|
|
1945
|
+
SELECT pg_advisory_xact_lock(hashtext('fabric_platform.schema_migrations'));
|
|
1946
|
+
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1947
|
+
CREATE TABLE IF NOT EXISTS fabric_platform.schema_migrations (
|
|
1948
|
+
version integer PRIMARY KEY,
|
|
1949
|
+
name text NOT NULL,
|
|
1950
|
+
checksum text NOT NULL,
|
|
1951
|
+
applied_at timestamptz NOT NULL DEFAULT now()
|
|
1952
|
+
);
|
|
1953
|
+
${statements.join("\n")}
|
|
1954
|
+
COMMIT;
|
|
1955
|
+
`);
|
|
1956
|
+
}
|
|
1957
|
+
function validateMigrations(migrations) {
|
|
1958
|
+
const ordered = [...migrations].sort((left, right) => left.version - right.version);
|
|
1959
|
+
const versions = /* @__PURE__ */ new Set();
|
|
1960
|
+
for (const migration of ordered) {
|
|
1961
|
+
if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
|
|
1962
|
+
throw new Error(`Migration version must be a positive safe integer: ${migration.version}`);
|
|
1963
|
+
}
|
|
1964
|
+
if (versions.has(migration.version)) {
|
|
1965
|
+
throw new Error(`Duplicate migration version: ${migration.version}`);
|
|
1966
|
+
}
|
|
1967
|
+
if (!/^[a-z][a-z0-9_]*$/.test(migration.name)) {
|
|
1968
|
+
throw new Error(`Invalid migration name: ${migration.name}`);
|
|
1969
|
+
}
|
|
1970
|
+
if (migration.sql.trim().length === 0) {
|
|
1971
|
+
throw new Error(`Migration ${migration.version} has no SQL`);
|
|
1972
|
+
}
|
|
1973
|
+
versions.add(migration.version);
|
|
1974
|
+
}
|
|
1975
|
+
return ordered;
|
|
1976
|
+
}
|
|
1977
|
+
function migrationStatement(migration) {
|
|
1978
|
+
const checksum = createHash("sha256").update(migration.sql).digest("hex");
|
|
1979
|
+
const delimiter = `$fabric_platform_migration_${migration.version}$`;
|
|
1980
|
+
if (migration.sql.includes(delimiter)) {
|
|
1981
|
+
throw new Error(`Migration ${migration.version} contains its reserved SQL delimiter`);
|
|
1982
|
+
}
|
|
1983
|
+
return `
|
|
1984
|
+
DO ${delimiter}
|
|
1985
|
+
DECLARE
|
|
1986
|
+
recorded_checksum text;
|
|
1987
|
+
BEGIN
|
|
1988
|
+
SELECT checksum INTO recorded_checksum
|
|
1989
|
+
FROM fabric_platform.schema_migrations
|
|
1990
|
+
WHERE version = ${migration.version};
|
|
1991
|
+
|
|
1992
|
+
IF FOUND THEN
|
|
1993
|
+
IF recorded_checksum <> ${sqlLiteral(checksum)} THEN
|
|
1994
|
+
RAISE EXCEPTION 'Migration checksum mismatch for version ${migration.version}';
|
|
1995
|
+
END IF;
|
|
1996
|
+
ELSE
|
|
1997
|
+
${migration.sql}
|
|
1998
|
+
INSERT INTO fabric_platform.schema_migrations (version, name, checksum)
|
|
1999
|
+
VALUES (${migration.version}, ${sqlLiteral(migration.name)}, ${sqlLiteral(checksum)});
|
|
2000
|
+
END IF;
|
|
2001
|
+
END
|
|
2002
|
+
${delimiter};`;
|
|
2003
|
+
}
|
|
2004
|
+
function sqlLiteral(value) {
|
|
2005
|
+
return `'${value.replaceAll("'", "''")}'`;
|
|
2006
|
+
}
|
|
1329
2007
|
|
|
1330
2008
|
// src/postgres-store.ts
|
|
1331
2009
|
var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
@@ -1338,11 +2016,23 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1338
2016
|
const scoped = new _PostgresPlatformHostStore(db2, sql2);
|
|
1339
2017
|
return run({
|
|
1340
2018
|
db: db2,
|
|
2019
|
+
getActionInvocationForUpdate: async (id, tenantId, spaceId) => {
|
|
2020
|
+
const result = await sql2.query(
|
|
2021
|
+
`SELECT * FROM fabric_platform.action_invocations
|
|
2022
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2023
|
+
FOR UPDATE`,
|
|
2024
|
+
[id, tenantId, spaceId]
|
|
2025
|
+
);
|
|
2026
|
+
return result.rows[0] ? toActionRecord(result.rows[0]) : void 0;
|
|
2027
|
+
},
|
|
2028
|
+
getEvent: (id, tenantId, spaceId) => scoped.getEvent(id, tenantId, spaceId),
|
|
1341
2029
|
appendEvent: (event) => scoped.appendEvent(event),
|
|
1342
2030
|
appendEventWithOutbox: (event, metadata) => scoped.appendEventWithOutbox(event, metadata),
|
|
1343
2031
|
nextEventSequence: (tenantId, spaceId) => scoped.nextEventSequence(tenantId, spaceId),
|
|
1344
2032
|
listEvents: (tenantId, spaceId) => scoped.listEvents(tenantId, spaceId),
|
|
1345
|
-
|
|
2033
|
+
getEntityState: (tenantId, spaceId, entityType, entityId) => scoped.getEntityState(tenantId, spaceId, entityType, entityId),
|
|
2034
|
+
updateActionInvocation: (id, tenantId, spaceId, patch) => scoped.updateActionInvocation(id, tenantId, spaceId, patch),
|
|
2035
|
+
updateLeasedActionInvocation: (input) => scoped.updateLeasedActionInvocation(input)
|
|
1346
2036
|
});
|
|
1347
2037
|
});
|
|
1348
2038
|
}
|
|
@@ -1352,7 +2042,10 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1352
2042
|
transactionWithEvents;
|
|
1353
2043
|
transactionalOutbox;
|
|
1354
2044
|
async ensureSchema() {
|
|
1355
|
-
await this.sql
|
|
2045
|
+
await applyPostgresMigrations(this.sql, [{
|
|
2046
|
+
version: 1,
|
|
2047
|
+
name: "host_ledger_baseline",
|
|
2048
|
+
sql: `
|
|
1356
2049
|
CREATE SCHEMA IF NOT EXISTS fabric_platform;
|
|
1357
2050
|
CREATE TABLE IF NOT EXISTS fabric_platform.action_invocations (
|
|
1358
2051
|
id text PRIMARY KEY, tenant_id text NOT NULL, space_id text NOT NULL,
|
|
@@ -1363,7 +2056,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1363
2056
|
parameter_digest text, parameter_digest_algorithm text, idempotency_actor_id text,
|
|
1364
2057
|
idempotency_authorization_binding_id text, invocation_provenance jsonb,
|
|
1365
2058
|
authorization_binding jsonb, execution_reason text, authorization_reconciliation jsonb,
|
|
1366
|
-
authorization_binding_id text, error text,
|
|
2059
|
+
adapter_reconciliation jsonb, authorization_binding_id text, error text,
|
|
1367
2060
|
attempt_count integer NOT NULL DEFAULT 0, lease_owner text,
|
|
1368
2061
|
lease_expires_at timestamptz, hitl_route text, hitl_risk_tier text,
|
|
1369
2062
|
hitl_reason text, hitl_policy_version text, approval_decision jsonb,
|
|
@@ -1380,6 +2073,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1380
2073
|
ADD COLUMN IF NOT EXISTS authorization_binding jsonb,
|
|
1381
2074
|
ADD COLUMN IF NOT EXISTS execution_reason text,
|
|
1382
2075
|
ADD COLUMN IF NOT EXISTS authorization_reconciliation jsonb,
|
|
2076
|
+
ADD COLUMN IF NOT EXISTS adapter_reconciliation jsonb,
|
|
1383
2077
|
ADD COLUMN IF NOT EXISTS authorization_binding_id text,
|
|
1384
2078
|
ADD COLUMN IF NOT EXISTS attempt_count integer NOT NULL DEFAULT 0,
|
|
1385
2079
|
ADD COLUMN IF NOT EXISTS lease_owner text,
|
|
@@ -1462,7 +2156,17 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1462
2156
|
);
|
|
1463
2157
|
CREATE INDEX IF NOT EXISTS event_outbox_claim_idx
|
|
1464
2158
|
ON fabric_platform.event_outbox (status, available_at, lease_expires_at, created_at);
|
|
1465
|
-
|
|
2159
|
+
`
|
|
2160
|
+
}, {
|
|
2161
|
+
version: 2,
|
|
2162
|
+
name: "lease_fencing",
|
|
2163
|
+
sql: `
|
|
2164
|
+
ALTER TABLE fabric_platform.action_invocations
|
|
2165
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2166
|
+
ALTER TABLE fabric_platform.event_outbox
|
|
2167
|
+
ADD COLUMN lease_token bigint NOT NULL DEFAULT 0;
|
|
2168
|
+
`
|
|
2169
|
+
}]);
|
|
1466
2170
|
}
|
|
1467
2171
|
async transaction(run) {
|
|
1468
2172
|
return run(this.db);
|
|
@@ -1523,6 +2227,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1523
2227
|
status=COALESCE($4,status), result=COALESCE($5::jsonb,result),
|
|
1524
2228
|
error=CASE WHEN $6::boolean THEN $7 ELSE error END,
|
|
1525
2229
|
authorization_reconciliation=CASE WHEN $8::boolean THEN $9::jsonb ELSE authorization_reconciliation END,
|
|
2230
|
+
adapter_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE adapter_reconciliation END,
|
|
1526
2231
|
lease_owner=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
1527
2232
|
THEN NULL ELSE lease_owner END,
|
|
1528
2233
|
lease_expires_at=CASE WHEN $4 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
@@ -1538,7 +2243,9 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1538
2243
|
Object.hasOwn(patch, "error"),
|
|
1539
2244
|
patch.error ?? null,
|
|
1540
2245
|
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
1541
|
-
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null
|
|
2246
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2247
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2248
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
1542
2249
|
]
|
|
1543
2250
|
);
|
|
1544
2251
|
}
|
|
@@ -1572,6 +2279,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1572
2279
|
error=CASE WHEN $5::boolean THEN NULL ELSE $6 END,
|
|
1573
2280
|
lease_owner=CASE WHEN $5::boolean THEN $7 ELSE NULL END,
|
|
1574
2281
|
lease_expires_at=CASE WHEN $5::boolean THEN $8::timestamptz ELSE NULL END,
|
|
2282
|
+
lease_token=CASE WHEN $5::boolean THEN lease_token+1 ELSE lease_token END,
|
|
1575
2283
|
attempt_count=CASE WHEN $5::boolean THEN GREATEST(attempt_count,1) ELSE attempt_count END,
|
|
1576
2284
|
updated_at=$9
|
|
1577
2285
|
WHERE id=$1 AND tenant_id=$2 AND space_id=$3 AND status='waiting_for_approval'
|
|
@@ -1809,28 +2517,48 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1809
2517
|
ORDER BY created_at,id FOR UPDATE SKIP LOCKED LIMIT $4
|
|
1810
2518
|
)
|
|
1811
2519
|
UPDATE fabric_platform.event_outbox AS item
|
|
1812
|
-
SET lease_owner=$5, lease_expires_at=$6,
|
|
2520
|
+
SET lease_owner=$5, lease_expires_at=$6, lease_token=lease_token+1,
|
|
2521
|
+
attempt_count=attempt_count+1
|
|
1813
2522
|
FROM claimable WHERE item.id=claimable.id RETURNING item.*`,
|
|
1814
2523
|
[current, input.tenantId ?? null, input.spaceId ?? null, Math.max(1, Math.min(input.limit ?? 100, 1e3)), input.workerId, leaseExpiresAt]
|
|
1815
2524
|
);
|
|
1816
2525
|
return result.rows.map(toOutboxRecord);
|
|
1817
2526
|
}
|
|
1818
|
-
async markOutboxPublished(id, workerId, publishedAt) {
|
|
2527
|
+
async markOutboxPublished(id, workerId, publishedAt, leaseToken) {
|
|
1819
2528
|
const result = await this.sql.query(
|
|
1820
2529
|
`UPDATE fabric_platform.event_outbox SET status='published',published_at=$3,
|
|
1821
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1822
|
-
|
|
2530
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2531
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($4,0) RETURNING id`,
|
|
2532
|
+
[id, workerId, publishedAt, leaseToken ?? null]
|
|
1823
2533
|
);
|
|
1824
2534
|
if (result.rows.length === 0) throw new Error(`Outbox record ${id} is not leased by ${workerId}.`);
|
|
1825
2535
|
}
|
|
1826
2536
|
async markOutboxFailed(input) {
|
|
1827
2537
|
const result = await this.sql.query(
|
|
1828
2538
|
`UPDATE fabric_platform.event_outbox SET status=$3,last_error=$4,available_at=$5,
|
|
1829
|
-
lease_owner=NULL,lease_expires_at=NULL
|
|
1830
|
-
|
|
2539
|
+
lease_owner=NULL,lease_expires_at=NULL
|
|
2540
|
+
WHERE id=$1 AND lease_owner=$2 AND lease_token=COALESCE($6,0) RETURNING id`,
|
|
2541
|
+
[input.id, input.workerId, input.deadLetter ? "dead_letter" : "pending", input.error, input.availableAt, input.leaseToken ?? null]
|
|
1831
2542
|
);
|
|
1832
2543
|
if (result.rows.length === 0) throw new Error(`Outbox record ${input.id} is not leased by ${input.workerId}.`);
|
|
1833
2544
|
}
|
|
2545
|
+
async renewOutboxLease(input) {
|
|
2546
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2547
|
+
const result = await this.sql.query(
|
|
2548
|
+
`UPDATE fabric_platform.event_outbox SET lease_expires_at=$5
|
|
2549
|
+
WHERE id=$1 AND status='pending' AND lease_owner=$2 AND lease_token=$3
|
|
2550
|
+
AND lease_expires_at > $4
|
|
2551
|
+
RETURNING id`,
|
|
2552
|
+
[
|
|
2553
|
+
input.id,
|
|
2554
|
+
input.workerId,
|
|
2555
|
+
input.leaseToken,
|
|
2556
|
+
current,
|
|
2557
|
+
new Date(current.getTime() + input.leaseDurationMs)
|
|
2558
|
+
]
|
|
2559
|
+
);
|
|
2560
|
+
return result.rows.length === 1;
|
|
2561
|
+
}
|
|
1834
2562
|
async listOutbox(input = {}) {
|
|
1835
2563
|
const result = await this.sql.query(
|
|
1836
2564
|
`SELECT * FROM fabric_platform.event_outbox
|
|
@@ -1879,6 +2607,55 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1879
2607
|
);
|
|
1880
2608
|
return result.rows.map(toActionRecord);
|
|
1881
2609
|
}
|
|
2610
|
+
async getHealthCounts(input) {
|
|
2611
|
+
const result = await this.sql.query(
|
|
2612
|
+
`SELECT
|
|
2613
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2614
|
+
WHERE status='pending'
|
|
2615
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2616
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_backlog,
|
|
2617
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2618
|
+
WHERE status='running'
|
|
2619
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2620
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_running,
|
|
2621
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2622
|
+
WHERE status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2623
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2624
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_expired_leases,
|
|
2625
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2626
|
+
WHERE status='waiting_for_approval'
|
|
2627
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2628
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_approval_waits,
|
|
2629
|
+
(SELECT COUNT(*) FROM fabric_platform.action_invocations
|
|
2630
|
+
WHERE status='reconciliation_required'
|
|
2631
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2632
|
+
AND ($2::text IS NULL OR space_id=$2)) AS invocation_reconciliation_required,
|
|
2633
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2634
|
+
WHERE status='pending'
|
|
2635
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2636
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_backlog,
|
|
2637
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2638
|
+
WHERE status='pending' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $3
|
|
2639
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2640
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_expired_leases,
|
|
2641
|
+
(SELECT COUNT(*) FROM fabric_platform.event_outbox
|
|
2642
|
+
WHERE status='dead_letter'
|
|
2643
|
+
AND ($1::text IS NULL OR tenant_id=$1)
|
|
2644
|
+
AND ($2::text IS NULL OR space_id=$2)) AS outbox_dead_letters`,
|
|
2645
|
+
[input.tenantId ?? null, input.spaceId ?? null, input.now]
|
|
2646
|
+
);
|
|
2647
|
+
const row = result.rows[0] ?? {};
|
|
2648
|
+
return {
|
|
2649
|
+
invocationBacklog: Number(row.invocation_backlog ?? 0),
|
|
2650
|
+
invocationRunning: Number(row.invocation_running ?? 0),
|
|
2651
|
+
invocationExpiredLeases: Number(row.invocation_expired_leases ?? 0),
|
|
2652
|
+
invocationApprovalWaits: Number(row.invocation_approval_waits ?? 0),
|
|
2653
|
+
invocationReconciliationRequired: Number(row.invocation_reconciliation_required ?? 0),
|
|
2654
|
+
outboxBacklog: Number(row.outbox_backlog ?? 0),
|
|
2655
|
+
outboxExpiredLeases: Number(row.outbox_expired_leases ?? 0),
|
|
2656
|
+
outboxDeadLetters: Number(row.outbox_dead_letters ?? 0)
|
|
2657
|
+
};
|
|
2658
|
+
}
|
|
1882
2659
|
async claimActionInvocations(input) {
|
|
1883
2660
|
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
1884
2661
|
const limit = Math.max(1, Math.min(input.limit ?? 10, 100));
|
|
@@ -1887,7 +2664,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1887
2664
|
`WITH claimable AS (
|
|
1888
2665
|
SELECT id FROM fabric_platform.action_invocations
|
|
1889
2666
|
WHERE (status='pending' OR
|
|
1890
|
-
(status='running' AND
|
|
2667
|
+
(status='running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1))
|
|
1891
2668
|
AND ($2::text IS NULL OR tenant_id=$2)
|
|
1892
2669
|
AND ($3::text IS NULL OR space_id=$3)
|
|
1893
2670
|
ORDER BY created_at
|
|
@@ -1896,7 +2673,7 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1896
2673
|
)
|
|
1897
2674
|
UPDATE fabric_platform.action_invocations AS invocation
|
|
1898
2675
|
SET status='running', lease_owner=$5, lease_expires_at=$6,
|
|
1899
|
-
attempt_count=attempt_count+1, updated_at=$1
|
|
2676
|
+
lease_token=lease_token+1, attempt_count=attempt_count+1, updated_at=$1
|
|
1900
2677
|
FROM claimable WHERE invocation.id=claimable.id
|
|
1901
2678
|
RETURNING invocation.*`,
|
|
1902
2679
|
[
|
|
@@ -1910,6 +2687,61 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1910
2687
|
);
|
|
1911
2688
|
return result.rows.map(toActionRecord);
|
|
1912
2689
|
}
|
|
2690
|
+
async updateLeasedActionInvocation(input) {
|
|
2691
|
+
const patch = input.patch;
|
|
2692
|
+
const result = await this.sql.query(
|
|
2693
|
+
`UPDATE fabric_platform.action_invocations SET
|
|
2694
|
+
status=COALESCE($6,status), result=COALESCE($7::jsonb,result),
|
|
2695
|
+
error=CASE WHEN $8::boolean THEN $9 ELSE error END,
|
|
2696
|
+
authorization_reconciliation=CASE WHEN $10::boolean THEN $11::jsonb ELSE authorization_reconciliation END,
|
|
2697
|
+
adapter_reconciliation=CASE WHEN $12::boolean THEN $13::jsonb ELSE adapter_reconciliation END,
|
|
2698
|
+
lease_owner=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2699
|
+
THEN NULL ELSE lease_owner END,
|
|
2700
|
+
lease_expires_at=CASE WHEN $6 IN ('waiting_for_approval','completed','failed','blocked_by_policy','reconciliation_required','validation_failed')
|
|
2701
|
+
THEN NULL ELSE lease_expires_at END,
|
|
2702
|
+
updated_at=now()
|
|
2703
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2704
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$5
|
|
2705
|
+
RETURNING id`,
|
|
2706
|
+
[
|
|
2707
|
+
input.id,
|
|
2708
|
+
input.tenantId,
|
|
2709
|
+
input.spaceId,
|
|
2710
|
+
input.workerId,
|
|
2711
|
+
input.leaseToken,
|
|
2712
|
+
patch.status ?? null,
|
|
2713
|
+
patch.result === void 0 ? null : JSON.stringify(patch.result),
|
|
2714
|
+
Object.hasOwn(patch, "error"),
|
|
2715
|
+
patch.error ?? null,
|
|
2716
|
+
Object.hasOwn(patch, "authorizationReconciliation"),
|
|
2717
|
+
patch.authorizationReconciliation ? JSON.stringify(patch.authorizationReconciliation) : null,
|
|
2718
|
+
Object.hasOwn(patch, "adapterReconciliation"),
|
|
2719
|
+
patch.adapterReconciliation ? JSON.stringify(patch.adapterReconciliation) : null
|
|
2720
|
+
]
|
|
2721
|
+
);
|
|
2722
|
+
return result.rows.length === 1;
|
|
2723
|
+
}
|
|
2724
|
+
async renewActionInvocationLease(input) {
|
|
2725
|
+
const current = input.now ?? /* @__PURE__ */ new Date();
|
|
2726
|
+
const result = await this.sql.query(
|
|
2727
|
+
`UPDATE fabric_platform.action_invocations
|
|
2728
|
+
SET lease_expires_at=$6,updated_at=$5
|
|
2729
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3
|
|
2730
|
+
AND status='running' AND lease_owner=$4 AND lease_token=$7
|
|
2731
|
+
AND lease_expires_at > $5
|
|
2732
|
+
RETURNING id`,
|
|
2733
|
+
[
|
|
2734
|
+
input.id,
|
|
2735
|
+
input.tenantId,
|
|
2736
|
+
input.spaceId,
|
|
2737
|
+
input.workerId,
|
|
2738
|
+
current,
|
|
2739
|
+
new Date(current.getTime() + input.leaseDurationMs),
|
|
2740
|
+
input.leaseToken
|
|
2741
|
+
]
|
|
2742
|
+
);
|
|
2743
|
+
return result.rows.length === 1;
|
|
2744
|
+
}
|
|
1913
2745
|
async listEvents(tenantId, spaceId) {
|
|
1914
2746
|
const result = await this.sql.query(
|
|
1915
2747
|
`SELECT * FROM fabric_platform.asset_events
|
|
@@ -1918,6 +2750,14 @@ var PostgresPlatformHostStore = class _PostgresPlatformHostStore {
|
|
|
1918
2750
|
);
|
|
1919
2751
|
return result.rows.map(toEventRecord);
|
|
1920
2752
|
}
|
|
2753
|
+
async getEvent(id, tenantId, spaceId) {
|
|
2754
|
+
const result = await this.sql.query(
|
|
2755
|
+
`SELECT * FROM fabric_platform.asset_events
|
|
2756
|
+
WHERE id=$1 AND tenant_id=$2 AND space_id=$3`,
|
|
2757
|
+
[id, tenantId, spaceId]
|
|
2758
|
+
);
|
|
2759
|
+
return result.rows[0] ? toEventRecord(result.rows[0]) : void 0;
|
|
2760
|
+
}
|
|
1921
2761
|
};
|
|
1922
2762
|
function toAdapterRecord(row) {
|
|
1923
2763
|
return {
|
|
@@ -1960,8 +2800,10 @@ function toActionRecord(row) {
|
|
|
1960
2800
|
...row.authorization_binding ? { authorizationBinding: row.authorization_binding } : {},
|
|
1961
2801
|
...row.execution_reason ? { executionReason: String(row.execution_reason) } : {},
|
|
1962
2802
|
...row.authorization_reconciliation ? { authorizationReconciliation: row.authorization_reconciliation } : {},
|
|
2803
|
+
...row.adapter_reconciliation ? { adapterReconciliation: row.adapter_reconciliation } : {},
|
|
1963
2804
|
...row.authorization_binding_id ? { authorizationBindingId: String(row.authorization_binding_id) } : {},
|
|
1964
2805
|
attemptCount: Number(row.attempt_count ?? 0),
|
|
2806
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
1965
2807
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
1966
2808
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
1967
2809
|
...row.hitl_route ? { hitlRoute: String(row.hitl_route) } : {},
|
|
@@ -2057,6 +2899,7 @@ function toOutboxRecord(row) {
|
|
|
2057
2899
|
correlationId: String(event.correlationId),
|
|
2058
2900
|
...event.causationId ? { causationId: String(event.causationId) } : {},
|
|
2059
2901
|
...event.provenance ? { provenance: event.provenance } : {},
|
|
2902
|
+
...event.consistency ? { consistency: event.consistency } : {},
|
|
2060
2903
|
occurredAt: new Date(event.occurredAt),
|
|
2061
2904
|
recordedAt: new Date(event.recordedAt),
|
|
2062
2905
|
producerModuleVersion: String(event.producerModuleVersion),
|
|
@@ -2069,6 +2912,7 @@ function toOutboxRecord(row) {
|
|
|
2069
2912
|
availableAt: new Date(row.available_at),
|
|
2070
2913
|
...row.lease_owner ? { leaseOwner: String(row.lease_owner) } : {},
|
|
2071
2914
|
...row.lease_expires_at ? { leaseExpiresAt: new Date(row.lease_expires_at) } : {},
|
|
2915
|
+
leaseToken: Number(row.lease_token ?? 0),
|
|
2072
2916
|
...row.last_error ? { lastError: String(row.last_error) } : {},
|
|
2073
2917
|
createdAt: new Date(row.created_at),
|
|
2074
2918
|
...row.published_at ? { publishedAt: new Date(row.published_at) } : {}
|
|
@@ -2085,38 +2929,93 @@ function createStoreBackedActionDispatcher() {
|
|
|
2085
2929
|
};
|
|
2086
2930
|
}
|
|
2087
2931
|
async function runPlatformActionWorkerCycle(options) {
|
|
2088
|
-
const
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2932
|
+
const leaseDurationMs = options.leaseDurationMs ?? DEFAULT_LEASE_DURATION_MS;
|
|
2933
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.started", options.workerId);
|
|
2934
|
+
let claimed;
|
|
2935
|
+
try {
|
|
2936
|
+
claimed = await options.store.claimActionInvocations({
|
|
2937
|
+
workerId: options.workerId,
|
|
2938
|
+
limit: options.batchSize ?? DEFAULT_BATCH_SIZE,
|
|
2939
|
+
leaseDurationMs,
|
|
2940
|
+
...options.tenantId ? { tenantId: options.tenantId } : {},
|
|
2941
|
+
...options.spaceId ? { spaceId: options.spaceId } : {}
|
|
2942
|
+
});
|
|
2943
|
+
} catch (error) {
|
|
2944
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.failed", options.workerId);
|
|
2945
|
+
throw error;
|
|
2946
|
+
}
|
|
2947
|
+
for (const invocation of claimed) {
|
|
2948
|
+
emitPlatformHostTelemetry(options.telemetry, {
|
|
2949
|
+
kind: "event",
|
|
2950
|
+
name: "invocation.lease_claimed",
|
|
2951
|
+
metricName: PLATFORM_HOST_METRIC_NAMES.invocationLeaseClaimed,
|
|
2952
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
2953
|
+
tenantId: invocation.tenantId,
|
|
2954
|
+
spaceId: invocation.spaceId,
|
|
2955
|
+
attributes: { attempt: invocation.attemptCount }
|
|
2956
|
+
});
|
|
2957
|
+
}
|
|
2095
2958
|
let completed = 0;
|
|
2096
2959
|
let failed = 0;
|
|
2097
2960
|
let waitingForApproval = 0;
|
|
2098
2961
|
for (const invocation of claimed) {
|
|
2962
|
+
const stopHeartbeat = startLeaseHeartbeat(options, invocation, leaseDurationMs);
|
|
2099
2963
|
try {
|
|
2100
|
-
const
|
|
2964
|
+
const result2 = await options.host.executeInvocation(
|
|
2101
2965
|
invocation.id,
|
|
2102
2966
|
invocation.tenantId,
|
|
2103
2967
|
invocation.spaceId,
|
|
2104
|
-
{ leaseOwner: options.workerId }
|
|
2968
|
+
{ leaseOwner: options.workerId, leaseToken: invocation.leaseToken }
|
|
2105
2969
|
);
|
|
2106
|
-
if (
|
|
2107
|
-
else if (
|
|
2970
|
+
if (result2.status === "completed") completed += 1;
|
|
2971
|
+
else if (result2.status === "waiting_for_approval") waitingForApproval += 1;
|
|
2108
2972
|
else failed += 1;
|
|
2109
2973
|
} catch (error) {
|
|
2110
2974
|
failed += 1;
|
|
2111
2975
|
options.onError?.(error, invocation);
|
|
2976
|
+
} finally {
|
|
2977
|
+
await stopHeartbeat();
|
|
2112
2978
|
}
|
|
2113
2979
|
}
|
|
2114
|
-
|
|
2980
|
+
const result = {
|
|
2115
2981
|
claimed: claimed.length,
|
|
2116
2982
|
completed,
|
|
2117
2983
|
failed,
|
|
2118
2984
|
...waitingForApproval > 0 ? { waitingForApproval } : {}
|
|
2119
2985
|
};
|
|
2986
|
+
emitWorkerTelemetry(options.telemetry, "worker.cycle.completed", options.workerId, result);
|
|
2987
|
+
return result;
|
|
2988
|
+
}
|
|
2989
|
+
function startLeaseHeartbeat(options, invocation, leaseDurationMs) {
|
|
2990
|
+
if (!invocation?.leaseToken) return async () => void 0;
|
|
2991
|
+
const intervalMs = options.leaseRenewalIntervalMs ?? Math.max(1, Math.floor(leaseDurationMs / 3));
|
|
2992
|
+
let stopped = false;
|
|
2993
|
+
let timer;
|
|
2994
|
+
let running;
|
|
2995
|
+
const schedule = () => {
|
|
2996
|
+
if (!stopped) timer = setTimeout(tick, intervalMs);
|
|
2997
|
+
};
|
|
2998
|
+
const tick = () => {
|
|
2999
|
+
running = options.store.renewActionInvocationLease({
|
|
3000
|
+
id: invocation.id,
|
|
3001
|
+
tenantId: invocation.tenantId,
|
|
3002
|
+
spaceId: invocation.spaceId,
|
|
3003
|
+
workerId: options.workerId,
|
|
3004
|
+
leaseToken: invocation.leaseToken,
|
|
3005
|
+
leaseDurationMs
|
|
3006
|
+
}).then((renewed) => {
|
|
3007
|
+
if (!renewed) options.onError?.(
|
|
3008
|
+
new Error(`Invocation lease lost: ${invocation.id}`),
|
|
3009
|
+
invocation
|
|
3010
|
+
);
|
|
3011
|
+
}).catch((error) => options.onError?.(error, invocation)).finally(schedule);
|
|
3012
|
+
};
|
|
3013
|
+
schedule();
|
|
3014
|
+
return async () => {
|
|
3015
|
+
stopped = true;
|
|
3016
|
+
if (timer) clearTimeout(timer);
|
|
3017
|
+
await running;
|
|
3018
|
+
};
|
|
2120
3019
|
}
|
|
2121
3020
|
async function runPlatformActionWorker(options) {
|
|
2122
3021
|
while (!options.signal?.aborted) {
|
|
@@ -2142,7 +3041,158 @@ async function abortableDelay(milliseconds, signal) {
|
|
|
2142
3041
|
);
|
|
2143
3042
|
});
|
|
2144
3043
|
}
|
|
3044
|
+
function emitWorkerTelemetry(telemetry, name, workerId, attributes) {
|
|
3045
|
+
const metricName = name === "worker.cycle.started" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesStarted : name === "worker.cycle.completed" ? PLATFORM_HOST_METRIC_NAMES.workerCyclesCompleted : PLATFORM_HOST_METRIC_NAMES.workerCyclesFailed;
|
|
3046
|
+
emitPlatformHostTelemetry(telemetry, {
|
|
3047
|
+
kind: "event",
|
|
3048
|
+
name,
|
|
3049
|
+
metricName,
|
|
3050
|
+
occurredAt: /* @__PURE__ */ new Date(),
|
|
3051
|
+
attributes: { workerId, ...attributes }
|
|
3052
|
+
});
|
|
3053
|
+
}
|
|
3054
|
+
|
|
3055
|
+
// src/saga-parent-lifecycle.ts
|
|
3056
|
+
var TERMINAL_PARENT_STATUSES = /* @__PURE__ */ new Set([
|
|
3057
|
+
"completed",
|
|
3058
|
+
"failed",
|
|
3059
|
+
"blocked_by_policy",
|
|
3060
|
+
"reconciliation_required",
|
|
3061
|
+
"validation_failed"
|
|
3062
|
+
]);
|
|
3063
|
+
function createDurableSagaParentLifecycle(options) {
|
|
3064
|
+
const store = options.store;
|
|
3065
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
3066
|
+
const transactionWithEvents = store.transactionWithEvents?.bind(store);
|
|
3067
|
+
if (!transactionWithEvents) {
|
|
3068
|
+
throw new Error(
|
|
3069
|
+
"Durable saga parent lifecycle requires store.transactionWithEvents for atomic event and parent updates."
|
|
3070
|
+
);
|
|
3071
|
+
}
|
|
3072
|
+
const runInTransaction = transactionWithEvents;
|
|
3073
|
+
async function applyLifecycleTransition(input, lifecycleId2, eventType, payload, patch) {
|
|
3074
|
+
const eventId = `sagapar_${digestParameters({
|
|
3075
|
+
parentInvocationId: input.parentInvocationId,
|
|
3076
|
+
lifecycleId: lifecycleId2
|
|
3077
|
+
}).slice(0, 40)}`;
|
|
3078
|
+
await runInTransaction(async (transaction) => {
|
|
3079
|
+
if (!transaction.getActionInvocationForUpdate) {
|
|
3080
|
+
throw new Error(
|
|
3081
|
+
"Durable saga parent lifecycle requires transaction.getActionInvocationForUpdate for serialization."
|
|
3082
|
+
);
|
|
3083
|
+
}
|
|
3084
|
+
if (!transaction.getEvent) {
|
|
3085
|
+
throw new Error(
|
|
3086
|
+
"Durable saga parent lifecycle requires transaction.getEvent for identity-scoped evidence lookup."
|
|
3087
|
+
);
|
|
3088
|
+
}
|
|
3089
|
+
const parent = await transaction.getActionInvocationForUpdate(
|
|
3090
|
+
input.parentInvocationId,
|
|
3091
|
+
input.tenantId,
|
|
3092
|
+
input.spaceId
|
|
3093
|
+
);
|
|
3094
|
+
if (!parent) {
|
|
3095
|
+
throw new Error(`Saga parent invocation not found: ${input.parentInvocationId}`);
|
|
3096
|
+
}
|
|
3097
|
+
const prior = await transaction.getEvent(eventId, input.tenantId, input.spaceId);
|
|
3098
|
+
if (prior) {
|
|
3099
|
+
if (prior.eventType !== eventType || digestParameters(prior.payload) !== digestParameters(payload)) {
|
|
3100
|
+
throw new Error(`Saga lifecycle identity ${lifecycleId2} has contradictory evidence.`);
|
|
3101
|
+
}
|
|
3102
|
+
return;
|
|
3103
|
+
}
|
|
3104
|
+
if (TERMINAL_PARENT_STATUSES.has(parent.status)) {
|
|
3105
|
+
throw new Error(
|
|
3106
|
+
`Saga parent invocation ${input.parentInvocationId} is already terminal (${parent.status}).`
|
|
3107
|
+
);
|
|
3108
|
+
}
|
|
3109
|
+
const sequence = await transaction.nextEventSequence(input.tenantId, input.spaceId);
|
|
3110
|
+
const timestamp = now();
|
|
3111
|
+
await transaction.appendEvent({
|
|
3112
|
+
id: eventId,
|
|
3113
|
+
tenantId: input.tenantId,
|
|
3114
|
+
spaceId: input.spaceId,
|
|
3115
|
+
eventType,
|
|
3116
|
+
eventSchemaVersion: 1,
|
|
3117
|
+
subjectType: "ActionInvocation",
|
|
3118
|
+
subjectId: input.parentInvocationId,
|
|
3119
|
+
actorId: "system",
|
|
3120
|
+
actorType: "system",
|
|
3121
|
+
actionInvocationId: input.parentInvocationId,
|
|
3122
|
+
payload,
|
|
3123
|
+
sequence,
|
|
3124
|
+
occurredAt: timestamp,
|
|
3125
|
+
recordedAt: timestamp,
|
|
3126
|
+
correlationId: input.parentInvocationId,
|
|
3127
|
+
causationId: lifecycleId2
|
|
3128
|
+
});
|
|
3129
|
+
if (patch) {
|
|
3130
|
+
await transaction.updateActionInvocation(
|
|
3131
|
+
input.parentInvocationId,
|
|
3132
|
+
input.tenantId,
|
|
3133
|
+
input.spaceId,
|
|
3134
|
+
patch
|
|
3135
|
+
);
|
|
3136
|
+
}
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
return {
|
|
3140
|
+
async recordProgress(input) {
|
|
3141
|
+
await applyLifecycleTransition(
|
|
3142
|
+
input,
|
|
3143
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:progress:${input.progress.stepId}:${input.progress.index}`,
|
|
3144
|
+
"SagaParentProgress",
|
|
3145
|
+
{
|
|
3146
|
+
stepId: input.progress.stepId,
|
|
3147
|
+
index: input.progress.index,
|
|
3148
|
+
total: input.progress.total,
|
|
3149
|
+
completedAt: input.progress.completedAt
|
|
3150
|
+
}
|
|
3151
|
+
);
|
|
3152
|
+
},
|
|
3153
|
+
async parkApproval(input) {
|
|
3154
|
+
await applyLifecycleTransition(
|
|
3155
|
+
input,
|
|
3156
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:approval:${input.approvalId}`,
|
|
3157
|
+
"SagaParentApprovalParked",
|
|
3158
|
+
{
|
|
3159
|
+
stepId: input.stepId,
|
|
3160
|
+
approvalId: input.approvalId,
|
|
3161
|
+
reason: input.reason
|
|
3162
|
+
},
|
|
3163
|
+
{ status: "waiting_for_approval" }
|
|
3164
|
+
);
|
|
3165
|
+
},
|
|
3166
|
+
async cancel(input) {
|
|
3167
|
+
await applyLifecycleTransition(
|
|
3168
|
+
input,
|
|
3169
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:cancel`,
|
|
3170
|
+
"SagaParentCancelled",
|
|
3171
|
+
{ reason: input.reason, actorId: input.actorId },
|
|
3172
|
+
{ status: "failed", error: `Saga cancelled: ${input.reason}` }
|
|
3173
|
+
);
|
|
3174
|
+
},
|
|
3175
|
+
async complete(input) {
|
|
3176
|
+
await applyLifecycleTransition(
|
|
3177
|
+
input,
|
|
3178
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:complete`,
|
|
3179
|
+
"SagaParentCompleted",
|
|
3180
|
+
{ result: input.result },
|
|
3181
|
+
{ status: "completed", result: input.result }
|
|
3182
|
+
);
|
|
3183
|
+
},
|
|
3184
|
+
async fail(input) {
|
|
3185
|
+
await applyLifecycleTransition(
|
|
3186
|
+
input,
|
|
3187
|
+
input.lifecycleId ?? `host:${input.parentInvocationId}:fail`,
|
|
3188
|
+
"SagaParentFailed",
|
|
3189
|
+
{ error: input.error, compensation: input.compensation },
|
|
3190
|
+
{ status: "failed", error: input.error }
|
|
3191
|
+
);
|
|
3192
|
+
}
|
|
3193
|
+
};
|
|
3194
|
+
}
|
|
2145
3195
|
|
|
2146
|
-
export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PostgresPlatformHostStore, canonicalJson, cloneOutboxRecord, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
3196
|
+
export { IdempotencyConflictError, MemoryPlatformHostStore, PARAMETER_DIGEST_ALGORITHM, PLATFORM_HOST_CONTRACT_VERSION, PLATFORM_HOST_HEALTH_CONTRACT_VERSION, PLATFORM_HOST_METRIC_NAMES, PostgresPlatformHostStore, applyPostgresMigrations, canonicalJson, cloneOutboxRecord, createDurableSagaParentLifecycle, createGovernedActionHost, createStoreBackedActionDispatcher, digestParameters, emitPlatformHostTelemetry, getPlatformHostHealthSnapshot, runOutboxRelayCycle, runPlatformActionWorker, runPlatformActionWorkerCycle, toEnterpriseEventEnvelope };
|
|
2147
3197
|
//# sourceMappingURL=index.js.map
|
|
2148
3198
|
//# sourceMappingURL=index.js.map
|