@bastani/atomic 0.9.44-alpha.5 → 0.9.44-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/builtin/intercom/CHANGELOG.md +6 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +9 -1
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2373 -703
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2245 -697
- package/dist/builtin/workflows/src/index.js +2244 -697
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2245 -697
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +130 -13
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -274,6 +274,12 @@ class ScopedDurableBackend {
|
|
|
274
274
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
275
275
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
276
276
|
}
|
|
277
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
278
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
279
|
+
}
|
|
280
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
281
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
282
|
+
}
|
|
277
283
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
278
284
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
279
285
|
}
|
|
@@ -3136,8 +3142,11 @@ var init_pending_stage_delivery = __esm(() => {
|
|
|
3136
3142
|
});
|
|
3137
3143
|
|
|
3138
3144
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
3145
|
+
function isSettledPendingStageMessage(entry) {
|
|
3146
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
3147
|
+
}
|
|
3139
3148
|
async function retainPendingStageMessages(messages, archive) {
|
|
3140
|
-
const settled = messages.filter(
|
|
3149
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
3141
3150
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
3142
3151
|
for (const entry of expired)
|
|
3143
3152
|
await archive(entry);
|
|
@@ -3145,53 +3154,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
3145
3154
|
}
|
|
3146
3155
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
3147
3156
|
|
|
3157
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
3158
|
+
class PendingStageTransitionLedger {
|
|
3159
|
+
compacted = [];
|
|
3160
|
+
settled = 0;
|
|
3161
|
+
stages = new Map;
|
|
3162
|
+
fixed = new WeakSet;
|
|
3163
|
+
reconciled = [];
|
|
3164
|
+
reconciledVersion;
|
|
3165
|
+
reconcile(backend, runId, messages) {
|
|
3166
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
3167
|
+
return messages;
|
|
3168
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
3169
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
3170
|
+
let recovered;
|
|
3171
|
+
for (let index = 0;index < messages.length; index++) {
|
|
3172
|
+
const entry = messages[index];
|
|
3173
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
3174
|
+
continue;
|
|
3175
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
3176
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
3177
|
+
continue;
|
|
3178
|
+
recovered ??= [...messages];
|
|
3179
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
3180
|
+
}
|
|
3181
|
+
const result = recovered ?? messages;
|
|
3182
|
+
this.reconciled = result;
|
|
3183
|
+
this.reconciledVersion = version;
|
|
3184
|
+
return result;
|
|
3185
|
+
}
|
|
3186
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
3187
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
3188
|
+
const endedStages = [];
|
|
3189
|
+
const dirty = new Set;
|
|
3190
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
3191
|
+
for (const [stageId, entries] of byStage) {
|
|
3192
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
3193
|
+
continue;
|
|
3194
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
3195
|
+
for (const entry of entries)
|
|
3196
|
+
dirty.add(entry);
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
const base = this.compacted;
|
|
3200
|
+
const output = [];
|
|
3201
|
+
const removed = [];
|
|
3202
|
+
const added = [];
|
|
3203
|
+
let settled = this.settled;
|
|
3204
|
+
const length = Math.max(base.length, messages.length);
|
|
3205
|
+
for (let index = 0;index < length; index++) {
|
|
3206
|
+
const previous = base[index];
|
|
3207
|
+
const entry = messages[index];
|
|
3208
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
3209
|
+
output.push(entry);
|
|
3210
|
+
continue;
|
|
3211
|
+
}
|
|
3212
|
+
if (previous !== undefined) {
|
|
3213
|
+
removed.push(previous);
|
|
3214
|
+
if (isSettledPendingStageMessage(previous))
|
|
3215
|
+
settled--;
|
|
3216
|
+
}
|
|
3217
|
+
if (entry === undefined)
|
|
3218
|
+
continue;
|
|
3219
|
+
if (isSettledPendingStageMessage(entry))
|
|
3220
|
+
settled++;
|
|
3221
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
3222
|
+
output.push(next);
|
|
3223
|
+
added.push(next);
|
|
3224
|
+
}
|
|
3225
|
+
let retained = output;
|
|
3226
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
3227
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
3228
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
3229
|
+
}
|
|
3230
|
+
return {
|
|
3231
|
+
messages: retained,
|
|
3232
|
+
commit: () => {
|
|
3233
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
3234
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
3235
|
+
byStage?.delete(stageId);
|
|
3236
|
+
if (byStage?.size === 0)
|
|
3237
|
+
this.stages.delete(deliveryRunId);
|
|
3238
|
+
}
|
|
3239
|
+
for (const entry of dirty)
|
|
3240
|
+
this.fixed.delete(entry);
|
|
3241
|
+
for (const entry of removed)
|
|
3242
|
+
this.unindex(entry);
|
|
3243
|
+
for (const entry of added)
|
|
3244
|
+
this.index(entry);
|
|
3245
|
+
this.compacted = retained;
|
|
3246
|
+
this.settled = settled;
|
|
3247
|
+
}
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
index(entry) {
|
|
3251
|
+
this.fixed.add(entry);
|
|
3252
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3253
|
+
let byStage = this.stages.get(delivery.runId);
|
|
3254
|
+
if (byStage === undefined) {
|
|
3255
|
+
byStage = new Map;
|
|
3256
|
+
this.stages.set(delivery.runId, byStage);
|
|
3257
|
+
}
|
|
3258
|
+
let entries = byStage.get(delivery.stageId);
|
|
3259
|
+
if (entries === undefined) {
|
|
3260
|
+
entries = new Set;
|
|
3261
|
+
byStage.set(delivery.stageId, entries);
|
|
3262
|
+
}
|
|
3263
|
+
entries.add(entry);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
unindex(entry) {
|
|
3267
|
+
this.fixed.delete(entry);
|
|
3268
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3269
|
+
const byStage = this.stages.get(delivery.runId);
|
|
3270
|
+
const entries = byStage?.get(delivery.stageId);
|
|
3271
|
+
if (entries === undefined)
|
|
3272
|
+
continue;
|
|
3273
|
+
entries.delete(entry);
|
|
3274
|
+
if (entries.size > 0)
|
|
3275
|
+
continue;
|
|
3276
|
+
byStage.delete(delivery.stageId);
|
|
3277
|
+
if (byStage.size === 0)
|
|
3278
|
+
this.stages.delete(delivery.runId);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
function isCompactingBackend(backend) {
|
|
3283
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
3284
|
+
}
|
|
3285
|
+
function createStageStatusLookup(runs) {
|
|
3286
|
+
let runById;
|
|
3287
|
+
return (runId, stageId) => {
|
|
3288
|
+
if (runById === undefined) {
|
|
3289
|
+
runById = new Map;
|
|
3290
|
+
for (const run of runs)
|
|
3291
|
+
if (!runById.has(run.id))
|
|
3292
|
+
runById.set(run.id, run);
|
|
3293
|
+
}
|
|
3294
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
3295
|
+
};
|
|
3296
|
+
}
|
|
3297
|
+
function isEndedStageStatus(status) {
|
|
3298
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
3299
|
+
}
|
|
3300
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
3301
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
3302
|
+
return entry;
|
|
3303
|
+
const latest = new Map;
|
|
3304
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
3305
|
+
latest.set(deliverySlot(delivery), index);
|
|
3306
|
+
});
|
|
3307
|
+
const deliveries = [];
|
|
3308
|
+
let archivedThrough = -1;
|
|
3309
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
3310
|
+
const delivery = entry.deliveries[index];
|
|
3311
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
3312
|
+
while (archivedThrough < index) {
|
|
3313
|
+
const prefixIndex = archivedThrough + 1;
|
|
3314
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
3315
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
3316
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
3317
|
+
messageRunId: entry.runId,
|
|
3318
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3319
|
+
});
|
|
3320
|
+
archivedThrough = prefixIndex;
|
|
3321
|
+
}
|
|
3322
|
+
} else {
|
|
3323
|
+
deliveries.push(delivery);
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
3327
|
+
}
|
|
3328
|
+
function deliverySlot(delivery) {
|
|
3329
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
3330
|
+
}
|
|
3331
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
3332
|
+
|
|
3148
3333
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
3149
3334
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
3150
|
-
const
|
|
3151
|
-
const
|
|
3152
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
3153
|
-
const result = previous.catch(() => {
|
|
3154
|
-
return;
|
|
3155
|
-
}).then(transition);
|
|
3156
|
-
const settled = result.then(() => {
|
|
3157
|
-
return;
|
|
3158
|
-
}, () => {
|
|
3159
|
-
return;
|
|
3160
|
-
});
|
|
3161
|
-
transitions.set(runId, settled);
|
|
3162
|
-
settled.finally(() => {
|
|
3163
|
-
if (transitions.get(runId) === settled)
|
|
3164
|
-
transitions.delete(runId);
|
|
3165
|
-
});
|
|
3166
|
-
return await result;
|
|
3167
|
-
};
|
|
3335
|
+
const ledgers = new PendingStageLedgers;
|
|
3336
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
3168
3337
|
return {
|
|
3169
3338
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
3170
|
-
return await
|
|
3339
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3171
3340
|
const run = context.findRun(input.runId);
|
|
3172
3341
|
if (run === undefined)
|
|
3173
3342
|
return;
|
|
3174
3343
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
3175
3344
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3176
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3345
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
3177
3346
|
if (result.ok && !result.deduplicated) {
|
|
3178
|
-
|
|
3179
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3180
|
-
live.pendingStageMessages = [...messages];
|
|
3181
|
-
context.bumpAndNotify();
|
|
3347
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3182
3348
|
}
|
|
3183
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3349
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3184
3350
|
});
|
|
3185
3351
|
},
|
|
3186
3352
|
async validateLiveStageMessage(input) {
|
|
3187
|
-
return await
|
|
3353
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
3188
3354
|
const run = context.findRun(input.runId);
|
|
3189
3355
|
if (run === undefined)
|
|
3190
3356
|
return;
|
|
3191
3357
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
3192
3358
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
3193
3359
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3194
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3360
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
3195
3361
|
if (!result.ok) {
|
|
3196
3362
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
3197
3363
|
}
|
|
@@ -3213,77 +3379,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3213
3379
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
3214
3380
|
},
|
|
3215
3381
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
3216
|
-
return await
|
|
3382
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3217
3383
|
const run = context.findRun(runId);
|
|
3218
3384
|
if (run === undefined)
|
|
3219
3385
|
return false;
|
|
3220
|
-
const current = run
|
|
3386
|
+
const current = batch.messages(run);
|
|
3221
3387
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
3222
3388
|
if (next === current)
|
|
3223
3389
|
return false;
|
|
3224
|
-
|
|
3225
|
-
const live = context.findRun(runId) ?? run;
|
|
3226
|
-
live.pendingStageMessages = [...messages];
|
|
3227
|
-
context.bumpAndNotify();
|
|
3390
|
+
await batch.apply(backend, runId, next);
|
|
3228
3391
|
return true;
|
|
3229
3392
|
});
|
|
3230
3393
|
},
|
|
3231
3394
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
3232
|
-
return await
|
|
3395
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3233
3396
|
const run = context.findRun(runId);
|
|
3234
3397
|
if (run === undefined)
|
|
3235
3398
|
return false;
|
|
3236
|
-
const current = run
|
|
3399
|
+
const current = batch.messages(run);
|
|
3237
3400
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
3238
3401
|
if (next === current)
|
|
3239
3402
|
return false;
|
|
3240
|
-
|
|
3241
|
-
const live = context.findRun(runId) ?? run;
|
|
3242
|
-
live.pendingStageMessages = [...messages];
|
|
3243
|
-
context.bumpAndNotify();
|
|
3403
|
+
await batch.apply(backend, runId, next);
|
|
3244
3404
|
return true;
|
|
3245
3405
|
});
|
|
3246
3406
|
},
|
|
3247
3407
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
3248
|
-
return await
|
|
3408
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3249
3409
|
const run = context.findRun(runId);
|
|
3250
3410
|
if (run === undefined)
|
|
3251
3411
|
return false;
|
|
3252
|
-
const current = run
|
|
3412
|
+
const current = batch.messages(run);
|
|
3253
3413
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
3254
3414
|
if (next === current)
|
|
3255
3415
|
return false;
|
|
3256
|
-
|
|
3257
|
-
const live = context.findRun(runId) ?? run;
|
|
3258
|
-
live.pendingStageMessages = [...messages];
|
|
3259
|
-
context.bumpAndNotify();
|
|
3416
|
+
await batch.apply(backend, runId, next);
|
|
3260
3417
|
return true;
|
|
3261
3418
|
});
|
|
3262
3419
|
},
|
|
3263
3420
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
3264
|
-
return await
|
|
3421
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3265
3422
|
const run = context.findRun(input.runId);
|
|
3266
3423
|
if (run === undefined)
|
|
3267
3424
|
return;
|
|
3268
3425
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3269
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
3426
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
3270
3427
|
if (result.ok && !result.deduplicated) {
|
|
3271
|
-
|
|
3272
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3273
|
-
live.pendingStageMessages = [...messages];
|
|
3274
|
-
context.bumpAndNotify();
|
|
3428
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3275
3429
|
}
|
|
3276
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3430
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3277
3431
|
});
|
|
3278
3432
|
},
|
|
3279
3433
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
3280
|
-
return await
|
|
3434
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3281
3435
|
const run = context.findRun(runId);
|
|
3282
3436
|
if (run === undefined)
|
|
3283
3437
|
return false;
|
|
3284
3438
|
if (records.length === 0)
|
|
3285
3439
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3286
|
-
let current =
|
|
3440
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3287
3441
|
const unconfirmedRecords = [];
|
|
3288
3442
|
for (const record of records) {
|
|
3289
3443
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -3291,35 +3445,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3291
3445
|
unconfirmedRecords.push(record);
|
|
3292
3446
|
}
|
|
3293
3447
|
}
|
|
3294
|
-
current =
|
|
3295
|
-
|
|
3296
|
-
run.pendingStageMessages = [...current];
|
|
3297
|
-
context.bumpAndNotify();
|
|
3298
|
-
}
|
|
3448
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
3449
|
+
batch.reconciled(run, current);
|
|
3299
3450
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
3300
3451
|
if (next === current)
|
|
3301
3452
|
return false;
|
|
3302
|
-
|
|
3303
|
-
const live = context.findRun(runId) ?? run;
|
|
3304
|
-
live.pendingStageMessages = [...messages];
|
|
3305
|
-
context.bumpAndNotify();
|
|
3453
|
+
await batch.apply(backend, runId, next);
|
|
3306
3454
|
return true;
|
|
3307
3455
|
});
|
|
3308
3456
|
},
|
|
3309
3457
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
3310
|
-
return await
|
|
3458
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3311
3459
|
const run = context.findRun(runId);
|
|
3312
3460
|
if (run === undefined)
|
|
3313
3461
|
return false;
|
|
3314
3462
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3315
|
-
const current =
|
|
3463
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3316
3464
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
3317
3465
|
if (next === current)
|
|
3318
3466
|
return false;
|
|
3319
|
-
|
|
3320
|
-
const live = context.findRun(runId) ?? run;
|
|
3321
|
-
live.pendingStageMessages = [...messages];
|
|
3322
|
-
context.bumpAndNotify();
|
|
3467
|
+
await batch.apply(backend, runId, next);
|
|
3323
3468
|
return true;
|
|
3324
3469
|
});
|
|
3325
3470
|
}
|
|
@@ -3337,78 +3482,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
3337
3482
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
3338
3483
|
};
|
|
3339
3484
|
}
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3485
|
+
|
|
3486
|
+
class PendingStageTransitionBatches {
|
|
3487
|
+
queues = new Map;
|
|
3488
|
+
context;
|
|
3489
|
+
ledgers;
|
|
3490
|
+
constructor(context, ledgers) {
|
|
3491
|
+
this.context = context;
|
|
3492
|
+
this.ledgers = ledgers;
|
|
3493
|
+
}
|
|
3494
|
+
run(runId, backend, step) {
|
|
3495
|
+
return new Promise((resolve, reject) => {
|
|
3496
|
+
const transition = {
|
|
3497
|
+
backend,
|
|
3498
|
+
step,
|
|
3499
|
+
resolve: (value2) => resolve(value2),
|
|
3500
|
+
reject
|
|
3501
|
+
};
|
|
3502
|
+
const queue = this.queues.get(runId);
|
|
3503
|
+
if (queue !== undefined) {
|
|
3504
|
+
queue.push(transition);
|
|
3505
|
+
return;
|
|
3506
|
+
}
|
|
3507
|
+
const created = [transition];
|
|
3508
|
+
this.queues.set(runId, created);
|
|
3509
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
3510
|
+
});
|
|
3511
|
+
}
|
|
3512
|
+
async drain(runId, queue) {
|
|
3513
|
+
while (queue.length > 0)
|
|
3514
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
3515
|
+
this.queues.delete(runId);
|
|
3516
|
+
}
|
|
3517
|
+
async runBatch(runId, transitions) {
|
|
3518
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
3519
|
+
const outcomes = [];
|
|
3520
|
+
for (const transition of transitions) {
|
|
3521
|
+
const changedBefore = batch.changed;
|
|
3522
|
+
try {
|
|
3523
|
+
const value2 = await transition.step(batch);
|
|
3524
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
3525
|
+
} catch (error) {
|
|
3526
|
+
outcomes.push({ ok: false, error });
|
|
3373
3527
|
}
|
|
3374
|
-
retained = compacted;
|
|
3375
3528
|
}
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3529
|
+
let failure;
|
|
3530
|
+
try {
|
|
3531
|
+
await batch.persist();
|
|
3532
|
+
} catch (error) {
|
|
3533
|
+
failure = { error };
|
|
3534
|
+
}
|
|
3535
|
+
transitions.forEach((transition, index) => {
|
|
3536
|
+
const outcome = outcomes[index];
|
|
3537
|
+
if (!outcome.ok)
|
|
3538
|
+
transition.reject(outcome.error);
|
|
3539
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
3540
|
+
transition.reject(failure.error);
|
|
3541
|
+
else
|
|
3542
|
+
transition.resolve(outcome.value);
|
|
3543
|
+
});
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
function takeBatch(queue) {
|
|
3547
|
+
let count = 1;
|
|
3548
|
+
let backend = queue[0].backend;
|
|
3549
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
3550
|
+
const next = queue[count].backend;
|
|
3551
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
3552
|
+
break;
|
|
3553
|
+
backend ??= next;
|
|
3554
|
+
count++;
|
|
3555
|
+
}
|
|
3556
|
+
return queue.splice(0, count);
|
|
3557
|
+
}
|
|
3558
|
+
|
|
3559
|
+
class PendingStageTransitionBatch {
|
|
3560
|
+
changed = false;
|
|
3561
|
+
list;
|
|
3562
|
+
pending;
|
|
3563
|
+
context;
|
|
3564
|
+
ledgers;
|
|
3565
|
+
runId;
|
|
3566
|
+
constructor(context, ledgers, runId) {
|
|
3567
|
+
this.context = context;
|
|
3568
|
+
this.ledgers = ledgers;
|
|
3569
|
+
this.runId = runId;
|
|
3570
|
+
}
|
|
3571
|
+
messages(run) {
|
|
3572
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
3573
|
+
}
|
|
3574
|
+
reconciled(run, messages) {
|
|
3575
|
+
if (this.changed) {
|
|
3576
|
+
this.list = messages;
|
|
3577
|
+
return;
|
|
3578
|
+
}
|
|
3579
|
+
if (messages !== run.pendingStageMessages) {
|
|
3580
|
+
run.pendingStageMessages = [...messages];
|
|
3581
|
+
this.context.bumpAndNotify();
|
|
3379
3582
|
}
|
|
3380
|
-
|
|
3381
|
-
|
|
3583
|
+
}
|
|
3584
|
+
async apply(backend, runId, messages) {
|
|
3585
|
+
const run = this.context.findRun(runId);
|
|
3586
|
+
try {
|
|
3587
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
3588
|
+
this.list = compacted.messages;
|
|
3589
|
+
this.changed = true;
|
|
3590
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
3591
|
+
return compacted.messages;
|
|
3592
|
+
} catch (error) {
|
|
3593
|
+
if (this.changed && this.list !== undefined) {
|
|
3594
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
3595
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
3596
|
+
} else
|
|
3597
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
3598
|
+
throw error;
|
|
3382
3599
|
}
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3600
|
+
}
|
|
3601
|
+
async persist() {
|
|
3602
|
+
if (this.pending === undefined)
|
|
3603
|
+
return;
|
|
3604
|
+
const { backend, messages, commit, run } = this.pending;
|
|
3605
|
+
const published = this.list ?? messages;
|
|
3606
|
+
try {
|
|
3607
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
3608
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
3392
3609
|
}
|
|
3393
|
-
|
|
3610
|
+
commit?.();
|
|
3611
|
+
} finally {
|
|
3612
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
3613
|
+
}
|
|
3614
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
3615
|
+
if (live !== undefined)
|
|
3616
|
+
live.pendingStageMessages = [...published];
|
|
3617
|
+
this.context.bumpAndNotify();
|
|
3618
|
+
}
|
|
3619
|
+
}
|
|
3620
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
3621
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
3622
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
3623
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
3624
|
+
if (isCompactingBackend(backend)) {
|
|
3625
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
3626
|
+
}
|
|
3627
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
3628
|
+
if (archiveSettled !== undefined)
|
|
3629
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
3630
|
+
if (run !== undefined)
|
|
3631
|
+
ledgers.reset(run);
|
|
3632
|
+
return { messages: retained };
|
|
3633
|
+
}
|
|
3634
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
3635
|
+
const current = context.findRun(runId);
|
|
3636
|
+
if (current === undefined)
|
|
3637
|
+
return;
|
|
3638
|
+
const messages = current.pendingStageMessages ?? [];
|
|
3639
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
3640
|
+
if (recovered !== messages) {
|
|
3641
|
+
current.pendingStageMessages = [...recovered];
|
|
3642
|
+
context.bumpAndNotify();
|
|
3394
3643
|
}
|
|
3395
3644
|
}
|
|
3396
|
-
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
return
|
|
3406
|
-
}
|
|
3407
|
-
|
|
3645
|
+
|
|
3646
|
+
class PendingStageLedgers {
|
|
3647
|
+
ledgers = new WeakMap;
|
|
3648
|
+
for(run) {
|
|
3649
|
+
let ledger = this.ledgers.get(run);
|
|
3650
|
+
if (ledger === undefined) {
|
|
3651
|
+
ledger = new PendingStageTransitionLedger;
|
|
3652
|
+
this.ledgers.set(run, ledger);
|
|
3653
|
+
}
|
|
3654
|
+
return ledger;
|
|
3655
|
+
}
|
|
3656
|
+
reset(run) {
|
|
3657
|
+
this.ledgers.delete(run);
|
|
3658
|
+
}
|
|
3408
3659
|
}
|
|
3660
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
3409
3661
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
3410
3662
|
init_run_owner_backend();
|
|
3411
3663
|
init_pending_stage_delivery();
|
|
3664
|
+
init_pending_stage_transition_ledger();
|
|
3412
3665
|
});
|
|
3413
3666
|
|
|
3414
3667
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -4998,6 +5251,204 @@ var init_resume_eligibility = () => {};
|
|
|
4998
5251
|
// dist/builtin/workflows/src/durable/types.ts
|
|
4999
5252
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
5000
5253
|
|
|
5254
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5255
|
+
function immutableStageGroupError(stages) {
|
|
5256
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5257
|
+
if (occurrenceError !== undefined)
|
|
5258
|
+
return occurrenceError;
|
|
5259
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
5260
|
+
for (const group of byReplayKey.values()) {
|
|
5261
|
+
const firstName = group[0]?.name;
|
|
5262
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
5263
|
+
return "stage name drift within one replay-key group";
|
|
5264
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5265
|
+
const first = topologies[0];
|
|
5266
|
+
if (first === undefined)
|
|
5267
|
+
continue;
|
|
5268
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5269
|
+
return "immutable stage identity drift within one replay-key group";
|
|
5270
|
+
}
|
|
5271
|
+
}
|
|
5272
|
+
return;
|
|
5273
|
+
}
|
|
5274
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5275
|
+
return directChildTopologyErrorFromRecords({
|
|
5276
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
5277
|
+
direct: (claiming) => {
|
|
5278
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5279
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5280
|
+
}
|
|
5281
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
5282
|
+
}
|
|
5283
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5284
|
+
const prefix = `${replayScope}:`;
|
|
5285
|
+
const claiming = records.claiming;
|
|
5286
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5287
|
+
return "child run topology exists outside its durable invocation scope";
|
|
5288
|
+
}
|
|
5289
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5290
|
+
return "child checkpoint ownership is not reciprocal";
|
|
5291
|
+
}
|
|
5292
|
+
const direct = records.direct(claiming);
|
|
5293
|
+
const immutableError = immutableStageGroupError(direct);
|
|
5294
|
+
if (immutableError !== undefined)
|
|
5295
|
+
return immutableError;
|
|
5296
|
+
const merged = [];
|
|
5297
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5298
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5299
|
+
if (topology === undefined)
|
|
5300
|
+
return "direct child stage group lacks owning-run topology";
|
|
5301
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5302
|
+
return "direct child stage topology lacks current source identity";
|
|
5303
|
+
}
|
|
5304
|
+
if (topology.stageId.length === 0)
|
|
5305
|
+
return "direct child stage id is empty";
|
|
5306
|
+
merged.push(topology);
|
|
5307
|
+
}
|
|
5308
|
+
const ids = new Set(allowedNodeIds);
|
|
5309
|
+
const stageIds = new Set;
|
|
5310
|
+
const orders = new Set;
|
|
5311
|
+
for (const topology of merged) {
|
|
5312
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5313
|
+
return "direct child stage ids are not unique";
|
|
5314
|
+
stageIds.add(topology.stageId);
|
|
5315
|
+
ids.add(topology.stageId);
|
|
5316
|
+
if (topology.sourceOrder !== undefined) {
|
|
5317
|
+
if (orders.has(topology.sourceOrder))
|
|
5318
|
+
return "direct child source orders are not unique";
|
|
5319
|
+
orders.add(topology.sourceOrder);
|
|
5320
|
+
}
|
|
5321
|
+
}
|
|
5322
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5323
|
+
return "direct child stage references an unknown parent";
|
|
5324
|
+
}
|
|
5325
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5326
|
+
}
|
|
5327
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
5328
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5329
|
+
const seen = new Map;
|
|
5330
|
+
for (const start of starts) {
|
|
5331
|
+
const stageId = start.topology.stageId;
|
|
5332
|
+
if (stageId.length === 0)
|
|
5333
|
+
return "owning-run boundary stage id is empty";
|
|
5334
|
+
const prior = seen.get(stageId);
|
|
5335
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
5336
|
+
return "owning-run boundary stage id is aliased";
|
|
5337
|
+
seen.set(stageId, start.replayKey);
|
|
5338
|
+
}
|
|
5339
|
+
return;
|
|
5340
|
+
}
|
|
5341
|
+
function sameRun(actual, expected) {
|
|
5342
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5343
|
+
}
|
|
5344
|
+
function sameStrings(left, right) {
|
|
5345
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5346
|
+
}
|
|
5347
|
+
function sameOptionalRun(actual, expected) {
|
|
5348
|
+
if (actual === undefined || expected === undefined)
|
|
5349
|
+
return actual === expected;
|
|
5350
|
+
return sameRun(actual, expected);
|
|
5351
|
+
}
|
|
5352
|
+
function promptOccurrenceIdentityError(stages) {
|
|
5353
|
+
const replayByOccurrence = new Map;
|
|
5354
|
+
const occurrenceByReplayStage = new Map;
|
|
5355
|
+
const stageByReplayOccurrence = new Map;
|
|
5356
|
+
for (const stage of stages) {
|
|
5357
|
+
const topology = stage.topology;
|
|
5358
|
+
const occurrence = topology?.occurrenceKey;
|
|
5359
|
+
if (topology === undefined || occurrence === undefined)
|
|
5360
|
+
continue;
|
|
5361
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5362
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5363
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
5364
|
+
}
|
|
5365
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5366
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5367
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5368
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5369
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
5370
|
+
}
|
|
5371
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5372
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5373
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5374
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5375
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
5376
|
+
}
|
|
5377
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5378
|
+
}
|
|
5379
|
+
return;
|
|
5380
|
+
}
|
|
5381
|
+
function durableStageGroupKey(stage) {
|
|
5382
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
5383
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5384
|
+
}
|
|
5385
|
+
function groupByDurableStageKey(stages) {
|
|
5386
|
+
const groups = new Map;
|
|
5387
|
+
for (const stage of stages) {
|
|
5388
|
+
const key = durableStageGroupKey(stage);
|
|
5389
|
+
const group = groups.get(key) ?? [];
|
|
5390
|
+
group.push(stage);
|
|
5391
|
+
groups.set(key, group);
|
|
5392
|
+
}
|
|
5393
|
+
return groups;
|
|
5394
|
+
}
|
|
5395
|
+
function compositeKey(domain, ...parts) {
|
|
5396
|
+
return JSON.stringify([domain, ...parts]);
|
|
5397
|
+
}
|
|
5398
|
+
function hasCycle(topologies) {
|
|
5399
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5400
|
+
const visiting = new Set;
|
|
5401
|
+
const visited = new Set;
|
|
5402
|
+
const cyclic = (stageId) => {
|
|
5403
|
+
if (visiting.has(stageId))
|
|
5404
|
+
return true;
|
|
5405
|
+
if (visited.has(stageId))
|
|
5406
|
+
return false;
|
|
5407
|
+
visiting.add(stageId);
|
|
5408
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5409
|
+
return true;
|
|
5410
|
+
visiting.delete(stageId);
|
|
5411
|
+
visited.add(stageId);
|
|
5412
|
+
return false;
|
|
5413
|
+
};
|
|
5414
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5415
|
+
}
|
|
5416
|
+
var init_stage_topology_validation = () => {};
|
|
5417
|
+
|
|
5418
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5419
|
+
function parseWorkflowChildResult(value2) {
|
|
5420
|
+
if (!isOrdinaryObject(value2))
|
|
5421
|
+
return;
|
|
5422
|
+
const workflow = value2.workflow;
|
|
5423
|
+
const runId = value2.runId;
|
|
5424
|
+
const status = value2.status;
|
|
5425
|
+
const exited = value2.exited;
|
|
5426
|
+
const outputs = value2.outputs;
|
|
5427
|
+
const exitReason = value2.exitReason;
|
|
5428
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5429
|
+
return;
|
|
5430
|
+
if (exited === false) {
|
|
5431
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
5432
|
+
return;
|
|
5433
|
+
return value2;
|
|
5434
|
+
}
|
|
5435
|
+
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5436
|
+
return;
|
|
5437
|
+
return value2;
|
|
5438
|
+
}
|
|
5439
|
+
function isWorkflowChildResult(value2) {
|
|
5440
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
5441
|
+
}
|
|
5442
|
+
function isWorkflowExitStatus(value2) {
|
|
5443
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5444
|
+
}
|
|
5445
|
+
function isOrdinaryObject(value2) {
|
|
5446
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5447
|
+
return false;
|
|
5448
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
5449
|
+
return prototype === Object.prototype || prototype === null;
|
|
5450
|
+
}
|
|
5451
|
+
|
|
5001
5452
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
5002
5453
|
import { createHash } from "node:crypto";
|
|
5003
5454
|
function durableHash(value2) {
|
|
@@ -5064,6 +5515,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
5064
5515
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
5065
5516
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
5066
5517
|
}
|
|
5518
|
+
|
|
5519
|
+
class StageReceipts {
|
|
5520
|
+
exact = new Map;
|
|
5521
|
+
anySession = new Map;
|
|
5522
|
+
has(logicalRunId, sessionId, admission) {
|
|
5523
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
5524
|
+
}
|
|
5525
|
+
hasAnySession(logicalRunId, admission) {
|
|
5526
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
5527
|
+
}
|
|
5528
|
+
add(logicalRunId, sessionId, admission) {
|
|
5529
|
+
let bySession = this.exact.get(logicalRunId);
|
|
5530
|
+
if (bySession === undefined) {
|
|
5531
|
+
bySession = new Map;
|
|
5532
|
+
this.exact.set(logicalRunId, bySession);
|
|
5533
|
+
}
|
|
5534
|
+
let admissions = bySession.get(sessionId);
|
|
5535
|
+
if (admissions === undefined) {
|
|
5536
|
+
admissions = new Set;
|
|
5537
|
+
bySession.set(sessionId, admissions);
|
|
5538
|
+
}
|
|
5539
|
+
if (admissions.has(admission))
|
|
5540
|
+
return false;
|
|
5541
|
+
admissions.add(admission);
|
|
5542
|
+
return true;
|
|
5543
|
+
}
|
|
5544
|
+
addAnySession(logicalRunId, admission) {
|
|
5545
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
5546
|
+
if (admissions === undefined) {
|
|
5547
|
+
admissions = new Set;
|
|
5548
|
+
this.anySession.set(logicalRunId, admissions);
|
|
5549
|
+
}
|
|
5550
|
+
if (admissions.has(admission))
|
|
5551
|
+
return false;
|
|
5552
|
+
admissions.add(admission);
|
|
5553
|
+
return true;
|
|
5554
|
+
}
|
|
5555
|
+
}
|
|
5556
|
+
|
|
5557
|
+
class PendingStageReceiptIndex {
|
|
5558
|
+
workflows = new Map;
|
|
5559
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
5560
|
+
let workflow = this.workflows.get(workflowId);
|
|
5561
|
+
if (workflow === undefined) {
|
|
5562
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5563
|
+
this.workflows.set(workflowId, workflow);
|
|
5564
|
+
}
|
|
5565
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
5566
|
+
if (byRun === undefined) {
|
|
5567
|
+
byRun = new Map;
|
|
5568
|
+
workflow.byMessage.set(messageId, byRun);
|
|
5569
|
+
}
|
|
5570
|
+
let byStage = byRun.get(delivery.runId);
|
|
5571
|
+
if (byStage === undefined) {
|
|
5572
|
+
byStage = new Map;
|
|
5573
|
+
byRun.set(delivery.runId, byStage);
|
|
5574
|
+
}
|
|
5575
|
+
let receipts = byStage.get(delivery.stageId);
|
|
5576
|
+
if (receipts === undefined) {
|
|
5577
|
+
receipts = new StageReceipts;
|
|
5578
|
+
byStage.set(delivery.stageId, receipts);
|
|
5579
|
+
}
|
|
5580
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
5581
|
+
let changed = added;
|
|
5582
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
5583
|
+
changed = true;
|
|
5584
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
5585
|
+
changed = true;
|
|
5586
|
+
if (changed)
|
|
5587
|
+
workflow.log.push(messageId);
|
|
5588
|
+
return added;
|
|
5589
|
+
}
|
|
5590
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
5591
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
5592
|
+
if (receipts === undefined)
|
|
5593
|
+
return false;
|
|
5594
|
+
if (anySession && delivery.admission === "context")
|
|
5595
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
5596
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
5597
|
+
}
|
|
5598
|
+
changesSince(workflowId, cursor) {
|
|
5599
|
+
let workflow = this.workflows.get(workflowId);
|
|
5600
|
+
if (workflow === undefined) {
|
|
5601
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5602
|
+
this.workflows.set(workflowId, workflow);
|
|
5603
|
+
}
|
|
5604
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
5605
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
5606
|
+
return { cursor: next, messageIds: undefined };
|
|
5607
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
5608
|
+
}
|
|
5609
|
+
delete(workflowId) {
|
|
5610
|
+
this.workflows.delete(workflowId);
|
|
5611
|
+
}
|
|
5612
|
+
clear() {
|
|
5613
|
+
this.workflows.clear();
|
|
5614
|
+
}
|
|
5615
|
+
}
|
|
5616
|
+
function nextVersion() {
|
|
5617
|
+
lastVersion += 1;
|
|
5618
|
+
return lastVersion;
|
|
5619
|
+
}
|
|
5620
|
+
|
|
5621
|
+
class NestedSnapshotVersion {
|
|
5622
|
+
rootWorkflowId;
|
|
5623
|
+
current = nextVersion();
|
|
5624
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
5625
|
+
replayGroups = new Map;
|
|
5626
|
+
boundaryReplayKeys = new Set;
|
|
5627
|
+
constructor(rootWorkflowId) {
|
|
5628
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
5629
|
+
}
|
|
5630
|
+
get version() {
|
|
5631
|
+
return this.current;
|
|
5632
|
+
}
|
|
5633
|
+
record(checkpoint) {
|
|
5634
|
+
if (!this.isInert(checkpoint))
|
|
5635
|
+
this.current = nextVersion();
|
|
5636
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
5637
|
+
if (checkpoint.kind !== "stage")
|
|
5638
|
+
return;
|
|
5639
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
5640
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
5641
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
5642
|
+
return;
|
|
5643
|
+
const identity = stageIdentity(checkpoint);
|
|
5644
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
5645
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5646
|
+
if (summary === undefined) {
|
|
5647
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
5648
|
+
name: checkpoint.name,
|
|
5649
|
+
identity,
|
|
5650
|
+
consistent: true,
|
|
5651
|
+
hasOutput: checkpoint.output !== undefined,
|
|
5652
|
+
hasExitedChildOutput: exited
|
|
5653
|
+
});
|
|
5654
|
+
return;
|
|
5655
|
+
}
|
|
5656
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
5657
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
5658
|
+
summary.hasExitedChildOutput ||= exited;
|
|
5659
|
+
}
|
|
5660
|
+
isInert(checkpoint) {
|
|
5661
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
5662
|
+
return false;
|
|
5663
|
+
}
|
|
5664
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5665
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
5666
|
+
return false;
|
|
5667
|
+
}
|
|
5668
|
+
const id = checkpoint.checkpointId;
|
|
5669
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
5670
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
5671
|
+
return false;
|
|
5672
|
+
}
|
|
5673
|
+
return true;
|
|
5674
|
+
}
|
|
5675
|
+
isRootOwned(checkpoint) {
|
|
5676
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
5677
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
5678
|
+
}
|
|
5679
|
+
}
|
|
5680
|
+
function stageIdentity(checkpoint) {
|
|
5681
|
+
const topology = checkpoint.topology;
|
|
5682
|
+
return JSON.stringify([
|
|
5683
|
+
durableStageGroupKey(checkpoint),
|
|
5684
|
+
topology?.version,
|
|
5685
|
+
topology?.stageId,
|
|
5686
|
+
topology?.parentIds,
|
|
5687
|
+
topology?.sourceOrder,
|
|
5688
|
+
topology?.run === undefined ? undefined : [
|
|
5689
|
+
topology.run.runId,
|
|
5690
|
+
topology.run.runName,
|
|
5691
|
+
topology.run.parentRunId,
|
|
5692
|
+
topology.run.parentStageId,
|
|
5693
|
+
topology.run.rootRunId
|
|
5694
|
+
]
|
|
5695
|
+
]);
|
|
5696
|
+
}
|
|
5067
5697
|
function checkpointKey(c) {
|
|
5068
5698
|
return `${c.kind}:${c.checkpointId}`;
|
|
5069
5699
|
}
|
|
@@ -5075,7 +5705,7 @@ class JournalProjection {
|
|
|
5075
5705
|
workflows = new Map;
|
|
5076
5706
|
deletedWorkflowIds = new Set;
|
|
5077
5707
|
pendingReceipts = new Map;
|
|
5078
|
-
stickyReceipts = new
|
|
5708
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
5079
5709
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
5080
5710
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
5081
5711
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -5123,7 +5753,8 @@ class JournalProjection {
|
|
|
5123
5753
|
toolByHash: new Map,
|
|
5124
5754
|
uiByHash: new Map,
|
|
5125
5755
|
stageOutputByReplayKey: new Map,
|
|
5126
|
-
stageSessionByReplayKey: new Map
|
|
5756
|
+
stageSessionByReplayKey: new Map,
|
|
5757
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
5127
5758
|
});
|
|
5128
5759
|
}
|
|
5129
5760
|
applyWorkflowSnapshot(handle) {
|
|
@@ -5149,39 +5780,43 @@ class JournalProjection {
|
|
|
5149
5780
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
5150
5781
|
}
|
|
5151
5782
|
stickyDeliveryCounts = new Map;
|
|
5783
|
+
stickyDeliveryCountVersions = new Map;
|
|
5152
5784
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
5153
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
5785
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
5786
|
+
}
|
|
5787
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
5788
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
5789
|
+
}
|
|
5790
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
5791
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
5154
5792
|
}
|
|
5155
5793
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
5156
|
-
|
|
5157
|
-
if (
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
5165
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
5166
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
5167
|
-
if (accounting !== undefined && !existing) {
|
|
5168
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
5794
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
5795
|
+
if (accounting !== undefined && added) {
|
|
5796
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
5797
|
+
if (byRun === undefined) {
|
|
5798
|
+
byRun = new Map;
|
|
5799
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
5800
|
+
}
|
|
5801
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
5169
5802
|
if (counts === undefined) {
|
|
5170
5803
|
counts = new Map;
|
|
5171
|
-
|
|
5804
|
+
byRun.set(accounting.messageRunId, counts);
|
|
5172
5805
|
}
|
|
5173
|
-
const
|
|
5174
|
-
counts.set(
|
|
5806
|
+
const previous = counts.get(messageId) ?? 0;
|
|
5807
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
5808
|
+
if (accounting.deliveryCount > previous)
|
|
5809
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5175
5810
|
}
|
|
5176
5811
|
}
|
|
5177
5812
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
5178
|
-
return
|
|
5813
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
5179
5814
|
}
|
|
5180
5815
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
5181
|
-
return
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5816
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
5817
|
+
}
|
|
5818
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
5819
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
5185
5820
|
}
|
|
5186
5821
|
recordCheckpoint(recorded) {
|
|
5187
5822
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -5200,6 +5835,7 @@ class JournalProjection {
|
|
|
5200
5835
|
if (rec.checkpoints.has(key))
|
|
5201
5836
|
return;
|
|
5202
5837
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
5838
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
5203
5839
|
if (currentCheckpoint.kind === "tool") {
|
|
5204
5840
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
5205
5841
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -5257,6 +5893,9 @@ class JournalProjection {
|
|
|
5257
5893
|
getWorkflow(workflowId) {
|
|
5258
5894
|
return this.workflows.get(workflowId)?.handle;
|
|
5259
5895
|
}
|
|
5896
|
+
nestedSnapshotVersion(workflowId) {
|
|
5897
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
5898
|
+
}
|
|
5260
5899
|
getLoadableWorkflow(workflowId) {
|
|
5261
5900
|
return this.getWorkflow(workflowId);
|
|
5262
5901
|
}
|
|
@@ -5334,6 +5973,7 @@ class JournalProjection {
|
|
|
5334
5973
|
this.pendingReceipts.delete(workflowId);
|
|
5335
5974
|
this.stickyReceipts.delete(workflowId);
|
|
5336
5975
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
5976
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5337
5977
|
this.deletedWorkflowIds.add(workflowId);
|
|
5338
5978
|
}
|
|
5339
5979
|
isWorkflowLoadable(workflowId) {
|
|
@@ -5344,6 +5984,8 @@ class JournalProjection {
|
|
|
5344
5984
|
this.pendingReceipts.clear();
|
|
5345
5985
|
this.stickyReceipts.clear();
|
|
5346
5986
|
this.stickyDeliveryCounts.clear();
|
|
5987
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
5988
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5347
5989
|
this.deletedWorkflowIds.clear();
|
|
5348
5990
|
}
|
|
5349
5991
|
}
|
|
@@ -5400,13 +6042,10 @@ function toResumableEntry(handle) {
|
|
|
5400
6042
|
updatedAt: handle.updatedAt
|
|
5401
6043
|
};
|
|
5402
6044
|
}
|
|
5403
|
-
|
|
5404
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
5405
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
5406
|
-
}
|
|
5407
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
6045
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
5408
6046
|
var init_backend = __esm(() => {
|
|
5409
6047
|
init_resume_eligibility();
|
|
6048
|
+
init_stage_topology_validation();
|
|
5410
6049
|
init_durable_hash();
|
|
5411
6050
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
5412
6051
|
});
|
|
@@ -5490,205 +6129,24 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
5490
6129
|
}
|
|
5491
6130
|
var init_stage_session_store = () => {};
|
|
5492
6131
|
|
|
5493
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5494
|
-
function immutableStageGroupError(stages) {
|
|
5495
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5496
|
-
if (occurrenceError !== undefined)
|
|
5497
|
-
return occurrenceError;
|
|
5498
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
5499
|
-
for (const group of byReplayKey.values()) {
|
|
5500
|
-
const firstName = group[0]?.name;
|
|
5501
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
5502
|
-
return "stage name drift within one replay-key group";
|
|
5503
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5504
|
-
const first = topologies[0];
|
|
5505
|
-
if (first === undefined)
|
|
5506
|
-
continue;
|
|
5507
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5508
|
-
return "immutable stage identity drift within one replay-key group";
|
|
5509
|
-
}
|
|
5510
|
-
}
|
|
5511
|
-
return;
|
|
5512
|
-
}
|
|
5513
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5514
|
-
const prefix = `${replayScope}:`;
|
|
5515
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
5516
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5517
|
-
return "child run topology exists outside its durable invocation scope";
|
|
5518
|
-
}
|
|
5519
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5520
|
-
return "child checkpoint ownership is not reciprocal";
|
|
5521
|
-
}
|
|
5522
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5523
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5524
|
-
const immutableError = immutableStageGroupError(direct);
|
|
5525
|
-
if (immutableError !== undefined)
|
|
5526
|
-
return immutableError;
|
|
5527
|
-
const merged = [];
|
|
5528
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5529
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5530
|
-
if (topology === undefined)
|
|
5531
|
-
return "direct child stage group lacks owning-run topology";
|
|
5532
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5533
|
-
return "direct child stage topology lacks current source identity";
|
|
5534
|
-
}
|
|
5535
|
-
if (topology.stageId.length === 0)
|
|
5536
|
-
return "direct child stage id is empty";
|
|
5537
|
-
merged.push(topology);
|
|
5538
|
-
}
|
|
5539
|
-
const ids = new Set(allowedNodeIds);
|
|
5540
|
-
const stageIds = new Set;
|
|
5541
|
-
const orders = new Set;
|
|
5542
|
-
for (const topology of merged) {
|
|
5543
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5544
|
-
return "direct child stage ids are not unique";
|
|
5545
|
-
stageIds.add(topology.stageId);
|
|
5546
|
-
ids.add(topology.stageId);
|
|
5547
|
-
if (topology.sourceOrder !== undefined) {
|
|
5548
|
-
if (orders.has(topology.sourceOrder))
|
|
5549
|
-
return "direct child source orders are not unique";
|
|
5550
|
-
orders.add(topology.sourceOrder);
|
|
5551
|
-
}
|
|
5552
|
-
}
|
|
5553
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5554
|
-
return "direct child stage references an unknown parent";
|
|
5555
|
-
}
|
|
5556
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5557
|
-
}
|
|
5558
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
5559
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5560
|
-
const seen = new Map;
|
|
5561
|
-
for (const start of starts) {
|
|
5562
|
-
const stageId = start.topology.stageId;
|
|
5563
|
-
if (stageId.length === 0)
|
|
5564
|
-
return "owning-run boundary stage id is empty";
|
|
5565
|
-
const prior = seen.get(stageId);
|
|
5566
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
5567
|
-
return "owning-run boundary stage id is aliased";
|
|
5568
|
-
seen.set(stageId, start.replayKey);
|
|
5569
|
-
}
|
|
5570
|
-
return;
|
|
5571
|
-
}
|
|
5572
|
-
function sameRun(actual, expected) {
|
|
5573
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5574
|
-
}
|
|
5575
|
-
function sameStrings(left, right) {
|
|
5576
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5577
|
-
}
|
|
5578
|
-
function sameOptionalRun(actual, expected) {
|
|
5579
|
-
if (actual === undefined || expected === undefined)
|
|
5580
|
-
return actual === expected;
|
|
5581
|
-
return sameRun(actual, expected);
|
|
5582
|
-
}
|
|
5583
|
-
function promptOccurrenceIdentityError(stages) {
|
|
5584
|
-
const replayByOccurrence = new Map;
|
|
5585
|
-
const occurrenceByReplayStage = new Map;
|
|
5586
|
-
const stageByReplayOccurrence = new Map;
|
|
5587
|
-
for (const stage of stages) {
|
|
5588
|
-
const topology = stage.topology;
|
|
5589
|
-
const occurrence = topology?.occurrenceKey;
|
|
5590
|
-
if (topology === undefined || occurrence === undefined)
|
|
5591
|
-
continue;
|
|
5592
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5593
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5594
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
5595
|
-
}
|
|
5596
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5597
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5598
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5599
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5600
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
5601
|
-
}
|
|
5602
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5603
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5604
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5605
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5606
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
5607
|
-
}
|
|
5608
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5609
|
-
}
|
|
5610
|
-
return;
|
|
5611
|
-
}
|
|
5612
|
-
function durableStageGroupKey(stage) {
|
|
5613
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
5614
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5615
|
-
}
|
|
5616
|
-
function groupByDurableStageKey(stages) {
|
|
5617
|
-
const groups = new Map;
|
|
5618
|
-
for (const stage of stages) {
|
|
5619
|
-
const key = durableStageGroupKey(stage);
|
|
5620
|
-
const group = groups.get(key) ?? [];
|
|
5621
|
-
group.push(stage);
|
|
5622
|
-
groups.set(key, group);
|
|
5623
|
-
}
|
|
5624
|
-
return groups;
|
|
5625
|
-
}
|
|
5626
|
-
function compositeKey(domain, ...parts) {
|
|
5627
|
-
return JSON.stringify([domain, ...parts]);
|
|
5628
|
-
}
|
|
5629
|
-
function hasCycle(topologies) {
|
|
5630
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5631
|
-
const visiting = new Set;
|
|
5632
|
-
const visited = new Set;
|
|
5633
|
-
const cyclic = (stageId) => {
|
|
5634
|
-
if (visiting.has(stageId))
|
|
5635
|
-
return true;
|
|
5636
|
-
if (visited.has(stageId))
|
|
5637
|
-
return false;
|
|
5638
|
-
visiting.add(stageId);
|
|
5639
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5640
|
-
return true;
|
|
5641
|
-
visiting.delete(stageId);
|
|
5642
|
-
visited.add(stageId);
|
|
5643
|
-
return false;
|
|
5644
|
-
};
|
|
5645
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5646
|
-
}
|
|
5647
|
-
var init_stage_topology_validation = () => {};
|
|
5648
|
-
|
|
5649
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5650
|
-
function parseWorkflowChildResult(value2) {
|
|
5651
|
-
if (!isOrdinaryObject(value2))
|
|
5652
|
-
return;
|
|
5653
|
-
const workflow = value2.workflow;
|
|
5654
|
-
const runId = value2.runId;
|
|
5655
|
-
const status = value2.status;
|
|
5656
|
-
const exited = value2.exited;
|
|
5657
|
-
const outputs = value2.outputs;
|
|
5658
|
-
const exitReason = value2.exitReason;
|
|
5659
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5660
|
-
return;
|
|
5661
|
-
if (exited === false) {
|
|
5662
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
5663
|
-
return;
|
|
5664
|
-
return value2;
|
|
5665
|
-
}
|
|
5666
|
-
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5667
|
-
return;
|
|
5668
|
-
return value2;
|
|
5669
|
-
}
|
|
5670
|
-
function isWorkflowChildResult(value2) {
|
|
5671
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
5672
|
-
}
|
|
5673
|
-
function isWorkflowExitStatus(value2) {
|
|
5674
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5675
|
-
}
|
|
5676
|
-
function isOrdinaryObject(value2) {
|
|
5677
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5678
|
-
return false;
|
|
5679
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
5680
|
-
return prototype === Object.prototype || prototype === null;
|
|
5681
|
-
}
|
|
5682
|
-
|
|
5683
6132
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
5684
6133
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
5685
6134
|
const startTopology = start.topology;
|
|
5686
6135
|
const startBoundary = startTopology?.boundary;
|
|
5687
|
-
const startRun = startTopology?.run;
|
|
5688
6136
|
const related = stages.filter((stage) => {
|
|
5689
6137
|
const boundary = stage.topology?.boundary;
|
|
5690
6138
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
5691
6139
|
});
|
|
6140
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
6141
|
+
related,
|
|
6142
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
6143
|
+
}, requireTerminal);
|
|
6144
|
+
}
|
|
6145
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
6146
|
+
const startTopology = start.topology;
|
|
6147
|
+
const startBoundary = startTopology?.boundary;
|
|
6148
|
+
const startRun = startTopology?.run;
|
|
6149
|
+
const related = records.related;
|
|
5692
6150
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
5693
6151
|
if (starts.length !== 1 || starts[0] !== start)
|
|
5694
6152
|
return failure("boundary has no unique start record");
|
|
@@ -5725,7 +6183,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
5725
6183
|
return failure("boundary has an illegal terminal transition");
|
|
5726
6184
|
}
|
|
5727
6185
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
5728
|
-
const outputRecords =
|
|
6186
|
+
const outputRecords = records.outputRecords();
|
|
5729
6187
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
5730
6188
|
return failure("boundary replay key has conflicting output records");
|
|
5731
6189
|
}
|
|
@@ -5755,10 +6213,120 @@ var init_boundary_lifecycle = __esm(() => {
|
|
|
5755
6213
|
init_stage_topology_validation();
|
|
5756
6214
|
});
|
|
5757
6215
|
|
|
6216
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
6217
|
+
class CheckpointCatalogIndex {
|
|
6218
|
+
checkpoints;
|
|
6219
|
+
stages;
|
|
6220
|
+
positions = new Map;
|
|
6221
|
+
groupKeys = new Map;
|
|
6222
|
+
groups = new Map;
|
|
6223
|
+
stagesByRunId = new Map;
|
|
6224
|
+
boundariesByReplayKey = new Map;
|
|
6225
|
+
boundariesByStageId = new Map;
|
|
6226
|
+
boundariesByChildRunId = new Map;
|
|
6227
|
+
outputsByReplayKey = new Map;
|
|
6228
|
+
evidence;
|
|
6229
|
+
constructor(checkpoints) {
|
|
6230
|
+
this.checkpoints = checkpoints;
|
|
6231
|
+
const stages = [];
|
|
6232
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
6233
|
+
if (!this.positions.has(checkpoint))
|
|
6234
|
+
this.positions.set(checkpoint, index);
|
|
6235
|
+
if (checkpoint.kind !== "stage")
|
|
6236
|
+
return;
|
|
6237
|
+
stages.push(checkpoint);
|
|
6238
|
+
const key = durableStageGroupKey(checkpoint);
|
|
6239
|
+
this.groupKeys.set(checkpoint, key);
|
|
6240
|
+
append(this.groups, key, checkpoint);
|
|
6241
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
6242
|
+
if (runId !== undefined)
|
|
6243
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
6244
|
+
if (checkpoint.output !== undefined)
|
|
6245
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6246
|
+
const boundary = checkpoint.topology?.boundary;
|
|
6247
|
+
if (boundary === undefined)
|
|
6248
|
+
return;
|
|
6249
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6250
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
6251
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
6252
|
+
});
|
|
6253
|
+
this.stages = stages;
|
|
6254
|
+
}
|
|
6255
|
+
sequence(checkpoint) {
|
|
6256
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
6257
|
+
}
|
|
6258
|
+
stageGroups() {
|
|
6259
|
+
return this.groups.values();
|
|
6260
|
+
}
|
|
6261
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
6262
|
+
const topology = start.topology;
|
|
6263
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
6264
|
+
const related = this.inCheckpointOrder([
|
|
6265
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
6266
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
6267
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
6268
|
+
]);
|
|
6269
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
6270
|
+
}
|
|
6271
|
+
directChildRecords(childRunId) {
|
|
6272
|
+
return {
|
|
6273
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
6274
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
6275
|
+
};
|
|
6276
|
+
}
|
|
6277
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
6278
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
6279
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
6280
|
+
}
|
|
6281
|
+
inCheckpointOrder(lists) {
|
|
6282
|
+
const present = lists.filter((list) => list !== undefined);
|
|
6283
|
+
if (present.length === 1)
|
|
6284
|
+
return present[0];
|
|
6285
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
6286
|
+
}
|
|
6287
|
+
}
|
|
6288
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
6289
|
+
const runIds = new Set;
|
|
6290
|
+
const identities = [];
|
|
6291
|
+
for (const checkpoint of checkpoints) {
|
|
6292
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
6293
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
6294
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
6295
|
+
}
|
|
6296
|
+
}
|
|
6297
|
+
identities.sort(compareCodeUnits);
|
|
6298
|
+
return { runIds, identities };
|
|
6299
|
+
}
|
|
6300
|
+
function hasPrefix(sorted, prefix) {
|
|
6301
|
+
let low = 0;
|
|
6302
|
+
let high = sorted.length;
|
|
6303
|
+
while (low < high) {
|
|
6304
|
+
const middle = low + high >>> 1;
|
|
6305
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
6306
|
+
low = middle + 1;
|
|
6307
|
+
else
|
|
6308
|
+
high = middle;
|
|
6309
|
+
}
|
|
6310
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
6311
|
+
}
|
|
6312
|
+
function compareCodeUnits(left, right) {
|
|
6313
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
6314
|
+
}
|
|
6315
|
+
function append(map, key, value2) {
|
|
6316
|
+
const list = map.get(key);
|
|
6317
|
+
if (list === undefined)
|
|
6318
|
+
map.set(key, [value2]);
|
|
6319
|
+
else
|
|
6320
|
+
list.push(value2);
|
|
6321
|
+
}
|
|
6322
|
+
var init_completed_catalog_index = __esm(() => {
|
|
6323
|
+
init_boundary_lifecycle();
|
|
6324
|
+
init_stage_topology_validation();
|
|
6325
|
+
});
|
|
6326
|
+
|
|
5758
6327
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
5759
|
-
function validBoundaryRecordSet(
|
|
5760
|
-
const
|
|
5761
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
6328
|
+
function validBoundaryRecordSet(index, strict) {
|
|
6329
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
5762
6330
|
const startsByReplay = new Map;
|
|
5763
6331
|
for (const checkpoint of boundaries) {
|
|
5764
6332
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -5773,7 +6341,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
5773
6341
|
return false;
|
|
5774
6342
|
return [...startsByReplay.values()].every((starts) => {
|
|
5775
6343
|
const start = starts[0];
|
|
5776
|
-
if (
|
|
6344
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
5777
6345
|
return false;
|
|
5778
6346
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
5779
6347
|
});
|
|
@@ -5837,8 +6405,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
5837
6405
|
function runTopologyFor(drafts, tools) {
|
|
5838
6406
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
5839
6407
|
}
|
|
5840
|
-
function validateRunGroups(grouped, rootRunId,
|
|
5841
|
-
const toolsFor = (runId) =>
|
|
6408
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
6409
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
5842
6410
|
const owners = new Map;
|
|
5843
6411
|
const syntheticChildIds = new Set;
|
|
5844
6412
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -5849,12 +6417,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
5849
6417
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
5850
6418
|
return false;
|
|
5851
6419
|
if (!grouped.has(childRunId)) {
|
|
5852
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
6420
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
5853
6421
|
syntheticChildIds.add(childRunId);
|
|
5854
6422
|
owners.set(childRunId, draft);
|
|
5855
6423
|
continue;
|
|
5856
6424
|
}
|
|
5857
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
6425
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
5858
6426
|
return false;
|
|
5859
6427
|
syntheticChildIds.add(childRunId);
|
|
5860
6428
|
owners.set(childRunId, draft);
|
|
@@ -5895,13 +6463,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
5895
6463
|
const boundary = draft.topology?.boundary;
|
|
5896
6464
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
5897
6465
|
}
|
|
5898
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
5899
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
5900
|
-
return checkpoints.some((checkpoint) => {
|
|
5901
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
5902
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
5903
|
-
});
|
|
5904
|
-
}
|
|
5905
6466
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
5906
6467
|
const reachable = new Set([rootRunId]);
|
|
5907
6468
|
const settled = new Set;
|
|
@@ -5942,14 +6503,6 @@ function childRunIdFromDraft(draft) {
|
|
|
5942
6503
|
return;
|
|
5943
6504
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
5944
6505
|
}
|
|
5945
|
-
function boundaryOwner(grouped, childRunId) {
|
|
5946
|
-
for (const drafts of grouped.values()) {
|
|
5947
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
5948
|
-
if (owner !== undefined)
|
|
5949
|
-
return owner;
|
|
5950
|
-
}
|
|
5951
|
-
return;
|
|
5952
|
-
}
|
|
5953
6506
|
function childRunStatus(owner) {
|
|
5954
6507
|
const child = workflowChildFromDraft(owner);
|
|
5955
6508
|
if (child !== undefined)
|
|
@@ -5957,15 +6510,15 @@ function childRunStatus(owner) {
|
|
|
5957
6510
|
const status = owner.topology?.boundary?.status;
|
|
5958
6511
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
5959
6512
|
}
|
|
5960
|
-
function mergeStageGroup(records,
|
|
6513
|
+
function mergeStageGroup(records, index) {
|
|
5961
6514
|
let merged;
|
|
5962
6515
|
for (const record of records)
|
|
5963
|
-
merged = mergeStageDraft(merged, record,
|
|
6516
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
5964
6517
|
const draft = merged;
|
|
5965
6518
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
5966
6519
|
if (start === undefined)
|
|
5967
6520
|
return draft;
|
|
5968
|
-
const resolved =
|
|
6521
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
5969
6522
|
if ("error" in resolved)
|
|
5970
6523
|
return draft;
|
|
5971
6524
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -6028,9 +6581,7 @@ function workflowChildFromDraft(draft) {
|
|
|
6028
6581
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
6029
6582
|
};
|
|
6030
6583
|
}
|
|
6031
|
-
var init_completed_catalog_stage_groups =
|
|
6032
|
-
init_boundary_lifecycle();
|
|
6033
|
-
});
|
|
6584
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
6034
6585
|
|
|
6035
6586
|
// dist/builtin/workflows/src/shared/abort.ts
|
|
6036
6587
|
function raceAbort(operation, signal) {
|
|
@@ -6564,13 +7115,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
6564
7115
|
function runJournalVersions(journal) {
|
|
6565
7116
|
return versionReaders.get(journal);
|
|
6566
7117
|
}
|
|
7118
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
7119
|
+
appendManyWriters.set(journal, appendMany);
|
|
7120
|
+
}
|
|
7121
|
+
function runJournalAppendMany(journal) {
|
|
7122
|
+
return appendManyWriters.get(journal);
|
|
7123
|
+
}
|
|
6567
7124
|
function registerSqlPoolStorage(journal, storage) {
|
|
6568
7125
|
poolStorage.set(journal, storage);
|
|
6569
7126
|
}
|
|
6570
7127
|
function sqlPoolStorage(journal) {
|
|
6571
7128
|
return poolStorage.get(journal);
|
|
6572
7129
|
}
|
|
6573
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
7130
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
6574
7131
|
var init_internal = __esm(() => {
|
|
6575
7132
|
init_types();
|
|
6576
7133
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -6579,6 +7136,7 @@ var init_internal = __esm(() => {
|
|
|
6579
7136
|
RUN_EVENT_TYPES.messageDelivered
|
|
6580
7137
|
];
|
|
6581
7138
|
versionReaders = new WeakMap;
|
|
7139
|
+
appendManyWriters = new WeakMap;
|
|
6582
7140
|
poolStorage = new WeakMap;
|
|
6583
7141
|
});
|
|
6584
7142
|
|
|
@@ -6872,9 +7430,19 @@ function* invoke(action) {
|
|
|
6872
7430
|
} catch (error) {
|
|
6873
7431
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
6874
7432
|
throw error;
|
|
6875
|
-
throw
|
|
7433
|
+
throw customJournalUnavailable(error);
|
|
6876
7434
|
}
|
|
6877
7435
|
}
|
|
7436
|
+
function customJournalUnavailable(failure) {
|
|
7437
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
7438
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
7439
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
7440
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
7441
|
+
}
|
|
7442
|
+
function hasErrorCode(error) {
|
|
7443
|
+
const code = error.code;
|
|
7444
|
+
return typeof code === "string" || typeof code === "number";
|
|
7445
|
+
}
|
|
6878
7446
|
async function execute(operation) {
|
|
6879
7447
|
let step = operation.next();
|
|
6880
7448
|
while (!step.done) {
|
|
@@ -6902,6 +7470,8 @@ class JournalDurableBackend {
|
|
|
6902
7470
|
executions = new Map;
|
|
6903
7471
|
executionContext = new AsyncLocalStorage;
|
|
6904
7472
|
pending = new Map;
|
|
7473
|
+
reconciledPendingStages = new Map;
|
|
7474
|
+
appendBatches = new Map;
|
|
6905
7475
|
catalogQueue = Symbol("catalog");
|
|
6906
7476
|
lost = new Map;
|
|
6907
7477
|
prompts = new Map;
|
|
@@ -6934,7 +7504,7 @@ class JournalDurableBackend {
|
|
|
6934
7504
|
await backend.close();
|
|
6935
7505
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
6936
7506
|
throw error;
|
|
6937
|
-
throw
|
|
7507
|
+
throw customJournalUnavailable(error);
|
|
6938
7508
|
}
|
|
6939
7509
|
}
|
|
6940
7510
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -6949,6 +7519,8 @@ class JournalDurableBackend {
|
|
|
6949
7519
|
return this.enqueue(runId, action, admitted);
|
|
6950
7520
|
}
|
|
6951
7521
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
7522
|
+
if (typeof runId === "string")
|
|
7523
|
+
this.appendBatches.delete(runId);
|
|
6952
7524
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
6953
7525
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
6954
7526
|
const settled = result.then(() => {}, () => {});
|
|
@@ -6963,6 +7535,7 @@ class JournalDurableBackend {
|
|
|
6963
7535
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
6964
7536
|
return;
|
|
6965
7537
|
const event = entry.event;
|
|
7538
|
+
let raisedCount;
|
|
6966
7539
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
6967
7540
|
const checkpoint = event.payload;
|
|
6968
7541
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -6971,7 +7544,10 @@ class JournalDurableBackend {
|
|
|
6971
7544
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
6972
7545
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
6973
7546
|
const payload = event.payload;
|
|
7547
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
6974
7548
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
7549
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
7550
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
6975
7551
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
6976
7552
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
6977
7553
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -6982,20 +7558,80 @@ class JournalDurableBackend {
|
|
|
6982
7558
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
6983
7559
|
this.prompts.delete(entry.runId);
|
|
6984
7560
|
}
|
|
6985
|
-
|
|
6986
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
6987
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
6988
|
-
if (message.sticky !== true)
|
|
6989
|
-
return message;
|
|
6990
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
6991
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
6992
|
-
});
|
|
6993
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
6994
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
6995
|
-
}
|
|
7561
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
6996
7562
|
this.sequences.set(entry.runId, entry.seq);
|
|
6997
7563
|
this.foldedAt.set(entry.runId, entry.at);
|
|
6998
7564
|
}
|
|
7565
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
7566
|
+
const handle = this.projection.getWorkflow(runId);
|
|
7567
|
+
const list = handle?.pendingStageMessages;
|
|
7568
|
+
if (handle === undefined || list === undefined) {
|
|
7569
|
+
this.reconciledPendingStages.delete(runId);
|
|
7570
|
+
return;
|
|
7571
|
+
}
|
|
7572
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
7573
|
+
let next;
|
|
7574
|
+
const raise = (index, count) => {
|
|
7575
|
+
const message = (next ?? list)[index];
|
|
7576
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
7577
|
+
return;
|
|
7578
|
+
next ??= [...list];
|
|
7579
|
+
next[index] = { ...message, deliveryCount: count };
|
|
7580
|
+
};
|
|
7581
|
+
const reconcile = (index) => {
|
|
7582
|
+
const message = list[index];
|
|
7583
|
+
if (message.sticky !== true)
|
|
7584
|
+
return;
|
|
7585
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
7586
|
+
};
|
|
7587
|
+
if (state?.list !== list) {
|
|
7588
|
+
const previous = state?.list ?? [];
|
|
7589
|
+
let aligned = list.length >= previous.length;
|
|
7590
|
+
const changed = [];
|
|
7591
|
+
for (let index = 0;index < list.length; index++) {
|
|
7592
|
+
const message = list[index];
|
|
7593
|
+
if (previous[index] === message)
|
|
7594
|
+
continue;
|
|
7595
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
7596
|
+
aligned = false;
|
|
7597
|
+
changed.push(index);
|
|
7598
|
+
}
|
|
7599
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
7600
|
+
for (const index of changed)
|
|
7601
|
+
if (!reconciled?.has(list[index]))
|
|
7602
|
+
reconcile(index);
|
|
7603
|
+
const positions = aligned ? state?.positions : undefined;
|
|
7604
|
+
if (positions !== undefined) {
|
|
7605
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
7606
|
+
const indexes = positions.get(list[index].id);
|
|
7607
|
+
if (indexes === undefined)
|
|
7608
|
+
positions.set(list[index].id, [index]);
|
|
7609
|
+
else
|
|
7610
|
+
indexes.push(index);
|
|
7611
|
+
}
|
|
7612
|
+
}
|
|
7613
|
+
state = { list, positions };
|
|
7614
|
+
}
|
|
7615
|
+
if (raisedCount !== undefined) {
|
|
7616
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
7617
|
+
state.positions ??= messagePositions(list);
|
|
7618
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
7619
|
+
const message = list[index];
|
|
7620
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
7621
|
+
raise(index, count);
|
|
7622
|
+
}
|
|
7623
|
+
}
|
|
7624
|
+
if (next === undefined) {
|
|
7625
|
+
this.reconciledPendingStages.set(runId, state);
|
|
7626
|
+
return;
|
|
7627
|
+
}
|
|
7628
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
7629
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
7630
|
+
this.reconciledPendingStages.set(runId, {
|
|
7631
|
+
list: applied,
|
|
7632
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
7633
|
+
});
|
|
7634
|
+
}
|
|
6999
7635
|
summarize(runId, tail) {
|
|
7000
7636
|
const event = tail.snapshot.event;
|
|
7001
7637
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -7019,6 +7655,7 @@ class JournalDurableBackend {
|
|
|
7019
7655
|
*rebuild(runId) {
|
|
7020
7656
|
const entries = yield* this.entries(runId);
|
|
7021
7657
|
this.projection.deleteWorkflow(runId);
|
|
7658
|
+
this.reconciledPendingStages.delete(runId);
|
|
7022
7659
|
this.sequences.delete(runId);
|
|
7023
7660
|
this.foldedAt.delete(runId);
|
|
7024
7661
|
this.summarized.delete(runId);
|
|
@@ -7114,9 +7751,9 @@ class JournalDurableBackend {
|
|
|
7114
7751
|
}
|
|
7115
7752
|
yield* this.write(runId, lease, event, options);
|
|
7116
7753
|
}
|
|
7117
|
-
*write(runId, lease, event, options) {
|
|
7754
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
7118
7755
|
try {
|
|
7119
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
7756
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
7120
7757
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
7121
7758
|
if (outcome.entry.seq > nextSeq) {
|
|
7122
7759
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -7300,13 +7937,80 @@ class JournalDurableBackend {
|
|
|
7300
7937
|
return true;
|
|
7301
7938
|
}
|
|
7302
7939
|
async archivePendingStageMessage(runId, message) {
|
|
7303
|
-
await this.
|
|
7940
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
7304
7941
|
}
|
|
7305
7942
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
7306
|
-
await this.
|
|
7943
|
+
await this.queueAppend(runId, {
|
|
7307
7944
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
7308
7945
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
7309
|
-
})
|
|
7946
|
+
});
|
|
7947
|
+
}
|
|
7948
|
+
queueAppend(runId, event, key) {
|
|
7949
|
+
try {
|
|
7950
|
+
this.ensureOpen();
|
|
7951
|
+
this.assertExecutionActive();
|
|
7952
|
+
} catch (error) {
|
|
7953
|
+
return Promise.reject(error);
|
|
7954
|
+
}
|
|
7955
|
+
const open = this.appendBatches.get(runId);
|
|
7956
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
7957
|
+
const queued = new Promise((resolve, reject) => {
|
|
7958
|
+
batch.push({
|
|
7959
|
+
event,
|
|
7960
|
+
...key === undefined ? {} : { key },
|
|
7961
|
+
context: AsyncLocalStorage.snapshot(),
|
|
7962
|
+
resolve,
|
|
7963
|
+
reject
|
|
7964
|
+
});
|
|
7965
|
+
});
|
|
7966
|
+
if (batch !== open) {
|
|
7967
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
7968
|
+
for (const append of batch)
|
|
7969
|
+
append.reject(error);
|
|
7970
|
+
});
|
|
7971
|
+
this.appendBatches.set(runId, batch);
|
|
7972
|
+
}
|
|
7973
|
+
return queued;
|
|
7974
|
+
}
|
|
7975
|
+
*appendBatch(runId, batch) {
|
|
7976
|
+
if (this.appendBatches.get(runId) === batch)
|
|
7977
|
+
this.appendBatches.delete(runId);
|
|
7978
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
7979
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
7980
|
+
for (const append of batch) {
|
|
7981
|
+
try {
|
|
7982
|
+
const lease = yield* this.ownership(runId);
|
|
7983
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
7984
|
+
append.resolve();
|
|
7985
|
+
} catch (error) {
|
|
7986
|
+
append.reject(error);
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7989
|
+
return;
|
|
7990
|
+
}
|
|
7991
|
+
let settled = 0;
|
|
7992
|
+
try {
|
|
7993
|
+
const lease = yield* this.ownership(runId);
|
|
7994
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
7995
|
+
for (const result of results) {
|
|
7996
|
+
const append = batch[settled];
|
|
7997
|
+
if (result.status === "rejected") {
|
|
7998
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
7999
|
+
} else {
|
|
8000
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
8001
|
+
if (result.value.entry.seq > nextSeq)
|
|
8002
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
8003
|
+
this.fold(entry);
|
|
8004
|
+
this.fold(result.value.entry);
|
|
8005
|
+
append.resolve();
|
|
8006
|
+
}
|
|
8007
|
+
settled++;
|
|
8008
|
+
}
|
|
8009
|
+
} catch (error) {
|
|
8010
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
8011
|
+
for (const append of batch.slice(settled))
|
|
8012
|
+
append.reject(failure);
|
|
8013
|
+
}
|
|
7310
8014
|
}
|
|
7311
8015
|
getToolOutput(...args) {
|
|
7312
8016
|
return this.projection.getToolOutput(...args);
|
|
@@ -7326,6 +8030,9 @@ class JournalDurableBackend {
|
|
|
7326
8030
|
listCheckpoints(runId) {
|
|
7327
8031
|
return this.projection.listCheckpoints(runId);
|
|
7328
8032
|
}
|
|
8033
|
+
nestedSnapshotVersion(runId) {
|
|
8034
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
8035
|
+
}
|
|
7329
8036
|
getWorkflow(runId) {
|
|
7330
8037
|
return this.projection.getWorkflow(runId);
|
|
7331
8038
|
}
|
|
@@ -7350,6 +8057,12 @@ class JournalDurableBackend {
|
|
|
7350
8057
|
getPendingStageMessageDeliveryCount(...args) {
|
|
7351
8058
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
7352
8059
|
}
|
|
8060
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
8061
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
8062
|
+
}
|
|
8063
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
8064
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
8065
|
+
}
|
|
7353
8066
|
hasPendingStageDeliveryReceipt(...args) {
|
|
7354
8067
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
7355
8068
|
}
|
|
@@ -7434,6 +8147,7 @@ class JournalDurableBackend {
|
|
|
7434
8147
|
this.foldedAt.delete(runId);
|
|
7435
8148
|
this.summarized.delete(runId);
|
|
7436
8149
|
this.prompts.delete(runId);
|
|
8150
|
+
this.reconciledPendingStages.delete(runId);
|
|
7437
8151
|
this.projection.deleteWorkflow(runId);
|
|
7438
8152
|
}
|
|
7439
8153
|
async deleteWorkflow(runId) {
|
|
@@ -7616,8 +8330,24 @@ class JournalDurableBackend {
|
|
|
7616
8330
|
}
|
|
7617
8331
|
}
|
|
7618
8332
|
}
|
|
8333
|
+
function messagePositions(list) {
|
|
8334
|
+
const positions = new Map;
|
|
8335
|
+
list.forEach((message, index) => {
|
|
8336
|
+
const indexes = positions.get(message.id);
|
|
8337
|
+
if (indexes === undefined)
|
|
8338
|
+
positions.set(message.id, [index]);
|
|
8339
|
+
else
|
|
8340
|
+
indexes.push(index);
|
|
8341
|
+
});
|
|
8342
|
+
return positions;
|
|
8343
|
+
}
|
|
7619
8344
|
function identity(message) {
|
|
7620
|
-
|
|
8345
|
+
let key = entryIdentity.get(message);
|
|
8346
|
+
if (key === undefined) {
|
|
8347
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
8348
|
+
entryIdentity.set(message, key);
|
|
8349
|
+
}
|
|
8350
|
+
return key;
|
|
7621
8351
|
}
|
|
7622
8352
|
function uniqueByIdentity(messages) {
|
|
7623
8353
|
const result = new Map;
|
|
@@ -7791,7 +8521,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
7791
8521
|
function hasPendingStageDeliveryDelta(messages) {
|
|
7792
8522
|
return messages?.some(isDeltaEncoded) === true;
|
|
7793
8523
|
}
|
|
7794
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
8524
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
7795
8525
|
var init_journal_backend = __esm(() => {
|
|
7796
8526
|
init_workflow_instance_ownership_error();
|
|
7797
8527
|
init_backend();
|
|
@@ -7803,6 +8533,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7803
8533
|
init_types();
|
|
7804
8534
|
PROCESS_HOLDER_ID = randomUUID4();
|
|
7805
8535
|
processExitCleanups = new Set;
|
|
8536
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
7806
8537
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
7807
8538
|
runId;
|
|
7808
8539
|
holderId;
|
|
@@ -7849,6 +8580,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7849
8580
|
this.name = "JournalMessageScopeError";
|
|
7850
8581
|
}
|
|
7851
8582
|
};
|
|
8583
|
+
entryIdentity = new WeakMap;
|
|
7852
8584
|
entryJson = new WeakMap;
|
|
7853
8585
|
});
|
|
7854
8586
|
|
|
@@ -7976,6 +8708,7 @@ class SqlRunJournal {
|
|
|
7976
8708
|
runVersions: () => this.runVersions(),
|
|
7977
8709
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
7978
8710
|
});
|
|
8711
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
7979
8712
|
registerSqlPoolStorage(this, {
|
|
7980
8713
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
7981
8714
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -8240,38 +8973,76 @@ class SqlRunJournal {
|
|
|
8240
8973
|
this.assertOpen();
|
|
8241
8974
|
validateText("runId", lease.runId);
|
|
8242
8975
|
validateText("holderId", lease.holderId);
|
|
8243
|
-
if (options?.key !== undefined)
|
|
8244
|
-
validateText("key", options.key);
|
|
8245
8976
|
const owned = { ...lease };
|
|
8246
|
-
const
|
|
8977
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
8247
8978
|
const expected = options?.expectSeq;
|
|
8979
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
8980
|
+
await this.checkLease(tx, owned);
|
|
8981
|
+
return this.insert(tx, owned, prepared, expected);
|
|
8982
|
+
}));
|
|
8983
|
+
}
|
|
8984
|
+
async appendMany(lease, appends) {
|
|
8985
|
+
this.assertOpen();
|
|
8986
|
+
validateText("runId", lease.runId);
|
|
8987
|
+
validateText("holderId", lease.holderId);
|
|
8988
|
+
const owned = { ...lease };
|
|
8989
|
+
const prepared = appends.map((append) => {
|
|
8990
|
+
try {
|
|
8991
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
8992
|
+
} catch (error) {
|
|
8993
|
+
return error;
|
|
8994
|
+
}
|
|
8995
|
+
});
|
|
8996
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
8997
|
+
await this.checkLease(tx, owned);
|
|
8998
|
+
const results = [];
|
|
8999
|
+
for (const append of prepared) {
|
|
9000
|
+
if (append instanceof Error) {
|
|
9001
|
+
results.push({ status: "rejected", reason: append });
|
|
9002
|
+
continue;
|
|
9003
|
+
}
|
|
9004
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
9005
|
+
try {
|
|
9006
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
9007
|
+
} catch (error) {
|
|
9008
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
9009
|
+
results.push({ status: "rejected", reason: error });
|
|
9010
|
+
}
|
|
9011
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
9012
|
+
}
|
|
9013
|
+
return results;
|
|
9014
|
+
}));
|
|
9015
|
+
}
|
|
9016
|
+
prepare(event, key, appendGuard) {
|
|
9017
|
+
if (key !== undefined)
|
|
9018
|
+
validateText("key", key);
|
|
8248
9019
|
const type = event.type;
|
|
8249
9020
|
const payload = this.snapshotJson(event.payload);
|
|
8250
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
9021
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
8251
9022
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
8252
9023
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
8253
9024
|
validateText("status", status);
|
|
8254
|
-
return
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
}
|
|
9025
|
+
return { key, type, payload, guard, status };
|
|
9026
|
+
}
|
|
9027
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
9028
|
+
await guard?.(tx, owned.runId);
|
|
9029
|
+
if (key !== undefined) {
|
|
9030
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
9031
|
+
if (row)
|
|
9032
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
9033
|
+
}
|
|
9034
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
9035
|
+
if (expected !== undefined && expected !== actual)
|
|
9036
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
9037
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
9038
|
+
if (payload instanceof StorageUnavailableError2)
|
|
9039
|
+
throw payload;
|
|
9040
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
9041
|
+
if (status !== undefined)
|
|
9042
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
9043
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
9044
|
+
await guard?.(tx, owned.runId);
|
|
9045
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
8275
9046
|
}
|
|
8276
9047
|
async* read(runId, fromSeq = 1) {
|
|
8277
9048
|
this.assertOpen();
|
|
@@ -8339,7 +9110,7 @@ class SqlRunJournal {
|
|
|
8339
9110
|
return this.closePromise;
|
|
8340
9111
|
}
|
|
8341
9112
|
}
|
|
8342
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
9113
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
8343
9114
|
var init_sql_run_journal = __esm(() => {
|
|
8344
9115
|
init_internal();
|
|
8345
9116
|
init_legacy_journal_tables();
|
|
@@ -8547,8 +9318,59 @@ function firstError(schema, value2) {
|
|
|
8547
9318
|
}
|
|
8548
9319
|
return;
|
|
8549
9320
|
}
|
|
9321
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
9322
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
9323
|
+
return false;
|
|
9324
|
+
const pending = [root];
|
|
9325
|
+
const ancestors = new Set;
|
|
9326
|
+
const exits = [undefined];
|
|
9327
|
+
while (pending.length > 0) {
|
|
9328
|
+
const exit = exits.pop();
|
|
9329
|
+
const value2 = pending.pop();
|
|
9330
|
+
if (exit !== undefined) {
|
|
9331
|
+
ancestors.delete(exit);
|
|
9332
|
+
continue;
|
|
9333
|
+
}
|
|
9334
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
9335
|
+
continue;
|
|
9336
|
+
if (typeof value2 === "number") {
|
|
9337
|
+
if (!Number.isFinite(value2))
|
|
9338
|
+
return false;
|
|
9339
|
+
continue;
|
|
9340
|
+
}
|
|
9341
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
9342
|
+
return false;
|
|
9343
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
9344
|
+
return false;
|
|
9345
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
9346
|
+
let children;
|
|
9347
|
+
if (Array.isArray(value2)) {
|
|
9348
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
9349
|
+
return false;
|
|
9350
|
+
children = [];
|
|
9351
|
+
for (let index = 0;index < value2.length; index++)
|
|
9352
|
+
children.push(value2[index]);
|
|
9353
|
+
} else {
|
|
9354
|
+
const proto = Object.getPrototypeOf(value2);
|
|
9355
|
+
if (proto !== Object.prototype && proto !== null)
|
|
9356
|
+
return false;
|
|
9357
|
+
const keys = Object.keys(value2);
|
|
9358
|
+
if (keys.length !== names.length)
|
|
9359
|
+
return false;
|
|
9360
|
+
children = keys.map((key) => value2[key]);
|
|
9361
|
+
}
|
|
9362
|
+
ancestors.add(value2);
|
|
9363
|
+
pending.push(value2);
|
|
9364
|
+
exits.push(value2);
|
|
9365
|
+
for (const child of children) {
|
|
9366
|
+
pending.push(child);
|
|
9367
|
+
exits.push(undefined);
|
|
9368
|
+
}
|
|
9369
|
+
}
|
|
9370
|
+
return true;
|
|
9371
|
+
}
|
|
8550
9372
|
function workflowSerializableValidationError(value2, label) {
|
|
8551
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
9373
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
8552
9374
|
return;
|
|
8553
9375
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
8554
9376
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -8557,7 +9379,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
8557
9379
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
8558
9380
|
}
|
|
8559
9381
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
8560
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
9382
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
8561
9383
|
return;
|
|
8562
9384
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
8563
9385
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -60991,39 +61813,406 @@ var init_pending_stage_route_readiness = __esm(() => {
|
|
|
60991
61813
|
processState3[routeOwnersKey] = owners;
|
|
60992
61814
|
});
|
|
60993
61815
|
|
|
60994
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
60995
|
-
function
|
|
60996
|
-
|
|
60997
|
-
|
|
60998
|
-
const
|
|
60999
|
-
|
|
61000
|
-
const
|
|
61001
|
-
|
|
61002
|
-
const
|
|
61003
|
-
|
|
61816
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
61817
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
61818
|
+
if (states.length === 0)
|
|
61819
|
+
return { deliveryStates: states };
|
|
61820
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
61821
|
+
counts.set("delivered", 0);
|
|
61822
|
+
for (const delivery of states)
|
|
61823
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
61824
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
61825
|
+
if (states.length <= limit)
|
|
61826
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
61827
|
+
const exemplars = [];
|
|
61828
|
+
for (const state of UNRESOLVED_STATES) {
|
|
61829
|
+
for (const delivery of states) {
|
|
61830
|
+
if (exemplars.length === limit)
|
|
61831
|
+
break;
|
|
61832
|
+
if (delivery.state === state)
|
|
61833
|
+
exemplars.push(delivery);
|
|
61834
|
+
}
|
|
61835
|
+
}
|
|
61836
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
61837
|
+
const delivery = states[index];
|
|
61838
|
+
if (delivery.state === "delivered")
|
|
61839
|
+
exemplars.push(delivery);
|
|
61840
|
+
}
|
|
61841
|
+
return {
|
|
61842
|
+
deliveryStates: exemplars,
|
|
61843
|
+
deliveryStateCounts,
|
|
61844
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
61845
|
+
};
|
|
61846
|
+
}
|
|
61847
|
+
function omittedDeliveryStatesLine(bounded) {
|
|
61848
|
+
const omitted = bounded.omittedDeliveryStates ?? 0;
|
|
61849
|
+
if (omitted === 0 || bounded.deliveryStateCounts === undefined)
|
|
61850
|
+
return;
|
|
61851
|
+
const counts = Object.entries(bounded.deliveryStateCounts).map(([state, count]) => `${count} ${state}`).join(", ");
|
|
61852
|
+
return `… ${omitted} more delivery state${omitted === 1 ? "" : "s"} omitted (all: ${counts})`;
|
|
61853
|
+
}
|
|
61854
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
61855
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
61856
|
+
UNRESOLVED_STATES = [
|
|
61857
|
+
"queued",
|
|
61858
|
+
"receipt-unverified",
|
|
61859
|
+
"session-replaced",
|
|
61860
|
+
"delivery-unavailable",
|
|
61861
|
+
"root-unresolved",
|
|
61862
|
+
"stage-terminal"
|
|
61863
|
+
];
|
|
61864
|
+
});
|
|
61865
|
+
|
|
61866
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
61867
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
61868
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
61869
|
+
return {
|
|
61870
|
+
...bounded,
|
|
61871
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
61872
|
+
};
|
|
61873
|
+
}
|
|
61874
|
+
|
|
61875
|
+
class WorkflowTopologyIndex {
|
|
61876
|
+
lineage = [];
|
|
61877
|
+
runById = new Map;
|
|
61878
|
+
roots = new Map;
|
|
61879
|
+
hops = new Map;
|
|
61880
|
+
refresh(runs) {
|
|
61881
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61882
|
+
const lineage = [];
|
|
61883
|
+
for (const run of runs) {
|
|
61884
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
61885
|
+
if (run.parentRunId === undefined)
|
|
61886
|
+
continue;
|
|
61887
|
+
const parent = runById.get(run.parentRunId);
|
|
61888
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
61889
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
61890
|
+
}
|
|
61891
|
+
if (sameValues(lineage, this.lineage))
|
|
61892
|
+
return;
|
|
61893
|
+
this.lineage = lineage;
|
|
61894
|
+
this.runById = runById;
|
|
61895
|
+
this.roots.clear();
|
|
61896
|
+
this.hops.clear();
|
|
61897
|
+
}
|
|
61898
|
+
rootOf(runId) {
|
|
61899
|
+
if (this.roots.has(runId))
|
|
61900
|
+
return this.roots.get(runId);
|
|
61901
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
61902
|
+
this.roots.set(runId, root);
|
|
61903
|
+
return root;
|
|
61904
|
+
}
|
|
61905
|
+
hopsOf(runId) {
|
|
61906
|
+
if (this.hops.has(runId))
|
|
61907
|
+
return this.hops.get(runId);
|
|
61908
|
+
const hops = boundaryHops(this.runById, runId);
|
|
61909
|
+
this.hops.set(runId, hops);
|
|
61910
|
+
return hops;
|
|
61911
|
+
}
|
|
61912
|
+
}
|
|
61913
|
+
function boundaryHops(runById, runId) {
|
|
61914
|
+
const hops = [];
|
|
61915
|
+
let current = runById.get(runId);
|
|
61916
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
61917
|
+
const parent = runById.get(current.parentRunId);
|
|
61918
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
61919
|
+
if (parent === undefined || boundary === undefined)
|
|
61920
|
+
return;
|
|
61921
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
61922
|
+
current = parent;
|
|
61923
|
+
}
|
|
61924
|
+
return current === undefined ? undefined : hops;
|
|
61925
|
+
}
|
|
61926
|
+
function sameValues(left, right) {
|
|
61927
|
+
if (left.length !== right.length)
|
|
61928
|
+
return false;
|
|
61929
|
+
for (let index = 0;index < left.length; index++)
|
|
61930
|
+
if (left[index] !== right[index])
|
|
61931
|
+
return false;
|
|
61932
|
+
return true;
|
|
61933
|
+
}
|
|
61934
|
+
|
|
61935
|
+
class PossibleStageRowsIndex {
|
|
61936
|
+
rootRunId;
|
|
61937
|
+
topology;
|
|
61938
|
+
targets = new Map;
|
|
61939
|
+
patterns = [];
|
|
61940
|
+
targetOf = new Map;
|
|
61941
|
+
targetsByMessage = new Map;
|
|
61942
|
+
list = [];
|
|
61943
|
+
stageFields = new WeakMap;
|
|
61944
|
+
backend;
|
|
61945
|
+
receiptCursor;
|
|
61946
|
+
scanMatches = new Map;
|
|
61947
|
+
rowCache = new Map;
|
|
61948
|
+
boundedStates = new WeakMap;
|
|
61949
|
+
deliveriesByEntry = new WeakMap;
|
|
61950
|
+
statusCache = new WeakMap;
|
|
61951
|
+
receiptGeneration = 0;
|
|
61952
|
+
receiptEpochs = new Map;
|
|
61953
|
+
constructor(rootRunId, topology) {
|
|
61954
|
+
this.rootRunId = rootRunId;
|
|
61955
|
+
this.topology = topology;
|
|
61956
|
+
}
|
|
61957
|
+
rows(runs, rootRun, availableBackend) {
|
|
61958
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
61959
|
+
return [];
|
|
61960
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
61961
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
61962
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
61963
|
+
this.refreshReceipts(backend);
|
|
61964
|
+
this.refreshStages(runs);
|
|
61965
|
+
for (const state of this.patterns)
|
|
61966
|
+
if (state.dirty)
|
|
61967
|
+
this.recompute(state, backend);
|
|
61968
|
+
return this.assemble(runs, rootRun);
|
|
61969
|
+
}
|
|
61970
|
+
targetFor(entry) {
|
|
61004
61971
|
const target = entry.targetPath ?? entry.stageKey;
|
|
61005
|
-
let
|
|
61006
|
-
if (
|
|
61972
|
+
let state = this.targets.get(target);
|
|
61973
|
+
if (state === undefined) {
|
|
61007
61974
|
const parsed = parseWorkflowStageTarget(target);
|
|
61008
|
-
|
|
61009
|
-
|
|
61010
|
-
|
|
61011
|
-
|
|
61975
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
61976
|
+
state = {
|
|
61977
|
+
target,
|
|
61978
|
+
segments: owned ? parsed.segments : undefined,
|
|
61979
|
+
pattern: owned ? this.patterns.length : -1,
|
|
61980
|
+
entries: [],
|
|
61981
|
+
positions: [],
|
|
61982
|
+
queued: 0,
|
|
61983
|
+
stages: [],
|
|
61984
|
+
statuses: [],
|
|
61985
|
+
dirty: true
|
|
61986
|
+
};
|
|
61987
|
+
this.targets.set(target, state);
|
|
61988
|
+
if (owned)
|
|
61989
|
+
this.patterns.push(state);
|
|
61012
61990
|
}
|
|
61013
|
-
|
|
61991
|
+
return state;
|
|
61992
|
+
}
|
|
61993
|
+
refreshEntries(list) {
|
|
61994
|
+
const previous = this.list;
|
|
61995
|
+
if (list === previous)
|
|
61014
61996
|
return;
|
|
61015
|
-
|
|
61016
|
-
|
|
61017
|
-
|
|
61018
|
-
|
|
61019
|
-
|
|
61020
|
-
|
|
61021
|
-
|
|
61022
|
-
|
|
61023
|
-
|
|
61997
|
+
this.list = list;
|
|
61998
|
+
let incremental = list.length >= previous.length;
|
|
61999
|
+
const replaced = [];
|
|
62000
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
62001
|
+
if (list[index] === previous[index])
|
|
62002
|
+
continue;
|
|
62003
|
+
const before = previous[index];
|
|
62004
|
+
const after = list[index];
|
|
62005
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
62006
|
+
continue;
|
|
62007
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
62008
|
+
incremental = false;
|
|
62009
|
+
break;
|
|
62010
|
+
}
|
|
62011
|
+
replaced.push(index);
|
|
62012
|
+
}
|
|
62013
|
+
if (!incremental) {
|
|
62014
|
+
this.rebuildEntries(list);
|
|
62015
|
+
return;
|
|
62016
|
+
}
|
|
62017
|
+
for (const index of replaced) {
|
|
62018
|
+
const before = previous[index];
|
|
62019
|
+
const after = list[index];
|
|
62020
|
+
const state = this.targetOf.get(before);
|
|
62021
|
+
const slot = state.entries.indexOf(before);
|
|
62022
|
+
state.entries[slot] = after;
|
|
62023
|
+
this.targetOf.delete(before);
|
|
62024
|
+
this.targetOf.set(after, state);
|
|
62025
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
62026
|
+
if (before.id !== after.id)
|
|
62027
|
+
this.rebuildMessageIndex();
|
|
62028
|
+
state.dirty = true;
|
|
62029
|
+
}
|
|
62030
|
+
for (let index = previous.length;index < list.length; index++)
|
|
62031
|
+
this.addEntry(list[index], index);
|
|
62032
|
+
}
|
|
62033
|
+
addEntry(entry, position) {
|
|
62034
|
+
if (entry.sticky !== true)
|
|
62035
|
+
return;
|
|
62036
|
+
const state = this.targetFor(entry);
|
|
62037
|
+
state.entries.push(entry);
|
|
62038
|
+
state.positions.push(position);
|
|
62039
|
+
if (entry.status === "queued")
|
|
62040
|
+
state.queued++;
|
|
62041
|
+
state.dirty = true;
|
|
62042
|
+
this.targetOf.set(entry, state);
|
|
62043
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62044
|
+
if (targets === undefined) {
|
|
62045
|
+
targets = new Set;
|
|
62046
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62047
|
+
}
|
|
62048
|
+
targets.add(state);
|
|
62049
|
+
}
|
|
62050
|
+
rebuildEntries(list) {
|
|
62051
|
+
const before = new Map;
|
|
62052
|
+
for (const state of this.targets.values()) {
|
|
62053
|
+
before.set(state, state.entries);
|
|
62054
|
+
state.entries = [];
|
|
62055
|
+
state.positions = [];
|
|
62056
|
+
state.queued = 0;
|
|
62057
|
+
}
|
|
62058
|
+
this.targetOf.clear();
|
|
62059
|
+
this.targetsByMessage.clear();
|
|
62060
|
+
list.forEach((entry, position) => {
|
|
62061
|
+
this.addEntry(entry, position);
|
|
62062
|
+
});
|
|
62063
|
+
for (const state of this.targets.values()) {
|
|
62064
|
+
const previous = before.get(state);
|
|
62065
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
62066
|
+
}
|
|
62067
|
+
}
|
|
62068
|
+
rebuildMessageIndex() {
|
|
62069
|
+
this.targetsByMessage.clear();
|
|
62070
|
+
for (const [entry, state] of this.targetOf) {
|
|
62071
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62072
|
+
if (targets === undefined) {
|
|
62073
|
+
targets = new Set;
|
|
62074
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62075
|
+
}
|
|
62076
|
+
targets.add(state);
|
|
62077
|
+
}
|
|
62078
|
+
}
|
|
62079
|
+
refreshReceipts(backend) {
|
|
62080
|
+
if (backend !== this.backend) {
|
|
62081
|
+
this.backend = backend;
|
|
62082
|
+
this.receiptCursor = undefined;
|
|
62083
|
+
this.invalidateReceipts();
|
|
62084
|
+
}
|
|
62085
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
62086
|
+
return;
|
|
62087
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
62088
|
+
if (changes === undefined) {
|
|
62089
|
+
this.invalidateReceipts();
|
|
62090
|
+
return;
|
|
62091
|
+
}
|
|
62092
|
+
const firstRead = this.receiptCursor === undefined;
|
|
62093
|
+
this.receiptCursor = changes.cursor;
|
|
62094
|
+
if (changes.messageIds === undefined) {
|
|
62095
|
+
if (!firstRead)
|
|
62096
|
+
this.invalidateReceipts();
|
|
62097
|
+
return;
|
|
62098
|
+
}
|
|
62099
|
+
for (const messageId of changes.messageIds) {
|
|
62100
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
62101
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
62102
|
+
state.dirty = true;
|
|
62103
|
+
}
|
|
62104
|
+
}
|
|
62105
|
+
invalidateReceipts() {
|
|
62106
|
+
this.receiptGeneration++;
|
|
62107
|
+
this.receiptEpochs.clear();
|
|
62108
|
+
for (const state of this.patterns)
|
|
62109
|
+
state.dirty = true;
|
|
62110
|
+
}
|
|
62111
|
+
refreshStages(runs) {
|
|
62112
|
+
const current = new Map;
|
|
62113
|
+
let ordinal = 0;
|
|
62114
|
+
for (const run of runs) {
|
|
62115
|
+
const root = this.topology.rootOf(run.id);
|
|
62116
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
62117
|
+
continue;
|
|
62118
|
+
const resolved = root === this.rootRunId;
|
|
62119
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62120
|
+
for (const stage of run.stages) {
|
|
62121
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
62122
|
+
continue;
|
|
62123
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
62124
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
62125
|
+
for (const state of fields.matched) {
|
|
62126
|
+
const stages = current.get(state);
|
|
62127
|
+
if (stages === undefined)
|
|
62128
|
+
current.set(state, [status]);
|
|
62129
|
+
else
|
|
62130
|
+
stages.push(status);
|
|
62131
|
+
}
|
|
62132
|
+
}
|
|
62133
|
+
}
|
|
62134
|
+
for (const state of this.patterns) {
|
|
62135
|
+
const stages = current.get(state) ?? [];
|
|
62136
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
62137
|
+
const previous = state.stages[index];
|
|
62138
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
62139
|
+
}))
|
|
62140
|
+
state.dirty = true;
|
|
62141
|
+
state.stages = stages;
|
|
62142
|
+
}
|
|
62143
|
+
}
|
|
62144
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
62145
|
+
let fields = this.stageFields.get(stage);
|
|
62146
|
+
const matchable = resolved && hops !== undefined;
|
|
62147
|
+
let changed = false;
|
|
62148
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
62149
|
+
const previous = fields;
|
|
62150
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
62151
|
+
fields = {
|
|
62152
|
+
run,
|
|
62153
|
+
status: stage.status,
|
|
62154
|
+
error: stage.error,
|
|
62155
|
+
sessionId: stage.sessionId,
|
|
62156
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
62157
|
+
resolved,
|
|
62158
|
+
hops,
|
|
62159
|
+
id: stage.id,
|
|
62160
|
+
name: stage.name,
|
|
62161
|
+
tested: sameMatching ? previous.tested : 0,
|
|
62162
|
+
matched: sameMatching ? previous.matched : []
|
|
62163
|
+
};
|
|
62164
|
+
this.stageFields.set(stage, fields);
|
|
62165
|
+
changed = true;
|
|
62166
|
+
}
|
|
62167
|
+
if (fields.tested < this.patterns.length) {
|
|
62168
|
+
const added = this.patterns.slice(fields.tested);
|
|
62169
|
+
if (!matchable)
|
|
62170
|
+
fields.matched.push(...added);
|
|
62171
|
+
else {
|
|
62172
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
62173
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
62174
|
+
fields.matched.push(added[position]);
|
|
62175
|
+
}
|
|
62176
|
+
fields.tested = this.patterns.length;
|
|
62177
|
+
}
|
|
62178
|
+
if (changed)
|
|
62179
|
+
for (const state of fields.matched)
|
|
62180
|
+
state.dirty = true;
|
|
62181
|
+
return fields;
|
|
62182
|
+
}
|
|
62183
|
+
recompute(state, backend) {
|
|
62184
|
+
state.dirty = false;
|
|
62185
|
+
const statuses = [];
|
|
62186
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
62187
|
+
const fields = this.stageFields.get(stage);
|
|
62188
|
+
for (const entry of state.entries) {
|
|
62189
|
+
let byStage = this.statusCache.get(entry);
|
|
62190
|
+
if (byStage === undefined) {
|
|
62191
|
+
byStage = new Map;
|
|
62192
|
+
this.statusCache.set(entry, byStage);
|
|
62193
|
+
}
|
|
62194
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
62195
|
+
let cached = byStage.get(stage);
|
|
62196
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
62197
|
+
cached = {
|
|
62198
|
+
fields,
|
|
62199
|
+
generation: this.receiptGeneration,
|
|
62200
|
+
epoch,
|
|
62201
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
62202
|
+
};
|
|
62203
|
+
byStage.set(stage, cached);
|
|
62204
|
+
}
|
|
62205
|
+
statuses.push(cached.status);
|
|
62206
|
+
}
|
|
62207
|
+
}
|
|
62208
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
62209
|
+
state.statuses = statuses;
|
|
62210
|
+
}
|
|
62211
|
+
receiptsOf(entry, runId, stageId) {
|
|
62212
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
61024
62213
|
if (byStage === undefined) {
|
|
61025
62214
|
byStage = new Map;
|
|
61026
|
-
for (const delivery of
|
|
62215
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
61027
62216
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
61028
62217
|
const list = byStage.get(key);
|
|
61029
62218
|
if (list === undefined)
|
|
@@ -61031,75 +62220,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
61031
62220
|
else
|
|
61032
62221
|
list.push(delivery);
|
|
61033
62222
|
}
|
|
61034
|
-
deliveriesByEntry.set(
|
|
62223
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
61035
62224
|
}
|
|
61036
62225
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
61037
|
-
}
|
|
61038
|
-
|
|
61039
|
-
|
|
61040
|
-
|
|
61041
|
-
|
|
61042
|
-
|
|
61043
|
-
|
|
61044
|
-
if (
|
|
61045
|
-
|
|
61046
|
-
|
|
61047
|
-
|
|
61048
|
-
|
|
61049
|
-
|
|
62226
|
+
}
|
|
62227
|
+
status(entry, run, stage, matchable, backend) {
|
|
62228
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
62229
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
62230
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
62231
|
+
let state;
|
|
62232
|
+
let reason;
|
|
62233
|
+
if (!matchable) {
|
|
62234
|
+
state = "root-unresolved";
|
|
62235
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
62236
|
+
} else if (delivered) {
|
|
62237
|
+
state = "delivered";
|
|
62238
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
62239
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
62240
|
+
state = "stage-terminal";
|
|
62241
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
62242
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
62243
|
+
state = "receipt-unverified";
|
|
62244
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
62245
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
62246
|
+
state = "delivery-unavailable";
|
|
62247
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
62248
|
+
} else if (receipts.length > 0) {
|
|
62249
|
+
state = "session-replaced";
|
|
62250
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
62251
|
+
} else {
|
|
62252
|
+
state = "queued";
|
|
62253
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
62254
|
+
}
|
|
62255
|
+
return {
|
|
62256
|
+
messageId: entry.message.id,
|
|
62257
|
+
target,
|
|
62258
|
+
runId: run.id,
|
|
62259
|
+
stageId: stage.id,
|
|
62260
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
62261
|
+
state,
|
|
62262
|
+
reason
|
|
62263
|
+
};
|
|
62264
|
+
}
|
|
62265
|
+
assemble(runs, rootRun) {
|
|
62266
|
+
const rows = [];
|
|
62267
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
62268
|
+
let liveStageMatches;
|
|
62269
|
+
const rowTargets = new Set;
|
|
62270
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
62271
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
62272
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61050
62273
|
continue;
|
|
61051
|
-
|
|
61052
|
-
|
|
61053
|
-
|
|
61054
|
-
|
|
61055
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
61056
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
61057
|
-
runId: run.id,
|
|
61058
|
-
stageId: stage.id,
|
|
61059
|
-
sessionId: stage.sessionId,
|
|
61060
|
-
admission: "context"
|
|
61061
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
61062
|
-
let state;
|
|
61063
|
-
let reason;
|
|
61064
|
-
if (!resolved || !hops) {
|
|
61065
|
-
state = "root-unresolved";
|
|
61066
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
61067
|
-
} else if (delivered) {
|
|
61068
|
-
state = "delivered";
|
|
61069
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
61070
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
61071
|
-
state = "stage-terminal";
|
|
61072
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
61073
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
61074
|
-
state = "receipt-unverified";
|
|
61075
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
61076
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
61077
|
-
state = "delivery-unavailable";
|
|
61078
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
61079
|
-
} else if (receipts.length > 0) {
|
|
61080
|
-
state = "session-replaced";
|
|
61081
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
61082
|
-
} else {
|
|
61083
|
-
state = "queued";
|
|
61084
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
61085
|
-
}
|
|
61086
|
-
result.push({
|
|
61087
|
-
messageId: entry.message.id,
|
|
61088
|
-
target,
|
|
61089
|
-
runId: run.id,
|
|
61090
|
-
stageId: stage.id,
|
|
61091
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
61092
|
-
state,
|
|
61093
|
-
reason
|
|
61094
|
-
});
|
|
62274
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
62275
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
62276
|
+
if (liveStageMatches(rowSegments))
|
|
62277
|
+
continue;
|
|
61095
62278
|
}
|
|
62279
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
62280
|
+
if (matches === undefined) {
|
|
62281
|
+
matches = new Map;
|
|
62282
|
+
this.scanMatches.set(scanEntry, matches);
|
|
62283
|
+
}
|
|
62284
|
+
let queuedCount = 0;
|
|
62285
|
+
for (const state of scanTargets) {
|
|
62286
|
+
let match = matches.get(state);
|
|
62287
|
+
if (match === undefined) {
|
|
62288
|
+
const segments = state.segments;
|
|
62289
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
62290
|
+
matches.set(state, match);
|
|
62291
|
+
}
|
|
62292
|
+
if (match)
|
|
62293
|
+
queuedCount += state.queued;
|
|
62294
|
+
}
|
|
62295
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
62296
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
62297
|
+
rowTargets.add(target);
|
|
62298
|
+
}
|
|
62299
|
+
let broadcastCount = 0;
|
|
62300
|
+
for (const state of this.targets.values()) {
|
|
62301
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
62302
|
+
broadcastCount += state.queued;
|
|
62303
|
+
}
|
|
62304
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
62305
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
62306
|
+
rowTargets.add(broadcast);
|
|
62307
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
62308
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
62309
|
+
for (const state of statusTargets) {
|
|
62310
|
+
if (rowTargets.has(state.target))
|
|
62311
|
+
continue;
|
|
62312
|
+
rowTargets.add(state.target);
|
|
62313
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
61096
62314
|
}
|
|
62315
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
62316
|
+
this.pruneRows(result);
|
|
62317
|
+
return result;
|
|
61097
62318
|
}
|
|
61098
|
-
|
|
62319
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
62320
|
+
const cached = this.rowCache.get(slot);
|
|
62321
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
62322
|
+
if (same && cached.statuses === deliveryStates)
|
|
62323
|
+
return cached.row;
|
|
62324
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
62325
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
62326
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
62327
|
+
return cached.row;
|
|
62328
|
+
}
|
|
62329
|
+
const row = {
|
|
62330
|
+
target,
|
|
62331
|
+
queuedCount,
|
|
62332
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
62333
|
+
...bounded
|
|
62334
|
+
};
|
|
62335
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
62336
|
+
return row;
|
|
62337
|
+
}
|
|
62338
|
+
bounded(states) {
|
|
62339
|
+
let bounded = this.boundedStates.get(states);
|
|
62340
|
+
if (bounded === undefined) {
|
|
62341
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
62342
|
+
this.boundedStates.set(states, bounded);
|
|
62343
|
+
}
|
|
62344
|
+
return bounded;
|
|
62345
|
+
}
|
|
62346
|
+
pruneRows(rows) {
|
|
62347
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
62348
|
+
return;
|
|
62349
|
+
const kept = new Set(rows);
|
|
62350
|
+
for (const [key, cached] of this.rowCache)
|
|
62351
|
+
if (!kept.has(cached.row))
|
|
62352
|
+
this.rowCache.delete(key);
|
|
62353
|
+
}
|
|
62354
|
+
materializedStageMatcher(runs) {
|
|
62355
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
62356
|
+
return (entrySegments) => {
|
|
62357
|
+
for (const run of invocationRuns) {
|
|
62358
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62359
|
+
if (hops === undefined)
|
|
62360
|
+
continue;
|
|
62361
|
+
for (const stage of run.stages) {
|
|
62362
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
62363
|
+
continue;
|
|
62364
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
62365
|
+
continue;
|
|
62366
|
+
}
|
|
62367
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
62368
|
+
return true;
|
|
62369
|
+
}
|
|
62370
|
+
}
|
|
62371
|
+
return false;
|
|
62372
|
+
};
|
|
62373
|
+
}
|
|
62374
|
+
}
|
|
62375
|
+
function sameBoundedRow(row, bounded) {
|
|
62376
|
+
if (bounded === undefined)
|
|
62377
|
+
return row.deliveryStates === undefined;
|
|
62378
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
62379
|
+
return false;
|
|
62380
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
62381
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
62382
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
62383
|
+
return false;
|
|
62384
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
61099
62385
|
}
|
|
61100
|
-
|
|
62386
|
+
function sameStatuses(left, right) {
|
|
62387
|
+
if (left.length !== right.length)
|
|
62388
|
+
return false;
|
|
62389
|
+
for (let index = 0;index < left.length; index++) {
|
|
62390
|
+
const a = left[index];
|
|
62391
|
+
const b = right[index];
|
|
62392
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
62393
|
+
return false;
|
|
62394
|
+
}
|
|
62395
|
+
return true;
|
|
62396
|
+
}
|
|
62397
|
+
var init_possible_stage_rows = __esm(() => {
|
|
61101
62398
|
init_run_owner_backend();
|
|
61102
62399
|
init_pending_stage_status();
|
|
62400
|
+
init_sticky_delivery_state_bound();
|
|
62401
|
+
init_store_internal();
|
|
61103
62402
|
init_workflow_stage_target();
|
|
61104
62403
|
});
|
|
61105
62404
|
|
|
@@ -61164,11 +62463,32 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61164
62463
|
announceIfPending();
|
|
61165
62464
|
return routeCompletions.get(runId);
|
|
61166
62465
|
});
|
|
62466
|
+
const topology = new WorkflowTopologyIndex;
|
|
62467
|
+
const possibleStageRowIndexes = new Map;
|
|
62468
|
+
const rowIdentities = new WeakMap;
|
|
62469
|
+
let nextRowIdentity = 0;
|
|
62470
|
+
const rowIdentity = (row) => {
|
|
62471
|
+
let identity = rowIdentities.get(row);
|
|
62472
|
+
if (identity === undefined) {
|
|
62473
|
+
identity = nextRowIdentity++;
|
|
62474
|
+
rowIdentities.set(row, identity);
|
|
62475
|
+
}
|
|
62476
|
+
return identity;
|
|
62477
|
+
};
|
|
62478
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
62479
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
62480
|
+
if (index === undefined) {
|
|
62481
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
62482
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
62483
|
+
}
|
|
62484
|
+
return index;
|
|
62485
|
+
};
|
|
61167
62486
|
const announceRoutes = () => {
|
|
61168
62487
|
if (disposed)
|
|
61169
62488
|
return;
|
|
61170
62489
|
const ownedRunIds = new Set;
|
|
61171
62490
|
const runs = activeStore.runs();
|
|
62491
|
+
topology.refresh(runs);
|
|
61172
62492
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61173
62493
|
const childrenByBoundary = new Map;
|
|
61174
62494
|
for (const run of runs) {
|
|
@@ -61183,7 +62503,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61183
62503
|
for (const run of runs) {
|
|
61184
62504
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
61185
62505
|
continue;
|
|
61186
|
-
const rootRunId =
|
|
62506
|
+
const rootRunId = topology.rootOf(run.id);
|
|
61187
62507
|
if (rootRunId === undefined)
|
|
61188
62508
|
continue;
|
|
61189
62509
|
ownedRunIds.add(run.id);
|
|
@@ -61193,6 +62513,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61193
62513
|
runId: parentRun.id,
|
|
61194
62514
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
61195
62515
|
};
|
|
62516
|
+
let possibleStages;
|
|
62517
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
62518
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62519
|
+
possibleStages = [];
|
|
62520
|
+
} else if (run.id === rootRunId) {
|
|
62521
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
62522
|
+
}
|
|
61196
62523
|
const announcement = {
|
|
61197
62524
|
runId: run.id,
|
|
61198
62525
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -61217,11 +62544,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61217
62544
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
61218
62545
|
}))
|
|
61219
62546
|
],
|
|
61220
|
-
...
|
|
61221
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
61222
|
-
} : {}
|
|
62547
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
61223
62548
|
};
|
|
61224
|
-
const signature = JSON.stringify(announcement)
|
|
62549
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
61225
62550
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
61226
62551
|
continue;
|
|
61227
62552
|
const entry = { signature };
|
|
@@ -61256,6 +62581,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61256
62581
|
if (!ownedRunIds.has(runId))
|
|
61257
62582
|
routeCompletions.delete(runId);
|
|
61258
62583
|
}
|
|
62584
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
62585
|
+
if (!ownedRunIds.has(rootRunId))
|
|
62586
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62587
|
+
}
|
|
61259
62588
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
61260
62589
|
return;
|
|
61261
62590
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
@@ -61416,102 +62745,6 @@ function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegmen
|
|
|
61416
62745
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
61417
62746
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
61418
62747
|
}
|
|
61419
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
61420
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
61421
|
-
return [];
|
|
61422
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
61423
|
-
const segmentsByTarget = new Map;
|
|
61424
|
-
const stickySegments = (entry) => {
|
|
61425
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61426
|
-
if (segmentsByTarget.has(target))
|
|
61427
|
-
return segmentsByTarget.get(target);
|
|
61428
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
61429
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
61430
|
-
segmentsByTarget.set(target, segments);
|
|
61431
|
-
return segments;
|
|
61432
|
-
};
|
|
61433
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
61434
|
-
const segments = stickySegments(entry);
|
|
61435
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
61436
|
-
});
|
|
61437
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
61438
|
-
const rows = [];
|
|
61439
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
61440
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
61441
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61442
|
-
continue;
|
|
61443
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
61444
|
-
continue;
|
|
61445
|
-
}
|
|
61446
|
-
rows.push({
|
|
61447
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
61448
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
61449
|
-
const segments = stickySegments(entry);
|
|
61450
|
-
if (segments === undefined)
|
|
61451
|
-
return false;
|
|
61452
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
61453
|
-
}).length
|
|
61454
|
-
});
|
|
61455
|
-
}
|
|
61456
|
-
rows.push({
|
|
61457
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
61458
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
61459
|
-
const segments = stickySegments(entry);
|
|
61460
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
61461
|
-
}).length
|
|
61462
|
-
});
|
|
61463
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
61464
|
-
const statusesByTarget = new Map;
|
|
61465
|
-
for (const status of statuses) {
|
|
61466
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
61467
|
-
if (forTarget === undefined)
|
|
61468
|
-
statusesByTarget.set(status.target, [status]);
|
|
61469
|
-
else
|
|
61470
|
-
forTarget.push(status);
|
|
61471
|
-
}
|
|
61472
|
-
const queuedByTarget = new Map;
|
|
61473
|
-
for (const entry of stickyQueued) {
|
|
61474
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61475
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
61476
|
-
}
|
|
61477
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
61478
|
-
for (const target of statusesByTarget.keys()) {
|
|
61479
|
-
if (rowTargets.has(target))
|
|
61480
|
-
continue;
|
|
61481
|
-
rowTargets.add(target);
|
|
61482
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
61483
|
-
}
|
|
61484
|
-
for (let index = 0;index < rows.length; index++) {
|
|
61485
|
-
const row = rows[index];
|
|
61486
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
61487
|
-
if (deliveryStates !== undefined)
|
|
61488
|
-
rows[index] = { ...row, deliveryStates };
|
|
61489
|
-
}
|
|
61490
|
-
return rows;
|
|
61491
|
-
}
|
|
61492
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
61493
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61494
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
61495
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
61496
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
61497
|
-
}
|
|
61498
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
61499
|
-
for (const run of invocationRuns) {
|
|
61500
|
-
const hops = hopsFor(run.id);
|
|
61501
|
-
if (hops === undefined)
|
|
61502
|
-
continue;
|
|
61503
|
-
for (const stage of run.stages) {
|
|
61504
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
61505
|
-
continue;
|
|
61506
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
61507
|
-
continue;
|
|
61508
|
-
}
|
|
61509
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
61510
|
-
return true;
|
|
61511
|
-
}
|
|
61512
|
-
}
|
|
61513
|
-
return false;
|
|
61514
|
-
}
|
|
61515
62748
|
function isPendingStageMessageEvent(value2) {
|
|
61516
62749
|
if (typeof value2 !== "object" || value2 === null)
|
|
61517
62750
|
return false;
|
|
@@ -61895,7 +63128,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
61895
63128
|
init_pending_stage_route_capability();
|
|
61896
63129
|
init_pending_stage_route_readiness();
|
|
61897
63130
|
init_pending_stage_status();
|
|
61898
|
-
|
|
63131
|
+
init_possible_stage_rows();
|
|
61899
63132
|
init_store_internal();
|
|
61900
63133
|
init_workflow_stage_target();
|
|
61901
63134
|
});
|
|
@@ -64766,9 +65999,146 @@ var init_definitions = __esm(() => {
|
|
|
64766
65999
|
workerDefinitionSources = new WeakMap;
|
|
64767
66000
|
});
|
|
64768
66001
|
|
|
66002
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
66003
|
+
function push(map, key, value2) {
|
|
66004
|
+
if (key === undefined)
|
|
66005
|
+
return;
|
|
66006
|
+
const list = map.get(key);
|
|
66007
|
+
if (list)
|
|
66008
|
+
list.push(value2);
|
|
66009
|
+
else
|
|
66010
|
+
map.set(key, [value2]);
|
|
66011
|
+
}
|
|
66012
|
+
|
|
66013
|
+
class Partitions {
|
|
66014
|
+
byRun = new Map;
|
|
66015
|
+
startsByChild = new Map;
|
|
66016
|
+
stagesByGroup = new Map;
|
|
66017
|
+
stagesByReplayKey = new Map;
|
|
66018
|
+
boundariesByStageId = new Map;
|
|
66019
|
+
boundariesByChild = new Map;
|
|
66020
|
+
add(entry) {
|
|
66021
|
+
const checkpoint = entry.checkpoint;
|
|
66022
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
66023
|
+
if (checkpoint.kind !== "stage")
|
|
66024
|
+
return;
|
|
66025
|
+
const stage = entry;
|
|
66026
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
66027
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
66028
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66029
|
+
if (!boundary)
|
|
66030
|
+
return;
|
|
66031
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
66032
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
66033
|
+
if (boundary.event === "start")
|
|
66034
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
66035
|
+
}
|
|
66036
|
+
}
|
|
66037
|
+
function inKeyOrder(entries) {
|
|
66038
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
66039
|
+
}
|
|
66040
|
+
|
|
66041
|
+
class PoolCheckpointView {
|
|
66042
|
+
base;
|
|
66043
|
+
from;
|
|
66044
|
+
delta;
|
|
66045
|
+
constructor(base, from, delta) {
|
|
66046
|
+
this.base = base;
|
|
66047
|
+
this.from = from;
|
|
66048
|
+
this.delta = delta;
|
|
66049
|
+
}
|
|
66050
|
+
entries(partition, key) {
|
|
66051
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
66052
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
66053
|
+
}
|
|
66054
|
+
}
|
|
66055
|
+
|
|
66056
|
+
class PoolCheckpointRead {
|
|
66057
|
+
index;
|
|
66058
|
+
root;
|
|
66059
|
+
pending;
|
|
66060
|
+
constructor(index, root) {
|
|
66061
|
+
this.index = index;
|
|
66062
|
+
this.root = root;
|
|
66063
|
+
}
|
|
66064
|
+
read(tx, events) {
|
|
66065
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
66066
|
+
this.pending = apply;
|
|
66067
|
+
});
|
|
66068
|
+
}
|
|
66069
|
+
commit() {
|
|
66070
|
+
this.pending?.();
|
|
66071
|
+
this.pending = undefined;
|
|
66072
|
+
}
|
|
66073
|
+
}
|
|
66074
|
+
|
|
66075
|
+
class PoolCheckpointIndex {
|
|
66076
|
+
roots = new Map;
|
|
66077
|
+
begin(root) {
|
|
66078
|
+
return new PoolCheckpointRead(this, root);
|
|
66079
|
+
}
|
|
66080
|
+
async read(tx, events, root, onCommit) {
|
|
66081
|
+
const current = this.roots.get(root);
|
|
66082
|
+
let base = current;
|
|
66083
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
66084
|
+
if (base && base.seq > 0) {
|
|
66085
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
66086
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
66087
|
+
base = undefined;
|
|
66088
|
+
}
|
|
66089
|
+
const from = base?.seq ?? 0;
|
|
66090
|
+
const to = head ? Number(head.seq) : 0;
|
|
66091
|
+
const delta = new Partitions;
|
|
66092
|
+
const added = [];
|
|
66093
|
+
if (to > from) {
|
|
66094
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
66095
|
+
for (const row of rows) {
|
|
66096
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
66097
|
+
if (checkpoint.kind === "ui")
|
|
66098
|
+
continue;
|
|
66099
|
+
const entry = {
|
|
66100
|
+
seq: Number(row.seq),
|
|
66101
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
66102
|
+
checkpoint
|
|
66103
|
+
};
|
|
66104
|
+
delta.add(entry);
|
|
66105
|
+
added.push(entry);
|
|
66106
|
+
}
|
|
66107
|
+
}
|
|
66108
|
+
onCommit(() => {
|
|
66109
|
+
const latest = this.roots.get(root);
|
|
66110
|
+
if (!base) {
|
|
66111
|
+
if (latest !== current)
|
|
66112
|
+
return;
|
|
66113
|
+
const fresh = {
|
|
66114
|
+
seq: to,
|
|
66115
|
+
key: head?.key ?? null,
|
|
66116
|
+
at: Number(head?.at ?? 0),
|
|
66117
|
+
partitions: delta
|
|
66118
|
+
};
|
|
66119
|
+
this.roots.set(root, fresh);
|
|
66120
|
+
return;
|
|
66121
|
+
}
|
|
66122
|
+
if (latest !== base || to <= base.seq)
|
|
66123
|
+
return;
|
|
66124
|
+
for (const entry of added)
|
|
66125
|
+
if (entry.seq > base.seq)
|
|
66126
|
+
base.partitions.add(entry);
|
|
66127
|
+
base.seq = to;
|
|
66128
|
+
base.key = head?.key ?? null;
|
|
66129
|
+
base.at = Number(head?.at ?? 0);
|
|
66130
|
+
});
|
|
66131
|
+
return new PoolCheckpointView(base, from, delta);
|
|
66132
|
+
}
|
|
66133
|
+
}
|
|
66134
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
66135
|
+
init_stage_topology_validation();
|
|
66136
|
+
init_types();
|
|
66137
|
+
});
|
|
66138
|
+
|
|
64769
66139
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
64770
66140
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
64771
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
66141
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
64772
66142
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
64773
66143
|
throw new Error("Invalid pool mutation");
|
|
64774
66144
|
const checkpoint = mutation.checkpoint;
|
|
@@ -64788,42 +66158,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
64788
66158
|
}
|
|
64789
66159
|
if (checkpoint.kind === "ui")
|
|
64790
66160
|
return;
|
|
64791
|
-
const
|
|
64792
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
64793
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
66161
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
64794
66162
|
const topology = checkpoint.topology;
|
|
64795
66163
|
const run = topology?.run;
|
|
64796
66164
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
64797
66165
|
throw new Error("Pool topology scope mismatch");
|
|
64798
|
-
const owner =
|
|
66166
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
64799
66167
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
64800
66168
|
throw new Error("Pool topology scope mismatch");
|
|
64801
66169
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
64802
66170
|
throw new Error("Pool session scope mismatch");
|
|
66171
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
64803
66172
|
if (checkpoint.kind === "tool") {
|
|
64804
66173
|
const tool = checkpoint.topology;
|
|
64805
|
-
const peers =
|
|
66174
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
64806
66175
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
64807
66176
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
64808
66177
|
throw new Error("Pool tool topology scope mismatch");
|
|
64809
66178
|
return;
|
|
64810
66179
|
}
|
|
64811
|
-
const
|
|
64812
|
-
const
|
|
66180
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
66181
|
+
const relevant = [...runStages];
|
|
66182
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
66183
|
+
for (const group of groups)
|
|
66184
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
66185
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66186
|
+
let start;
|
|
66187
|
+
if (boundary) {
|
|
66188
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
66189
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
66190
|
+
relevant.push(...replayed);
|
|
66191
|
+
if (start) {
|
|
66192
|
+
const startTopology = start.topology;
|
|
66193
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
66194
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
66195
|
+
}
|
|
66196
|
+
}
|
|
66197
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
66198
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
64813
66199
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
64814
66200
|
if (stageError)
|
|
64815
66201
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
64816
|
-
const boundary = checkpoint.topology?.boundary;
|
|
64817
66202
|
if (!boundary) {
|
|
64818
66203
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
64819
66204
|
throw new Error("Pool boundary output without ownership");
|
|
64820
66205
|
return;
|
|
64821
66206
|
}
|
|
64822
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
64823
66207
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
64824
66208
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
64825
66209
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
64826
66210
|
}
|
|
66211
|
+
function stagesExcept(entries, checkpoint) {
|
|
66212
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
66213
|
+
}
|
|
64827
66214
|
async function sessionStore(tx, prefix) {
|
|
64828
66215
|
const database = {
|
|
64829
66216
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -64872,20 +66259,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
64872
66259
|
init_boundary_lifecycle();
|
|
64873
66260
|
init_boundary_topology();
|
|
64874
66261
|
init_stage_topology_validation();
|
|
64875
|
-
|
|
66262
|
+
init_pool_checkpoint_index();
|
|
64876
66263
|
});
|
|
64877
66264
|
|
|
64878
66265
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
64879
66266
|
function poolUsageTable(runs) {
|
|
64880
66267
|
return `${runs}_pool_usage`;
|
|
64881
66268
|
}
|
|
66269
|
+
function poolUsageTotalsTable(runs) {
|
|
66270
|
+
return `${runs}_pool_usage_totals`;
|
|
66271
|
+
}
|
|
66272
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
66273
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
66274
|
+
const total = field.toLowerCase();
|
|
66275
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
66276
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
66277
|
+
});
|
|
66278
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
66279
|
+
}
|
|
64882
66280
|
function poolUsageSchema(runs) {
|
|
64883
|
-
|
|
66281
|
+
const usage = poolUsageTable(runs);
|
|
66282
|
+
const totals = poolUsageTotalsTable(runs);
|
|
66283
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66284
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
64884
66285
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
64885
66286
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
64886
66287
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
64887
66288
|
PRIMARY KEY(work, generation, key)
|
|
64888
|
-
)
|
|
66289
|
+
);
|
|
66290
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
66291
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
66292
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
66293
|
+
);
|
|
66294
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
66295
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66296
|
+
END;
|
|
66297
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
66298
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
66299
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
66300
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66301
|
+
END;
|
|
66302
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
66303
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
66304
|
+
END;
|
|
66305
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
66306
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
66307
|
+
END;`;
|
|
64889
66308
|
}
|
|
64890
66309
|
function poolStageUsageKey(runId, stageId) {
|
|
64891
66310
|
return `stage:${runId}:${stageId}`;
|
|
@@ -64912,7 +66331,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
64912
66331
|
|
|
64913
66332
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
64914
66333
|
import { createHash as createHash4, randomUUID as randomUUID16 } from "node:crypto";
|
|
64915
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
66334
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
64916
66335
|
function requestIdentity(request) {
|
|
64917
66336
|
return createHash4("sha256").update(request).digest("hex");
|
|
64918
66337
|
}
|
|
@@ -64951,6 +66370,7 @@ class SqlPoolQueue {
|
|
|
64951
66370
|
operations;
|
|
64952
66371
|
transaction;
|
|
64953
66372
|
journal;
|
|
66373
|
+
checkpoints = new PoolCheckpointIndex;
|
|
64954
66374
|
constructor(storage) {
|
|
64955
66375
|
this.storage = storage;
|
|
64956
66376
|
const { transaction, tables: journal } = storage;
|
|
@@ -65062,17 +66482,23 @@ class SqlPoolQueue {
|
|
|
65062
66482
|
lease = snapshotLease(lease);
|
|
65063
66483
|
token = { ...token };
|
|
65064
66484
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
65065
|
-
|
|
65066
|
-
|
|
65067
|
-
|
|
65068
|
-
|
|
65069
|
-
|
|
65070
|
-
|
|
65071
|
-
|
|
65072
|
-
|
|
65073
|
-
|
|
65074
|
-
|
|
65075
|
-
|
|
66485
|
+
try {
|
|
66486
|
+
if (runId !== lease.runId)
|
|
66487
|
+
throw new Error("Pool delivery cannot write another root");
|
|
66488
|
+
await this.authority(tx, lease).catch((error) => {
|
|
66489
|
+
if (error instanceof StaleLeaseError)
|
|
66490
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
66491
|
+
throw error;
|
|
66492
|
+
});
|
|
66493
|
+
const row = await this.current(tx, lease, token);
|
|
66494
|
+
if (row.epoch !== lease.epoch)
|
|
66495
|
+
throw new Error(STALE_WORK_LEASE);
|
|
66496
|
+
assertSession();
|
|
66497
|
+
} catch (error) {
|
|
66498
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
66499
|
+
throw error;
|
|
66500
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
66501
|
+
}
|
|
65076
66502
|
}, action);
|
|
65077
66503
|
}
|
|
65078
66504
|
async assertCurrent(lease, token) {
|
|
@@ -65185,7 +66611,8 @@ class SqlPoolQueue {
|
|
|
65185
66611
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
65186
66612
|
const identity = requestIdentity(request);
|
|
65187
66613
|
mutation = JSON.parse(request).mutation;
|
|
65188
|
-
|
|
66614
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
66615
|
+
const receipt = await this.transaction(async (tx) => {
|
|
65189
66616
|
const { now, status } = await this.authority(tx, lease);
|
|
65190
66617
|
const row = await this.current(tx, lease, token);
|
|
65191
66618
|
if (row.epoch !== lease.epoch)
|
|
@@ -65200,24 +66627,26 @@ class SqlPoolQueue {
|
|
|
65200
66627
|
throw new Error(STALE_WORK_LEASE);
|
|
65201
66628
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
65202
66629
|
throw new Error("Pool root is not running");
|
|
65203
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
66630
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
65204
66631
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
65205
66632
|
return receipt;
|
|
65206
66633
|
});
|
|
66634
|
+
checkpoints.commit();
|
|
66635
|
+
return receipt;
|
|
65207
66636
|
}
|
|
65208
66637
|
async appendSession(tx, row, mutation, now) {
|
|
65209
66638
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
65210
66639
|
}
|
|
65211
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
66640
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
65212
66641
|
const checkpoint = mutation.checkpoint;
|
|
65213
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
66642
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
65214
66643
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
65215
66644
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
65216
66645
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
65217
66646
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
65218
66647
|
const payload = serializeJson(checkpoint);
|
|
65219
66648
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
65220
|
-
if (existing && existing.payload
|
|
66649
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
65221
66650
|
throw new Error("Pool checkpoint identity conflict");
|
|
65222
66651
|
if (existing && mutation.kind !== "checkpoint")
|
|
65223
66652
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -65315,22 +66744,37 @@ class SqlPoolQueue {
|
|
|
65315
66744
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
65316
66745
|
}
|
|
65317
66746
|
}
|
|
66747
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
66748
|
+
if (stored === payload)
|
|
66749
|
+
return true;
|
|
66750
|
+
const { completedAt } = JSON.parse(stored);
|
|
66751
|
+
if (typeof completedAt !== "number")
|
|
66752
|
+
return false;
|
|
66753
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
66754
|
+
}
|
|
65318
66755
|
function openSqlPoolQueue(journal) {
|
|
65319
66756
|
const storage = sqlPoolStorage(journal);
|
|
65320
66757
|
if (!storage)
|
|
65321
66758
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
65322
66759
|
return SqlPoolQueue.open(storage);
|
|
65323
66760
|
}
|
|
65324
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
66761
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
65325
66762
|
var init_sql_pool_queue = __esm(() => {
|
|
65326
66763
|
init_serializable();
|
|
65327
66764
|
init_boundary_topology();
|
|
65328
66765
|
init_durable_hash();
|
|
65329
66766
|
init_internal();
|
|
66767
|
+
init_pool_checkpoint_index();
|
|
65330
66768
|
init_pool_mutations();
|
|
65331
66769
|
init_pool_stage_usage();
|
|
65332
66770
|
init_types();
|
|
65333
66771
|
init_values();
|
|
66772
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
66773
|
+
constructor(message, options) {
|
|
66774
|
+
super(message, options);
|
|
66775
|
+
this.name = "PoolAppendRefusedError";
|
|
66776
|
+
}
|
|
66777
|
+
};
|
|
65334
66778
|
});
|
|
65335
66779
|
|
|
65336
66780
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
@@ -65353,15 +66797,22 @@ function validate(report) {
|
|
|
65353
66797
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
65354
66798
|
}
|
|
65355
66799
|
}
|
|
66800
|
+
function addCompensated(total, value2) {
|
|
66801
|
+
const sum = total.sum + value2;
|
|
66802
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
66803
|
+
total.sum = sum;
|
|
66804
|
+
}
|
|
65356
66805
|
|
|
65357
66806
|
class SqlPoolUsage {
|
|
65358
66807
|
transaction;
|
|
65359
66808
|
journal;
|
|
65360
66809
|
table;
|
|
66810
|
+
totals;
|
|
65361
66811
|
constructor(transaction, journal) {
|
|
65362
66812
|
this.transaction = transaction;
|
|
65363
66813
|
this.journal = journal;
|
|
65364
66814
|
this.table = poolUsageTable(journal.runs);
|
|
66815
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
65365
66816
|
}
|
|
65366
66817
|
static async open(transaction, journal) {
|
|
65367
66818
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -65369,9 +66820,27 @@ class SqlPoolUsage {
|
|
|
65369
66820
|
return usage;
|
|
65370
66821
|
}
|
|
65371
66822
|
async totalIn(tx, root) {
|
|
66823
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66824
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
66825
|
+
if (kept)
|
|
66826
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
66827
|
+
field,
|
|
66828
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
66829
|
+
]));
|
|
65372
66830
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
65373
66831
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
65374
|
-
|
|
66832
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
66833
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
66834
|
+
return total;
|
|
66835
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
66836
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
66837
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
66838
|
+
for (const usage of rows)
|
|
66839
|
+
for (const [index, column] of columns.entries())
|
|
66840
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
66841
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
66842
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
66843
|
+
return total;
|
|
65375
66844
|
}
|
|
65376
66845
|
async report(lease, token, reports) {
|
|
65377
66846
|
validateText("runId", lease.runId);
|
|
@@ -68724,29 +70193,23 @@ var init_run_durable_stage_session = __esm(() => {
|
|
|
68724
70193
|
|
|
68725
70194
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
68726
70195
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
68727
|
-
const
|
|
68728
|
-
if (!
|
|
70196
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
70197
|
+
if (!catalog.positionsById.has(childRunId))
|
|
68729
70198
|
return;
|
|
68730
|
-
const intentionalExitStatuses = new Map;
|
|
68731
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
68732
|
-
if (checkpoint.kind !== "stage")
|
|
68733
|
-
continue;
|
|
68734
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
68735
|
-
if (child?.exited === true)
|
|
68736
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
68737
|
-
}
|
|
68738
70199
|
const included = new Set([childRunId]);
|
|
68739
|
-
|
|
68740
|
-
|
|
68741
|
-
|
|
68742
|
-
|
|
68743
|
-
|
|
68744
|
-
|
|
70200
|
+
const pending = [childRunId];
|
|
70201
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
70202
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
70203
|
+
if (included.has(childId))
|
|
70204
|
+
continue;
|
|
70205
|
+
included.add(childId);
|
|
70206
|
+
pending.push(childId);
|
|
68745
70207
|
}
|
|
68746
70208
|
}
|
|
68747
|
-
const
|
|
68748
|
-
const
|
|
68749
|
-
|
|
70209
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
70210
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
70211
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
70212
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
68750
70213
|
}
|
|
68751
70214
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
68752
70215
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -68867,7 +70330,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
68867
70330
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
68868
70331
|
kind: "stage",
|
|
68869
70332
|
workflowId: input.run.id,
|
|
68870
|
-
checkpointId:
|
|
70333
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
68871
70334
|
name,
|
|
68872
70335
|
replayKey,
|
|
68873
70336
|
completedAt: Date.now(),
|
|
@@ -77288,11 +78751,68 @@ function runResultFields(backend, handle) {
|
|
|
77288
78751
|
const result = durableRunResult(backend, handle.workflowId);
|
|
77289
78752
|
return result === undefined ? {} : { result };
|
|
77290
78753
|
}
|
|
77291
|
-
function
|
|
78754
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
77292
78755
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
77293
78756
|
if (handle === undefined)
|
|
77294
|
-
return
|
|
77295
|
-
|
|
78757
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
78758
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
78759
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
78760
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
78761
|
+
return cached.catalog;
|
|
78762
|
+
}
|
|
78763
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
78764
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
78765
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
78766
|
+
if (version !== undefined) {
|
|
78767
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
78768
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
78769
|
+
byRoot.delete(rootWorkflowId);
|
|
78770
|
+
byRoot.set(rootWorkflowId, {
|
|
78771
|
+
version,
|
|
78772
|
+
name: handle.name,
|
|
78773
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
78774
|
+
catalog
|
|
78775
|
+
});
|
|
78776
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
78777
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
78778
|
+
}
|
|
78779
|
+
return catalog;
|
|
78780
|
+
}
|
|
78781
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
78782
|
+
const positionsById = new Map;
|
|
78783
|
+
const childIdsByParent = new Map;
|
|
78784
|
+
runs.forEach((run, position) => {
|
|
78785
|
+
const positions = positionsById.get(run.id);
|
|
78786
|
+
if (positions === undefined)
|
|
78787
|
+
positionsById.set(run.id, [position]);
|
|
78788
|
+
else
|
|
78789
|
+
positions.push(position);
|
|
78790
|
+
if (run.parentRunId === undefined)
|
|
78791
|
+
return;
|
|
78792
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
78793
|
+
if (children === undefined)
|
|
78794
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
78795
|
+
else
|
|
78796
|
+
children.push(run.id);
|
|
78797
|
+
});
|
|
78798
|
+
const intentionalExitStatuses = new Map;
|
|
78799
|
+
for (const checkpoint of checkpoints) {
|
|
78800
|
+
if (checkpoint.kind !== "stage")
|
|
78801
|
+
continue;
|
|
78802
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
78803
|
+
if (child?.exited === true)
|
|
78804
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
78805
|
+
}
|
|
78806
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
78807
|
+
}
|
|
78808
|
+
function readsFallbackCompletedAt(run) {
|
|
78809
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
78810
|
+
}
|
|
78811
|
+
function cloneRunSnapshot(run) {
|
|
78812
|
+
const copy = structuredClone(run);
|
|
78813
|
+
for (const tool of copy.toolNodes)
|
|
78814
|
+
Object.freeze(tool.parentIds);
|
|
78815
|
+
return copy;
|
|
77296
78816
|
}
|
|
77297
78817
|
async function validatedStageTranscript(stage, store) {
|
|
77298
78818
|
if (stage.sessionId !== undefined && await isReopenableSession(store, stage.sessionId))
|
|
@@ -77322,11 +78842,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
77322
78842
|
tools: [...toolByHash.values()]
|
|
77323
78843
|
};
|
|
77324
78844
|
}
|
|
77325
|
-
function completedToolNodes(
|
|
77326
|
-
return
|
|
77327
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
77328
|
-
if (topologyRunId !== runId)
|
|
77329
|
-
return [];
|
|
78845
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
78846
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
77330
78847
|
const topology = checkpoint.topology;
|
|
77331
78848
|
if (topology === undefined)
|
|
77332
78849
|
return [];
|
|
@@ -77363,18 +78880,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
77363
78880
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
77364
78881
|
}
|
|
77365
78882
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
77366
|
-
|
|
78883
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
78884
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
77367
78885
|
return [];
|
|
77368
|
-
const stageRecords =
|
|
78886
|
+
const stageRecords = index.stages;
|
|
77369
78887
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
77370
78888
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
77371
78889
|
return [];
|
|
77372
|
-
const
|
|
77373
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
78890
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
77374
78891
|
const drafts = checkpointDrafts(checkpoints);
|
|
77375
78892
|
const toolCheckpoints = drafts.tools;
|
|
77376
78893
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
77377
78894
|
return [];
|
|
78895
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
78896
|
+
const toolsByRun = new Map;
|
|
78897
|
+
for (const checkpoint of toolCheckpoints) {
|
|
78898
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
78899
|
+
if (claimedRunId !== undefined) {
|
|
78900
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
78901
|
+
ids.add(checkpoint.topology.nodeId);
|
|
78902
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
78903
|
+
}
|
|
78904
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
78905
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
78906
|
+
if (owned === undefined)
|
|
78907
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
78908
|
+
else
|
|
78909
|
+
owned.push(checkpoint);
|
|
78910
|
+
}
|
|
77378
78911
|
for (const draft of candidates) {
|
|
77379
78912
|
const boundary = draft.topology?.boundary;
|
|
77380
78913
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -77388,8 +78921,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77388
78921
|
parentStageId: boundary.child.parentStageId,
|
|
77389
78922
|
rootRunId: boundary.child.rootRunId
|
|
77390
78923
|
};
|
|
77391
|
-
const childToolIds =
|
|
77392
|
-
if (
|
|
78924
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
78925
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
77393
78926
|
return [];
|
|
77394
78927
|
}
|
|
77395
78928
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -77417,7 +78950,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77417
78950
|
group.sort(compareDraftSourceOrder);
|
|
77418
78951
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
77419
78952
|
return [];
|
|
77420
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
78953
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
77421
78954
|
return [];
|
|
77422
78955
|
const syntheticChildren = [];
|
|
77423
78956
|
const pendingChildren = [];
|
|
@@ -77431,7 +78964,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77431
78964
|
const child = workflowChildFromDraft(draft);
|
|
77432
78965
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
77433
78966
|
continue;
|
|
77434
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
78967
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
77435
78968
|
return [];
|
|
77436
78969
|
}
|
|
77437
78970
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -77439,7 +78972,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77439
78972
|
}
|
|
77440
78973
|
const idMaps = new Map;
|
|
77441
78974
|
for (const [runId, runDrafts] of grouped) {
|
|
77442
|
-
const ownedTools =
|
|
78975
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77443
78976
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
77444
78977
|
return [];
|
|
77445
78978
|
const ids = new Map;
|
|
@@ -77457,32 +78990,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77457
78990
|
});
|
|
77458
78991
|
if (hasIdentityConflict)
|
|
77459
78992
|
return [];
|
|
77460
|
-
for (const checkpoint of
|
|
77461
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
78993
|
+
for (const checkpoint of ownedTools) {
|
|
77462
78994
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
77463
|
-
|
|
77464
|
-
ids.set(sourceId, sourceId);
|
|
78995
|
+
ids.set(sourceId, sourceId);
|
|
77465
78996
|
}
|
|
77466
78997
|
idMaps.set(runId, ids);
|
|
77467
78998
|
}
|
|
78999
|
+
const boundaryOwners = new Map;
|
|
79000
|
+
const boundarySourcesByParent = new Map;
|
|
79001
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
79002
|
+
const sources = new Map;
|
|
79003
|
+
for (const draft of runDrafts) {
|
|
79004
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
79005
|
+
if (childRunId === undefined)
|
|
79006
|
+
continue;
|
|
79007
|
+
if (!boundaryOwners.has(childRunId))
|
|
79008
|
+
boundaryOwners.set(childRunId, draft);
|
|
79009
|
+
if (!sources.has(childRunId))
|
|
79010
|
+
sources.set(childRunId, draft);
|
|
79011
|
+
}
|
|
79012
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
79013
|
+
}
|
|
77468
79014
|
const runs = [];
|
|
77469
79015
|
const emittedRunIds = new Set;
|
|
77470
79016
|
for (const [runId, runDrafts] of grouped) {
|
|
77471
79017
|
const ids = idMaps.get(runId);
|
|
77472
|
-
const ownedTools =
|
|
79018
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77473
79019
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
77474
79020
|
if (hasUnknownParents)
|
|
77475
79021
|
return [];
|
|
77476
79022
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
77477
|
-
const toolNodes = completedToolNodes(
|
|
79023
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
77478
79024
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
77479
79025
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
77480
79026
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
77481
|
-
const owner = runId === rootRunId ? undefined :
|
|
79027
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
77482
79028
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
77483
79029
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
77484
79030
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
77485
|
-
const boundarySourceId =
|
|
79031
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
77486
79032
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
77487
79033
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
77488
79034
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -77618,10 +79164,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
77618
79164
|
resumable: false
|
|
77619
79165
|
};
|
|
77620
79166
|
}
|
|
77621
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
79167
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
77622
79168
|
var init_completed_catalog = __esm(() => {
|
|
77623
79169
|
init_stage_session_store();
|
|
77624
79170
|
init_tool_payload_bounds();
|
|
79171
|
+
init_completed_catalog_index();
|
|
77625
79172
|
init_completed_catalog_stage_groups();
|
|
77626
79173
|
init_resume_eligibility();
|
|
77627
79174
|
init_resume_runtime();
|
|
@@ -77630,6 +79177,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
77630
79177
|
init_workflow_heartbeat_anchor();
|
|
77631
79178
|
init_stage_topology_validation();
|
|
77632
79179
|
init_tool_outcome();
|
|
79180
|
+
nestedRunCatalogs = new WeakMap;
|
|
79181
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
79182
|
+
runs: [],
|
|
79183
|
+
positionsById: new Map,
|
|
79184
|
+
childIdsByParent: new Map,
|
|
79185
|
+
intentionalExitStatuses: new Map
|
|
79186
|
+
};
|
|
77633
79187
|
COMPLETED_RECONSTRUCTION = {
|
|
77634
79188
|
requireTerminal: true,
|
|
77635
79189
|
retainActive: false,
|
|
@@ -89860,7 +91414,119 @@ function observeRunExecution(run, now = Date.now()) {
|
|
|
89860
91414
|
}
|
|
89861
91415
|
|
|
89862
91416
|
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89863
|
-
|
|
91417
|
+
init_sticky_delivery_state_bound();
|
|
91418
|
+
|
|
91419
|
+
// dist/builtin/workflows/src/shared/sticky-stage-delivery-status.ts
|
|
91420
|
+
init_run_owner_backend();
|
|
91421
|
+
init_pending_stage_status();
|
|
91422
|
+
init_workflow_stage_target();
|
|
91423
|
+
function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
91424
|
+
const root = runs.find((run) => run.id === rootRunId);
|
|
91425
|
+
const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
|
|
91426
|
+
const result = [];
|
|
91427
|
+
const patternByTarget = new Map;
|
|
91428
|
+
const patterns = [];
|
|
91429
|
+
const entriesByPattern = [];
|
|
91430
|
+
const ownEntries = [];
|
|
91431
|
+
entries.forEach((entry, index) => {
|
|
91432
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91433
|
+
let pattern = patternByTarget.get(target);
|
|
91434
|
+
if (pattern === undefined) {
|
|
91435
|
+
const parsed = parseWorkflowStageTarget(target);
|
|
91436
|
+
pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
|
|
91437
|
+
if (pattern >= 0)
|
|
91438
|
+
entriesByPattern.push([]);
|
|
91439
|
+
patternByTarget.set(target, pattern);
|
|
91440
|
+
}
|
|
91441
|
+
if (pattern < 0)
|
|
91442
|
+
return;
|
|
91443
|
+
entriesByPattern[pattern].push(index);
|
|
91444
|
+
ownEntries.push(index);
|
|
91445
|
+
});
|
|
91446
|
+
if (ownEntries.length === 0)
|
|
91447
|
+
return result;
|
|
91448
|
+
const patternIndex = createStagePathPatternIndex(patterns);
|
|
91449
|
+
const deliveriesByEntry = new Map;
|
|
91450
|
+
const entryReceipts = (index, runId, stageId) => {
|
|
91451
|
+
let byStage = deliveriesByEntry.get(index);
|
|
91452
|
+
if (byStage === undefined) {
|
|
91453
|
+
byStage = new Map;
|
|
91454
|
+
for (const delivery of entries[index].deliveries ?? []) {
|
|
91455
|
+
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
91456
|
+
const list = byStage.get(key);
|
|
91457
|
+
if (list === undefined)
|
|
91458
|
+
byStage.set(key, [delivery]);
|
|
91459
|
+
else
|
|
91460
|
+
list.push(delivery);
|
|
91461
|
+
}
|
|
91462
|
+
deliveriesByEntry.set(index, byStage);
|
|
91463
|
+
}
|
|
91464
|
+
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
91465
|
+
};
|
|
91466
|
+
availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
|
|
91467
|
+
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
|
|
91468
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
91469
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
91470
|
+
for (const run of runs) {
|
|
91471
|
+
const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
91472
|
+
if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
|
|
91473
|
+
continue;
|
|
91474
|
+
const resolved = durableRootRunId === rootRunId;
|
|
91475
|
+
const hops = hopsFor(run.id);
|
|
91476
|
+
for (const stage of run.stages) {
|
|
91477
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
91478
|
+
continue;
|
|
91479
|
+
const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
|
|
91480
|
+
for (const index of matchedEntries) {
|
|
91481
|
+
const entry = entries[index];
|
|
91482
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91483
|
+
const receipts = entryReceipts(index, run.id, stage.id);
|
|
91484
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
91485
|
+
runId: run.id,
|
|
91486
|
+
stageId: stage.id,
|
|
91487
|
+
sessionId: stage.sessionId,
|
|
91488
|
+
admission: "context"
|
|
91489
|
+
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
91490
|
+
let state;
|
|
91491
|
+
let reason;
|
|
91492
|
+
if (!resolved || !hops) {
|
|
91493
|
+
state = "root-unresolved";
|
|
91494
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
91495
|
+
} else if (delivered) {
|
|
91496
|
+
state = "delivered";
|
|
91497
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
91498
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
91499
|
+
state = "stage-terminal";
|
|
91500
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
91501
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
91502
|
+
state = "receipt-unverified";
|
|
91503
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
91504
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
91505
|
+
state = "delivery-unavailable";
|
|
91506
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
91507
|
+
} else if (receipts.length > 0) {
|
|
91508
|
+
state = "session-replaced";
|
|
91509
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
91510
|
+
} else {
|
|
91511
|
+
state = "queued";
|
|
91512
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
91513
|
+
}
|
|
91514
|
+
result.push({
|
|
91515
|
+
messageId: entry.message.id,
|
|
91516
|
+
target,
|
|
91517
|
+
runId: run.id,
|
|
91518
|
+
stageId: stage.id,
|
|
91519
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
91520
|
+
state,
|
|
91521
|
+
reason
|
|
91522
|
+
});
|
|
91523
|
+
}
|
|
91524
|
+
}
|
|
91525
|
+
}
|
|
91526
|
+
return result;
|
|
91527
|
+
}
|
|
91528
|
+
|
|
91529
|
+
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89864
91530
|
init_store();
|
|
89865
91531
|
|
|
89866
91532
|
// dist/builtin/workflows/src/shared/workflow-identity.ts
|
|
@@ -89903,7 +91569,7 @@ function inspectRun(runId, opts) {
|
|
|
89903
91569
|
inputs: copy.inputs,
|
|
89904
91570
|
stages: expandedStages.map((stage) => structuredClone(stage)),
|
|
89905
91571
|
tools: expandedGraph.tools.map((tool) => structuredClone(tool)),
|
|
89906
|
-
|
|
91572
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(activeStore.runs(), copy.rootRunId ?? copy.id, availableStoreDurableBackend(activeStore, copy.id)).filter((delivery) => copy.id === (copy.rootRunId ?? copy.id) || delivery.runId === copy.id)),
|
|
89907
91573
|
result: copy.result,
|
|
89908
91574
|
error: copy.error ?? (strandedRoot ? IMPOSSIBLE_ROOT_LIVENESS_MESSAGE : effectiveRunStatus(copy) === copy.status ? undefined : structuredRecoverableWorkflowFailureText(copy) ?? actionableReturnedStatusText(copy.result)),
|
|
89909
91575
|
exited: copy.exited,
|
|
@@ -97628,7 +99294,7 @@ function buildWorkflowRunStatusDigest(detail) {
|
|
|
97628
99294
|
// dist/builtin/workflows/src/extension/workflow-status-summary.ts
|
|
97629
99295
|
init_pending_stage_status();
|
|
97630
99296
|
init_returned_run_status();
|
|
97631
|
-
|
|
99297
|
+
init_sticky_delivery_state_bound();
|
|
97632
99298
|
var workflowStatusRenderRuns = new WeakMap;
|
|
97633
99299
|
function setWorkflowStatusRenderRuns(result, runs) {
|
|
97634
99300
|
workflowStatusRenderRuns.set(result, runs);
|
|
@@ -97735,7 +99401,7 @@ function summarizeRunSnapshot(run, now = Date.now(), options) {
|
|
|
97735
99401
|
...stage.rateLimits !== undefined ? { rateLimits: stage.rateLimits } : {}
|
|
97736
99402
|
})),
|
|
97737
99403
|
pendingStages: pendingWorkflowStageStatuses(run, options?.owningRunStatus, options?.resolveBoundarySegments),
|
|
97738
|
-
|
|
99404
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(options?.allRuns ?? [run], run.id, options?.durableBackend)),
|
|
97739
99405
|
tools: run.toolNodes.map((tool) => {
|
|
97740
99406
|
const owner = tool;
|
|
97741
99407
|
return {
|
|
@@ -97937,7 +99603,7 @@ function formatStageStartup(startup, now = Date.now()) {
|
|
|
97937
99603
|
}
|
|
97938
99604
|
|
|
97939
99605
|
// dist/builtin/workflows/src/extension/workflow-tool-content.ts
|
|
97940
|
-
|
|
99606
|
+
init_sticky_delivery_state_bound();
|
|
97941
99607
|
function stringifyWorkflowToolResult(result) {
|
|
97942
99608
|
return JSON.stringify(result, null, 2);
|
|
97943
99609
|
}
|
|
@@ -98036,9 +99702,13 @@ function renderStatusToolContent(result) {
|
|
|
98036
99702
|
}
|
|
98037
99703
|
for (const entry of run.awaitingInput)
|
|
98038
99704
|
lines.push(statusAwaitingInputLine(entry));
|
|
98039
|
-
|
|
99705
|
+
const deliveries = run.deliveryStates === undefined ? boundStickyDeliveryStates(stickyStageDeliveryStatuses(allRuns, run.runId, undefined)) : run;
|
|
99706
|
+
for (const delivery of deliveries.deliveryStates ?? []) {
|
|
98040
99707
|
lines.push(` Intercom delivery ${delivery.messageId} target=${delivery.target} stage=${delivery.runId}/${delivery.stageId}${delivery.sessionId === undefined ? "" : ` session=${delivery.sessionId}`} state=${delivery.state} — ${delivery.reason}`);
|
|
98041
99708
|
}
|
|
99709
|
+
const omittedDeliveries = omittedDeliveryStatesLine(deliveries);
|
|
99710
|
+
if (omittedDeliveries !== undefined)
|
|
99711
|
+
lines.push(` ${omittedDeliveries}`);
|
|
98042
99712
|
for (const stage of run.pendingStages.slice(0, STATUS_PENDING_STAGE_LIMIT)) {
|
|
98043
99713
|
lines.push(statusPendingStageLine(stage));
|
|
98044
99714
|
}
|