@bastani/atomic 0.9.44-alpha.4 → 0.9.44-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/builtin/intercom/CHANGELOG.md +12 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +9 -1
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +26 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2457 -736
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2329 -730
- package/dist/builtin/workflows/src/index.js +2328 -730
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2329 -730
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/errors.d.ts +5 -1
- package/dist/core/storage/errors.d.ts.map +1 -1
- package/dist/core/storage/errors.js +8 -2
- package/dist/core/storage/errors.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts +2 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +213 -20
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -274,6 +274,12 @@ class ScopedDurableBackend {
|
|
|
274
274
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
275
275
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
276
276
|
}
|
|
277
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
278
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
279
|
+
}
|
|
280
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
281
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
282
|
+
}
|
|
277
283
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
278
284
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
279
285
|
}
|
|
@@ -3136,8 +3142,11 @@ var init_pending_stage_delivery = __esm(() => {
|
|
|
3136
3142
|
});
|
|
3137
3143
|
|
|
3138
3144
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
3145
|
+
function isSettledPendingStageMessage(entry) {
|
|
3146
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
3147
|
+
}
|
|
3139
3148
|
async function retainPendingStageMessages(messages, archive) {
|
|
3140
|
-
const settled = messages.filter(
|
|
3149
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
3141
3150
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
3142
3151
|
for (const entry of expired)
|
|
3143
3152
|
await archive(entry);
|
|
@@ -3145,53 +3154,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
3145
3154
|
}
|
|
3146
3155
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
3147
3156
|
|
|
3157
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
3158
|
+
class PendingStageTransitionLedger {
|
|
3159
|
+
compacted = [];
|
|
3160
|
+
settled = 0;
|
|
3161
|
+
stages = new Map;
|
|
3162
|
+
fixed = new WeakSet;
|
|
3163
|
+
reconciled = [];
|
|
3164
|
+
reconciledVersion;
|
|
3165
|
+
reconcile(backend, runId, messages) {
|
|
3166
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
3167
|
+
return messages;
|
|
3168
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
3169
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
3170
|
+
let recovered;
|
|
3171
|
+
for (let index = 0;index < messages.length; index++) {
|
|
3172
|
+
const entry = messages[index];
|
|
3173
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
3174
|
+
continue;
|
|
3175
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
3176
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
3177
|
+
continue;
|
|
3178
|
+
recovered ??= [...messages];
|
|
3179
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
3180
|
+
}
|
|
3181
|
+
const result = recovered ?? messages;
|
|
3182
|
+
this.reconciled = result;
|
|
3183
|
+
this.reconciledVersion = version;
|
|
3184
|
+
return result;
|
|
3185
|
+
}
|
|
3186
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
3187
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
3188
|
+
const endedStages = [];
|
|
3189
|
+
const dirty = new Set;
|
|
3190
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
3191
|
+
for (const [stageId, entries] of byStage) {
|
|
3192
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
3193
|
+
continue;
|
|
3194
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
3195
|
+
for (const entry of entries)
|
|
3196
|
+
dirty.add(entry);
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
3199
|
+
const base = this.compacted;
|
|
3200
|
+
const output = [];
|
|
3201
|
+
const removed = [];
|
|
3202
|
+
const added = [];
|
|
3203
|
+
let settled = this.settled;
|
|
3204
|
+
const length = Math.max(base.length, messages.length);
|
|
3205
|
+
for (let index = 0;index < length; index++) {
|
|
3206
|
+
const previous = base[index];
|
|
3207
|
+
const entry = messages[index];
|
|
3208
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
3209
|
+
output.push(entry);
|
|
3210
|
+
continue;
|
|
3211
|
+
}
|
|
3212
|
+
if (previous !== undefined) {
|
|
3213
|
+
removed.push(previous);
|
|
3214
|
+
if (isSettledPendingStageMessage(previous))
|
|
3215
|
+
settled--;
|
|
3216
|
+
}
|
|
3217
|
+
if (entry === undefined)
|
|
3218
|
+
continue;
|
|
3219
|
+
if (isSettledPendingStageMessage(entry))
|
|
3220
|
+
settled++;
|
|
3221
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
3222
|
+
output.push(next);
|
|
3223
|
+
added.push(next);
|
|
3224
|
+
}
|
|
3225
|
+
let retained = output;
|
|
3226
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
3227
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
3228
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
3229
|
+
}
|
|
3230
|
+
return {
|
|
3231
|
+
messages: retained,
|
|
3232
|
+
commit: () => {
|
|
3233
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
3234
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
3235
|
+
byStage?.delete(stageId);
|
|
3236
|
+
if (byStage?.size === 0)
|
|
3237
|
+
this.stages.delete(deliveryRunId);
|
|
3238
|
+
}
|
|
3239
|
+
for (const entry of dirty)
|
|
3240
|
+
this.fixed.delete(entry);
|
|
3241
|
+
for (const entry of removed)
|
|
3242
|
+
this.unindex(entry);
|
|
3243
|
+
for (const entry of added)
|
|
3244
|
+
this.index(entry);
|
|
3245
|
+
this.compacted = retained;
|
|
3246
|
+
this.settled = settled;
|
|
3247
|
+
}
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
index(entry) {
|
|
3251
|
+
this.fixed.add(entry);
|
|
3252
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3253
|
+
let byStage = this.stages.get(delivery.runId);
|
|
3254
|
+
if (byStage === undefined) {
|
|
3255
|
+
byStage = new Map;
|
|
3256
|
+
this.stages.set(delivery.runId, byStage);
|
|
3257
|
+
}
|
|
3258
|
+
let entries = byStage.get(delivery.stageId);
|
|
3259
|
+
if (entries === undefined) {
|
|
3260
|
+
entries = new Set;
|
|
3261
|
+
byStage.set(delivery.stageId, entries);
|
|
3262
|
+
}
|
|
3263
|
+
entries.add(entry);
|
|
3264
|
+
}
|
|
3265
|
+
}
|
|
3266
|
+
unindex(entry) {
|
|
3267
|
+
this.fixed.delete(entry);
|
|
3268
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3269
|
+
const byStage = this.stages.get(delivery.runId);
|
|
3270
|
+
const entries = byStage?.get(delivery.stageId);
|
|
3271
|
+
if (entries === undefined)
|
|
3272
|
+
continue;
|
|
3273
|
+
entries.delete(entry);
|
|
3274
|
+
if (entries.size > 0)
|
|
3275
|
+
continue;
|
|
3276
|
+
byStage.delete(delivery.stageId);
|
|
3277
|
+
if (byStage.size === 0)
|
|
3278
|
+
this.stages.delete(delivery.runId);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
function isCompactingBackend(backend) {
|
|
3283
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
3284
|
+
}
|
|
3285
|
+
function createStageStatusLookup(runs) {
|
|
3286
|
+
let runById;
|
|
3287
|
+
return (runId, stageId) => {
|
|
3288
|
+
if (runById === undefined) {
|
|
3289
|
+
runById = new Map;
|
|
3290
|
+
for (const run of runs)
|
|
3291
|
+
if (!runById.has(run.id))
|
|
3292
|
+
runById.set(run.id, run);
|
|
3293
|
+
}
|
|
3294
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
3295
|
+
};
|
|
3296
|
+
}
|
|
3297
|
+
function isEndedStageStatus(status) {
|
|
3298
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
3299
|
+
}
|
|
3300
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
3301
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
3302
|
+
return entry;
|
|
3303
|
+
const latest = new Map;
|
|
3304
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
3305
|
+
latest.set(deliverySlot(delivery), index);
|
|
3306
|
+
});
|
|
3307
|
+
const deliveries = [];
|
|
3308
|
+
let archivedThrough = -1;
|
|
3309
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
3310
|
+
const delivery = entry.deliveries[index];
|
|
3311
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
3312
|
+
while (archivedThrough < index) {
|
|
3313
|
+
const prefixIndex = archivedThrough + 1;
|
|
3314
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
3315
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
3316
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
3317
|
+
messageRunId: entry.runId,
|
|
3318
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3319
|
+
});
|
|
3320
|
+
archivedThrough = prefixIndex;
|
|
3321
|
+
}
|
|
3322
|
+
} else {
|
|
3323
|
+
deliveries.push(delivery);
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
3327
|
+
}
|
|
3328
|
+
function deliverySlot(delivery) {
|
|
3329
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
3330
|
+
}
|
|
3331
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
3332
|
+
|
|
3148
3333
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
3149
3334
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
3150
|
-
const
|
|
3151
|
-
const
|
|
3152
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
3153
|
-
const result = previous.catch(() => {
|
|
3154
|
-
return;
|
|
3155
|
-
}).then(transition);
|
|
3156
|
-
const settled = result.then(() => {
|
|
3157
|
-
return;
|
|
3158
|
-
}, () => {
|
|
3159
|
-
return;
|
|
3160
|
-
});
|
|
3161
|
-
transitions.set(runId, settled);
|
|
3162
|
-
settled.finally(() => {
|
|
3163
|
-
if (transitions.get(runId) === settled)
|
|
3164
|
-
transitions.delete(runId);
|
|
3165
|
-
});
|
|
3166
|
-
return await result;
|
|
3167
|
-
};
|
|
3335
|
+
const ledgers = new PendingStageLedgers;
|
|
3336
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
3168
3337
|
return {
|
|
3169
3338
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
3170
|
-
return await
|
|
3339
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3171
3340
|
const run = context.findRun(input.runId);
|
|
3172
3341
|
if (run === undefined)
|
|
3173
3342
|
return;
|
|
3174
3343
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
3175
3344
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3176
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3345
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
3177
3346
|
if (result.ok && !result.deduplicated) {
|
|
3178
|
-
|
|
3179
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3180
|
-
live.pendingStageMessages = [...messages];
|
|
3181
|
-
context.bumpAndNotify();
|
|
3347
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3182
3348
|
}
|
|
3183
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3349
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3184
3350
|
});
|
|
3185
3351
|
},
|
|
3186
3352
|
async validateLiveStageMessage(input) {
|
|
3187
|
-
return await
|
|
3353
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
3188
3354
|
const run = context.findRun(input.runId);
|
|
3189
3355
|
if (run === undefined)
|
|
3190
3356
|
return;
|
|
3191
3357
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
3192
3358
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
3193
3359
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3194
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3360
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
3195
3361
|
if (!result.ok) {
|
|
3196
3362
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
3197
3363
|
}
|
|
@@ -3213,77 +3379,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3213
3379
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
3214
3380
|
},
|
|
3215
3381
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
3216
|
-
return await
|
|
3382
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3217
3383
|
const run = context.findRun(runId);
|
|
3218
3384
|
if (run === undefined)
|
|
3219
3385
|
return false;
|
|
3220
|
-
const current = run
|
|
3386
|
+
const current = batch.messages(run);
|
|
3221
3387
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
3222
3388
|
if (next === current)
|
|
3223
3389
|
return false;
|
|
3224
|
-
|
|
3225
|
-
const live = context.findRun(runId) ?? run;
|
|
3226
|
-
live.pendingStageMessages = [...messages];
|
|
3227
|
-
context.bumpAndNotify();
|
|
3390
|
+
await batch.apply(backend, runId, next);
|
|
3228
3391
|
return true;
|
|
3229
3392
|
});
|
|
3230
3393
|
},
|
|
3231
3394
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
3232
|
-
return await
|
|
3395
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3233
3396
|
const run = context.findRun(runId);
|
|
3234
3397
|
if (run === undefined)
|
|
3235
3398
|
return false;
|
|
3236
|
-
const current = run
|
|
3399
|
+
const current = batch.messages(run);
|
|
3237
3400
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
3238
3401
|
if (next === current)
|
|
3239
3402
|
return false;
|
|
3240
|
-
|
|
3241
|
-
const live = context.findRun(runId) ?? run;
|
|
3242
|
-
live.pendingStageMessages = [...messages];
|
|
3243
|
-
context.bumpAndNotify();
|
|
3403
|
+
await batch.apply(backend, runId, next);
|
|
3244
3404
|
return true;
|
|
3245
3405
|
});
|
|
3246
3406
|
},
|
|
3247
3407
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
3248
|
-
return await
|
|
3408
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3249
3409
|
const run = context.findRun(runId);
|
|
3250
3410
|
if (run === undefined)
|
|
3251
3411
|
return false;
|
|
3252
|
-
const current = run
|
|
3412
|
+
const current = batch.messages(run);
|
|
3253
3413
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
3254
3414
|
if (next === current)
|
|
3255
3415
|
return false;
|
|
3256
|
-
|
|
3257
|
-
const live = context.findRun(runId) ?? run;
|
|
3258
|
-
live.pendingStageMessages = [...messages];
|
|
3259
|
-
context.bumpAndNotify();
|
|
3416
|
+
await batch.apply(backend, runId, next);
|
|
3260
3417
|
return true;
|
|
3261
3418
|
});
|
|
3262
3419
|
},
|
|
3263
3420
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
3264
|
-
return await
|
|
3421
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3265
3422
|
const run = context.findRun(input.runId);
|
|
3266
3423
|
if (run === undefined)
|
|
3267
3424
|
return;
|
|
3268
3425
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3269
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
3426
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
3270
3427
|
if (result.ok && !result.deduplicated) {
|
|
3271
|
-
|
|
3272
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3273
|
-
live.pendingStageMessages = [...messages];
|
|
3274
|
-
context.bumpAndNotify();
|
|
3428
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3275
3429
|
}
|
|
3276
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3430
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3277
3431
|
});
|
|
3278
3432
|
},
|
|
3279
3433
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
3280
|
-
return await
|
|
3434
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3281
3435
|
const run = context.findRun(runId);
|
|
3282
3436
|
if (run === undefined)
|
|
3283
3437
|
return false;
|
|
3284
3438
|
if (records.length === 0)
|
|
3285
3439
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3286
|
-
let current =
|
|
3440
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3287
3441
|
const unconfirmedRecords = [];
|
|
3288
3442
|
for (const record of records) {
|
|
3289
3443
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -3291,35 +3445,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3291
3445
|
unconfirmedRecords.push(record);
|
|
3292
3446
|
}
|
|
3293
3447
|
}
|
|
3294
|
-
current =
|
|
3295
|
-
|
|
3296
|
-
run.pendingStageMessages = [...current];
|
|
3297
|
-
context.bumpAndNotify();
|
|
3298
|
-
}
|
|
3448
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
3449
|
+
batch.reconciled(run, current);
|
|
3299
3450
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
3300
3451
|
if (next === current)
|
|
3301
3452
|
return false;
|
|
3302
|
-
|
|
3303
|
-
const live = context.findRun(runId) ?? run;
|
|
3304
|
-
live.pendingStageMessages = [...messages];
|
|
3305
|
-
context.bumpAndNotify();
|
|
3453
|
+
await batch.apply(backend, runId, next);
|
|
3306
3454
|
return true;
|
|
3307
3455
|
});
|
|
3308
3456
|
},
|
|
3309
3457
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
3310
|
-
return await
|
|
3458
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3311
3459
|
const run = context.findRun(runId);
|
|
3312
3460
|
if (run === undefined)
|
|
3313
3461
|
return false;
|
|
3314
3462
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3315
|
-
const current =
|
|
3463
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3316
3464
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
3317
3465
|
if (next === current)
|
|
3318
3466
|
return false;
|
|
3319
|
-
|
|
3320
|
-
const live = context.findRun(runId) ?? run;
|
|
3321
|
-
live.pendingStageMessages = [...messages];
|
|
3322
|
-
context.bumpAndNotify();
|
|
3467
|
+
await batch.apply(backend, runId, next);
|
|
3323
3468
|
return true;
|
|
3324
3469
|
});
|
|
3325
3470
|
}
|
|
@@ -3337,78 +3482,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
3337
3482
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
3338
3483
|
};
|
|
3339
3484
|
}
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
|
|
3485
|
+
|
|
3486
|
+
class PendingStageTransitionBatches {
|
|
3487
|
+
queues = new Map;
|
|
3488
|
+
context;
|
|
3489
|
+
ledgers;
|
|
3490
|
+
constructor(context, ledgers) {
|
|
3491
|
+
this.context = context;
|
|
3492
|
+
this.ledgers = ledgers;
|
|
3493
|
+
}
|
|
3494
|
+
run(runId, backend, step) {
|
|
3495
|
+
return new Promise((resolve, reject) => {
|
|
3496
|
+
const transition = {
|
|
3497
|
+
backend,
|
|
3498
|
+
step,
|
|
3499
|
+
resolve: (value2) => resolve(value2),
|
|
3500
|
+
reject
|
|
3501
|
+
};
|
|
3502
|
+
const queue = this.queues.get(runId);
|
|
3503
|
+
if (queue !== undefined) {
|
|
3504
|
+
queue.push(transition);
|
|
3505
|
+
return;
|
|
3506
|
+
}
|
|
3507
|
+
const created = [transition];
|
|
3508
|
+
this.queues.set(runId, created);
|
|
3509
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
3510
|
+
});
|
|
3511
|
+
}
|
|
3512
|
+
async drain(runId, queue) {
|
|
3513
|
+
while (queue.length > 0)
|
|
3514
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
3515
|
+
this.queues.delete(runId);
|
|
3516
|
+
}
|
|
3517
|
+
async runBatch(runId, transitions) {
|
|
3518
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
3519
|
+
const outcomes = [];
|
|
3520
|
+
for (const transition of transitions) {
|
|
3521
|
+
const changedBefore = batch.changed;
|
|
3522
|
+
try {
|
|
3523
|
+
const value2 = await transition.step(batch);
|
|
3524
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
3525
|
+
} catch (error) {
|
|
3526
|
+
outcomes.push({ ok: false, error });
|
|
3373
3527
|
}
|
|
3374
|
-
retained = compacted;
|
|
3375
3528
|
}
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3529
|
+
let failure;
|
|
3530
|
+
try {
|
|
3531
|
+
await batch.persist();
|
|
3532
|
+
} catch (error) {
|
|
3533
|
+
failure = { error };
|
|
3534
|
+
}
|
|
3535
|
+
transitions.forEach((transition, index) => {
|
|
3536
|
+
const outcome = outcomes[index];
|
|
3537
|
+
if (!outcome.ok)
|
|
3538
|
+
transition.reject(outcome.error);
|
|
3539
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
3540
|
+
transition.reject(failure.error);
|
|
3541
|
+
else
|
|
3542
|
+
transition.resolve(outcome.value);
|
|
3543
|
+
});
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
function takeBatch(queue) {
|
|
3547
|
+
let count = 1;
|
|
3548
|
+
let backend = queue[0].backend;
|
|
3549
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
3550
|
+
const next = queue[count].backend;
|
|
3551
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
3552
|
+
break;
|
|
3553
|
+
backend ??= next;
|
|
3554
|
+
count++;
|
|
3555
|
+
}
|
|
3556
|
+
return queue.splice(0, count);
|
|
3557
|
+
}
|
|
3558
|
+
|
|
3559
|
+
class PendingStageTransitionBatch {
|
|
3560
|
+
changed = false;
|
|
3561
|
+
list;
|
|
3562
|
+
pending;
|
|
3563
|
+
context;
|
|
3564
|
+
ledgers;
|
|
3565
|
+
runId;
|
|
3566
|
+
constructor(context, ledgers, runId) {
|
|
3567
|
+
this.context = context;
|
|
3568
|
+
this.ledgers = ledgers;
|
|
3569
|
+
this.runId = runId;
|
|
3570
|
+
}
|
|
3571
|
+
messages(run) {
|
|
3572
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
3573
|
+
}
|
|
3574
|
+
reconciled(run, messages) {
|
|
3575
|
+
if (this.changed) {
|
|
3576
|
+
this.list = messages;
|
|
3577
|
+
return;
|
|
3379
3578
|
}
|
|
3380
|
-
if (
|
|
3381
|
-
|
|
3579
|
+
if (messages !== run.pendingStageMessages) {
|
|
3580
|
+
run.pendingStageMessages = [...messages];
|
|
3581
|
+
this.context.bumpAndNotify();
|
|
3382
3582
|
}
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
const run = context.findRun(runId);
|
|
3386
|
-
|
|
3387
|
-
const
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3392
|
-
|
|
3583
|
+
}
|
|
3584
|
+
async apply(backend, runId, messages) {
|
|
3585
|
+
const run = this.context.findRun(runId);
|
|
3586
|
+
try {
|
|
3587
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
3588
|
+
this.list = compacted.messages;
|
|
3589
|
+
this.changed = true;
|
|
3590
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
3591
|
+
return compacted.messages;
|
|
3592
|
+
} catch (error) {
|
|
3593
|
+
if (this.changed && this.list !== undefined) {
|
|
3594
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
3595
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
3596
|
+
} else
|
|
3597
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
3598
|
+
throw error;
|
|
3393
3599
|
}
|
|
3394
3600
|
}
|
|
3601
|
+
async persist() {
|
|
3602
|
+
if (this.pending === undefined)
|
|
3603
|
+
return;
|
|
3604
|
+
const { backend, messages, commit, run } = this.pending;
|
|
3605
|
+
const published = this.list ?? messages;
|
|
3606
|
+
try {
|
|
3607
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
3608
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
3609
|
+
}
|
|
3610
|
+
commit?.();
|
|
3611
|
+
} finally {
|
|
3612
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
3613
|
+
}
|
|
3614
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
3615
|
+
if (live !== undefined)
|
|
3616
|
+
live.pendingStageMessages = [...published];
|
|
3617
|
+
this.context.bumpAndNotify();
|
|
3618
|
+
}
|
|
3619
|
+
}
|
|
3620
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
3621
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
3622
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
3623
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
3624
|
+
if (isCompactingBackend(backend)) {
|
|
3625
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
3626
|
+
}
|
|
3627
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
3628
|
+
if (archiveSettled !== undefined)
|
|
3629
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
3630
|
+
if (run !== undefined)
|
|
3631
|
+
ledgers.reset(run);
|
|
3632
|
+
return { messages: retained };
|
|
3633
|
+
}
|
|
3634
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
3635
|
+
const current = context.findRun(runId);
|
|
3636
|
+
if (current === undefined)
|
|
3637
|
+
return;
|
|
3638
|
+
const messages = current.pendingStageMessages ?? [];
|
|
3639
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
3640
|
+
if (recovered !== messages) {
|
|
3641
|
+
current.pendingStageMessages = [...recovered];
|
|
3642
|
+
context.bumpAndNotify();
|
|
3643
|
+
}
|
|
3395
3644
|
}
|
|
3396
|
-
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
return
|
|
3406
|
-
}
|
|
3407
|
-
|
|
3645
|
+
|
|
3646
|
+
class PendingStageLedgers {
|
|
3647
|
+
ledgers = new WeakMap;
|
|
3648
|
+
for(run) {
|
|
3649
|
+
let ledger = this.ledgers.get(run);
|
|
3650
|
+
if (ledger === undefined) {
|
|
3651
|
+
ledger = new PendingStageTransitionLedger;
|
|
3652
|
+
this.ledgers.set(run, ledger);
|
|
3653
|
+
}
|
|
3654
|
+
return ledger;
|
|
3655
|
+
}
|
|
3656
|
+
reset(run) {
|
|
3657
|
+
this.ledgers.delete(run);
|
|
3658
|
+
}
|
|
3408
3659
|
}
|
|
3660
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
3409
3661
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
3410
3662
|
init_run_owner_backend();
|
|
3411
3663
|
init_pending_stage_delivery();
|
|
3664
|
+
init_pending_stage_transition_ledger();
|
|
3412
3665
|
});
|
|
3413
3666
|
|
|
3414
3667
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -4998,6 +5251,204 @@ var init_resume_eligibility = () => {};
|
|
|
4998
5251
|
// dist/builtin/workflows/src/durable/types.ts
|
|
4999
5252
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
5000
5253
|
|
|
5254
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5255
|
+
function immutableStageGroupError(stages) {
|
|
5256
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5257
|
+
if (occurrenceError !== undefined)
|
|
5258
|
+
return occurrenceError;
|
|
5259
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
5260
|
+
for (const group of byReplayKey.values()) {
|
|
5261
|
+
const firstName = group[0]?.name;
|
|
5262
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
5263
|
+
return "stage name drift within one replay-key group";
|
|
5264
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5265
|
+
const first = topologies[0];
|
|
5266
|
+
if (first === undefined)
|
|
5267
|
+
continue;
|
|
5268
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5269
|
+
return "immutable stage identity drift within one replay-key group";
|
|
5270
|
+
}
|
|
5271
|
+
}
|
|
5272
|
+
return;
|
|
5273
|
+
}
|
|
5274
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5275
|
+
return directChildTopologyErrorFromRecords({
|
|
5276
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
5277
|
+
direct: (claiming) => {
|
|
5278
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5279
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5280
|
+
}
|
|
5281
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
5282
|
+
}
|
|
5283
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5284
|
+
const prefix = `${replayScope}:`;
|
|
5285
|
+
const claiming = records.claiming;
|
|
5286
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5287
|
+
return "child run topology exists outside its durable invocation scope";
|
|
5288
|
+
}
|
|
5289
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5290
|
+
return "child checkpoint ownership is not reciprocal";
|
|
5291
|
+
}
|
|
5292
|
+
const direct = records.direct(claiming);
|
|
5293
|
+
const immutableError = immutableStageGroupError(direct);
|
|
5294
|
+
if (immutableError !== undefined)
|
|
5295
|
+
return immutableError;
|
|
5296
|
+
const merged = [];
|
|
5297
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5298
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5299
|
+
if (topology === undefined)
|
|
5300
|
+
return "direct child stage group lacks owning-run topology";
|
|
5301
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5302
|
+
return "direct child stage topology lacks current source identity";
|
|
5303
|
+
}
|
|
5304
|
+
if (topology.stageId.length === 0)
|
|
5305
|
+
return "direct child stage id is empty";
|
|
5306
|
+
merged.push(topology);
|
|
5307
|
+
}
|
|
5308
|
+
const ids = new Set(allowedNodeIds);
|
|
5309
|
+
const stageIds = new Set;
|
|
5310
|
+
const orders = new Set;
|
|
5311
|
+
for (const topology of merged) {
|
|
5312
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5313
|
+
return "direct child stage ids are not unique";
|
|
5314
|
+
stageIds.add(topology.stageId);
|
|
5315
|
+
ids.add(topology.stageId);
|
|
5316
|
+
if (topology.sourceOrder !== undefined) {
|
|
5317
|
+
if (orders.has(topology.sourceOrder))
|
|
5318
|
+
return "direct child source orders are not unique";
|
|
5319
|
+
orders.add(topology.sourceOrder);
|
|
5320
|
+
}
|
|
5321
|
+
}
|
|
5322
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5323
|
+
return "direct child stage references an unknown parent";
|
|
5324
|
+
}
|
|
5325
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5326
|
+
}
|
|
5327
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
5328
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5329
|
+
const seen = new Map;
|
|
5330
|
+
for (const start of starts) {
|
|
5331
|
+
const stageId = start.topology.stageId;
|
|
5332
|
+
if (stageId.length === 0)
|
|
5333
|
+
return "owning-run boundary stage id is empty";
|
|
5334
|
+
const prior = seen.get(stageId);
|
|
5335
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
5336
|
+
return "owning-run boundary stage id is aliased";
|
|
5337
|
+
seen.set(stageId, start.replayKey);
|
|
5338
|
+
}
|
|
5339
|
+
return;
|
|
5340
|
+
}
|
|
5341
|
+
function sameRun(actual, expected) {
|
|
5342
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5343
|
+
}
|
|
5344
|
+
function sameStrings(left, right) {
|
|
5345
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5346
|
+
}
|
|
5347
|
+
function sameOptionalRun(actual, expected) {
|
|
5348
|
+
if (actual === undefined || expected === undefined)
|
|
5349
|
+
return actual === expected;
|
|
5350
|
+
return sameRun(actual, expected);
|
|
5351
|
+
}
|
|
5352
|
+
function promptOccurrenceIdentityError(stages) {
|
|
5353
|
+
const replayByOccurrence = new Map;
|
|
5354
|
+
const occurrenceByReplayStage = new Map;
|
|
5355
|
+
const stageByReplayOccurrence = new Map;
|
|
5356
|
+
for (const stage of stages) {
|
|
5357
|
+
const topology = stage.topology;
|
|
5358
|
+
const occurrence = topology?.occurrenceKey;
|
|
5359
|
+
if (topology === undefined || occurrence === undefined)
|
|
5360
|
+
continue;
|
|
5361
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5362
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5363
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
5364
|
+
}
|
|
5365
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5366
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5367
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5368
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5369
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
5370
|
+
}
|
|
5371
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5372
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5373
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5374
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5375
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
5376
|
+
}
|
|
5377
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5378
|
+
}
|
|
5379
|
+
return;
|
|
5380
|
+
}
|
|
5381
|
+
function durableStageGroupKey(stage) {
|
|
5382
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
5383
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5384
|
+
}
|
|
5385
|
+
function groupByDurableStageKey(stages) {
|
|
5386
|
+
const groups = new Map;
|
|
5387
|
+
for (const stage of stages) {
|
|
5388
|
+
const key = durableStageGroupKey(stage);
|
|
5389
|
+
const group = groups.get(key) ?? [];
|
|
5390
|
+
group.push(stage);
|
|
5391
|
+
groups.set(key, group);
|
|
5392
|
+
}
|
|
5393
|
+
return groups;
|
|
5394
|
+
}
|
|
5395
|
+
function compositeKey(domain, ...parts) {
|
|
5396
|
+
return JSON.stringify([domain, ...parts]);
|
|
5397
|
+
}
|
|
5398
|
+
function hasCycle(topologies) {
|
|
5399
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5400
|
+
const visiting = new Set;
|
|
5401
|
+
const visited = new Set;
|
|
5402
|
+
const cyclic = (stageId) => {
|
|
5403
|
+
if (visiting.has(stageId))
|
|
5404
|
+
return true;
|
|
5405
|
+
if (visited.has(stageId))
|
|
5406
|
+
return false;
|
|
5407
|
+
visiting.add(stageId);
|
|
5408
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5409
|
+
return true;
|
|
5410
|
+
visiting.delete(stageId);
|
|
5411
|
+
visited.add(stageId);
|
|
5412
|
+
return false;
|
|
5413
|
+
};
|
|
5414
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5415
|
+
}
|
|
5416
|
+
var init_stage_topology_validation = () => {};
|
|
5417
|
+
|
|
5418
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5419
|
+
function parseWorkflowChildResult(value2) {
|
|
5420
|
+
if (!isOrdinaryObject(value2))
|
|
5421
|
+
return;
|
|
5422
|
+
const workflow = value2.workflow;
|
|
5423
|
+
const runId = value2.runId;
|
|
5424
|
+
const status = value2.status;
|
|
5425
|
+
const exited = value2.exited;
|
|
5426
|
+
const outputs = value2.outputs;
|
|
5427
|
+
const exitReason = value2.exitReason;
|
|
5428
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5429
|
+
return;
|
|
5430
|
+
if (exited === false) {
|
|
5431
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
5432
|
+
return;
|
|
5433
|
+
return value2;
|
|
5434
|
+
}
|
|
5435
|
+
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5436
|
+
return;
|
|
5437
|
+
return value2;
|
|
5438
|
+
}
|
|
5439
|
+
function isWorkflowChildResult(value2) {
|
|
5440
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
5441
|
+
}
|
|
5442
|
+
function isWorkflowExitStatus(value2) {
|
|
5443
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5444
|
+
}
|
|
5445
|
+
function isOrdinaryObject(value2) {
|
|
5446
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5447
|
+
return false;
|
|
5448
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
5449
|
+
return prototype === Object.prototype || prototype === null;
|
|
5450
|
+
}
|
|
5451
|
+
|
|
5001
5452
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
5002
5453
|
import { createHash } from "node:crypto";
|
|
5003
5454
|
function durableHash(value2) {
|
|
@@ -5064,6 +5515,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
5064
5515
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
5065
5516
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
5066
5517
|
}
|
|
5518
|
+
|
|
5519
|
+
class StageReceipts {
|
|
5520
|
+
exact = new Map;
|
|
5521
|
+
anySession = new Map;
|
|
5522
|
+
has(logicalRunId, sessionId, admission) {
|
|
5523
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
5524
|
+
}
|
|
5525
|
+
hasAnySession(logicalRunId, admission) {
|
|
5526
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
5527
|
+
}
|
|
5528
|
+
add(logicalRunId, sessionId, admission) {
|
|
5529
|
+
let bySession = this.exact.get(logicalRunId);
|
|
5530
|
+
if (bySession === undefined) {
|
|
5531
|
+
bySession = new Map;
|
|
5532
|
+
this.exact.set(logicalRunId, bySession);
|
|
5533
|
+
}
|
|
5534
|
+
let admissions = bySession.get(sessionId);
|
|
5535
|
+
if (admissions === undefined) {
|
|
5536
|
+
admissions = new Set;
|
|
5537
|
+
bySession.set(sessionId, admissions);
|
|
5538
|
+
}
|
|
5539
|
+
if (admissions.has(admission))
|
|
5540
|
+
return false;
|
|
5541
|
+
admissions.add(admission);
|
|
5542
|
+
return true;
|
|
5543
|
+
}
|
|
5544
|
+
addAnySession(logicalRunId, admission) {
|
|
5545
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
5546
|
+
if (admissions === undefined) {
|
|
5547
|
+
admissions = new Set;
|
|
5548
|
+
this.anySession.set(logicalRunId, admissions);
|
|
5549
|
+
}
|
|
5550
|
+
if (admissions.has(admission))
|
|
5551
|
+
return false;
|
|
5552
|
+
admissions.add(admission);
|
|
5553
|
+
return true;
|
|
5554
|
+
}
|
|
5555
|
+
}
|
|
5556
|
+
|
|
5557
|
+
class PendingStageReceiptIndex {
|
|
5558
|
+
workflows = new Map;
|
|
5559
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
5560
|
+
let workflow = this.workflows.get(workflowId);
|
|
5561
|
+
if (workflow === undefined) {
|
|
5562
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5563
|
+
this.workflows.set(workflowId, workflow);
|
|
5564
|
+
}
|
|
5565
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
5566
|
+
if (byRun === undefined) {
|
|
5567
|
+
byRun = new Map;
|
|
5568
|
+
workflow.byMessage.set(messageId, byRun);
|
|
5569
|
+
}
|
|
5570
|
+
let byStage = byRun.get(delivery.runId);
|
|
5571
|
+
if (byStage === undefined) {
|
|
5572
|
+
byStage = new Map;
|
|
5573
|
+
byRun.set(delivery.runId, byStage);
|
|
5574
|
+
}
|
|
5575
|
+
let receipts = byStage.get(delivery.stageId);
|
|
5576
|
+
if (receipts === undefined) {
|
|
5577
|
+
receipts = new StageReceipts;
|
|
5578
|
+
byStage.set(delivery.stageId, receipts);
|
|
5579
|
+
}
|
|
5580
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
5581
|
+
let changed = added;
|
|
5582
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
5583
|
+
changed = true;
|
|
5584
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
5585
|
+
changed = true;
|
|
5586
|
+
if (changed)
|
|
5587
|
+
workflow.log.push(messageId);
|
|
5588
|
+
return added;
|
|
5589
|
+
}
|
|
5590
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
5591
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
5592
|
+
if (receipts === undefined)
|
|
5593
|
+
return false;
|
|
5594
|
+
if (anySession && delivery.admission === "context")
|
|
5595
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
5596
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
5597
|
+
}
|
|
5598
|
+
changesSince(workflowId, cursor) {
|
|
5599
|
+
let workflow = this.workflows.get(workflowId);
|
|
5600
|
+
if (workflow === undefined) {
|
|
5601
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5602
|
+
this.workflows.set(workflowId, workflow);
|
|
5603
|
+
}
|
|
5604
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
5605
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
5606
|
+
return { cursor: next, messageIds: undefined };
|
|
5607
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
5608
|
+
}
|
|
5609
|
+
delete(workflowId) {
|
|
5610
|
+
this.workflows.delete(workflowId);
|
|
5611
|
+
}
|
|
5612
|
+
clear() {
|
|
5613
|
+
this.workflows.clear();
|
|
5614
|
+
}
|
|
5615
|
+
}
|
|
5616
|
+
function nextVersion() {
|
|
5617
|
+
lastVersion += 1;
|
|
5618
|
+
return lastVersion;
|
|
5619
|
+
}
|
|
5620
|
+
|
|
5621
|
+
class NestedSnapshotVersion {
|
|
5622
|
+
rootWorkflowId;
|
|
5623
|
+
current = nextVersion();
|
|
5624
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
5625
|
+
replayGroups = new Map;
|
|
5626
|
+
boundaryReplayKeys = new Set;
|
|
5627
|
+
constructor(rootWorkflowId) {
|
|
5628
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
5629
|
+
}
|
|
5630
|
+
get version() {
|
|
5631
|
+
return this.current;
|
|
5632
|
+
}
|
|
5633
|
+
record(checkpoint) {
|
|
5634
|
+
if (!this.isInert(checkpoint))
|
|
5635
|
+
this.current = nextVersion();
|
|
5636
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
5637
|
+
if (checkpoint.kind !== "stage")
|
|
5638
|
+
return;
|
|
5639
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
5640
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
5641
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
5642
|
+
return;
|
|
5643
|
+
const identity = stageIdentity(checkpoint);
|
|
5644
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
5645
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5646
|
+
if (summary === undefined) {
|
|
5647
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
5648
|
+
name: checkpoint.name,
|
|
5649
|
+
identity,
|
|
5650
|
+
consistent: true,
|
|
5651
|
+
hasOutput: checkpoint.output !== undefined,
|
|
5652
|
+
hasExitedChildOutput: exited
|
|
5653
|
+
});
|
|
5654
|
+
return;
|
|
5655
|
+
}
|
|
5656
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
5657
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
5658
|
+
summary.hasExitedChildOutput ||= exited;
|
|
5659
|
+
}
|
|
5660
|
+
isInert(checkpoint) {
|
|
5661
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
5662
|
+
return false;
|
|
5663
|
+
}
|
|
5664
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5665
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
5666
|
+
return false;
|
|
5667
|
+
}
|
|
5668
|
+
const id = checkpoint.checkpointId;
|
|
5669
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
5670
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
5671
|
+
return false;
|
|
5672
|
+
}
|
|
5673
|
+
return true;
|
|
5674
|
+
}
|
|
5675
|
+
isRootOwned(checkpoint) {
|
|
5676
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
5677
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
5678
|
+
}
|
|
5679
|
+
}
|
|
5680
|
+
function stageIdentity(checkpoint) {
|
|
5681
|
+
const topology = checkpoint.topology;
|
|
5682
|
+
return JSON.stringify([
|
|
5683
|
+
durableStageGroupKey(checkpoint),
|
|
5684
|
+
topology?.version,
|
|
5685
|
+
topology?.stageId,
|
|
5686
|
+
topology?.parentIds,
|
|
5687
|
+
topology?.sourceOrder,
|
|
5688
|
+
topology?.run === undefined ? undefined : [
|
|
5689
|
+
topology.run.runId,
|
|
5690
|
+
topology.run.runName,
|
|
5691
|
+
topology.run.parentRunId,
|
|
5692
|
+
topology.run.parentStageId,
|
|
5693
|
+
topology.run.rootRunId
|
|
5694
|
+
]
|
|
5695
|
+
]);
|
|
5696
|
+
}
|
|
5067
5697
|
function checkpointKey(c) {
|
|
5068
5698
|
return `${c.kind}:${c.checkpointId}`;
|
|
5069
5699
|
}
|
|
@@ -5075,7 +5705,7 @@ class JournalProjection {
|
|
|
5075
5705
|
workflows = new Map;
|
|
5076
5706
|
deletedWorkflowIds = new Set;
|
|
5077
5707
|
pendingReceipts = new Map;
|
|
5078
|
-
stickyReceipts = new
|
|
5708
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
5079
5709
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
5080
5710
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
5081
5711
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -5123,7 +5753,8 @@ class JournalProjection {
|
|
|
5123
5753
|
toolByHash: new Map,
|
|
5124
5754
|
uiByHash: new Map,
|
|
5125
5755
|
stageOutputByReplayKey: new Map,
|
|
5126
|
-
stageSessionByReplayKey: new Map
|
|
5756
|
+
stageSessionByReplayKey: new Map,
|
|
5757
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
5127
5758
|
});
|
|
5128
5759
|
}
|
|
5129
5760
|
applyWorkflowSnapshot(handle) {
|
|
@@ -5149,39 +5780,43 @@ class JournalProjection {
|
|
|
5149
5780
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
5150
5781
|
}
|
|
5151
5782
|
stickyDeliveryCounts = new Map;
|
|
5783
|
+
stickyDeliveryCountVersions = new Map;
|
|
5152
5784
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
5153
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
5785
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
5786
|
+
}
|
|
5787
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
5788
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
5789
|
+
}
|
|
5790
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
5791
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
5154
5792
|
}
|
|
5155
5793
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
5156
|
-
|
|
5157
|
-
if (
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
5165
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
5166
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
5167
|
-
if (accounting !== undefined && !existing) {
|
|
5168
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
5794
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
5795
|
+
if (accounting !== undefined && added) {
|
|
5796
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
5797
|
+
if (byRun === undefined) {
|
|
5798
|
+
byRun = new Map;
|
|
5799
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
5800
|
+
}
|
|
5801
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
5169
5802
|
if (counts === undefined) {
|
|
5170
5803
|
counts = new Map;
|
|
5171
|
-
|
|
5804
|
+
byRun.set(accounting.messageRunId, counts);
|
|
5172
5805
|
}
|
|
5173
|
-
const
|
|
5174
|
-
counts.set(
|
|
5806
|
+
const previous = counts.get(messageId) ?? 0;
|
|
5807
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
5808
|
+
if (accounting.deliveryCount > previous)
|
|
5809
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5175
5810
|
}
|
|
5176
5811
|
}
|
|
5177
5812
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
5178
|
-
return
|
|
5813
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
5179
5814
|
}
|
|
5180
5815
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
5181
|
-
return
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5816
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
5817
|
+
}
|
|
5818
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
5819
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
5185
5820
|
}
|
|
5186
5821
|
recordCheckpoint(recorded) {
|
|
5187
5822
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -5200,6 +5835,7 @@ class JournalProjection {
|
|
|
5200
5835
|
if (rec.checkpoints.has(key))
|
|
5201
5836
|
return;
|
|
5202
5837
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
5838
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
5203
5839
|
if (currentCheckpoint.kind === "tool") {
|
|
5204
5840
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
5205
5841
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -5257,6 +5893,9 @@ class JournalProjection {
|
|
|
5257
5893
|
getWorkflow(workflowId) {
|
|
5258
5894
|
return this.workflows.get(workflowId)?.handle;
|
|
5259
5895
|
}
|
|
5896
|
+
nestedSnapshotVersion(workflowId) {
|
|
5897
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
5898
|
+
}
|
|
5260
5899
|
getLoadableWorkflow(workflowId) {
|
|
5261
5900
|
return this.getWorkflow(workflowId);
|
|
5262
5901
|
}
|
|
@@ -5334,6 +5973,7 @@ class JournalProjection {
|
|
|
5334
5973
|
this.pendingReceipts.delete(workflowId);
|
|
5335
5974
|
this.stickyReceipts.delete(workflowId);
|
|
5336
5975
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
5976
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5337
5977
|
this.deletedWorkflowIds.add(workflowId);
|
|
5338
5978
|
}
|
|
5339
5979
|
isWorkflowLoadable(workflowId) {
|
|
@@ -5344,6 +5984,8 @@ class JournalProjection {
|
|
|
5344
5984
|
this.pendingReceipts.clear();
|
|
5345
5985
|
this.stickyReceipts.clear();
|
|
5346
5986
|
this.stickyDeliveryCounts.clear();
|
|
5987
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
5988
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5347
5989
|
this.deletedWorkflowIds.clear();
|
|
5348
5990
|
}
|
|
5349
5991
|
}
|
|
@@ -5400,13 +6042,10 @@ function toResumableEntry(handle) {
|
|
|
5400
6042
|
updatedAt: handle.updatedAt
|
|
5401
6043
|
};
|
|
5402
6044
|
}
|
|
5403
|
-
|
|
5404
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
5405
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
5406
|
-
}
|
|
5407
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
6045
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
5408
6046
|
var init_backend = __esm(() => {
|
|
5409
6047
|
init_resume_eligibility();
|
|
6048
|
+
init_stage_topology_validation();
|
|
5410
6049
|
init_durable_hash();
|
|
5411
6050
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
5412
6051
|
});
|
|
@@ -5490,205 +6129,24 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
5490
6129
|
}
|
|
5491
6130
|
var init_stage_session_store = () => {};
|
|
5492
6131
|
|
|
5493
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5494
|
-
function immutableStageGroupError(stages) {
|
|
5495
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5496
|
-
if (occurrenceError !== undefined)
|
|
5497
|
-
return occurrenceError;
|
|
5498
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
5499
|
-
for (const group of byReplayKey.values()) {
|
|
5500
|
-
const firstName = group[0]?.name;
|
|
5501
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
5502
|
-
return "stage name drift within one replay-key group";
|
|
5503
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5504
|
-
const first = topologies[0];
|
|
5505
|
-
if (first === undefined)
|
|
5506
|
-
continue;
|
|
5507
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5508
|
-
return "immutable stage identity drift within one replay-key group";
|
|
5509
|
-
}
|
|
5510
|
-
}
|
|
5511
|
-
return;
|
|
5512
|
-
}
|
|
5513
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5514
|
-
const prefix = `${replayScope}:`;
|
|
5515
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
5516
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5517
|
-
return "child run topology exists outside its durable invocation scope";
|
|
5518
|
-
}
|
|
5519
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5520
|
-
return "child checkpoint ownership is not reciprocal";
|
|
5521
|
-
}
|
|
5522
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5523
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5524
|
-
const immutableError = immutableStageGroupError(direct);
|
|
5525
|
-
if (immutableError !== undefined)
|
|
5526
|
-
return immutableError;
|
|
5527
|
-
const merged = [];
|
|
5528
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5529
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5530
|
-
if (topology === undefined)
|
|
5531
|
-
return "direct child stage group lacks owning-run topology";
|
|
5532
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5533
|
-
return "direct child stage topology lacks current source identity";
|
|
5534
|
-
}
|
|
5535
|
-
if (topology.stageId.length === 0)
|
|
5536
|
-
return "direct child stage id is empty";
|
|
5537
|
-
merged.push(topology);
|
|
5538
|
-
}
|
|
5539
|
-
const ids = new Set(allowedNodeIds);
|
|
5540
|
-
const stageIds = new Set;
|
|
5541
|
-
const orders = new Set;
|
|
5542
|
-
for (const topology of merged) {
|
|
5543
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5544
|
-
return "direct child stage ids are not unique";
|
|
5545
|
-
stageIds.add(topology.stageId);
|
|
5546
|
-
ids.add(topology.stageId);
|
|
5547
|
-
if (topology.sourceOrder !== undefined) {
|
|
5548
|
-
if (orders.has(topology.sourceOrder))
|
|
5549
|
-
return "direct child source orders are not unique";
|
|
5550
|
-
orders.add(topology.sourceOrder);
|
|
5551
|
-
}
|
|
5552
|
-
}
|
|
5553
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5554
|
-
return "direct child stage references an unknown parent";
|
|
5555
|
-
}
|
|
5556
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5557
|
-
}
|
|
5558
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
5559
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5560
|
-
const seen = new Map;
|
|
5561
|
-
for (const start of starts) {
|
|
5562
|
-
const stageId = start.topology.stageId;
|
|
5563
|
-
if (stageId.length === 0)
|
|
5564
|
-
return "owning-run boundary stage id is empty";
|
|
5565
|
-
const prior = seen.get(stageId);
|
|
5566
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
5567
|
-
return "owning-run boundary stage id is aliased";
|
|
5568
|
-
seen.set(stageId, start.replayKey);
|
|
5569
|
-
}
|
|
5570
|
-
return;
|
|
5571
|
-
}
|
|
5572
|
-
function sameRun(actual, expected) {
|
|
5573
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5574
|
-
}
|
|
5575
|
-
function sameStrings(left, right) {
|
|
5576
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5577
|
-
}
|
|
5578
|
-
function sameOptionalRun(actual, expected) {
|
|
5579
|
-
if (actual === undefined || expected === undefined)
|
|
5580
|
-
return actual === expected;
|
|
5581
|
-
return sameRun(actual, expected);
|
|
5582
|
-
}
|
|
5583
|
-
function promptOccurrenceIdentityError(stages) {
|
|
5584
|
-
const replayByOccurrence = new Map;
|
|
5585
|
-
const occurrenceByReplayStage = new Map;
|
|
5586
|
-
const stageByReplayOccurrence = new Map;
|
|
5587
|
-
for (const stage of stages) {
|
|
5588
|
-
const topology = stage.topology;
|
|
5589
|
-
const occurrence = topology?.occurrenceKey;
|
|
5590
|
-
if (topology === undefined || occurrence === undefined)
|
|
5591
|
-
continue;
|
|
5592
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5593
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5594
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
5595
|
-
}
|
|
5596
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5597
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5598
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5599
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5600
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
5601
|
-
}
|
|
5602
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5603
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5604
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5605
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5606
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
5607
|
-
}
|
|
5608
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5609
|
-
}
|
|
5610
|
-
return;
|
|
5611
|
-
}
|
|
5612
|
-
function durableStageGroupKey(stage) {
|
|
5613
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
5614
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5615
|
-
}
|
|
5616
|
-
function groupByDurableStageKey(stages) {
|
|
5617
|
-
const groups = new Map;
|
|
5618
|
-
for (const stage of stages) {
|
|
5619
|
-
const key = durableStageGroupKey(stage);
|
|
5620
|
-
const group = groups.get(key) ?? [];
|
|
5621
|
-
group.push(stage);
|
|
5622
|
-
groups.set(key, group);
|
|
5623
|
-
}
|
|
5624
|
-
return groups;
|
|
5625
|
-
}
|
|
5626
|
-
function compositeKey(domain, ...parts) {
|
|
5627
|
-
return JSON.stringify([domain, ...parts]);
|
|
5628
|
-
}
|
|
5629
|
-
function hasCycle(topologies) {
|
|
5630
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5631
|
-
const visiting = new Set;
|
|
5632
|
-
const visited = new Set;
|
|
5633
|
-
const cyclic = (stageId) => {
|
|
5634
|
-
if (visiting.has(stageId))
|
|
5635
|
-
return true;
|
|
5636
|
-
if (visited.has(stageId))
|
|
5637
|
-
return false;
|
|
5638
|
-
visiting.add(stageId);
|
|
5639
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5640
|
-
return true;
|
|
5641
|
-
visiting.delete(stageId);
|
|
5642
|
-
visited.add(stageId);
|
|
5643
|
-
return false;
|
|
5644
|
-
};
|
|
5645
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5646
|
-
}
|
|
5647
|
-
var init_stage_topology_validation = () => {};
|
|
5648
|
-
|
|
5649
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5650
|
-
function parseWorkflowChildResult(value2) {
|
|
5651
|
-
if (!isOrdinaryObject(value2))
|
|
5652
|
-
return;
|
|
5653
|
-
const workflow = value2.workflow;
|
|
5654
|
-
const runId = value2.runId;
|
|
5655
|
-
const status = value2.status;
|
|
5656
|
-
const exited = value2.exited;
|
|
5657
|
-
const outputs = value2.outputs;
|
|
5658
|
-
const exitReason = value2.exitReason;
|
|
5659
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5660
|
-
return;
|
|
5661
|
-
if (exited === false) {
|
|
5662
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
5663
|
-
return;
|
|
5664
|
-
return value2;
|
|
5665
|
-
}
|
|
5666
|
-
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5667
|
-
return;
|
|
5668
|
-
return value2;
|
|
5669
|
-
}
|
|
5670
|
-
function isWorkflowChildResult(value2) {
|
|
5671
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
5672
|
-
}
|
|
5673
|
-
function isWorkflowExitStatus(value2) {
|
|
5674
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5675
|
-
}
|
|
5676
|
-
function isOrdinaryObject(value2) {
|
|
5677
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5678
|
-
return false;
|
|
5679
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
5680
|
-
return prototype === Object.prototype || prototype === null;
|
|
5681
|
-
}
|
|
5682
|
-
|
|
5683
6132
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
5684
6133
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
5685
6134
|
const startTopology = start.topology;
|
|
5686
6135
|
const startBoundary = startTopology?.boundary;
|
|
5687
|
-
const startRun = startTopology?.run;
|
|
5688
6136
|
const related = stages.filter((stage) => {
|
|
5689
6137
|
const boundary = stage.topology?.boundary;
|
|
5690
6138
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
5691
6139
|
});
|
|
6140
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
6141
|
+
related,
|
|
6142
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
6143
|
+
}, requireTerminal);
|
|
6144
|
+
}
|
|
6145
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
6146
|
+
const startTopology = start.topology;
|
|
6147
|
+
const startBoundary = startTopology?.boundary;
|
|
6148
|
+
const startRun = startTopology?.run;
|
|
6149
|
+
const related = records.related;
|
|
5692
6150
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
5693
6151
|
if (starts.length !== 1 || starts[0] !== start)
|
|
5694
6152
|
return failure("boundary has no unique start record");
|
|
@@ -5725,7 +6183,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
5725
6183
|
return failure("boundary has an illegal terminal transition");
|
|
5726
6184
|
}
|
|
5727
6185
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
5728
|
-
const outputRecords =
|
|
6186
|
+
const outputRecords = records.outputRecords();
|
|
5729
6187
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
5730
6188
|
return failure("boundary replay key has conflicting output records");
|
|
5731
6189
|
}
|
|
@@ -5755,10 +6213,120 @@ var init_boundary_lifecycle = __esm(() => {
|
|
|
5755
6213
|
init_stage_topology_validation();
|
|
5756
6214
|
});
|
|
5757
6215
|
|
|
6216
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
6217
|
+
class CheckpointCatalogIndex {
|
|
6218
|
+
checkpoints;
|
|
6219
|
+
stages;
|
|
6220
|
+
positions = new Map;
|
|
6221
|
+
groupKeys = new Map;
|
|
6222
|
+
groups = new Map;
|
|
6223
|
+
stagesByRunId = new Map;
|
|
6224
|
+
boundariesByReplayKey = new Map;
|
|
6225
|
+
boundariesByStageId = new Map;
|
|
6226
|
+
boundariesByChildRunId = new Map;
|
|
6227
|
+
outputsByReplayKey = new Map;
|
|
6228
|
+
evidence;
|
|
6229
|
+
constructor(checkpoints) {
|
|
6230
|
+
this.checkpoints = checkpoints;
|
|
6231
|
+
const stages = [];
|
|
6232
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
6233
|
+
if (!this.positions.has(checkpoint))
|
|
6234
|
+
this.positions.set(checkpoint, index);
|
|
6235
|
+
if (checkpoint.kind !== "stage")
|
|
6236
|
+
return;
|
|
6237
|
+
stages.push(checkpoint);
|
|
6238
|
+
const key = durableStageGroupKey(checkpoint);
|
|
6239
|
+
this.groupKeys.set(checkpoint, key);
|
|
6240
|
+
append(this.groups, key, checkpoint);
|
|
6241
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
6242
|
+
if (runId !== undefined)
|
|
6243
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
6244
|
+
if (checkpoint.output !== undefined)
|
|
6245
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6246
|
+
const boundary = checkpoint.topology?.boundary;
|
|
6247
|
+
if (boundary === undefined)
|
|
6248
|
+
return;
|
|
6249
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6250
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
6251
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
6252
|
+
});
|
|
6253
|
+
this.stages = stages;
|
|
6254
|
+
}
|
|
6255
|
+
sequence(checkpoint) {
|
|
6256
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
6257
|
+
}
|
|
6258
|
+
stageGroups() {
|
|
6259
|
+
return this.groups.values();
|
|
6260
|
+
}
|
|
6261
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
6262
|
+
const topology = start.topology;
|
|
6263
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
6264
|
+
const related = this.inCheckpointOrder([
|
|
6265
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
6266
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
6267
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
6268
|
+
]);
|
|
6269
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
6270
|
+
}
|
|
6271
|
+
directChildRecords(childRunId) {
|
|
6272
|
+
return {
|
|
6273
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
6274
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
6275
|
+
};
|
|
6276
|
+
}
|
|
6277
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
6278
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
6279
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
6280
|
+
}
|
|
6281
|
+
inCheckpointOrder(lists) {
|
|
6282
|
+
const present = lists.filter((list) => list !== undefined);
|
|
6283
|
+
if (present.length === 1)
|
|
6284
|
+
return present[0];
|
|
6285
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
6286
|
+
}
|
|
6287
|
+
}
|
|
6288
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
6289
|
+
const runIds = new Set;
|
|
6290
|
+
const identities = [];
|
|
6291
|
+
for (const checkpoint of checkpoints) {
|
|
6292
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
6293
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
6294
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
6295
|
+
}
|
|
6296
|
+
}
|
|
6297
|
+
identities.sort(compareCodeUnits);
|
|
6298
|
+
return { runIds, identities };
|
|
6299
|
+
}
|
|
6300
|
+
function hasPrefix(sorted, prefix) {
|
|
6301
|
+
let low = 0;
|
|
6302
|
+
let high = sorted.length;
|
|
6303
|
+
while (low < high) {
|
|
6304
|
+
const middle = low + high >>> 1;
|
|
6305
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
6306
|
+
low = middle + 1;
|
|
6307
|
+
else
|
|
6308
|
+
high = middle;
|
|
6309
|
+
}
|
|
6310
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
6311
|
+
}
|
|
6312
|
+
function compareCodeUnits(left, right) {
|
|
6313
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
6314
|
+
}
|
|
6315
|
+
function append(map, key, value2) {
|
|
6316
|
+
const list = map.get(key);
|
|
6317
|
+
if (list === undefined)
|
|
6318
|
+
map.set(key, [value2]);
|
|
6319
|
+
else
|
|
6320
|
+
list.push(value2);
|
|
6321
|
+
}
|
|
6322
|
+
var init_completed_catalog_index = __esm(() => {
|
|
6323
|
+
init_boundary_lifecycle();
|
|
6324
|
+
init_stage_topology_validation();
|
|
6325
|
+
});
|
|
6326
|
+
|
|
5758
6327
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
5759
|
-
function validBoundaryRecordSet(
|
|
5760
|
-
const
|
|
5761
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
6328
|
+
function validBoundaryRecordSet(index, strict) {
|
|
6329
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
5762
6330
|
const startsByReplay = new Map;
|
|
5763
6331
|
for (const checkpoint of boundaries) {
|
|
5764
6332
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -5773,7 +6341,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
5773
6341
|
return false;
|
|
5774
6342
|
return [...startsByReplay.values()].every((starts) => {
|
|
5775
6343
|
const start = starts[0];
|
|
5776
|
-
if (
|
|
6344
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
5777
6345
|
return false;
|
|
5778
6346
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
5779
6347
|
});
|
|
@@ -5837,8 +6405,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
5837
6405
|
function runTopologyFor(drafts, tools) {
|
|
5838
6406
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
5839
6407
|
}
|
|
5840
|
-
function validateRunGroups(grouped, rootRunId,
|
|
5841
|
-
const toolsFor = (runId) =>
|
|
6408
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
6409
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
5842
6410
|
const owners = new Map;
|
|
5843
6411
|
const syntheticChildIds = new Set;
|
|
5844
6412
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -5849,12 +6417,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
5849
6417
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
5850
6418
|
return false;
|
|
5851
6419
|
if (!grouped.has(childRunId)) {
|
|
5852
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
6420
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
5853
6421
|
syntheticChildIds.add(childRunId);
|
|
5854
6422
|
owners.set(childRunId, draft);
|
|
5855
6423
|
continue;
|
|
5856
6424
|
}
|
|
5857
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
6425
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
5858
6426
|
return false;
|
|
5859
6427
|
syntheticChildIds.add(childRunId);
|
|
5860
6428
|
owners.set(childRunId, draft);
|
|
@@ -5895,13 +6463,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
5895
6463
|
const boundary = draft.topology?.boundary;
|
|
5896
6464
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
5897
6465
|
}
|
|
5898
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
5899
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
5900
|
-
return checkpoints.some((checkpoint) => {
|
|
5901
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
5902
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
5903
|
-
});
|
|
5904
|
-
}
|
|
5905
6466
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
5906
6467
|
const reachable = new Set([rootRunId]);
|
|
5907
6468
|
const settled = new Set;
|
|
@@ -5942,14 +6503,6 @@ function childRunIdFromDraft(draft) {
|
|
|
5942
6503
|
return;
|
|
5943
6504
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
5944
6505
|
}
|
|
5945
|
-
function boundaryOwner(grouped, childRunId) {
|
|
5946
|
-
for (const drafts of grouped.values()) {
|
|
5947
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
5948
|
-
if (owner !== undefined)
|
|
5949
|
-
return owner;
|
|
5950
|
-
}
|
|
5951
|
-
return;
|
|
5952
|
-
}
|
|
5953
6506
|
function childRunStatus(owner) {
|
|
5954
6507
|
const child = workflowChildFromDraft(owner);
|
|
5955
6508
|
if (child !== undefined)
|
|
@@ -5957,15 +6510,15 @@ function childRunStatus(owner) {
|
|
|
5957
6510
|
const status = owner.topology?.boundary?.status;
|
|
5958
6511
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
5959
6512
|
}
|
|
5960
|
-
function mergeStageGroup(records,
|
|
6513
|
+
function mergeStageGroup(records, index) {
|
|
5961
6514
|
let merged;
|
|
5962
6515
|
for (const record of records)
|
|
5963
|
-
merged = mergeStageDraft(merged, record,
|
|
6516
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
5964
6517
|
const draft = merged;
|
|
5965
6518
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
5966
6519
|
if (start === undefined)
|
|
5967
6520
|
return draft;
|
|
5968
|
-
const resolved =
|
|
6521
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
5969
6522
|
if ("error" in resolved)
|
|
5970
6523
|
return draft;
|
|
5971
6524
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -6028,9 +6581,7 @@ function workflowChildFromDraft(draft) {
|
|
|
6028
6581
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
6029
6582
|
};
|
|
6030
6583
|
}
|
|
6031
|
-
var init_completed_catalog_stage_groups =
|
|
6032
|
-
init_boundary_lifecycle();
|
|
6033
|
-
});
|
|
6584
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
6034
6585
|
|
|
6035
6586
|
// dist/builtin/workflows/src/shared/abort.ts
|
|
6036
6587
|
function raceAbort(operation, signal) {
|
|
@@ -6564,13 +7115,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
6564
7115
|
function runJournalVersions(journal) {
|
|
6565
7116
|
return versionReaders.get(journal);
|
|
6566
7117
|
}
|
|
7118
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
7119
|
+
appendManyWriters.set(journal, appendMany);
|
|
7120
|
+
}
|
|
7121
|
+
function runJournalAppendMany(journal) {
|
|
7122
|
+
return appendManyWriters.get(journal);
|
|
7123
|
+
}
|
|
6567
7124
|
function registerSqlPoolStorage(journal, storage) {
|
|
6568
7125
|
poolStorage.set(journal, storage);
|
|
6569
7126
|
}
|
|
6570
7127
|
function sqlPoolStorage(journal) {
|
|
6571
7128
|
return poolStorage.get(journal);
|
|
6572
7129
|
}
|
|
6573
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
7130
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
6574
7131
|
var init_internal = __esm(() => {
|
|
6575
7132
|
init_types();
|
|
6576
7133
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -6579,6 +7136,7 @@ var init_internal = __esm(() => {
|
|
|
6579
7136
|
RUN_EVENT_TYPES.messageDelivered
|
|
6580
7137
|
];
|
|
6581
7138
|
versionReaders = new WeakMap;
|
|
7139
|
+
appendManyWriters = new WeakMap;
|
|
6582
7140
|
poolStorage = new WeakMap;
|
|
6583
7141
|
});
|
|
6584
7142
|
|
|
@@ -6872,9 +7430,19 @@ function* invoke(action) {
|
|
|
6872
7430
|
} catch (error) {
|
|
6873
7431
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
6874
7432
|
throw error;
|
|
6875
|
-
throw
|
|
7433
|
+
throw customJournalUnavailable(error);
|
|
6876
7434
|
}
|
|
6877
7435
|
}
|
|
7436
|
+
function customJournalUnavailable(failure) {
|
|
7437
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
7438
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
7439
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
7440
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
7441
|
+
}
|
|
7442
|
+
function hasErrorCode(error) {
|
|
7443
|
+
const code = error.code;
|
|
7444
|
+
return typeof code === "string" || typeof code === "number";
|
|
7445
|
+
}
|
|
6878
7446
|
async function execute(operation) {
|
|
6879
7447
|
let step = operation.next();
|
|
6880
7448
|
while (!step.done) {
|
|
@@ -6902,6 +7470,8 @@ class JournalDurableBackend {
|
|
|
6902
7470
|
executions = new Map;
|
|
6903
7471
|
executionContext = new AsyncLocalStorage;
|
|
6904
7472
|
pending = new Map;
|
|
7473
|
+
reconciledPendingStages = new Map;
|
|
7474
|
+
appendBatches = new Map;
|
|
6905
7475
|
catalogQueue = Symbol("catalog");
|
|
6906
7476
|
lost = new Map;
|
|
6907
7477
|
prompts = new Map;
|
|
@@ -6934,7 +7504,7 @@ class JournalDurableBackend {
|
|
|
6934
7504
|
await backend.close();
|
|
6935
7505
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
6936
7506
|
throw error;
|
|
6937
|
-
throw
|
|
7507
|
+
throw customJournalUnavailable(error);
|
|
6938
7508
|
}
|
|
6939
7509
|
}
|
|
6940
7510
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -6949,6 +7519,8 @@ class JournalDurableBackend {
|
|
|
6949
7519
|
return this.enqueue(runId, action, admitted);
|
|
6950
7520
|
}
|
|
6951
7521
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
7522
|
+
if (typeof runId === "string")
|
|
7523
|
+
this.appendBatches.delete(runId);
|
|
6952
7524
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
6953
7525
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
6954
7526
|
const settled = result.then(() => {}, () => {});
|
|
@@ -6963,6 +7535,7 @@ class JournalDurableBackend {
|
|
|
6963
7535
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
6964
7536
|
return;
|
|
6965
7537
|
const event = entry.event;
|
|
7538
|
+
let raisedCount;
|
|
6966
7539
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
6967
7540
|
const checkpoint = event.payload;
|
|
6968
7541
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -6971,7 +7544,10 @@ class JournalDurableBackend {
|
|
|
6971
7544
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
6972
7545
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
6973
7546
|
const payload = event.payload;
|
|
7547
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
6974
7548
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
7549
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
7550
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
6975
7551
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
6976
7552
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
6977
7553
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -6982,20 +7558,80 @@ class JournalDurableBackend {
|
|
|
6982
7558
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
6983
7559
|
this.prompts.delete(entry.runId);
|
|
6984
7560
|
}
|
|
6985
|
-
|
|
6986
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
6987
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
6988
|
-
if (message.sticky !== true)
|
|
6989
|
-
return message;
|
|
6990
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
6991
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
6992
|
-
});
|
|
6993
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
6994
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
6995
|
-
}
|
|
7561
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
6996
7562
|
this.sequences.set(entry.runId, entry.seq);
|
|
6997
7563
|
this.foldedAt.set(entry.runId, entry.at);
|
|
6998
7564
|
}
|
|
7565
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
7566
|
+
const handle = this.projection.getWorkflow(runId);
|
|
7567
|
+
const list = handle?.pendingStageMessages;
|
|
7568
|
+
if (handle === undefined || list === undefined) {
|
|
7569
|
+
this.reconciledPendingStages.delete(runId);
|
|
7570
|
+
return;
|
|
7571
|
+
}
|
|
7572
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
7573
|
+
let next;
|
|
7574
|
+
const raise = (index, count) => {
|
|
7575
|
+
const message = (next ?? list)[index];
|
|
7576
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
7577
|
+
return;
|
|
7578
|
+
next ??= [...list];
|
|
7579
|
+
next[index] = { ...message, deliveryCount: count };
|
|
7580
|
+
};
|
|
7581
|
+
const reconcile = (index) => {
|
|
7582
|
+
const message = list[index];
|
|
7583
|
+
if (message.sticky !== true)
|
|
7584
|
+
return;
|
|
7585
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
7586
|
+
};
|
|
7587
|
+
if (state?.list !== list) {
|
|
7588
|
+
const previous = state?.list ?? [];
|
|
7589
|
+
let aligned = list.length >= previous.length;
|
|
7590
|
+
const changed = [];
|
|
7591
|
+
for (let index = 0;index < list.length; index++) {
|
|
7592
|
+
const message = list[index];
|
|
7593
|
+
if (previous[index] === message)
|
|
7594
|
+
continue;
|
|
7595
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
7596
|
+
aligned = false;
|
|
7597
|
+
changed.push(index);
|
|
7598
|
+
}
|
|
7599
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
7600
|
+
for (const index of changed)
|
|
7601
|
+
if (!reconciled?.has(list[index]))
|
|
7602
|
+
reconcile(index);
|
|
7603
|
+
const positions = aligned ? state?.positions : undefined;
|
|
7604
|
+
if (positions !== undefined) {
|
|
7605
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
7606
|
+
const indexes = positions.get(list[index].id);
|
|
7607
|
+
if (indexes === undefined)
|
|
7608
|
+
positions.set(list[index].id, [index]);
|
|
7609
|
+
else
|
|
7610
|
+
indexes.push(index);
|
|
7611
|
+
}
|
|
7612
|
+
}
|
|
7613
|
+
state = { list, positions };
|
|
7614
|
+
}
|
|
7615
|
+
if (raisedCount !== undefined) {
|
|
7616
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
7617
|
+
state.positions ??= messagePositions(list);
|
|
7618
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
7619
|
+
const message = list[index];
|
|
7620
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
7621
|
+
raise(index, count);
|
|
7622
|
+
}
|
|
7623
|
+
}
|
|
7624
|
+
if (next === undefined) {
|
|
7625
|
+
this.reconciledPendingStages.set(runId, state);
|
|
7626
|
+
return;
|
|
7627
|
+
}
|
|
7628
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
7629
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
7630
|
+
this.reconciledPendingStages.set(runId, {
|
|
7631
|
+
list: applied,
|
|
7632
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
7633
|
+
});
|
|
7634
|
+
}
|
|
6999
7635
|
summarize(runId, tail) {
|
|
7000
7636
|
const event = tail.snapshot.event;
|
|
7001
7637
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -7019,6 +7655,7 @@ class JournalDurableBackend {
|
|
|
7019
7655
|
*rebuild(runId) {
|
|
7020
7656
|
const entries = yield* this.entries(runId);
|
|
7021
7657
|
this.projection.deleteWorkflow(runId);
|
|
7658
|
+
this.reconciledPendingStages.delete(runId);
|
|
7022
7659
|
this.sequences.delete(runId);
|
|
7023
7660
|
this.foldedAt.delete(runId);
|
|
7024
7661
|
this.summarized.delete(runId);
|
|
@@ -7114,9 +7751,9 @@ class JournalDurableBackend {
|
|
|
7114
7751
|
}
|
|
7115
7752
|
yield* this.write(runId, lease, event, options);
|
|
7116
7753
|
}
|
|
7117
|
-
*write(runId, lease, event, options) {
|
|
7754
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
7118
7755
|
try {
|
|
7119
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
7756
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
7120
7757
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
7121
7758
|
if (outcome.entry.seq > nextSeq) {
|
|
7122
7759
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -7300,13 +7937,80 @@ class JournalDurableBackend {
|
|
|
7300
7937
|
return true;
|
|
7301
7938
|
}
|
|
7302
7939
|
async archivePendingStageMessage(runId, message) {
|
|
7303
|
-
await this.
|
|
7940
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
7304
7941
|
}
|
|
7305
7942
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
7306
|
-
await this.
|
|
7943
|
+
await this.queueAppend(runId, {
|
|
7307
7944
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
7308
7945
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
7309
|
-
})
|
|
7946
|
+
});
|
|
7947
|
+
}
|
|
7948
|
+
queueAppend(runId, event, key) {
|
|
7949
|
+
try {
|
|
7950
|
+
this.ensureOpen();
|
|
7951
|
+
this.assertExecutionActive();
|
|
7952
|
+
} catch (error) {
|
|
7953
|
+
return Promise.reject(error);
|
|
7954
|
+
}
|
|
7955
|
+
const open = this.appendBatches.get(runId);
|
|
7956
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
7957
|
+
const queued = new Promise((resolve, reject) => {
|
|
7958
|
+
batch.push({
|
|
7959
|
+
event,
|
|
7960
|
+
...key === undefined ? {} : { key },
|
|
7961
|
+
context: AsyncLocalStorage.snapshot(),
|
|
7962
|
+
resolve,
|
|
7963
|
+
reject
|
|
7964
|
+
});
|
|
7965
|
+
});
|
|
7966
|
+
if (batch !== open) {
|
|
7967
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
7968
|
+
for (const append of batch)
|
|
7969
|
+
append.reject(error);
|
|
7970
|
+
});
|
|
7971
|
+
this.appendBatches.set(runId, batch);
|
|
7972
|
+
}
|
|
7973
|
+
return queued;
|
|
7974
|
+
}
|
|
7975
|
+
*appendBatch(runId, batch) {
|
|
7976
|
+
if (this.appendBatches.get(runId) === batch)
|
|
7977
|
+
this.appendBatches.delete(runId);
|
|
7978
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
7979
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
7980
|
+
for (const append of batch) {
|
|
7981
|
+
try {
|
|
7982
|
+
const lease = yield* this.ownership(runId);
|
|
7983
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
7984
|
+
append.resolve();
|
|
7985
|
+
} catch (error) {
|
|
7986
|
+
append.reject(error);
|
|
7987
|
+
}
|
|
7988
|
+
}
|
|
7989
|
+
return;
|
|
7990
|
+
}
|
|
7991
|
+
let settled = 0;
|
|
7992
|
+
try {
|
|
7993
|
+
const lease = yield* this.ownership(runId);
|
|
7994
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
7995
|
+
for (const result of results) {
|
|
7996
|
+
const append = batch[settled];
|
|
7997
|
+
if (result.status === "rejected") {
|
|
7998
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
7999
|
+
} else {
|
|
8000
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
8001
|
+
if (result.value.entry.seq > nextSeq)
|
|
8002
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
8003
|
+
this.fold(entry);
|
|
8004
|
+
this.fold(result.value.entry);
|
|
8005
|
+
append.resolve();
|
|
8006
|
+
}
|
|
8007
|
+
settled++;
|
|
8008
|
+
}
|
|
8009
|
+
} catch (error) {
|
|
8010
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
8011
|
+
for (const append of batch.slice(settled))
|
|
8012
|
+
append.reject(failure);
|
|
8013
|
+
}
|
|
7310
8014
|
}
|
|
7311
8015
|
getToolOutput(...args) {
|
|
7312
8016
|
return this.projection.getToolOutput(...args);
|
|
@@ -7326,6 +8030,9 @@ class JournalDurableBackend {
|
|
|
7326
8030
|
listCheckpoints(runId) {
|
|
7327
8031
|
return this.projection.listCheckpoints(runId);
|
|
7328
8032
|
}
|
|
8033
|
+
nestedSnapshotVersion(runId) {
|
|
8034
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
8035
|
+
}
|
|
7329
8036
|
getWorkflow(runId) {
|
|
7330
8037
|
return this.projection.getWorkflow(runId);
|
|
7331
8038
|
}
|
|
@@ -7350,6 +8057,12 @@ class JournalDurableBackend {
|
|
|
7350
8057
|
getPendingStageMessageDeliveryCount(...args) {
|
|
7351
8058
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
7352
8059
|
}
|
|
8060
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
8061
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
8062
|
+
}
|
|
8063
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
8064
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
8065
|
+
}
|
|
7353
8066
|
hasPendingStageDeliveryReceipt(...args) {
|
|
7354
8067
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
7355
8068
|
}
|
|
@@ -7434,6 +8147,7 @@ class JournalDurableBackend {
|
|
|
7434
8147
|
this.foldedAt.delete(runId);
|
|
7435
8148
|
this.summarized.delete(runId);
|
|
7436
8149
|
this.prompts.delete(runId);
|
|
8150
|
+
this.reconciledPendingStages.delete(runId);
|
|
7437
8151
|
this.projection.deleteWorkflow(runId);
|
|
7438
8152
|
}
|
|
7439
8153
|
async deleteWorkflow(runId) {
|
|
@@ -7616,8 +8330,24 @@ class JournalDurableBackend {
|
|
|
7616
8330
|
}
|
|
7617
8331
|
}
|
|
7618
8332
|
}
|
|
8333
|
+
function messagePositions(list) {
|
|
8334
|
+
const positions = new Map;
|
|
8335
|
+
list.forEach((message, index) => {
|
|
8336
|
+
const indexes = positions.get(message.id);
|
|
8337
|
+
if (indexes === undefined)
|
|
8338
|
+
positions.set(message.id, [index]);
|
|
8339
|
+
else
|
|
8340
|
+
indexes.push(index);
|
|
8341
|
+
});
|
|
8342
|
+
return positions;
|
|
8343
|
+
}
|
|
7619
8344
|
function identity(message) {
|
|
7620
|
-
|
|
8345
|
+
let key = entryIdentity.get(message);
|
|
8346
|
+
if (key === undefined) {
|
|
8347
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
8348
|
+
entryIdentity.set(message, key);
|
|
8349
|
+
}
|
|
8350
|
+
return key;
|
|
7621
8351
|
}
|
|
7622
8352
|
function uniqueByIdentity(messages) {
|
|
7623
8353
|
const result = new Map;
|
|
@@ -7791,7 +8521,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
7791
8521
|
function hasPendingStageDeliveryDelta(messages) {
|
|
7792
8522
|
return messages?.some(isDeltaEncoded) === true;
|
|
7793
8523
|
}
|
|
7794
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
8524
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
7795
8525
|
var init_journal_backend = __esm(() => {
|
|
7796
8526
|
init_workflow_instance_ownership_error();
|
|
7797
8527
|
init_backend();
|
|
@@ -7803,6 +8533,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7803
8533
|
init_types();
|
|
7804
8534
|
PROCESS_HOLDER_ID = randomUUID4();
|
|
7805
8535
|
processExitCleanups = new Set;
|
|
8536
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
7806
8537
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
7807
8538
|
runId;
|
|
7808
8539
|
holderId;
|
|
@@ -7849,6 +8580,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7849
8580
|
this.name = "JournalMessageScopeError";
|
|
7850
8581
|
}
|
|
7851
8582
|
};
|
|
8583
|
+
entryIdentity = new WeakMap;
|
|
7852
8584
|
entryJson = new WeakMap;
|
|
7853
8585
|
});
|
|
7854
8586
|
|
|
@@ -7976,6 +8708,7 @@ class SqlRunJournal {
|
|
|
7976
8708
|
runVersions: () => this.runVersions(),
|
|
7977
8709
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
7978
8710
|
});
|
|
8711
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
7979
8712
|
registerSqlPoolStorage(this, {
|
|
7980
8713
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
7981
8714
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -8240,38 +8973,76 @@ class SqlRunJournal {
|
|
|
8240
8973
|
this.assertOpen();
|
|
8241
8974
|
validateText("runId", lease.runId);
|
|
8242
8975
|
validateText("holderId", lease.holderId);
|
|
8243
|
-
if (options?.key !== undefined)
|
|
8244
|
-
validateText("key", options.key);
|
|
8245
8976
|
const owned = { ...lease };
|
|
8246
|
-
const
|
|
8977
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
8247
8978
|
const expected = options?.expectSeq;
|
|
8979
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
8980
|
+
await this.checkLease(tx, owned);
|
|
8981
|
+
return this.insert(tx, owned, prepared, expected);
|
|
8982
|
+
}));
|
|
8983
|
+
}
|
|
8984
|
+
async appendMany(lease, appends) {
|
|
8985
|
+
this.assertOpen();
|
|
8986
|
+
validateText("runId", lease.runId);
|
|
8987
|
+
validateText("holderId", lease.holderId);
|
|
8988
|
+
const owned = { ...lease };
|
|
8989
|
+
const prepared = appends.map((append) => {
|
|
8990
|
+
try {
|
|
8991
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
8992
|
+
} catch (error) {
|
|
8993
|
+
return error;
|
|
8994
|
+
}
|
|
8995
|
+
});
|
|
8996
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
8997
|
+
await this.checkLease(tx, owned);
|
|
8998
|
+
const results = [];
|
|
8999
|
+
for (const append of prepared) {
|
|
9000
|
+
if (append instanceof Error) {
|
|
9001
|
+
results.push({ status: "rejected", reason: append });
|
|
9002
|
+
continue;
|
|
9003
|
+
}
|
|
9004
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
9005
|
+
try {
|
|
9006
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
9007
|
+
} catch (error) {
|
|
9008
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
9009
|
+
results.push({ status: "rejected", reason: error });
|
|
9010
|
+
}
|
|
9011
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
9012
|
+
}
|
|
9013
|
+
return results;
|
|
9014
|
+
}));
|
|
9015
|
+
}
|
|
9016
|
+
prepare(event, key, appendGuard) {
|
|
9017
|
+
if (key !== undefined)
|
|
9018
|
+
validateText("key", key);
|
|
8248
9019
|
const type = event.type;
|
|
8249
9020
|
const payload = this.snapshotJson(event.payload);
|
|
8250
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
9021
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
8251
9022
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
8252
9023
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
8253
9024
|
validateText("status", status);
|
|
8254
|
-
return
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
}
|
|
9025
|
+
return { key, type, payload, guard, status };
|
|
9026
|
+
}
|
|
9027
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
9028
|
+
await guard?.(tx, owned.runId);
|
|
9029
|
+
if (key !== undefined) {
|
|
9030
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
9031
|
+
if (row)
|
|
9032
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
9033
|
+
}
|
|
9034
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
9035
|
+
if (expected !== undefined && expected !== actual)
|
|
9036
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
9037
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
9038
|
+
if (payload instanceof StorageUnavailableError2)
|
|
9039
|
+
throw payload;
|
|
9040
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
9041
|
+
if (status !== undefined)
|
|
9042
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
9043
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
9044
|
+
await guard?.(tx, owned.runId);
|
|
9045
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
8275
9046
|
}
|
|
8276
9047
|
async* read(runId, fromSeq = 1) {
|
|
8277
9048
|
this.assertOpen();
|
|
@@ -8339,7 +9110,7 @@ class SqlRunJournal {
|
|
|
8339
9110
|
return this.closePromise;
|
|
8340
9111
|
}
|
|
8341
9112
|
}
|
|
8342
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
9113
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
8343
9114
|
var init_sql_run_journal = __esm(() => {
|
|
8344
9115
|
init_internal();
|
|
8345
9116
|
init_legacy_journal_tables();
|
|
@@ -8547,8 +9318,59 @@ function firstError(schema, value2) {
|
|
|
8547
9318
|
}
|
|
8548
9319
|
return;
|
|
8549
9320
|
}
|
|
9321
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
9322
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
9323
|
+
return false;
|
|
9324
|
+
const pending = [root];
|
|
9325
|
+
const ancestors = new Set;
|
|
9326
|
+
const exits = [undefined];
|
|
9327
|
+
while (pending.length > 0) {
|
|
9328
|
+
const exit = exits.pop();
|
|
9329
|
+
const value2 = pending.pop();
|
|
9330
|
+
if (exit !== undefined) {
|
|
9331
|
+
ancestors.delete(exit);
|
|
9332
|
+
continue;
|
|
9333
|
+
}
|
|
9334
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
9335
|
+
continue;
|
|
9336
|
+
if (typeof value2 === "number") {
|
|
9337
|
+
if (!Number.isFinite(value2))
|
|
9338
|
+
return false;
|
|
9339
|
+
continue;
|
|
9340
|
+
}
|
|
9341
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
9342
|
+
return false;
|
|
9343
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
9344
|
+
return false;
|
|
9345
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
9346
|
+
let children;
|
|
9347
|
+
if (Array.isArray(value2)) {
|
|
9348
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
9349
|
+
return false;
|
|
9350
|
+
children = [];
|
|
9351
|
+
for (let index = 0;index < value2.length; index++)
|
|
9352
|
+
children.push(value2[index]);
|
|
9353
|
+
} else {
|
|
9354
|
+
const proto = Object.getPrototypeOf(value2);
|
|
9355
|
+
if (proto !== Object.prototype && proto !== null)
|
|
9356
|
+
return false;
|
|
9357
|
+
const keys = Object.keys(value2);
|
|
9358
|
+
if (keys.length !== names.length)
|
|
9359
|
+
return false;
|
|
9360
|
+
children = keys.map((key) => value2[key]);
|
|
9361
|
+
}
|
|
9362
|
+
ancestors.add(value2);
|
|
9363
|
+
pending.push(value2);
|
|
9364
|
+
exits.push(value2);
|
|
9365
|
+
for (const child of children) {
|
|
9366
|
+
pending.push(child);
|
|
9367
|
+
exits.push(undefined);
|
|
9368
|
+
}
|
|
9369
|
+
}
|
|
9370
|
+
return true;
|
|
9371
|
+
}
|
|
8550
9372
|
function workflowSerializableValidationError(value2, label) {
|
|
8551
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
9373
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
8552
9374
|
return;
|
|
8553
9375
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
8554
9376
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -8557,7 +9379,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
8557
9379
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
8558
9380
|
}
|
|
8559
9381
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
8560
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
9382
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
8561
9383
|
return;
|
|
8562
9384
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
8563
9385
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -60863,6 +61685,27 @@ class GraphFrontierTracker {
|
|
|
60863
61685
|
}
|
|
60864
61686
|
}
|
|
60865
61687
|
|
|
61688
|
+
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
61689
|
+
import { isStaleExtensionContextError as isStaleExtensionContextError6 } from "@bastani/atomic";
|
|
61690
|
+
function reportsClosedSession(error) {
|
|
61691
|
+
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError6(error);
|
|
61692
|
+
}
|
|
61693
|
+
function isSessionClosedError(error) {
|
|
61694
|
+
let current = error;
|
|
61695
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined; depth++) {
|
|
61696
|
+
try {
|
|
61697
|
+
if (reportsClosedSession(current))
|
|
61698
|
+
return true;
|
|
61699
|
+
current = current.cause instanceof Error ? current.cause : undefined;
|
|
61700
|
+
} catch {
|
|
61701
|
+
return false;
|
|
61702
|
+
}
|
|
61703
|
+
}
|
|
61704
|
+
return false;
|
|
61705
|
+
}
|
|
61706
|
+
var MAX_CAUSE_DEPTH = 8;
|
|
61707
|
+
var init_session_closed_error = () => {};
|
|
61708
|
+
|
|
60866
61709
|
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
60867
61710
|
function claimExternalRunRoute(store, runId, owner) {
|
|
60868
61711
|
let owners = registry.owners.get(store);
|
|
@@ -60970,39 +61813,406 @@ var init_pending_stage_route_readiness = __esm(() => {
|
|
|
60970
61813
|
processState3[routeOwnersKey] = owners;
|
|
60971
61814
|
});
|
|
60972
61815
|
|
|
60973
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
60974
|
-
function
|
|
60975
|
-
|
|
60976
|
-
|
|
60977
|
-
const
|
|
60978
|
-
|
|
60979
|
-
const
|
|
60980
|
-
|
|
60981
|
-
const
|
|
60982
|
-
|
|
61816
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
61817
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
61818
|
+
if (states.length === 0)
|
|
61819
|
+
return { deliveryStates: states };
|
|
61820
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
61821
|
+
counts.set("delivered", 0);
|
|
61822
|
+
for (const delivery of states)
|
|
61823
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
61824
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
61825
|
+
if (states.length <= limit)
|
|
61826
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
61827
|
+
const exemplars = [];
|
|
61828
|
+
for (const state of UNRESOLVED_STATES) {
|
|
61829
|
+
for (const delivery of states) {
|
|
61830
|
+
if (exemplars.length === limit)
|
|
61831
|
+
break;
|
|
61832
|
+
if (delivery.state === state)
|
|
61833
|
+
exemplars.push(delivery);
|
|
61834
|
+
}
|
|
61835
|
+
}
|
|
61836
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
61837
|
+
const delivery = states[index];
|
|
61838
|
+
if (delivery.state === "delivered")
|
|
61839
|
+
exemplars.push(delivery);
|
|
61840
|
+
}
|
|
61841
|
+
return {
|
|
61842
|
+
deliveryStates: exemplars,
|
|
61843
|
+
deliveryStateCounts,
|
|
61844
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
61845
|
+
};
|
|
61846
|
+
}
|
|
61847
|
+
function omittedDeliveryStatesLine(bounded) {
|
|
61848
|
+
const omitted = bounded.omittedDeliveryStates ?? 0;
|
|
61849
|
+
if (omitted === 0 || bounded.deliveryStateCounts === undefined)
|
|
61850
|
+
return;
|
|
61851
|
+
const counts = Object.entries(bounded.deliveryStateCounts).map(([state, count]) => `${count} ${state}`).join(", ");
|
|
61852
|
+
return `… ${omitted} more delivery state${omitted === 1 ? "" : "s"} omitted (all: ${counts})`;
|
|
61853
|
+
}
|
|
61854
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
61855
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
61856
|
+
UNRESOLVED_STATES = [
|
|
61857
|
+
"queued",
|
|
61858
|
+
"receipt-unverified",
|
|
61859
|
+
"session-replaced",
|
|
61860
|
+
"delivery-unavailable",
|
|
61861
|
+
"root-unresolved",
|
|
61862
|
+
"stage-terminal"
|
|
61863
|
+
];
|
|
61864
|
+
});
|
|
61865
|
+
|
|
61866
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
61867
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
61868
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
61869
|
+
return {
|
|
61870
|
+
...bounded,
|
|
61871
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
61872
|
+
};
|
|
61873
|
+
}
|
|
61874
|
+
|
|
61875
|
+
class WorkflowTopologyIndex {
|
|
61876
|
+
lineage = [];
|
|
61877
|
+
runById = new Map;
|
|
61878
|
+
roots = new Map;
|
|
61879
|
+
hops = new Map;
|
|
61880
|
+
refresh(runs) {
|
|
61881
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61882
|
+
const lineage = [];
|
|
61883
|
+
for (const run of runs) {
|
|
61884
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
61885
|
+
if (run.parentRunId === undefined)
|
|
61886
|
+
continue;
|
|
61887
|
+
const parent = runById.get(run.parentRunId);
|
|
61888
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
61889
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
61890
|
+
}
|
|
61891
|
+
if (sameValues(lineage, this.lineage))
|
|
61892
|
+
return;
|
|
61893
|
+
this.lineage = lineage;
|
|
61894
|
+
this.runById = runById;
|
|
61895
|
+
this.roots.clear();
|
|
61896
|
+
this.hops.clear();
|
|
61897
|
+
}
|
|
61898
|
+
rootOf(runId) {
|
|
61899
|
+
if (this.roots.has(runId))
|
|
61900
|
+
return this.roots.get(runId);
|
|
61901
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
61902
|
+
this.roots.set(runId, root);
|
|
61903
|
+
return root;
|
|
61904
|
+
}
|
|
61905
|
+
hopsOf(runId) {
|
|
61906
|
+
if (this.hops.has(runId))
|
|
61907
|
+
return this.hops.get(runId);
|
|
61908
|
+
const hops = boundaryHops(this.runById, runId);
|
|
61909
|
+
this.hops.set(runId, hops);
|
|
61910
|
+
return hops;
|
|
61911
|
+
}
|
|
61912
|
+
}
|
|
61913
|
+
function boundaryHops(runById, runId) {
|
|
61914
|
+
const hops = [];
|
|
61915
|
+
let current = runById.get(runId);
|
|
61916
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
61917
|
+
const parent = runById.get(current.parentRunId);
|
|
61918
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
61919
|
+
if (parent === undefined || boundary === undefined)
|
|
61920
|
+
return;
|
|
61921
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
61922
|
+
current = parent;
|
|
61923
|
+
}
|
|
61924
|
+
return current === undefined ? undefined : hops;
|
|
61925
|
+
}
|
|
61926
|
+
function sameValues(left, right) {
|
|
61927
|
+
if (left.length !== right.length)
|
|
61928
|
+
return false;
|
|
61929
|
+
for (let index = 0;index < left.length; index++)
|
|
61930
|
+
if (left[index] !== right[index])
|
|
61931
|
+
return false;
|
|
61932
|
+
return true;
|
|
61933
|
+
}
|
|
61934
|
+
|
|
61935
|
+
class PossibleStageRowsIndex {
|
|
61936
|
+
rootRunId;
|
|
61937
|
+
topology;
|
|
61938
|
+
targets = new Map;
|
|
61939
|
+
patterns = [];
|
|
61940
|
+
targetOf = new Map;
|
|
61941
|
+
targetsByMessage = new Map;
|
|
61942
|
+
list = [];
|
|
61943
|
+
stageFields = new WeakMap;
|
|
61944
|
+
backend;
|
|
61945
|
+
receiptCursor;
|
|
61946
|
+
scanMatches = new Map;
|
|
61947
|
+
rowCache = new Map;
|
|
61948
|
+
boundedStates = new WeakMap;
|
|
61949
|
+
deliveriesByEntry = new WeakMap;
|
|
61950
|
+
statusCache = new WeakMap;
|
|
61951
|
+
receiptGeneration = 0;
|
|
61952
|
+
receiptEpochs = new Map;
|
|
61953
|
+
constructor(rootRunId, topology) {
|
|
61954
|
+
this.rootRunId = rootRunId;
|
|
61955
|
+
this.topology = topology;
|
|
61956
|
+
}
|
|
61957
|
+
rows(runs, rootRun, availableBackend) {
|
|
61958
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
61959
|
+
return [];
|
|
61960
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
61961
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
61962
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
61963
|
+
this.refreshReceipts(backend);
|
|
61964
|
+
this.refreshStages(runs);
|
|
61965
|
+
for (const state of this.patterns)
|
|
61966
|
+
if (state.dirty)
|
|
61967
|
+
this.recompute(state, backend);
|
|
61968
|
+
return this.assemble(runs, rootRun);
|
|
61969
|
+
}
|
|
61970
|
+
targetFor(entry) {
|
|
60983
61971
|
const target = entry.targetPath ?? entry.stageKey;
|
|
60984
|
-
let
|
|
60985
|
-
if (
|
|
61972
|
+
let state = this.targets.get(target);
|
|
61973
|
+
if (state === undefined) {
|
|
60986
61974
|
const parsed = parseWorkflowStageTarget(target);
|
|
60987
|
-
|
|
60988
|
-
|
|
60989
|
-
|
|
60990
|
-
|
|
61975
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
61976
|
+
state = {
|
|
61977
|
+
target,
|
|
61978
|
+
segments: owned ? parsed.segments : undefined,
|
|
61979
|
+
pattern: owned ? this.patterns.length : -1,
|
|
61980
|
+
entries: [],
|
|
61981
|
+
positions: [],
|
|
61982
|
+
queued: 0,
|
|
61983
|
+
stages: [],
|
|
61984
|
+
statuses: [],
|
|
61985
|
+
dirty: true
|
|
61986
|
+
};
|
|
61987
|
+
this.targets.set(target, state);
|
|
61988
|
+
if (owned)
|
|
61989
|
+
this.patterns.push(state);
|
|
60991
61990
|
}
|
|
60992
|
-
|
|
61991
|
+
return state;
|
|
61992
|
+
}
|
|
61993
|
+
refreshEntries(list) {
|
|
61994
|
+
const previous = this.list;
|
|
61995
|
+
if (list === previous)
|
|
60993
61996
|
return;
|
|
60994
|
-
|
|
60995
|
-
|
|
60996
|
-
|
|
60997
|
-
|
|
60998
|
-
|
|
60999
|
-
|
|
61000
|
-
|
|
61001
|
-
|
|
61002
|
-
|
|
61997
|
+
this.list = list;
|
|
61998
|
+
let incremental = list.length >= previous.length;
|
|
61999
|
+
const replaced = [];
|
|
62000
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
62001
|
+
if (list[index] === previous[index])
|
|
62002
|
+
continue;
|
|
62003
|
+
const before = previous[index];
|
|
62004
|
+
const after = list[index];
|
|
62005
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
62006
|
+
continue;
|
|
62007
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
62008
|
+
incremental = false;
|
|
62009
|
+
break;
|
|
62010
|
+
}
|
|
62011
|
+
replaced.push(index);
|
|
62012
|
+
}
|
|
62013
|
+
if (!incremental) {
|
|
62014
|
+
this.rebuildEntries(list);
|
|
62015
|
+
return;
|
|
62016
|
+
}
|
|
62017
|
+
for (const index of replaced) {
|
|
62018
|
+
const before = previous[index];
|
|
62019
|
+
const after = list[index];
|
|
62020
|
+
const state = this.targetOf.get(before);
|
|
62021
|
+
const slot = state.entries.indexOf(before);
|
|
62022
|
+
state.entries[slot] = after;
|
|
62023
|
+
this.targetOf.delete(before);
|
|
62024
|
+
this.targetOf.set(after, state);
|
|
62025
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
62026
|
+
if (before.id !== after.id)
|
|
62027
|
+
this.rebuildMessageIndex();
|
|
62028
|
+
state.dirty = true;
|
|
62029
|
+
}
|
|
62030
|
+
for (let index = previous.length;index < list.length; index++)
|
|
62031
|
+
this.addEntry(list[index], index);
|
|
62032
|
+
}
|
|
62033
|
+
addEntry(entry, position) {
|
|
62034
|
+
if (entry.sticky !== true)
|
|
62035
|
+
return;
|
|
62036
|
+
const state = this.targetFor(entry);
|
|
62037
|
+
state.entries.push(entry);
|
|
62038
|
+
state.positions.push(position);
|
|
62039
|
+
if (entry.status === "queued")
|
|
62040
|
+
state.queued++;
|
|
62041
|
+
state.dirty = true;
|
|
62042
|
+
this.targetOf.set(entry, state);
|
|
62043
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62044
|
+
if (targets === undefined) {
|
|
62045
|
+
targets = new Set;
|
|
62046
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62047
|
+
}
|
|
62048
|
+
targets.add(state);
|
|
62049
|
+
}
|
|
62050
|
+
rebuildEntries(list) {
|
|
62051
|
+
const before = new Map;
|
|
62052
|
+
for (const state of this.targets.values()) {
|
|
62053
|
+
before.set(state, state.entries);
|
|
62054
|
+
state.entries = [];
|
|
62055
|
+
state.positions = [];
|
|
62056
|
+
state.queued = 0;
|
|
62057
|
+
}
|
|
62058
|
+
this.targetOf.clear();
|
|
62059
|
+
this.targetsByMessage.clear();
|
|
62060
|
+
list.forEach((entry, position) => {
|
|
62061
|
+
this.addEntry(entry, position);
|
|
62062
|
+
});
|
|
62063
|
+
for (const state of this.targets.values()) {
|
|
62064
|
+
const previous = before.get(state);
|
|
62065
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
62066
|
+
}
|
|
62067
|
+
}
|
|
62068
|
+
rebuildMessageIndex() {
|
|
62069
|
+
this.targetsByMessage.clear();
|
|
62070
|
+
for (const [entry, state] of this.targetOf) {
|
|
62071
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62072
|
+
if (targets === undefined) {
|
|
62073
|
+
targets = new Set;
|
|
62074
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62075
|
+
}
|
|
62076
|
+
targets.add(state);
|
|
62077
|
+
}
|
|
62078
|
+
}
|
|
62079
|
+
refreshReceipts(backend) {
|
|
62080
|
+
if (backend !== this.backend) {
|
|
62081
|
+
this.backend = backend;
|
|
62082
|
+
this.receiptCursor = undefined;
|
|
62083
|
+
this.invalidateReceipts();
|
|
62084
|
+
}
|
|
62085
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
62086
|
+
return;
|
|
62087
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
62088
|
+
if (changes === undefined) {
|
|
62089
|
+
this.invalidateReceipts();
|
|
62090
|
+
return;
|
|
62091
|
+
}
|
|
62092
|
+
const firstRead = this.receiptCursor === undefined;
|
|
62093
|
+
this.receiptCursor = changes.cursor;
|
|
62094
|
+
if (changes.messageIds === undefined) {
|
|
62095
|
+
if (!firstRead)
|
|
62096
|
+
this.invalidateReceipts();
|
|
62097
|
+
return;
|
|
62098
|
+
}
|
|
62099
|
+
for (const messageId of changes.messageIds) {
|
|
62100
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
62101
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
62102
|
+
state.dirty = true;
|
|
62103
|
+
}
|
|
62104
|
+
}
|
|
62105
|
+
invalidateReceipts() {
|
|
62106
|
+
this.receiptGeneration++;
|
|
62107
|
+
this.receiptEpochs.clear();
|
|
62108
|
+
for (const state of this.patterns)
|
|
62109
|
+
state.dirty = true;
|
|
62110
|
+
}
|
|
62111
|
+
refreshStages(runs) {
|
|
62112
|
+
const current = new Map;
|
|
62113
|
+
let ordinal = 0;
|
|
62114
|
+
for (const run of runs) {
|
|
62115
|
+
const root = this.topology.rootOf(run.id);
|
|
62116
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
62117
|
+
continue;
|
|
62118
|
+
const resolved = root === this.rootRunId;
|
|
62119
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62120
|
+
for (const stage of run.stages) {
|
|
62121
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
62122
|
+
continue;
|
|
62123
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
62124
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
62125
|
+
for (const state of fields.matched) {
|
|
62126
|
+
const stages = current.get(state);
|
|
62127
|
+
if (stages === undefined)
|
|
62128
|
+
current.set(state, [status]);
|
|
62129
|
+
else
|
|
62130
|
+
stages.push(status);
|
|
62131
|
+
}
|
|
62132
|
+
}
|
|
62133
|
+
}
|
|
62134
|
+
for (const state of this.patterns) {
|
|
62135
|
+
const stages = current.get(state) ?? [];
|
|
62136
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
62137
|
+
const previous = state.stages[index];
|
|
62138
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
62139
|
+
}))
|
|
62140
|
+
state.dirty = true;
|
|
62141
|
+
state.stages = stages;
|
|
62142
|
+
}
|
|
62143
|
+
}
|
|
62144
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
62145
|
+
let fields = this.stageFields.get(stage);
|
|
62146
|
+
const matchable = resolved && hops !== undefined;
|
|
62147
|
+
let changed = false;
|
|
62148
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
62149
|
+
const previous = fields;
|
|
62150
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
62151
|
+
fields = {
|
|
62152
|
+
run,
|
|
62153
|
+
status: stage.status,
|
|
62154
|
+
error: stage.error,
|
|
62155
|
+
sessionId: stage.sessionId,
|
|
62156
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
62157
|
+
resolved,
|
|
62158
|
+
hops,
|
|
62159
|
+
id: stage.id,
|
|
62160
|
+
name: stage.name,
|
|
62161
|
+
tested: sameMatching ? previous.tested : 0,
|
|
62162
|
+
matched: sameMatching ? previous.matched : []
|
|
62163
|
+
};
|
|
62164
|
+
this.stageFields.set(stage, fields);
|
|
62165
|
+
changed = true;
|
|
62166
|
+
}
|
|
62167
|
+
if (fields.tested < this.patterns.length) {
|
|
62168
|
+
const added = this.patterns.slice(fields.tested);
|
|
62169
|
+
if (!matchable)
|
|
62170
|
+
fields.matched.push(...added);
|
|
62171
|
+
else {
|
|
62172
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
62173
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
62174
|
+
fields.matched.push(added[position]);
|
|
62175
|
+
}
|
|
62176
|
+
fields.tested = this.patterns.length;
|
|
62177
|
+
}
|
|
62178
|
+
if (changed)
|
|
62179
|
+
for (const state of fields.matched)
|
|
62180
|
+
state.dirty = true;
|
|
62181
|
+
return fields;
|
|
62182
|
+
}
|
|
62183
|
+
recompute(state, backend) {
|
|
62184
|
+
state.dirty = false;
|
|
62185
|
+
const statuses = [];
|
|
62186
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
62187
|
+
const fields = this.stageFields.get(stage);
|
|
62188
|
+
for (const entry of state.entries) {
|
|
62189
|
+
let byStage = this.statusCache.get(entry);
|
|
62190
|
+
if (byStage === undefined) {
|
|
62191
|
+
byStage = new Map;
|
|
62192
|
+
this.statusCache.set(entry, byStage);
|
|
62193
|
+
}
|
|
62194
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
62195
|
+
let cached = byStage.get(stage);
|
|
62196
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
62197
|
+
cached = {
|
|
62198
|
+
fields,
|
|
62199
|
+
generation: this.receiptGeneration,
|
|
62200
|
+
epoch,
|
|
62201
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
62202
|
+
};
|
|
62203
|
+
byStage.set(stage, cached);
|
|
62204
|
+
}
|
|
62205
|
+
statuses.push(cached.status);
|
|
62206
|
+
}
|
|
62207
|
+
}
|
|
62208
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
62209
|
+
state.statuses = statuses;
|
|
62210
|
+
}
|
|
62211
|
+
receiptsOf(entry, runId, stageId) {
|
|
62212
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
61003
62213
|
if (byStage === undefined) {
|
|
61004
62214
|
byStage = new Map;
|
|
61005
|
-
for (const delivery of
|
|
62215
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
61006
62216
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
61007
62217
|
const list = byStage.get(key);
|
|
61008
62218
|
if (list === undefined)
|
|
@@ -61010,75 +62220,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
61010
62220
|
else
|
|
61011
62221
|
list.push(delivery);
|
|
61012
62222
|
}
|
|
61013
|
-
deliveriesByEntry.set(
|
|
62223
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
61014
62224
|
}
|
|
61015
62225
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
61016
|
-
}
|
|
61017
|
-
|
|
61018
|
-
|
|
61019
|
-
|
|
61020
|
-
|
|
61021
|
-
|
|
61022
|
-
|
|
61023
|
-
if (
|
|
61024
|
-
|
|
61025
|
-
|
|
61026
|
-
|
|
61027
|
-
|
|
61028
|
-
|
|
62226
|
+
}
|
|
62227
|
+
status(entry, run, stage, matchable, backend) {
|
|
62228
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
62229
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
62230
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
62231
|
+
let state;
|
|
62232
|
+
let reason;
|
|
62233
|
+
if (!matchable) {
|
|
62234
|
+
state = "root-unresolved";
|
|
62235
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
62236
|
+
} else if (delivered) {
|
|
62237
|
+
state = "delivered";
|
|
62238
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
62239
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
62240
|
+
state = "stage-terminal";
|
|
62241
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
62242
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
62243
|
+
state = "receipt-unverified";
|
|
62244
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
62245
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
62246
|
+
state = "delivery-unavailable";
|
|
62247
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
62248
|
+
} else if (receipts.length > 0) {
|
|
62249
|
+
state = "session-replaced";
|
|
62250
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
62251
|
+
} else {
|
|
62252
|
+
state = "queued";
|
|
62253
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
62254
|
+
}
|
|
62255
|
+
return {
|
|
62256
|
+
messageId: entry.message.id,
|
|
62257
|
+
target,
|
|
62258
|
+
runId: run.id,
|
|
62259
|
+
stageId: stage.id,
|
|
62260
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
62261
|
+
state,
|
|
62262
|
+
reason
|
|
62263
|
+
};
|
|
62264
|
+
}
|
|
62265
|
+
assemble(runs, rootRun) {
|
|
62266
|
+
const rows = [];
|
|
62267
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
62268
|
+
let liveStageMatches;
|
|
62269
|
+
const rowTargets = new Set;
|
|
62270
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
62271
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
62272
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61029
62273
|
continue;
|
|
61030
|
-
|
|
61031
|
-
|
|
61032
|
-
|
|
61033
|
-
|
|
61034
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
61035
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
61036
|
-
runId: run.id,
|
|
61037
|
-
stageId: stage.id,
|
|
61038
|
-
sessionId: stage.sessionId,
|
|
61039
|
-
admission: "context"
|
|
61040
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
61041
|
-
let state;
|
|
61042
|
-
let reason;
|
|
61043
|
-
if (!resolved || !hops) {
|
|
61044
|
-
state = "root-unresolved";
|
|
61045
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
61046
|
-
} else if (delivered) {
|
|
61047
|
-
state = "delivered";
|
|
61048
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
61049
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
61050
|
-
state = "stage-terminal";
|
|
61051
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
61052
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
61053
|
-
state = "receipt-unverified";
|
|
61054
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
61055
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
61056
|
-
state = "delivery-unavailable";
|
|
61057
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
61058
|
-
} else if (receipts.length > 0) {
|
|
61059
|
-
state = "session-replaced";
|
|
61060
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
61061
|
-
} else {
|
|
61062
|
-
state = "queued";
|
|
61063
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
61064
|
-
}
|
|
61065
|
-
result.push({
|
|
61066
|
-
messageId: entry.message.id,
|
|
61067
|
-
target,
|
|
61068
|
-
runId: run.id,
|
|
61069
|
-
stageId: stage.id,
|
|
61070
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
61071
|
-
state,
|
|
61072
|
-
reason
|
|
61073
|
-
});
|
|
62274
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
62275
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
62276
|
+
if (liveStageMatches(rowSegments))
|
|
62277
|
+
continue;
|
|
61074
62278
|
}
|
|
62279
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
62280
|
+
if (matches === undefined) {
|
|
62281
|
+
matches = new Map;
|
|
62282
|
+
this.scanMatches.set(scanEntry, matches);
|
|
62283
|
+
}
|
|
62284
|
+
let queuedCount = 0;
|
|
62285
|
+
for (const state of scanTargets) {
|
|
62286
|
+
let match = matches.get(state);
|
|
62287
|
+
if (match === undefined) {
|
|
62288
|
+
const segments = state.segments;
|
|
62289
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
62290
|
+
matches.set(state, match);
|
|
62291
|
+
}
|
|
62292
|
+
if (match)
|
|
62293
|
+
queuedCount += state.queued;
|
|
62294
|
+
}
|
|
62295
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
62296
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
62297
|
+
rowTargets.add(target);
|
|
62298
|
+
}
|
|
62299
|
+
let broadcastCount = 0;
|
|
62300
|
+
for (const state of this.targets.values()) {
|
|
62301
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
62302
|
+
broadcastCount += state.queued;
|
|
62303
|
+
}
|
|
62304
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
62305
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
62306
|
+
rowTargets.add(broadcast);
|
|
62307
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
62308
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
62309
|
+
for (const state of statusTargets) {
|
|
62310
|
+
if (rowTargets.has(state.target))
|
|
62311
|
+
continue;
|
|
62312
|
+
rowTargets.add(state.target);
|
|
62313
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
61075
62314
|
}
|
|
62315
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
62316
|
+
this.pruneRows(result);
|
|
62317
|
+
return result;
|
|
62318
|
+
}
|
|
62319
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
62320
|
+
const cached = this.rowCache.get(slot);
|
|
62321
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
62322
|
+
if (same && cached.statuses === deliveryStates)
|
|
62323
|
+
return cached.row;
|
|
62324
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
62325
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
62326
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
62327
|
+
return cached.row;
|
|
62328
|
+
}
|
|
62329
|
+
const row = {
|
|
62330
|
+
target,
|
|
62331
|
+
queuedCount,
|
|
62332
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
62333
|
+
...bounded
|
|
62334
|
+
};
|
|
62335
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
62336
|
+
return row;
|
|
62337
|
+
}
|
|
62338
|
+
bounded(states) {
|
|
62339
|
+
let bounded = this.boundedStates.get(states);
|
|
62340
|
+
if (bounded === undefined) {
|
|
62341
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
62342
|
+
this.boundedStates.set(states, bounded);
|
|
62343
|
+
}
|
|
62344
|
+
return bounded;
|
|
62345
|
+
}
|
|
62346
|
+
pruneRows(rows) {
|
|
62347
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
62348
|
+
return;
|
|
62349
|
+
const kept = new Set(rows);
|
|
62350
|
+
for (const [key, cached] of this.rowCache)
|
|
62351
|
+
if (!kept.has(cached.row))
|
|
62352
|
+
this.rowCache.delete(key);
|
|
62353
|
+
}
|
|
62354
|
+
materializedStageMatcher(runs) {
|
|
62355
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
62356
|
+
return (entrySegments) => {
|
|
62357
|
+
for (const run of invocationRuns) {
|
|
62358
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62359
|
+
if (hops === undefined)
|
|
62360
|
+
continue;
|
|
62361
|
+
for (const stage of run.stages) {
|
|
62362
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
62363
|
+
continue;
|
|
62364
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
62365
|
+
continue;
|
|
62366
|
+
}
|
|
62367
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
62368
|
+
return true;
|
|
62369
|
+
}
|
|
62370
|
+
}
|
|
62371
|
+
return false;
|
|
62372
|
+
};
|
|
61076
62373
|
}
|
|
61077
|
-
return result;
|
|
61078
62374
|
}
|
|
61079
|
-
|
|
62375
|
+
function sameBoundedRow(row, bounded) {
|
|
62376
|
+
if (bounded === undefined)
|
|
62377
|
+
return row.deliveryStates === undefined;
|
|
62378
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
62379
|
+
return false;
|
|
62380
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
62381
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
62382
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
62383
|
+
return false;
|
|
62384
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
62385
|
+
}
|
|
62386
|
+
function sameStatuses(left, right) {
|
|
62387
|
+
if (left.length !== right.length)
|
|
62388
|
+
return false;
|
|
62389
|
+
for (let index = 0;index < left.length; index++) {
|
|
62390
|
+
const a = left[index];
|
|
62391
|
+
const b = right[index];
|
|
62392
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
62393
|
+
return false;
|
|
62394
|
+
}
|
|
62395
|
+
return true;
|
|
62396
|
+
}
|
|
62397
|
+
var init_possible_stage_rows = __esm(() => {
|
|
61080
62398
|
init_run_owner_backend();
|
|
61081
62399
|
init_pending_stage_status();
|
|
62400
|
+
init_sticky_delivery_state_bound();
|
|
62401
|
+
init_store_internal();
|
|
61082
62402
|
init_workflow_stage_target();
|
|
61083
62403
|
});
|
|
61084
62404
|
|
|
@@ -61139,12 +62459,36 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61139
62459
|
};
|
|
61140
62460
|
const announcedRoutes = new Map;
|
|
61141
62461
|
const routeCompletions = new Map;
|
|
61142
|
-
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) =>
|
|
62462
|
+
const disposeReadiness = registerWorkflowPendingStageRouteReadiness(activeStore, (runId) => {
|
|
62463
|
+
announceIfPending();
|
|
62464
|
+
return routeCompletions.get(runId);
|
|
62465
|
+
});
|
|
62466
|
+
const topology = new WorkflowTopologyIndex;
|
|
62467
|
+
const possibleStageRowIndexes = new Map;
|
|
62468
|
+
const rowIdentities = new WeakMap;
|
|
62469
|
+
let nextRowIdentity = 0;
|
|
62470
|
+
const rowIdentity = (row) => {
|
|
62471
|
+
let identity = rowIdentities.get(row);
|
|
62472
|
+
if (identity === undefined) {
|
|
62473
|
+
identity = nextRowIdentity++;
|
|
62474
|
+
rowIdentities.set(row, identity);
|
|
62475
|
+
}
|
|
62476
|
+
return identity;
|
|
62477
|
+
};
|
|
62478
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
62479
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
62480
|
+
if (index === undefined) {
|
|
62481
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
62482
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
62483
|
+
}
|
|
62484
|
+
return index;
|
|
62485
|
+
};
|
|
61143
62486
|
const announceRoutes = () => {
|
|
61144
62487
|
if (disposed)
|
|
61145
62488
|
return;
|
|
61146
62489
|
const ownedRunIds = new Set;
|
|
61147
62490
|
const runs = activeStore.runs();
|
|
62491
|
+
topology.refresh(runs);
|
|
61148
62492
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61149
62493
|
const childrenByBoundary = new Map;
|
|
61150
62494
|
for (const run of runs) {
|
|
@@ -61159,7 +62503,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61159
62503
|
for (const run of runs) {
|
|
61160
62504
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
61161
62505
|
continue;
|
|
61162
|
-
const rootRunId =
|
|
62506
|
+
const rootRunId = topology.rootOf(run.id);
|
|
61163
62507
|
if (rootRunId === undefined)
|
|
61164
62508
|
continue;
|
|
61165
62509
|
ownedRunIds.add(run.id);
|
|
@@ -61169,6 +62513,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61169
62513
|
runId: parentRun.id,
|
|
61170
62514
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
61171
62515
|
};
|
|
62516
|
+
let possibleStages;
|
|
62517
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
62518
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62519
|
+
possibleStages = [];
|
|
62520
|
+
} else if (run.id === rootRunId) {
|
|
62521
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
62522
|
+
}
|
|
61172
62523
|
const announcement = {
|
|
61173
62524
|
runId: run.id,
|
|
61174
62525
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -61193,11 +62544,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61193
62544
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
61194
62545
|
}))
|
|
61195
62546
|
],
|
|
61196
|
-
...
|
|
61197
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
61198
|
-
} : {}
|
|
62547
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
61199
62548
|
};
|
|
61200
|
-
const signature = JSON.stringify(announcement)
|
|
62549
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
61201
62550
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
61202
62551
|
continue;
|
|
61203
62552
|
const entry = { signature };
|
|
@@ -61232,15 +62581,61 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61232
62581
|
if (!ownedRunIds.has(runId))
|
|
61233
62582
|
routeCompletions.delete(runId);
|
|
61234
62583
|
}
|
|
62584
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
62585
|
+
if (!ownedRunIds.has(rootRunId))
|
|
62586
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62587
|
+
}
|
|
61235
62588
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
61236
62589
|
return;
|
|
61237
62590
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
61238
62591
|
};
|
|
61239
|
-
|
|
61240
|
-
|
|
62592
|
+
let announcementPending = false;
|
|
62593
|
+
let announcementTimer;
|
|
62594
|
+
let announcedAt = Number.NEGATIVE_INFINITY;
|
|
62595
|
+
let announcementCost = 0;
|
|
62596
|
+
const announceNow = () => {
|
|
62597
|
+
if (announcementTimer !== undefined)
|
|
62598
|
+
clearTimeout(announcementTimer);
|
|
62599
|
+
announcementTimer = undefined;
|
|
62600
|
+
announcementPending = false;
|
|
62601
|
+
const started = performance.now();
|
|
62602
|
+
announceRoutes();
|
|
62603
|
+
announcedAt = performance.now();
|
|
62604
|
+
announcementCost = announcedAt - started;
|
|
62605
|
+
};
|
|
62606
|
+
function announceIfPending() {
|
|
62607
|
+
if (announcementPending)
|
|
62608
|
+
announceNow();
|
|
62609
|
+
}
|
|
62610
|
+
const announceBatch = () => {
|
|
62611
|
+
try {
|
|
62612
|
+
announceNow();
|
|
62613
|
+
} catch (error) {
|
|
62614
|
+
if (error instanceof Error && isSessionClosedError(error)) {
|
|
62615
|
+
dispose();
|
|
62616
|
+
return;
|
|
62617
|
+
}
|
|
62618
|
+
try {
|
|
62619
|
+
reportWarning(`atomic-workflows: pending stage route announcement failed: ${String(error)}`);
|
|
62620
|
+
} catch {
|
|
62621
|
+
dispose();
|
|
62622
|
+
}
|
|
62623
|
+
}
|
|
62624
|
+
};
|
|
62625
|
+
const scheduleAnnouncement = () => {
|
|
62626
|
+
if (disposed || announcementTimer !== undefined)
|
|
62627
|
+
return;
|
|
62628
|
+
const idle = announcementCost / ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE - announcementCost;
|
|
62629
|
+
const wait = Math.max(ROUTE_ANNOUNCEMENT_BATCH_MS, announcedAt + idle - performance.now());
|
|
62630
|
+
announcementPending = true;
|
|
62631
|
+
announcementTimer = setTimeout(announceBatch, wait);
|
|
62632
|
+
announcementTimer.unref?.();
|
|
62633
|
+
};
|
|
62634
|
+
const unsubscribeStore = activeStore.subscribeInvalidation(scheduleAnnouncement);
|
|
62635
|
+
announceNow();
|
|
61241
62636
|
pi.on?.("session_start", () => {
|
|
61242
62637
|
announcedRoutes.clear();
|
|
61243
|
-
|
|
62638
|
+
announceNow();
|
|
61244
62639
|
});
|
|
61245
62640
|
const routePendingStageMessage = (payload) => {
|
|
61246
62641
|
if (disposed || !isPendingStageMessageEvent(payload) || payload.handled)
|
|
@@ -61322,6 +62717,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61322
62717
|
});
|
|
61323
62718
|
const dispose = () => {
|
|
61324
62719
|
disposed = true;
|
|
62720
|
+
if (announcementTimer !== undefined)
|
|
62721
|
+
clearTimeout(announcementTimer);
|
|
62722
|
+
announcementTimer = undefined;
|
|
62723
|
+
announcementPending = false;
|
|
61325
62724
|
disposeReadiness();
|
|
61326
62725
|
disposeRouter();
|
|
61327
62726
|
routeCompletions.clear();
|
|
@@ -61335,112 +62734,17 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61335
62734
|
pi.on?.("session_shutdown", dispose);
|
|
61336
62735
|
return Object.assign(dispose, {
|
|
61337
62736
|
async flush() {
|
|
62737
|
+
announceIfPending();
|
|
61338
62738
|
await Promise.allSettled(routeCompletions.values());
|
|
61339
62739
|
await sweepPromise;
|
|
61340
62740
|
},
|
|
61341
|
-
refresh:
|
|
62741
|
+
refresh: announceNow
|
|
61342
62742
|
});
|
|
61343
62743
|
}
|
|
61344
62744
|
function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegments = (id) => workflowBoundarySegments(runs, id)) {
|
|
61345
62745
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
61346
62746
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
61347
62747
|
}
|
|
61348
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
61349
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
61350
|
-
return [];
|
|
61351
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
61352
|
-
const segmentsByTarget = new Map;
|
|
61353
|
-
const stickySegments = (entry) => {
|
|
61354
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61355
|
-
if (segmentsByTarget.has(target))
|
|
61356
|
-
return segmentsByTarget.get(target);
|
|
61357
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
61358
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
61359
|
-
segmentsByTarget.set(target, segments);
|
|
61360
|
-
return segments;
|
|
61361
|
-
};
|
|
61362
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
61363
|
-
const segments = stickySegments(entry);
|
|
61364
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
61365
|
-
});
|
|
61366
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
61367
|
-
const rows = [];
|
|
61368
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
61369
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
61370
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61371
|
-
continue;
|
|
61372
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
61373
|
-
continue;
|
|
61374
|
-
}
|
|
61375
|
-
rows.push({
|
|
61376
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
61377
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
61378
|
-
const segments = stickySegments(entry);
|
|
61379
|
-
if (segments === undefined)
|
|
61380
|
-
return false;
|
|
61381
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
61382
|
-
}).length
|
|
61383
|
-
});
|
|
61384
|
-
}
|
|
61385
|
-
rows.push({
|
|
61386
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
61387
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
61388
|
-
const segments = stickySegments(entry);
|
|
61389
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
61390
|
-
}).length
|
|
61391
|
-
});
|
|
61392
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
61393
|
-
const statusesByTarget = new Map;
|
|
61394
|
-
for (const status of statuses) {
|
|
61395
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
61396
|
-
if (forTarget === undefined)
|
|
61397
|
-
statusesByTarget.set(status.target, [status]);
|
|
61398
|
-
else
|
|
61399
|
-
forTarget.push(status);
|
|
61400
|
-
}
|
|
61401
|
-
const queuedByTarget = new Map;
|
|
61402
|
-
for (const entry of stickyQueued) {
|
|
61403
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61404
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
61405
|
-
}
|
|
61406
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
61407
|
-
for (const target of statusesByTarget.keys()) {
|
|
61408
|
-
if (rowTargets.has(target))
|
|
61409
|
-
continue;
|
|
61410
|
-
rowTargets.add(target);
|
|
61411
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
61412
|
-
}
|
|
61413
|
-
for (let index = 0;index < rows.length; index++) {
|
|
61414
|
-
const row = rows[index];
|
|
61415
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
61416
|
-
if (deliveryStates !== undefined)
|
|
61417
|
-
rows[index] = { ...row, deliveryStates };
|
|
61418
|
-
}
|
|
61419
|
-
return rows;
|
|
61420
|
-
}
|
|
61421
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
61422
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61423
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
61424
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
61425
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
61426
|
-
}
|
|
61427
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
61428
|
-
for (const run of invocationRuns) {
|
|
61429
|
-
const hops = hopsFor(run.id);
|
|
61430
|
-
if (hops === undefined)
|
|
61431
|
-
continue;
|
|
61432
|
-
for (const stage of run.stages) {
|
|
61433
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
61434
|
-
continue;
|
|
61435
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
61436
|
-
continue;
|
|
61437
|
-
}
|
|
61438
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
61439
|
-
return true;
|
|
61440
|
-
}
|
|
61441
|
-
}
|
|
61442
|
-
return false;
|
|
61443
|
-
}
|
|
61444
62748
|
function isPendingStageMessageEvent(value2) {
|
|
61445
62749
|
if (typeof value2 !== "object" || value2 === null)
|
|
61446
62750
|
return false;
|
|
@@ -61813,17 +63117,18 @@ async function settleUndeliverablePendingStageMessages(activeStore, notify, owns
|
|
|
61813
63117
|
}
|
|
61814
63118
|
return settled;
|
|
61815
63119
|
}
|
|
61816
|
-
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.";
|
|
63120
|
+
var PENDING_STAGE_ROUTE_EVENT = "atomic:workflow-pending-stage-route", PENDING_STAGE_MESSAGE_EVENT = "atomic:workflow-pending-stage-message", PENDING_STAGE_UNDELIVERABLE_EVENT = "atomic:workflow-pending-stage-undeliverable", STICKY_LIVE_DELIVERED_EVENT = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_ASK_REFUSAL = "Cannot ask a workflow stage whose session has not initialized. Use send; Atomic will queue the message until the stage session initializes.", ROUTE_ANNOUNCEMENT_BATCH_MS = 25, ROUTE_ANNOUNCEMENT_EVENT_LOOP_SHARE = 0.2;
|
|
61817
63121
|
var init_pending_stage_intercom = __esm(() => {
|
|
61818
63122
|
init_execution_context_boundary();
|
|
61819
63123
|
init_run_owner_backend();
|
|
61820
63124
|
init_store_backend();
|
|
63125
|
+
init_session_closed_error();
|
|
61821
63126
|
init_external_run_routes();
|
|
61822
63127
|
init_intercom_group();
|
|
61823
63128
|
init_pending_stage_route_capability();
|
|
61824
63129
|
init_pending_stage_route_readiness();
|
|
61825
63130
|
init_pending_stage_status();
|
|
61826
|
-
|
|
63131
|
+
init_possible_stage_rows();
|
|
61827
63132
|
init_store_internal();
|
|
61828
63133
|
init_workflow_stage_target();
|
|
61829
63134
|
});
|
|
@@ -64374,7 +65679,7 @@ function isInfrastructureError(value2) {
|
|
|
64374
65679
|
}
|
|
64375
65680
|
function isRetryableWorkflowError(error) {
|
|
64376
65681
|
let current = error;
|
|
64377
|
-
for (let depth = 0;depth <
|
|
65682
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
|
|
64378
65683
|
if (isInfrastructureError(current))
|
|
64379
65684
|
return true;
|
|
64380
65685
|
current = current instanceof Error ? current.cause : undefined;
|
|
@@ -64392,7 +65697,7 @@ function asWorkflowInfrastructureError(error) {
|
|
|
64392
65697
|
const code = workflowInfrastructureFault(error.message);
|
|
64393
65698
|
return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
|
|
64394
65699
|
}
|
|
64395
|
-
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError",
|
|
65700
|
+
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
|
|
64396
65701
|
var init_workflow_infrastructure_error = __esm(() => {
|
|
64397
65702
|
FAULT_PATTERNS = [
|
|
64398
65703
|
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
@@ -64694,9 +65999,146 @@ var init_definitions = __esm(() => {
|
|
|
64694
65999
|
workerDefinitionSources = new WeakMap;
|
|
64695
66000
|
});
|
|
64696
66001
|
|
|
66002
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
66003
|
+
function push(map, key, value2) {
|
|
66004
|
+
if (key === undefined)
|
|
66005
|
+
return;
|
|
66006
|
+
const list = map.get(key);
|
|
66007
|
+
if (list)
|
|
66008
|
+
list.push(value2);
|
|
66009
|
+
else
|
|
66010
|
+
map.set(key, [value2]);
|
|
66011
|
+
}
|
|
66012
|
+
|
|
66013
|
+
class Partitions {
|
|
66014
|
+
byRun = new Map;
|
|
66015
|
+
startsByChild = new Map;
|
|
66016
|
+
stagesByGroup = new Map;
|
|
66017
|
+
stagesByReplayKey = new Map;
|
|
66018
|
+
boundariesByStageId = new Map;
|
|
66019
|
+
boundariesByChild = new Map;
|
|
66020
|
+
add(entry) {
|
|
66021
|
+
const checkpoint = entry.checkpoint;
|
|
66022
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
66023
|
+
if (checkpoint.kind !== "stage")
|
|
66024
|
+
return;
|
|
66025
|
+
const stage = entry;
|
|
66026
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
66027
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
66028
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66029
|
+
if (!boundary)
|
|
66030
|
+
return;
|
|
66031
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
66032
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
66033
|
+
if (boundary.event === "start")
|
|
66034
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
66035
|
+
}
|
|
66036
|
+
}
|
|
66037
|
+
function inKeyOrder(entries) {
|
|
66038
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
66039
|
+
}
|
|
66040
|
+
|
|
66041
|
+
class PoolCheckpointView {
|
|
66042
|
+
base;
|
|
66043
|
+
from;
|
|
66044
|
+
delta;
|
|
66045
|
+
constructor(base, from, delta) {
|
|
66046
|
+
this.base = base;
|
|
66047
|
+
this.from = from;
|
|
66048
|
+
this.delta = delta;
|
|
66049
|
+
}
|
|
66050
|
+
entries(partition, key) {
|
|
66051
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
66052
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
66053
|
+
}
|
|
66054
|
+
}
|
|
66055
|
+
|
|
66056
|
+
class PoolCheckpointRead {
|
|
66057
|
+
index;
|
|
66058
|
+
root;
|
|
66059
|
+
pending;
|
|
66060
|
+
constructor(index, root) {
|
|
66061
|
+
this.index = index;
|
|
66062
|
+
this.root = root;
|
|
66063
|
+
}
|
|
66064
|
+
read(tx, events) {
|
|
66065
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
66066
|
+
this.pending = apply;
|
|
66067
|
+
});
|
|
66068
|
+
}
|
|
66069
|
+
commit() {
|
|
66070
|
+
this.pending?.();
|
|
66071
|
+
this.pending = undefined;
|
|
66072
|
+
}
|
|
66073
|
+
}
|
|
66074
|
+
|
|
66075
|
+
class PoolCheckpointIndex {
|
|
66076
|
+
roots = new Map;
|
|
66077
|
+
begin(root) {
|
|
66078
|
+
return new PoolCheckpointRead(this, root);
|
|
66079
|
+
}
|
|
66080
|
+
async read(tx, events, root, onCommit) {
|
|
66081
|
+
const current = this.roots.get(root);
|
|
66082
|
+
let base = current;
|
|
66083
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
66084
|
+
if (base && base.seq > 0) {
|
|
66085
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
66086
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
66087
|
+
base = undefined;
|
|
66088
|
+
}
|
|
66089
|
+
const from = base?.seq ?? 0;
|
|
66090
|
+
const to = head ? Number(head.seq) : 0;
|
|
66091
|
+
const delta = new Partitions;
|
|
66092
|
+
const added = [];
|
|
66093
|
+
if (to > from) {
|
|
66094
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
66095
|
+
for (const row of rows) {
|
|
66096
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
66097
|
+
if (checkpoint.kind === "ui")
|
|
66098
|
+
continue;
|
|
66099
|
+
const entry = {
|
|
66100
|
+
seq: Number(row.seq),
|
|
66101
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
66102
|
+
checkpoint
|
|
66103
|
+
};
|
|
66104
|
+
delta.add(entry);
|
|
66105
|
+
added.push(entry);
|
|
66106
|
+
}
|
|
66107
|
+
}
|
|
66108
|
+
onCommit(() => {
|
|
66109
|
+
const latest = this.roots.get(root);
|
|
66110
|
+
if (!base) {
|
|
66111
|
+
if (latest !== current)
|
|
66112
|
+
return;
|
|
66113
|
+
const fresh = {
|
|
66114
|
+
seq: to,
|
|
66115
|
+
key: head?.key ?? null,
|
|
66116
|
+
at: Number(head?.at ?? 0),
|
|
66117
|
+
partitions: delta
|
|
66118
|
+
};
|
|
66119
|
+
this.roots.set(root, fresh);
|
|
66120
|
+
return;
|
|
66121
|
+
}
|
|
66122
|
+
if (latest !== base || to <= base.seq)
|
|
66123
|
+
return;
|
|
66124
|
+
for (const entry of added)
|
|
66125
|
+
if (entry.seq > base.seq)
|
|
66126
|
+
base.partitions.add(entry);
|
|
66127
|
+
base.seq = to;
|
|
66128
|
+
base.key = head?.key ?? null;
|
|
66129
|
+
base.at = Number(head?.at ?? 0);
|
|
66130
|
+
});
|
|
66131
|
+
return new PoolCheckpointView(base, from, delta);
|
|
66132
|
+
}
|
|
66133
|
+
}
|
|
66134
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
66135
|
+
init_stage_topology_validation();
|
|
66136
|
+
init_types();
|
|
66137
|
+
});
|
|
66138
|
+
|
|
64697
66139
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
64698
66140
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
64699
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
66141
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
64700
66142
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
64701
66143
|
throw new Error("Invalid pool mutation");
|
|
64702
66144
|
const checkpoint = mutation.checkpoint;
|
|
@@ -64716,42 +66158,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
64716
66158
|
}
|
|
64717
66159
|
if (checkpoint.kind === "ui")
|
|
64718
66160
|
return;
|
|
64719
|
-
const
|
|
64720
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
64721
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
66161
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
64722
66162
|
const topology = checkpoint.topology;
|
|
64723
66163
|
const run = topology?.run;
|
|
64724
66164
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
64725
66165
|
throw new Error("Pool topology scope mismatch");
|
|
64726
|
-
const owner =
|
|
66166
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
64727
66167
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
64728
66168
|
throw new Error("Pool topology scope mismatch");
|
|
64729
66169
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
64730
66170
|
throw new Error("Pool session scope mismatch");
|
|
66171
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
64731
66172
|
if (checkpoint.kind === "tool") {
|
|
64732
66173
|
const tool = checkpoint.topology;
|
|
64733
|
-
const peers =
|
|
66174
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
64734
66175
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
64735
66176
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
64736
66177
|
throw new Error("Pool tool topology scope mismatch");
|
|
64737
66178
|
return;
|
|
64738
66179
|
}
|
|
64739
|
-
const
|
|
64740
|
-
const
|
|
66180
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
66181
|
+
const relevant = [...runStages];
|
|
66182
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
66183
|
+
for (const group of groups)
|
|
66184
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
66185
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66186
|
+
let start;
|
|
66187
|
+
if (boundary) {
|
|
66188
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
66189
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
66190
|
+
relevant.push(...replayed);
|
|
66191
|
+
if (start) {
|
|
66192
|
+
const startTopology = start.topology;
|
|
66193
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
66194
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
66195
|
+
}
|
|
66196
|
+
}
|
|
66197
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
66198
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
64741
66199
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
64742
66200
|
if (stageError)
|
|
64743
66201
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
64744
|
-
const boundary = checkpoint.topology?.boundary;
|
|
64745
66202
|
if (!boundary) {
|
|
64746
66203
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
64747
66204
|
throw new Error("Pool boundary output without ownership");
|
|
64748
66205
|
return;
|
|
64749
66206
|
}
|
|
64750
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
64751
66207
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
64752
66208
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
64753
66209
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
64754
66210
|
}
|
|
66211
|
+
function stagesExcept(entries, checkpoint) {
|
|
66212
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
66213
|
+
}
|
|
64755
66214
|
async function sessionStore(tx, prefix) {
|
|
64756
66215
|
const database = {
|
|
64757
66216
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -64800,20 +66259,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
64800
66259
|
init_boundary_lifecycle();
|
|
64801
66260
|
init_boundary_topology();
|
|
64802
66261
|
init_stage_topology_validation();
|
|
64803
|
-
|
|
66262
|
+
init_pool_checkpoint_index();
|
|
64804
66263
|
});
|
|
64805
66264
|
|
|
64806
66265
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
64807
66266
|
function poolUsageTable(runs) {
|
|
64808
66267
|
return `${runs}_pool_usage`;
|
|
64809
66268
|
}
|
|
66269
|
+
function poolUsageTotalsTable(runs) {
|
|
66270
|
+
return `${runs}_pool_usage_totals`;
|
|
66271
|
+
}
|
|
66272
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
66273
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
66274
|
+
const total = field.toLowerCase();
|
|
66275
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
66276
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
66277
|
+
});
|
|
66278
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
66279
|
+
}
|
|
64810
66280
|
function poolUsageSchema(runs) {
|
|
64811
|
-
|
|
66281
|
+
const usage = poolUsageTable(runs);
|
|
66282
|
+
const totals = poolUsageTotalsTable(runs);
|
|
66283
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66284
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
64812
66285
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
64813
66286
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
64814
66287
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
64815
66288
|
PRIMARY KEY(work, generation, key)
|
|
64816
|
-
)
|
|
66289
|
+
);
|
|
66290
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
66291
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
66292
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
66293
|
+
);
|
|
66294
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
66295
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66296
|
+
END;
|
|
66297
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
66298
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
66299
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
66300
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66301
|
+
END;
|
|
66302
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
66303
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
66304
|
+
END;
|
|
66305
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
66306
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
66307
|
+
END;`;
|
|
64817
66308
|
}
|
|
64818
66309
|
function poolStageUsageKey(runId, stageId) {
|
|
64819
66310
|
return `stage:${runId}:${stageId}`;
|
|
@@ -64840,7 +66331,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
64840
66331
|
|
|
64841
66332
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
64842
66333
|
import { createHash as createHash4, randomUUID as randomUUID16 } from "node:crypto";
|
|
64843
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
66334
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
64844
66335
|
function requestIdentity(request) {
|
|
64845
66336
|
return createHash4("sha256").update(request).digest("hex");
|
|
64846
66337
|
}
|
|
@@ -64879,6 +66370,7 @@ class SqlPoolQueue {
|
|
|
64879
66370
|
operations;
|
|
64880
66371
|
transaction;
|
|
64881
66372
|
journal;
|
|
66373
|
+
checkpoints = new PoolCheckpointIndex;
|
|
64882
66374
|
constructor(storage) {
|
|
64883
66375
|
this.storage = storage;
|
|
64884
66376
|
const { transaction, tables: journal } = storage;
|
|
@@ -64990,17 +66482,23 @@ class SqlPoolQueue {
|
|
|
64990
66482
|
lease = snapshotLease(lease);
|
|
64991
66483
|
token = { ...token };
|
|
64992
66484
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
64993
|
-
|
|
64994
|
-
|
|
64995
|
-
|
|
64996
|
-
|
|
64997
|
-
|
|
64998
|
-
|
|
64999
|
-
|
|
65000
|
-
|
|
65001
|
-
|
|
65002
|
-
|
|
65003
|
-
|
|
66485
|
+
try {
|
|
66486
|
+
if (runId !== lease.runId)
|
|
66487
|
+
throw new Error("Pool delivery cannot write another root");
|
|
66488
|
+
await this.authority(tx, lease).catch((error) => {
|
|
66489
|
+
if (error instanceof StaleLeaseError)
|
|
66490
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
66491
|
+
throw error;
|
|
66492
|
+
});
|
|
66493
|
+
const row = await this.current(tx, lease, token);
|
|
66494
|
+
if (row.epoch !== lease.epoch)
|
|
66495
|
+
throw new Error(STALE_WORK_LEASE);
|
|
66496
|
+
assertSession();
|
|
66497
|
+
} catch (error) {
|
|
66498
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
66499
|
+
throw error;
|
|
66500
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
66501
|
+
}
|
|
65004
66502
|
}, action);
|
|
65005
66503
|
}
|
|
65006
66504
|
async assertCurrent(lease, token) {
|
|
@@ -65113,7 +66611,8 @@ class SqlPoolQueue {
|
|
|
65113
66611
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
65114
66612
|
const identity = requestIdentity(request);
|
|
65115
66613
|
mutation = JSON.parse(request).mutation;
|
|
65116
|
-
|
|
66614
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
66615
|
+
const receipt = await this.transaction(async (tx) => {
|
|
65117
66616
|
const { now, status } = await this.authority(tx, lease);
|
|
65118
66617
|
const row = await this.current(tx, lease, token);
|
|
65119
66618
|
if (row.epoch !== lease.epoch)
|
|
@@ -65128,24 +66627,26 @@ class SqlPoolQueue {
|
|
|
65128
66627
|
throw new Error(STALE_WORK_LEASE);
|
|
65129
66628
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
65130
66629
|
throw new Error("Pool root is not running");
|
|
65131
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
66630
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
65132
66631
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
65133
66632
|
return receipt;
|
|
65134
66633
|
});
|
|
66634
|
+
checkpoints.commit();
|
|
66635
|
+
return receipt;
|
|
65135
66636
|
}
|
|
65136
66637
|
async appendSession(tx, row, mutation, now) {
|
|
65137
66638
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
65138
66639
|
}
|
|
65139
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
66640
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
65140
66641
|
const checkpoint = mutation.checkpoint;
|
|
65141
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
66642
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
65142
66643
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
65143
66644
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
65144
66645
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
65145
66646
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
65146
66647
|
const payload = serializeJson(checkpoint);
|
|
65147
66648
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
65148
|
-
if (existing && existing.payload
|
|
66649
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
65149
66650
|
throw new Error("Pool checkpoint identity conflict");
|
|
65150
66651
|
if (existing && mutation.kind !== "checkpoint")
|
|
65151
66652
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -65243,22 +66744,37 @@ class SqlPoolQueue {
|
|
|
65243
66744
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
65244
66745
|
}
|
|
65245
66746
|
}
|
|
66747
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
66748
|
+
if (stored === payload)
|
|
66749
|
+
return true;
|
|
66750
|
+
const { completedAt } = JSON.parse(stored);
|
|
66751
|
+
if (typeof completedAt !== "number")
|
|
66752
|
+
return false;
|
|
66753
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
66754
|
+
}
|
|
65246
66755
|
function openSqlPoolQueue(journal) {
|
|
65247
66756
|
const storage = sqlPoolStorage(journal);
|
|
65248
66757
|
if (!storage)
|
|
65249
66758
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
65250
66759
|
return SqlPoolQueue.open(storage);
|
|
65251
66760
|
}
|
|
65252
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
66761
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
65253
66762
|
var init_sql_pool_queue = __esm(() => {
|
|
65254
66763
|
init_serializable();
|
|
65255
66764
|
init_boundary_topology();
|
|
65256
66765
|
init_durable_hash();
|
|
65257
66766
|
init_internal();
|
|
66767
|
+
init_pool_checkpoint_index();
|
|
65258
66768
|
init_pool_mutations();
|
|
65259
66769
|
init_pool_stage_usage();
|
|
65260
66770
|
init_types();
|
|
65261
66771
|
init_values();
|
|
66772
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
66773
|
+
constructor(message, options) {
|
|
66774
|
+
super(message, options);
|
|
66775
|
+
this.name = "PoolAppendRefusedError";
|
|
66776
|
+
}
|
|
66777
|
+
};
|
|
65262
66778
|
});
|
|
65263
66779
|
|
|
65264
66780
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
@@ -65281,15 +66797,22 @@ function validate(report) {
|
|
|
65281
66797
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
65282
66798
|
}
|
|
65283
66799
|
}
|
|
66800
|
+
function addCompensated(total, value2) {
|
|
66801
|
+
const sum = total.sum + value2;
|
|
66802
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
66803
|
+
total.sum = sum;
|
|
66804
|
+
}
|
|
65284
66805
|
|
|
65285
66806
|
class SqlPoolUsage {
|
|
65286
66807
|
transaction;
|
|
65287
66808
|
journal;
|
|
65288
66809
|
table;
|
|
66810
|
+
totals;
|
|
65289
66811
|
constructor(transaction, journal) {
|
|
65290
66812
|
this.transaction = transaction;
|
|
65291
66813
|
this.journal = journal;
|
|
65292
66814
|
this.table = poolUsageTable(journal.runs);
|
|
66815
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
65293
66816
|
}
|
|
65294
66817
|
static async open(transaction, journal) {
|
|
65295
66818
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -65297,9 +66820,27 @@ class SqlPoolUsage {
|
|
|
65297
66820
|
return usage;
|
|
65298
66821
|
}
|
|
65299
66822
|
async totalIn(tx, root) {
|
|
66823
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66824
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
66825
|
+
if (kept)
|
|
66826
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
66827
|
+
field,
|
|
66828
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
66829
|
+
]));
|
|
65300
66830
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
65301
66831
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
65302
|
-
|
|
66832
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
66833
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
66834
|
+
return total;
|
|
66835
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
66836
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
66837
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
66838
|
+
for (const usage of rows)
|
|
66839
|
+
for (const [index, column] of columns.entries())
|
|
66840
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
66841
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
66842
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
66843
|
+
return total;
|
|
65303
66844
|
}
|
|
65304
66845
|
async report(lease, token, reports) {
|
|
65305
66846
|
validateText("runId", lease.runId);
|
|
@@ -68438,7 +69979,7 @@ function describeErrorWithCauses(error) {
|
|
|
68438
69979
|
const messages = [];
|
|
68439
69980
|
const visited = new Set;
|
|
68440
69981
|
let current = error;
|
|
68441
|
-
for (let depth = 0;current !== undefined && depth <
|
|
69982
|
+
for (let depth = 0;current !== undefined && depth < MAX_CAUSE_DEPTH3 && !visited.has(current); depth++) {
|
|
68442
69983
|
visited.add(current);
|
|
68443
69984
|
const message = current instanceof Error ? current.message : String(current);
|
|
68444
69985
|
if (!messages.includes(message))
|
|
@@ -68447,7 +69988,7 @@ function describeErrorWithCauses(error) {
|
|
|
68447
69988
|
}
|
|
68448
69989
|
return messages.join(": ");
|
|
68449
69990
|
}
|
|
68450
|
-
var
|
|
69991
|
+
var MAX_CAUSE_DEPTH3 = 5;
|
|
68451
69992
|
|
|
68452
69993
|
// dist/builtin/workflows/src/engine/run-durable-finalize.ts
|
|
68453
69994
|
async function finalizeDurableTerminalStatus(input) {
|
|
@@ -68652,29 +70193,23 @@ var init_run_durable_stage_session = __esm(() => {
|
|
|
68652
70193
|
|
|
68653
70194
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
68654
70195
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
68655
|
-
const
|
|
68656
|
-
if (!
|
|
70196
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
70197
|
+
if (!catalog.positionsById.has(childRunId))
|
|
68657
70198
|
return;
|
|
68658
|
-
const intentionalExitStatuses = new Map;
|
|
68659
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
68660
|
-
if (checkpoint.kind !== "stage")
|
|
68661
|
-
continue;
|
|
68662
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
68663
|
-
if (child?.exited === true)
|
|
68664
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
68665
|
-
}
|
|
68666
70199
|
const included = new Set([childRunId]);
|
|
68667
|
-
|
|
68668
|
-
|
|
68669
|
-
|
|
68670
|
-
|
|
68671
|
-
|
|
68672
|
-
|
|
70200
|
+
const pending = [childRunId];
|
|
70201
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
70202
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
70203
|
+
if (included.has(childId))
|
|
70204
|
+
continue;
|
|
70205
|
+
included.add(childId);
|
|
70206
|
+
pending.push(childId);
|
|
68673
70207
|
}
|
|
68674
70208
|
}
|
|
68675
|
-
const
|
|
68676
|
-
const
|
|
68677
|
-
|
|
70209
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
70210
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
70211
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
70212
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
68678
70213
|
}
|
|
68679
70214
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
68680
70215
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -68795,7 +70330,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
68795
70330
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
68796
70331
|
kind: "stage",
|
|
68797
70332
|
workflowId: input.run.id,
|
|
68798
|
-
checkpointId:
|
|
70333
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
68799
70334
|
name,
|
|
68800
70335
|
replayKey,
|
|
68801
70336
|
completedAt: Date.now(),
|
|
@@ -68967,7 +70502,7 @@ function recordedOwners(boundary, reason) {
|
|
|
68967
70502
|
const pending = [{ value: reason, depth: 0 }];
|
|
68968
70503
|
while (pending.length > 0) {
|
|
68969
70504
|
const { value: value2, depth } = pending.pop();
|
|
68970
|
-
if (depth >
|
|
70505
|
+
if (depth > MAX_CAUSE_DEPTH4)
|
|
68971
70506
|
continue;
|
|
68972
70507
|
if (isPrimitive(value2)) {
|
|
68973
70508
|
if (depth === 0 || value2 !== undefined && value2 !== null) {
|
|
@@ -69289,7 +70824,7 @@ function containUiRejections(ui, scope) {
|
|
|
69289
70824
|
custom: (factory, options) => scope.watch(ui.custom(factory, options))
|
|
69290
70825
|
};
|
|
69291
70826
|
}
|
|
69292
|
-
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections",
|
|
70827
|
+
var EVENT = "unhandledRejection", BOUNDARY_KEY, FLAG = "--unhandled-rejections", MAX_CAUSE_DEPTH4 = 8, MAX_PRIMITIVES_PER_RUN = 64, MAX_STOP_STRINGS = 64, ENDED_RUN_GRACE_MS = 30000, NATIVE_REPORTING_MODES, WATCHED_STAGE_METHODS;
|
|
69293
70828
|
var init_run_rejection_boundary = __esm(() => {
|
|
69294
70829
|
init_prompt_callsite();
|
|
69295
70830
|
init_workflow_source();
|
|
@@ -75135,27 +76670,6 @@ var init_runtime = __esm(() => {
|
|
|
75135
76670
|
init_intercom_group();
|
|
75136
76671
|
});
|
|
75137
76672
|
|
|
75138
|
-
// dist/builtin/workflows/src/engine/session-closed-error.ts
|
|
75139
|
-
import { isStaleExtensionContextError as isStaleExtensionContextError6 } from "@bastani/atomic";
|
|
75140
|
-
function reportsClosedSession(error) {
|
|
75141
|
-
return "code" in error && error.code === "SessionClosed" || isStaleExtensionContextError6(error);
|
|
75142
|
-
}
|
|
75143
|
-
function isSessionClosedError(error) {
|
|
75144
|
-
let current = error;
|
|
75145
|
-
for (let depth = 0;depth < MAX_CAUSE_DEPTH4 && current !== undefined; depth++) {
|
|
75146
|
-
try {
|
|
75147
|
-
if (reportsClosedSession(current))
|
|
75148
|
-
return true;
|
|
75149
|
-
current = current.cause instanceof Error ? current.cause : undefined;
|
|
75150
|
-
} catch {
|
|
75151
|
-
return false;
|
|
75152
|
-
}
|
|
75153
|
-
}
|
|
75154
|
-
return false;
|
|
75155
|
-
}
|
|
75156
|
-
var MAX_CAUSE_DEPTH4 = 8;
|
|
75157
|
-
var init_session_closed_error = () => {};
|
|
75158
|
-
|
|
75159
76673
|
// dist/builtin/workflows/src/engine/workflow-activity.ts
|
|
75160
76674
|
import { runCallback as runCallback4 } from "@bastani/atomic";
|
|
75161
76675
|
function nextEventLoopTurn() {
|
|
@@ -77237,11 +78751,68 @@ function runResultFields(backend, handle) {
|
|
|
77237
78751
|
const result = durableRunResult(backend, handle.workflowId);
|
|
77238
78752
|
return result === undefined ? {} : { result };
|
|
77239
78753
|
}
|
|
77240
|
-
function
|
|
78754
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
77241
78755
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
77242
78756
|
if (handle === undefined)
|
|
77243
|
-
return
|
|
77244
|
-
|
|
78757
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
78758
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
78759
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
78760
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
78761
|
+
return cached.catalog;
|
|
78762
|
+
}
|
|
78763
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
78764
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
78765
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
78766
|
+
if (version !== undefined) {
|
|
78767
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
78768
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
78769
|
+
byRoot.delete(rootWorkflowId);
|
|
78770
|
+
byRoot.set(rootWorkflowId, {
|
|
78771
|
+
version,
|
|
78772
|
+
name: handle.name,
|
|
78773
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
78774
|
+
catalog
|
|
78775
|
+
});
|
|
78776
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
78777
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
78778
|
+
}
|
|
78779
|
+
return catalog;
|
|
78780
|
+
}
|
|
78781
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
78782
|
+
const positionsById = new Map;
|
|
78783
|
+
const childIdsByParent = new Map;
|
|
78784
|
+
runs.forEach((run, position) => {
|
|
78785
|
+
const positions = positionsById.get(run.id);
|
|
78786
|
+
if (positions === undefined)
|
|
78787
|
+
positionsById.set(run.id, [position]);
|
|
78788
|
+
else
|
|
78789
|
+
positions.push(position);
|
|
78790
|
+
if (run.parentRunId === undefined)
|
|
78791
|
+
return;
|
|
78792
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
78793
|
+
if (children === undefined)
|
|
78794
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
78795
|
+
else
|
|
78796
|
+
children.push(run.id);
|
|
78797
|
+
});
|
|
78798
|
+
const intentionalExitStatuses = new Map;
|
|
78799
|
+
for (const checkpoint of checkpoints) {
|
|
78800
|
+
if (checkpoint.kind !== "stage")
|
|
78801
|
+
continue;
|
|
78802
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
78803
|
+
if (child?.exited === true)
|
|
78804
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
78805
|
+
}
|
|
78806
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
78807
|
+
}
|
|
78808
|
+
function readsFallbackCompletedAt(run) {
|
|
78809
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
78810
|
+
}
|
|
78811
|
+
function cloneRunSnapshot(run) {
|
|
78812
|
+
const copy = structuredClone(run);
|
|
78813
|
+
for (const tool of copy.toolNodes)
|
|
78814
|
+
Object.freeze(tool.parentIds);
|
|
78815
|
+
return copy;
|
|
77245
78816
|
}
|
|
77246
78817
|
async function validatedStageTranscript(stage, store) {
|
|
77247
78818
|
if (stage.sessionId !== undefined && await isReopenableSession(store, stage.sessionId))
|
|
@@ -77271,11 +78842,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
77271
78842
|
tools: [...toolByHash.values()]
|
|
77272
78843
|
};
|
|
77273
78844
|
}
|
|
77274
|
-
function completedToolNodes(
|
|
77275
|
-
return
|
|
77276
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
77277
|
-
if (topologyRunId !== runId)
|
|
77278
|
-
return [];
|
|
78845
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
78846
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
77279
78847
|
const topology = checkpoint.topology;
|
|
77280
78848
|
if (topology === undefined)
|
|
77281
78849
|
return [];
|
|
@@ -77312,18 +78880,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
77312
78880
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
77313
78881
|
}
|
|
77314
78882
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
77315
|
-
|
|
78883
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
78884
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
77316
78885
|
return [];
|
|
77317
|
-
const stageRecords =
|
|
78886
|
+
const stageRecords = index.stages;
|
|
77318
78887
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
77319
78888
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
77320
78889
|
return [];
|
|
77321
|
-
const
|
|
77322
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
78890
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
77323
78891
|
const drafts = checkpointDrafts(checkpoints);
|
|
77324
78892
|
const toolCheckpoints = drafts.tools;
|
|
77325
78893
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
77326
78894
|
return [];
|
|
78895
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
78896
|
+
const toolsByRun = new Map;
|
|
78897
|
+
for (const checkpoint of toolCheckpoints) {
|
|
78898
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
78899
|
+
if (claimedRunId !== undefined) {
|
|
78900
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
78901
|
+
ids.add(checkpoint.topology.nodeId);
|
|
78902
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
78903
|
+
}
|
|
78904
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
78905
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
78906
|
+
if (owned === undefined)
|
|
78907
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
78908
|
+
else
|
|
78909
|
+
owned.push(checkpoint);
|
|
78910
|
+
}
|
|
77327
78911
|
for (const draft of candidates) {
|
|
77328
78912
|
const boundary = draft.topology?.boundary;
|
|
77329
78913
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -77337,8 +78921,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77337
78921
|
parentStageId: boundary.child.parentStageId,
|
|
77338
78922
|
rootRunId: boundary.child.rootRunId
|
|
77339
78923
|
};
|
|
77340
|
-
const childToolIds =
|
|
77341
|
-
if (
|
|
78924
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
78925
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
77342
78926
|
return [];
|
|
77343
78927
|
}
|
|
77344
78928
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -77366,7 +78950,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77366
78950
|
group.sort(compareDraftSourceOrder);
|
|
77367
78951
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
77368
78952
|
return [];
|
|
77369
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
78953
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
77370
78954
|
return [];
|
|
77371
78955
|
const syntheticChildren = [];
|
|
77372
78956
|
const pendingChildren = [];
|
|
@@ -77380,7 +78964,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77380
78964
|
const child = workflowChildFromDraft(draft);
|
|
77381
78965
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
77382
78966
|
continue;
|
|
77383
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
78967
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
77384
78968
|
return [];
|
|
77385
78969
|
}
|
|
77386
78970
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -77388,7 +78972,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77388
78972
|
}
|
|
77389
78973
|
const idMaps = new Map;
|
|
77390
78974
|
for (const [runId, runDrafts] of grouped) {
|
|
77391
|
-
const ownedTools =
|
|
78975
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77392
78976
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
77393
78977
|
return [];
|
|
77394
78978
|
const ids = new Map;
|
|
@@ -77406,32 +78990,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77406
78990
|
});
|
|
77407
78991
|
if (hasIdentityConflict)
|
|
77408
78992
|
return [];
|
|
77409
|
-
for (const checkpoint of
|
|
77410
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
78993
|
+
for (const checkpoint of ownedTools) {
|
|
77411
78994
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
77412
|
-
|
|
77413
|
-
ids.set(sourceId, sourceId);
|
|
78995
|
+
ids.set(sourceId, sourceId);
|
|
77414
78996
|
}
|
|
77415
78997
|
idMaps.set(runId, ids);
|
|
77416
78998
|
}
|
|
78999
|
+
const boundaryOwners = new Map;
|
|
79000
|
+
const boundarySourcesByParent = new Map;
|
|
79001
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
79002
|
+
const sources = new Map;
|
|
79003
|
+
for (const draft of runDrafts) {
|
|
79004
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
79005
|
+
if (childRunId === undefined)
|
|
79006
|
+
continue;
|
|
79007
|
+
if (!boundaryOwners.has(childRunId))
|
|
79008
|
+
boundaryOwners.set(childRunId, draft);
|
|
79009
|
+
if (!sources.has(childRunId))
|
|
79010
|
+
sources.set(childRunId, draft);
|
|
79011
|
+
}
|
|
79012
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
79013
|
+
}
|
|
77417
79014
|
const runs = [];
|
|
77418
79015
|
const emittedRunIds = new Set;
|
|
77419
79016
|
for (const [runId, runDrafts] of grouped) {
|
|
77420
79017
|
const ids = idMaps.get(runId);
|
|
77421
|
-
const ownedTools =
|
|
79018
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77422
79019
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
77423
79020
|
if (hasUnknownParents)
|
|
77424
79021
|
return [];
|
|
77425
79022
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
77426
|
-
const toolNodes = completedToolNodes(
|
|
79023
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
77427
79024
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
77428
79025
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
77429
79026
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
77430
|
-
const owner = runId === rootRunId ? undefined :
|
|
79027
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
77431
79028
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
77432
79029
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
77433
79030
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
77434
|
-
const boundarySourceId =
|
|
79031
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
77435
79032
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
77436
79033
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
77437
79034
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -77567,10 +79164,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
77567
79164
|
resumable: false
|
|
77568
79165
|
};
|
|
77569
79166
|
}
|
|
77570
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
79167
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
77571
79168
|
var init_completed_catalog = __esm(() => {
|
|
77572
79169
|
init_stage_session_store();
|
|
77573
79170
|
init_tool_payload_bounds();
|
|
79171
|
+
init_completed_catalog_index();
|
|
77574
79172
|
init_completed_catalog_stage_groups();
|
|
77575
79173
|
init_resume_eligibility();
|
|
77576
79174
|
init_resume_runtime();
|
|
@@ -77579,6 +79177,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
77579
79177
|
init_workflow_heartbeat_anchor();
|
|
77580
79178
|
init_stage_topology_validation();
|
|
77581
79179
|
init_tool_outcome();
|
|
79180
|
+
nestedRunCatalogs = new WeakMap;
|
|
79181
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
79182
|
+
runs: [],
|
|
79183
|
+
positionsById: new Map,
|
|
79184
|
+
childIdsByParent: new Map,
|
|
79185
|
+
intentionalExitStatuses: new Map
|
|
79186
|
+
};
|
|
77582
79187
|
COMPLETED_RECONSTRUCTION = {
|
|
77583
79188
|
requireTerminal: true,
|
|
77584
79189
|
retainActive: false,
|
|
@@ -89809,7 +91414,119 @@ function observeRunExecution(run, now = Date.now()) {
|
|
|
89809
91414
|
}
|
|
89810
91415
|
|
|
89811
91416
|
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89812
|
-
|
|
91417
|
+
init_sticky_delivery_state_bound();
|
|
91418
|
+
|
|
91419
|
+
// dist/builtin/workflows/src/shared/sticky-stage-delivery-status.ts
|
|
91420
|
+
init_run_owner_backend();
|
|
91421
|
+
init_pending_stage_status();
|
|
91422
|
+
init_workflow_stage_target();
|
|
91423
|
+
function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
91424
|
+
const root = runs.find((run) => run.id === rootRunId);
|
|
91425
|
+
const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
|
|
91426
|
+
const result = [];
|
|
91427
|
+
const patternByTarget = new Map;
|
|
91428
|
+
const patterns = [];
|
|
91429
|
+
const entriesByPattern = [];
|
|
91430
|
+
const ownEntries = [];
|
|
91431
|
+
entries.forEach((entry, index) => {
|
|
91432
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91433
|
+
let pattern = patternByTarget.get(target);
|
|
91434
|
+
if (pattern === undefined) {
|
|
91435
|
+
const parsed = parseWorkflowStageTarget(target);
|
|
91436
|
+
pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
|
|
91437
|
+
if (pattern >= 0)
|
|
91438
|
+
entriesByPattern.push([]);
|
|
91439
|
+
patternByTarget.set(target, pattern);
|
|
91440
|
+
}
|
|
91441
|
+
if (pattern < 0)
|
|
91442
|
+
return;
|
|
91443
|
+
entriesByPattern[pattern].push(index);
|
|
91444
|
+
ownEntries.push(index);
|
|
91445
|
+
});
|
|
91446
|
+
if (ownEntries.length === 0)
|
|
91447
|
+
return result;
|
|
91448
|
+
const patternIndex = createStagePathPatternIndex(patterns);
|
|
91449
|
+
const deliveriesByEntry = new Map;
|
|
91450
|
+
const entryReceipts = (index, runId, stageId) => {
|
|
91451
|
+
let byStage = deliveriesByEntry.get(index);
|
|
91452
|
+
if (byStage === undefined) {
|
|
91453
|
+
byStage = new Map;
|
|
91454
|
+
for (const delivery of entries[index].deliveries ?? []) {
|
|
91455
|
+
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
91456
|
+
const list = byStage.get(key);
|
|
91457
|
+
if (list === undefined)
|
|
91458
|
+
byStage.set(key, [delivery]);
|
|
91459
|
+
else
|
|
91460
|
+
list.push(delivery);
|
|
91461
|
+
}
|
|
91462
|
+
deliveriesByEntry.set(index, byStage);
|
|
91463
|
+
}
|
|
91464
|
+
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
91465
|
+
};
|
|
91466
|
+
availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
|
|
91467
|
+
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
|
|
91468
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
91469
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
91470
|
+
for (const run of runs) {
|
|
91471
|
+
const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
91472
|
+
if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
|
|
91473
|
+
continue;
|
|
91474
|
+
const resolved = durableRootRunId === rootRunId;
|
|
91475
|
+
const hops = hopsFor(run.id);
|
|
91476
|
+
for (const stage of run.stages) {
|
|
91477
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
91478
|
+
continue;
|
|
91479
|
+
const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
|
|
91480
|
+
for (const index of matchedEntries) {
|
|
91481
|
+
const entry = entries[index];
|
|
91482
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91483
|
+
const receipts = entryReceipts(index, run.id, stage.id);
|
|
91484
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
91485
|
+
runId: run.id,
|
|
91486
|
+
stageId: stage.id,
|
|
91487
|
+
sessionId: stage.sessionId,
|
|
91488
|
+
admission: "context"
|
|
91489
|
+
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
91490
|
+
let state;
|
|
91491
|
+
let reason;
|
|
91492
|
+
if (!resolved || !hops) {
|
|
91493
|
+
state = "root-unresolved";
|
|
91494
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
91495
|
+
} else if (delivered) {
|
|
91496
|
+
state = "delivered";
|
|
91497
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
91498
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
91499
|
+
state = "stage-terminal";
|
|
91500
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
91501
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
91502
|
+
state = "receipt-unverified";
|
|
91503
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
91504
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
91505
|
+
state = "delivery-unavailable";
|
|
91506
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
91507
|
+
} else if (receipts.length > 0) {
|
|
91508
|
+
state = "session-replaced";
|
|
91509
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
91510
|
+
} else {
|
|
91511
|
+
state = "queued";
|
|
91512
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
91513
|
+
}
|
|
91514
|
+
result.push({
|
|
91515
|
+
messageId: entry.message.id,
|
|
91516
|
+
target,
|
|
91517
|
+
runId: run.id,
|
|
91518
|
+
stageId: stage.id,
|
|
91519
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
91520
|
+
state,
|
|
91521
|
+
reason
|
|
91522
|
+
});
|
|
91523
|
+
}
|
|
91524
|
+
}
|
|
91525
|
+
}
|
|
91526
|
+
return result;
|
|
91527
|
+
}
|
|
91528
|
+
|
|
91529
|
+
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89813
91530
|
init_store();
|
|
89814
91531
|
|
|
89815
91532
|
// dist/builtin/workflows/src/shared/workflow-identity.ts
|
|
@@ -89852,7 +91569,7 @@ function inspectRun(runId, opts) {
|
|
|
89852
91569
|
inputs: copy.inputs,
|
|
89853
91570
|
stages: expandedStages.map((stage) => structuredClone(stage)),
|
|
89854
91571
|
tools: expandedGraph.tools.map((tool) => structuredClone(tool)),
|
|
89855
|
-
|
|
91572
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(activeStore.runs(), copy.rootRunId ?? copy.id, availableStoreDurableBackend(activeStore, copy.id)).filter((delivery) => copy.id === (copy.rootRunId ?? copy.id) || delivery.runId === copy.id)),
|
|
89856
91573
|
result: copy.result,
|
|
89857
91574
|
error: copy.error ?? (strandedRoot ? IMPOSSIBLE_ROOT_LIVENESS_MESSAGE : effectiveRunStatus(copy) === copy.status ? undefined : structuredRecoverableWorkflowFailureText(copy) ?? actionableReturnedStatusText(copy.result)),
|
|
89858
91575
|
exited: copy.exited,
|
|
@@ -97577,7 +99294,7 @@ function buildWorkflowRunStatusDigest(detail) {
|
|
|
97577
99294
|
// dist/builtin/workflows/src/extension/workflow-status-summary.ts
|
|
97578
99295
|
init_pending_stage_status();
|
|
97579
99296
|
init_returned_run_status();
|
|
97580
|
-
|
|
99297
|
+
init_sticky_delivery_state_bound();
|
|
97581
99298
|
var workflowStatusRenderRuns = new WeakMap;
|
|
97582
99299
|
function setWorkflowStatusRenderRuns(result, runs) {
|
|
97583
99300
|
workflowStatusRenderRuns.set(result, runs);
|
|
@@ -97684,7 +99401,7 @@ function summarizeRunSnapshot(run, now = Date.now(), options) {
|
|
|
97684
99401
|
...stage.rateLimits !== undefined ? { rateLimits: stage.rateLimits } : {}
|
|
97685
99402
|
})),
|
|
97686
99403
|
pendingStages: pendingWorkflowStageStatuses(run, options?.owningRunStatus, options?.resolveBoundarySegments),
|
|
97687
|
-
|
|
99404
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(options?.allRuns ?? [run], run.id, options?.durableBackend)),
|
|
97688
99405
|
tools: run.toolNodes.map((tool) => {
|
|
97689
99406
|
const owner = tool;
|
|
97690
99407
|
return {
|
|
@@ -97886,7 +99603,7 @@ function formatStageStartup(startup, now = Date.now()) {
|
|
|
97886
99603
|
}
|
|
97887
99604
|
|
|
97888
99605
|
// dist/builtin/workflows/src/extension/workflow-tool-content.ts
|
|
97889
|
-
|
|
99606
|
+
init_sticky_delivery_state_bound();
|
|
97890
99607
|
function stringifyWorkflowToolResult(result) {
|
|
97891
99608
|
return JSON.stringify(result, null, 2);
|
|
97892
99609
|
}
|
|
@@ -97985,9 +99702,13 @@ function renderStatusToolContent(result) {
|
|
|
97985
99702
|
}
|
|
97986
99703
|
for (const entry of run.awaitingInput)
|
|
97987
99704
|
lines.push(statusAwaitingInputLine(entry));
|
|
97988
|
-
|
|
99705
|
+
const deliveries = run.deliveryStates === undefined ? boundStickyDeliveryStates(stickyStageDeliveryStatuses(allRuns, run.runId, undefined)) : run;
|
|
99706
|
+
for (const delivery of deliveries.deliveryStates ?? []) {
|
|
97989
99707
|
lines.push(` Intercom delivery ${delivery.messageId} target=${delivery.target} stage=${delivery.runId}/${delivery.stageId}${delivery.sessionId === undefined ? "" : ` session=${delivery.sessionId}`} state=${delivery.state} — ${delivery.reason}`);
|
|
97990
99708
|
}
|
|
99709
|
+
const omittedDeliveries = omittedDeliveryStatesLine(deliveries);
|
|
99710
|
+
if (omittedDeliveries !== undefined)
|
|
99711
|
+
lines.push(` ${omittedDeliveries}`);
|
|
97991
99712
|
for (const stage of run.pendingStages.slice(0, STATUS_PENDING_STAGE_LIMIT)) {
|
|
97992
99713
|
lines.push(statusPendingStageLine(stage));
|
|
97993
99714
|
}
|