@bastani/atomic 0.9.44-alpha.5 → 0.9.44-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/builtin/intercom/CHANGELOG.md +6 -0
- package/dist/builtin/intercom/broker/broker.ts +13 -1
- package/dist/builtin/intercom/index.bundle.mjs +41 -5
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/intercom/types.ts +7 -0
- package/dist/builtin/intercom/workflow-delivery-states.ts +48 -0
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +20 -0
- package/dist/builtin/workflows/builtin/{chunk-zjecnjxx.js → chunk-6901nsd6.js} +1 -1
- package/dist/builtin/workflows/builtin/{chunk-k8d7ehjx.js → chunk-jvna28mq.js} +381 -130
- package/dist/builtin/workflows/builtin/{chunk-qxpymnaf.js → chunk-vwx023ge.js} +1 -1
- package/dist/builtin/workflows/builtin/goal.js +2 -2
- package/dist/builtin/workflows/builtin/index.js +3 -3
- package/dist/builtin/workflows/builtin/ralph.js +2 -2
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/durable/journal/internal.d.ts +15 -1
- package/dist/builtin/workflows/src/durable/journal/sql-run-journal.d.ts +11 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +2514 -741
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +2384 -727
- package/dist/builtin/workflows/src/index.js +2383 -727
- package/dist/builtin/workflows/src/local-worker-entry.mjs +2384 -727
- package/dist/core/agent-session-custom-message-commit.d.ts.map +1 -1
- package/dist/core/agent-session-custom-message-commit.js +16 -0
- package/dist/core/agent-session-custom-message-commit.js.map +1 -1
- package/dist/core/storage/node-sqlite.d.ts.map +1 -1
- package/dist/core/storage/node-sqlite.js +130 -13
- package/dist/core/storage/node-sqlite.js.map +1 -1
- package/docs/intercom/reference.md +2 -0
- package/docs/workflows/operations.md +1 -1
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
|
@@ -274,6 +274,12 @@ class ScopedDurableBackend {
|
|
|
274
274
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
275
275
|
return this.inner.getPendingStageMessageDeliveryCount?.(this.scope.rootWorkflowId, messageId, logicalRunId) ?? 0;
|
|
276
276
|
}
|
|
277
|
+
pendingStageDeliveryReceiptChanges(_workflowId, cursor) {
|
|
278
|
+
return this.inner.pendingStageDeliveryReceiptChanges?.(this.scope.rootWorkflowId, cursor);
|
|
279
|
+
}
|
|
280
|
+
pendingStageDeliveryCountVersion(_workflowId) {
|
|
281
|
+
return this.inner.pendingStageDeliveryCountVersion?.(this.scope.rootWorkflowId);
|
|
282
|
+
}
|
|
277
283
|
async readPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
278
284
|
return this.inner.readPendingStageMessageDeliveryCount === undefined ? this.getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId) : await this.inner.readPendingStageMessageDeliveryCount(this.scope.rootWorkflowId, messageId, logicalRunId);
|
|
279
285
|
}
|
|
@@ -2066,6 +2072,9 @@ var init_workflow_stage_target = __esm(() => {
|
|
|
2066
2072
|
});
|
|
2067
2073
|
|
|
2068
2074
|
// dist/builtin/workflows/src/shared/pending-stage-status.ts
|
|
2075
|
+
function isAddressableStageStatus(status) {
|
|
2076
|
+
return ADDRESSABLE_STAGE_STATUSES.has(status);
|
|
2077
|
+
}
|
|
2069
2078
|
function workflowBoundarySegments(runs, runId) {
|
|
2070
2079
|
return createWorkflowBoundarySegmentsResolver(runs)(runId);
|
|
2071
2080
|
}
|
|
@@ -2214,9 +2223,17 @@ function pendingWorkflowStageStatuses(run, resolveOwningRunStatus, resolveBounda
|
|
|
2214
2223
|
return pending === undefined ? [] : [pending];
|
|
2215
2224
|
});
|
|
2216
2225
|
}
|
|
2226
|
+
var ADDRESSABLE_STAGE_STATUSES;
|
|
2217
2227
|
var init_pending_stage_status = __esm(() => {
|
|
2218
2228
|
init_store_internal();
|
|
2219
2229
|
init_workflow_stage_target();
|
|
2230
|
+
ADDRESSABLE_STAGE_STATUSES = new Set([
|
|
2231
|
+
"pending",
|
|
2232
|
+
"running",
|
|
2233
|
+
"awaiting_input",
|
|
2234
|
+
"paused",
|
|
2235
|
+
"blocked"
|
|
2236
|
+
]);
|
|
2220
2237
|
});
|
|
2221
2238
|
|
|
2222
2239
|
// dist/builtin/workflows/src/shared/timing.ts
|
|
@@ -3136,8 +3153,11 @@ var init_pending_stage_delivery = __esm(() => {
|
|
|
3136
3153
|
});
|
|
3137
3154
|
|
|
3138
3155
|
// dist/builtin/workflows/src/shared/pending-stage-retention.ts
|
|
3156
|
+
function isSettledPendingStageMessage(entry) {
|
|
3157
|
+
return entry.status === "delivered" || entry.status === "undeliverable" && entry.undeliverableNotifiedAt !== undefined;
|
|
3158
|
+
}
|
|
3139
3159
|
async function retainPendingStageMessages(messages, archive) {
|
|
3140
|
-
const settled = messages.filter(
|
|
3160
|
+
const settled = messages.filter(isSettledPendingStageMessage);
|
|
3141
3161
|
const expired = new Set(settled.slice(0, Math.max(0, settled.length - PENDING_STAGE_SETTLED_MESSAGE_LIMIT)));
|
|
3142
3162
|
for (const entry of expired)
|
|
3143
3163
|
await archive(entry);
|
|
@@ -3145,53 +3165,210 @@ async function retainPendingStageMessages(messages, archive) {
|
|
|
3145
3165
|
}
|
|
3146
3166
|
var PENDING_STAGE_SETTLED_MESSAGE_LIMIT = 50;
|
|
3147
3167
|
|
|
3168
|
+
// dist/builtin/workflows/src/shared/pending-stage-transition-ledger.ts
|
|
3169
|
+
class PendingStageTransitionLedger {
|
|
3170
|
+
compacted = [];
|
|
3171
|
+
settled = 0;
|
|
3172
|
+
stages = new Map;
|
|
3173
|
+
fixed = new WeakSet;
|
|
3174
|
+
reconciled = [];
|
|
3175
|
+
reconciledVersion;
|
|
3176
|
+
reconcile(backend, runId, messages) {
|
|
3177
|
+
if (backend.getPendingStageMessageDeliveryCount === undefined)
|
|
3178
|
+
return messages;
|
|
3179
|
+
const version = backend.pendingStageDeliveryCountVersion?.(runId);
|
|
3180
|
+
const base = version !== undefined && version === this.reconciledVersion ? this.reconciled : undefined;
|
|
3181
|
+
let recovered;
|
|
3182
|
+
for (let index = 0;index < messages.length; index++) {
|
|
3183
|
+
const entry = messages[index];
|
|
3184
|
+
if (entry.sticky !== true || base !== undefined && base[index] === entry)
|
|
3185
|
+
continue;
|
|
3186
|
+
const count = backend.getPendingStageMessageDeliveryCount(runId, entry.id, entry.runId);
|
|
3187
|
+
if (count <= (entry.deliveryCount ?? entry.deliveries?.length ?? 0))
|
|
3188
|
+
continue;
|
|
3189
|
+
recovered ??= [...messages];
|
|
3190
|
+
recovered[index] = { ...entry, deliveryCount: count };
|
|
3191
|
+
}
|
|
3192
|
+
const result = recovered ?? messages;
|
|
3193
|
+
this.reconciled = result;
|
|
3194
|
+
this.reconciledVersion = version;
|
|
3195
|
+
return result;
|
|
3196
|
+
}
|
|
3197
|
+
async compact(backend, runId, messages, runs, archiveSettled) {
|
|
3198
|
+
const stageStatus = createStageStatusLookup(runs);
|
|
3199
|
+
const endedStages = [];
|
|
3200
|
+
const dirty = new Set;
|
|
3201
|
+
for (const [deliveryRunId, byStage] of this.stages) {
|
|
3202
|
+
for (const [stageId, entries] of byStage) {
|
|
3203
|
+
if (!isEndedStageStatus(stageStatus(deliveryRunId, stageId)))
|
|
3204
|
+
continue;
|
|
3205
|
+
endedStages.push([deliveryRunId, stageId]);
|
|
3206
|
+
for (const entry of entries)
|
|
3207
|
+
dirty.add(entry);
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
const base = this.compacted;
|
|
3211
|
+
const output = [];
|
|
3212
|
+
const removed = [];
|
|
3213
|
+
const added = [];
|
|
3214
|
+
let settled = this.settled;
|
|
3215
|
+
const length = Math.max(base.length, messages.length);
|
|
3216
|
+
for (let index = 0;index < length; index++) {
|
|
3217
|
+
const previous = base[index];
|
|
3218
|
+
const entry = messages[index];
|
|
3219
|
+
if (previous === entry && !dirty.has(entry)) {
|
|
3220
|
+
output.push(entry);
|
|
3221
|
+
continue;
|
|
3222
|
+
}
|
|
3223
|
+
if (previous !== undefined) {
|
|
3224
|
+
removed.push(previous);
|
|
3225
|
+
if (isSettledPendingStageMessage(previous))
|
|
3226
|
+
settled--;
|
|
3227
|
+
}
|
|
3228
|
+
if (entry === undefined)
|
|
3229
|
+
continue;
|
|
3230
|
+
if (isSettledPendingStageMessage(entry))
|
|
3231
|
+
settled++;
|
|
3232
|
+
const next = this.fixed.has(entry) && !dirty.has(entry) ? entry : await compactEntryDeliveries(backend, runId, entry, stageStatus);
|
|
3233
|
+
output.push(next);
|
|
3234
|
+
added.push(next);
|
|
3235
|
+
}
|
|
3236
|
+
let retained = output;
|
|
3237
|
+
if (archiveSettled !== undefined && settled > PENDING_STAGE_SETTLED_MESSAGE_LIMIT) {
|
|
3238
|
+
retained = await retainPendingStageMessages(output, archiveSettled);
|
|
3239
|
+
settled = PENDING_STAGE_SETTLED_MESSAGE_LIMIT;
|
|
3240
|
+
}
|
|
3241
|
+
return {
|
|
3242
|
+
messages: retained,
|
|
3243
|
+
commit: () => {
|
|
3244
|
+
for (const [deliveryRunId, stageId] of endedStages) {
|
|
3245
|
+
const byStage = this.stages.get(deliveryRunId);
|
|
3246
|
+
byStage?.delete(stageId);
|
|
3247
|
+
if (byStage?.size === 0)
|
|
3248
|
+
this.stages.delete(deliveryRunId);
|
|
3249
|
+
}
|
|
3250
|
+
for (const entry of dirty)
|
|
3251
|
+
this.fixed.delete(entry);
|
|
3252
|
+
for (const entry of removed)
|
|
3253
|
+
this.unindex(entry);
|
|
3254
|
+
for (const entry of added)
|
|
3255
|
+
this.index(entry);
|
|
3256
|
+
this.compacted = retained;
|
|
3257
|
+
this.settled = settled;
|
|
3258
|
+
}
|
|
3259
|
+
};
|
|
3260
|
+
}
|
|
3261
|
+
index(entry) {
|
|
3262
|
+
this.fixed.add(entry);
|
|
3263
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3264
|
+
let byStage = this.stages.get(delivery.runId);
|
|
3265
|
+
if (byStage === undefined) {
|
|
3266
|
+
byStage = new Map;
|
|
3267
|
+
this.stages.set(delivery.runId, byStage);
|
|
3268
|
+
}
|
|
3269
|
+
let entries = byStage.get(delivery.stageId);
|
|
3270
|
+
if (entries === undefined) {
|
|
3271
|
+
entries = new Set;
|
|
3272
|
+
byStage.set(delivery.stageId, entries);
|
|
3273
|
+
}
|
|
3274
|
+
entries.add(entry);
|
|
3275
|
+
}
|
|
3276
|
+
}
|
|
3277
|
+
unindex(entry) {
|
|
3278
|
+
this.fixed.delete(entry);
|
|
3279
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
3280
|
+
const byStage = this.stages.get(delivery.runId);
|
|
3281
|
+
const entries = byStage?.get(delivery.stageId);
|
|
3282
|
+
if (entries === undefined)
|
|
3283
|
+
continue;
|
|
3284
|
+
entries.delete(entry);
|
|
3285
|
+
if (entries.size > 0)
|
|
3286
|
+
continue;
|
|
3287
|
+
byStage.delete(delivery.stageId);
|
|
3288
|
+
if (byStage.size === 0)
|
|
3289
|
+
this.stages.delete(delivery.runId);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
}
|
|
3293
|
+
function isCompactingBackend(backend) {
|
|
3294
|
+
return backend.archivePendingStageDeliveryReceipt !== undefined && backend.hasPendingStageDeliveryReceipt !== undefined;
|
|
3295
|
+
}
|
|
3296
|
+
function createStageStatusLookup(runs) {
|
|
3297
|
+
let runById;
|
|
3298
|
+
return (runId, stageId) => {
|
|
3299
|
+
if (runById === undefined) {
|
|
3300
|
+
runById = new Map;
|
|
3301
|
+
for (const run of runs)
|
|
3302
|
+
if (!runById.has(run.id))
|
|
3303
|
+
runById.set(run.id, run);
|
|
3304
|
+
}
|
|
3305
|
+
return runById.get(runId)?.stages.find((stage) => stage.id === stageId)?.status;
|
|
3306
|
+
};
|
|
3307
|
+
}
|
|
3308
|
+
function isEndedStageStatus(status) {
|
|
3309
|
+
return status === "completed" || status === "skipped" || status === "failed";
|
|
3310
|
+
}
|
|
3311
|
+
async function compactEntryDeliveries(backend, runId, entry, stageStatus) {
|
|
3312
|
+
if (entry.sticky !== true || entry.deliveries === undefined)
|
|
3313
|
+
return entry;
|
|
3314
|
+
const latest = new Map;
|
|
3315
|
+
entry.deliveries.forEach((delivery, index) => {
|
|
3316
|
+
latest.set(deliverySlot(delivery), index);
|
|
3317
|
+
});
|
|
3318
|
+
const deliveries = [];
|
|
3319
|
+
let archivedThrough = -1;
|
|
3320
|
+
for (let index = 0;index < entry.deliveries.length; index++) {
|
|
3321
|
+
const delivery = entry.deliveries[index];
|
|
3322
|
+
if (entry.status !== "queued" || isEndedStageStatus(stageStatus(delivery.runId, delivery.stageId)) || latest.get(deliverySlot(delivery)) !== index && delivery.sessionId !== undefined) {
|
|
3323
|
+
while (archivedThrough < index) {
|
|
3324
|
+
const prefixIndex = archivedThrough + 1;
|
|
3325
|
+
const receipt = entry.deliveries[prefixIndex];
|
|
3326
|
+
if (!await backend.hasPendingStageDeliveryReceipt(runId, entry.id, receipt, entry.runId))
|
|
3327
|
+
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, receipt, {
|
|
3328
|
+
messageRunId: entry.runId,
|
|
3329
|
+
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3330
|
+
});
|
|
3331
|
+
archivedThrough = prefixIndex;
|
|
3332
|
+
}
|
|
3333
|
+
} else {
|
|
3334
|
+
deliveries.push(delivery);
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
return deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries };
|
|
3338
|
+
}
|
|
3339
|
+
function deliverySlot(delivery) {
|
|
3340
|
+
return `${delivery.runId.length}:${delivery.runId}${delivery.stageId.length}:${delivery.stageId}${delivery.admission === undefined ? "-" : `+${delivery.admission}`}`;
|
|
3341
|
+
}
|
|
3342
|
+
var init_pending_stage_transition_ledger = () => {};
|
|
3343
|
+
|
|
3148
3344
|
// dist/builtin/workflows/src/shared/store-pending-stage-delivery-methods.ts
|
|
3149
3345
|
function createPendingStageDeliveryStoreMethods(context) {
|
|
3150
|
-
const
|
|
3151
|
-
const
|
|
3152
|
-
const previous = transitions.get(runId) ?? Promise.resolve();
|
|
3153
|
-
const result = previous.catch(() => {
|
|
3154
|
-
return;
|
|
3155
|
-
}).then(transition);
|
|
3156
|
-
const settled = result.then(() => {
|
|
3157
|
-
return;
|
|
3158
|
-
}, () => {
|
|
3159
|
-
return;
|
|
3160
|
-
});
|
|
3161
|
-
transitions.set(runId, settled);
|
|
3162
|
-
settled.finally(() => {
|
|
3163
|
-
if (transitions.get(runId) === settled)
|
|
3164
|
-
transitions.delete(runId);
|
|
3165
|
-
});
|
|
3166
|
-
return await result;
|
|
3167
|
-
};
|
|
3346
|
+
const ledgers = new PendingStageLedgers;
|
|
3347
|
+
const batches = new PendingStageTransitionBatches(context, ledgers);
|
|
3168
3348
|
return {
|
|
3169
3349
|
async queueStageMessage(input, senderGroup, runGroup, backend) {
|
|
3170
|
-
return await
|
|
3350
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3171
3351
|
const run = context.findRun(input.runId);
|
|
3172
3352
|
if (run === undefined)
|
|
3173
3353
|
return;
|
|
3174
3354
|
const stageIdentity = resolvePendingStageIdentity(run, input.stageKey);
|
|
3175
3355
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3176
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3356
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup, stageIdentity);
|
|
3177
3357
|
if (result.ok && !result.deduplicated) {
|
|
3178
|
-
|
|
3179
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3180
|
-
live.pendingStageMessages = [...messages];
|
|
3181
|
-
context.bumpAndNotify();
|
|
3358
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3182
3359
|
}
|
|
3183
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3360
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3184
3361
|
});
|
|
3185
3362
|
},
|
|
3186
3363
|
async validateLiveStageMessage(input) {
|
|
3187
|
-
return await
|
|
3364
|
+
return await batches.run(input.runId, undefined, async (batch) => {
|
|
3188
3365
|
const run = context.findRun(input.runId);
|
|
3189
3366
|
if (run === undefined)
|
|
3190
3367
|
return;
|
|
3191
3368
|
const availableBackend = availableRunDurableBackend(context.state.runs, input.runId) ?? context.backend;
|
|
3192
3369
|
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, context.state.runs, input.runId);
|
|
3193
3370
|
const receipt = await backend?.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3194
|
-
const result = queueStageMessage(receipt === undefined ? run
|
|
3371
|
+
const result = queueStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, undefined, undefined, resolvePendingStageIdentity(run, input.stageKey));
|
|
3195
3372
|
if (!result.ok) {
|
|
3196
3373
|
return result.reason === "message_id_conflict" ? { outcome: "message_id_conflict", messageId: result.messageId } : { outcome: "forward" };
|
|
3197
3374
|
}
|
|
@@ -3213,77 +3390,65 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3213
3390
|
return pendingStageMessagesFor(run?.pendingStageMessages ?? [], runId, stageKey, run === undefined ? undefined : resolvePendingStageIdentity(run, stageKey));
|
|
3214
3391
|
},
|
|
3215
3392
|
async markPendingStageMessageDelivered(runId, stageKey, messageId, deliveredAt, backend) {
|
|
3216
|
-
return await
|
|
3393
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3217
3394
|
const run = context.findRun(runId);
|
|
3218
3395
|
if (run === undefined)
|
|
3219
3396
|
return false;
|
|
3220
|
-
const current = run
|
|
3397
|
+
const current = batch.messages(run);
|
|
3221
3398
|
const next = markPendingStageMessageDelivered(current, runId, stageKey, messageId, deliveredAt, resolvePendingStageIdentity(run, stageKey));
|
|
3222
3399
|
if (next === current)
|
|
3223
3400
|
return false;
|
|
3224
|
-
|
|
3225
|
-
const live = context.findRun(runId) ?? run;
|
|
3226
|
-
live.pendingStageMessages = [...messages];
|
|
3227
|
-
context.bumpAndNotify();
|
|
3401
|
+
await batch.apply(backend, runId, next);
|
|
3228
3402
|
return true;
|
|
3229
3403
|
});
|
|
3230
3404
|
},
|
|
3231
3405
|
async markPendingStageMessageUndeliverable(runId, stageKey, messageId, reason, backend) {
|
|
3232
|
-
return await
|
|
3406
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3233
3407
|
const run = context.findRun(runId);
|
|
3234
3408
|
if (run === undefined)
|
|
3235
3409
|
return false;
|
|
3236
|
-
const current = run
|
|
3410
|
+
const current = batch.messages(run);
|
|
3237
3411
|
const next = markPendingStageMessageUndeliverable(current, runId, stageKey, messageId, reason, resolvePendingStageIdentity(run, stageKey));
|
|
3238
3412
|
if (next === current)
|
|
3239
3413
|
return false;
|
|
3240
|
-
|
|
3241
|
-
const live = context.findRun(runId) ?? run;
|
|
3242
|
-
live.pendingStageMessages = [...messages];
|
|
3243
|
-
context.bumpAndNotify();
|
|
3414
|
+
await batch.apply(backend, runId, next);
|
|
3244
3415
|
return true;
|
|
3245
3416
|
});
|
|
3246
3417
|
},
|
|
3247
3418
|
async markPendingStageMessageUndeliverableNotified(runId, stageKey, messageId, notificationId, notifiedAt, backend) {
|
|
3248
|
-
return await
|
|
3419
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3249
3420
|
const run = context.findRun(runId);
|
|
3250
3421
|
if (run === undefined)
|
|
3251
3422
|
return false;
|
|
3252
|
-
const current = run
|
|
3423
|
+
const current = batch.messages(run);
|
|
3253
3424
|
const next = markPendingStageMessageUndeliverableNotified(current, runId, stageKey, messageId, notificationId, notifiedAt);
|
|
3254
3425
|
if (next === current)
|
|
3255
3426
|
return false;
|
|
3256
|
-
|
|
3257
|
-
const live = context.findRun(runId) ?? run;
|
|
3258
|
-
live.pendingStageMessages = [...messages];
|
|
3259
|
-
context.bumpAndNotify();
|
|
3427
|
+
await batch.apply(backend, runId, next);
|
|
3260
3428
|
return true;
|
|
3261
3429
|
});
|
|
3262
3430
|
},
|
|
3263
3431
|
async queueStickyStageMessage(input, senderGroup, runGroup, backend) {
|
|
3264
|
-
return await
|
|
3432
|
+
return await batches.run(input.runId, backend, async (batch) => {
|
|
3265
3433
|
const run = context.findRun(input.runId);
|
|
3266
3434
|
if (run === undefined)
|
|
3267
3435
|
return;
|
|
3268
3436
|
const receipt = await backend.readSettledPendingStageMessage?.(input.runId, input.message.id);
|
|
3269
|
-
const result = queueStickyStageMessage(receipt === undefined ? run
|
|
3437
|
+
const result = queueStickyStageMessage(receipt === undefined ? batch.messages(run) : [...batch.messages(run), receipt], input, senderGroup, runGroup);
|
|
3270
3438
|
if (result.ok && !result.deduplicated) {
|
|
3271
|
-
|
|
3272
|
-
const live = context.findRun(input.runId) ?? run;
|
|
3273
|
-
live.pendingStageMessages = [...messages];
|
|
3274
|
-
context.bumpAndNotify();
|
|
3439
|
+
await batch.apply(backend, input.runId, result.messages);
|
|
3275
3440
|
}
|
|
3276
|
-
return result.ok ? { ...result, messages: (context.findRun(input.runId) ?? run)
|
|
3441
|
+
return result.ok ? { ...result, messages: batch.messages(context.findRun(input.runId) ?? run) } : result;
|
|
3277
3442
|
});
|
|
3278
3443
|
},
|
|
3279
3444
|
async recordPendingStageMessageDeliveries(runId, messageId, records, deliveredAt, backend) {
|
|
3280
|
-
return await
|
|
3445
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3281
3446
|
const run = context.findRun(runId);
|
|
3282
3447
|
if (run === undefined)
|
|
3283
3448
|
return false;
|
|
3284
3449
|
if (records.length === 0)
|
|
3285
3450
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3286
|
-
let current =
|
|
3451
|
+
let current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3287
3452
|
const unconfirmedRecords = [];
|
|
3288
3453
|
for (const record of records) {
|
|
3289
3454
|
const contextReceipt = record.admission === "transport" ? { ...record, admission: "context" } : undefined;
|
|
@@ -3291,35 +3456,26 @@ function createPendingStageDeliveryStoreMethods(context) {
|
|
|
3291
3456
|
unconfirmedRecords.push(record);
|
|
3292
3457
|
}
|
|
3293
3458
|
}
|
|
3294
|
-
current =
|
|
3295
|
-
|
|
3296
|
-
run.pendingStageMessages = [...current];
|
|
3297
|
-
context.bumpAndNotify();
|
|
3298
|
-
}
|
|
3459
|
+
current = ledgers.for(run).reconcile(backend, runId, current);
|
|
3460
|
+
batch.reconciled(run, current);
|
|
3299
3461
|
const next = recordPendingStageMessageDeliveries(current, runId, messageId, unconfirmedRecords, deliveredAt);
|
|
3300
3462
|
if (next === current)
|
|
3301
3463
|
return false;
|
|
3302
|
-
|
|
3303
|
-
const live = context.findRun(runId) ?? run;
|
|
3304
|
-
live.pendingStageMessages = [...messages];
|
|
3305
|
-
context.bumpAndNotify();
|
|
3464
|
+
await batch.apply(backend, runId, next);
|
|
3306
3465
|
return true;
|
|
3307
3466
|
});
|
|
3308
3467
|
},
|
|
3309
3468
|
async settleStickyPendingStageMessageDelivered(runId, messageId, settledAt, backend) {
|
|
3310
|
-
return await
|
|
3469
|
+
return await batches.run(runId, backend, async (batch) => {
|
|
3311
3470
|
const run = context.findRun(runId);
|
|
3312
3471
|
if (run === undefined)
|
|
3313
3472
|
return false;
|
|
3314
3473
|
await backend.readPendingStageMessageDeliveryCount?.(runId, messageId, runId);
|
|
3315
|
-
const current =
|
|
3474
|
+
const current = ledgers.for(run).reconcile(backend, runId, batch.messages(run));
|
|
3316
3475
|
const next = settleStickyPendingStageMessageDelivered(current, runId, messageId, settledAt);
|
|
3317
3476
|
if (next === current)
|
|
3318
3477
|
return false;
|
|
3319
|
-
|
|
3320
|
-
const live = context.findRun(runId) ?? run;
|
|
3321
|
-
live.pendingStageMessages = [...messages];
|
|
3322
|
-
context.bumpAndNotify();
|
|
3478
|
+
await batch.apply(backend, runId, next);
|
|
3323
3479
|
return true;
|
|
3324
3480
|
});
|
|
3325
3481
|
}
|
|
@@ -3337,78 +3493,186 @@ function resolvePendingStageIdentity(run, stageKey) {
|
|
|
3337
3493
|
aliases: stage.id === stage.name ? [stage.id] : [stage.id, stage.name]
|
|
3338
3494
|
};
|
|
3339
3495
|
}
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
const prefixIndex = archivedThrough + 1;
|
|
3362
|
-
await backend.archivePendingStageDeliveryReceipt(runId, entry.id, entry.deliveries[prefixIndex], {
|
|
3363
|
-
messageRunId: entry.runId,
|
|
3364
|
-
deliveryCount: (entry.deliveryCount ?? entry.deliveries.length) - entry.deliveries.length + prefixIndex + 1
|
|
3365
|
-
});
|
|
3366
|
-
archivedThrough = prefixIndex;
|
|
3367
|
-
}
|
|
3368
|
-
} else {
|
|
3369
|
-
deliveries.push(delivery);
|
|
3370
|
-
}
|
|
3371
|
-
}
|
|
3372
|
-
compacted.push(deliveries.length === entry.deliveries.length ? entry : { ...entry, deliveries });
|
|
3496
|
+
|
|
3497
|
+
class PendingStageTransitionBatches {
|
|
3498
|
+
queues = new Map;
|
|
3499
|
+
context;
|
|
3500
|
+
ledgers;
|
|
3501
|
+
constructor(context, ledgers) {
|
|
3502
|
+
this.context = context;
|
|
3503
|
+
this.ledgers = ledgers;
|
|
3504
|
+
}
|
|
3505
|
+
run(runId, backend, step) {
|
|
3506
|
+
return new Promise((resolve, reject) => {
|
|
3507
|
+
const transition = {
|
|
3508
|
+
backend,
|
|
3509
|
+
step,
|
|
3510
|
+
resolve: (value2) => resolve(value2),
|
|
3511
|
+
reject
|
|
3512
|
+
};
|
|
3513
|
+
const queue = this.queues.get(runId);
|
|
3514
|
+
if (queue !== undefined) {
|
|
3515
|
+
queue.push(transition);
|
|
3516
|
+
return;
|
|
3373
3517
|
}
|
|
3374
|
-
|
|
3518
|
+
const created = [transition];
|
|
3519
|
+
this.queues.set(runId, created);
|
|
3520
|
+
Promise.resolve().then(() => this.drain(runId, created));
|
|
3521
|
+
});
|
|
3522
|
+
}
|
|
3523
|
+
async drain(runId, queue) {
|
|
3524
|
+
while (queue.length > 0)
|
|
3525
|
+
await this.runBatch(runId, takeBatch(queue));
|
|
3526
|
+
this.queues.delete(runId);
|
|
3527
|
+
}
|
|
3528
|
+
async runBatch(runId, transitions) {
|
|
3529
|
+
const batch = new PendingStageTransitionBatch(this.context, this.ledgers, runId);
|
|
3530
|
+
const outcomes = [];
|
|
3531
|
+
for (const transition of transitions) {
|
|
3532
|
+
const changedBefore = batch.changed;
|
|
3533
|
+
try {
|
|
3534
|
+
const value2 = await transition.step(batch);
|
|
3535
|
+
outcomes.push({ ok: true, value: value2, uncommitted: changedBefore || batch.changed });
|
|
3536
|
+
} catch (error) {
|
|
3537
|
+
outcomes.push({ ok: false, error });
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3540
|
+
let failure;
|
|
3541
|
+
try {
|
|
3542
|
+
await batch.persist();
|
|
3543
|
+
} catch (error) {
|
|
3544
|
+
failure = { error };
|
|
3545
|
+
}
|
|
3546
|
+
transitions.forEach((transition, index) => {
|
|
3547
|
+
const outcome = outcomes[index];
|
|
3548
|
+
if (!outcome.ok)
|
|
3549
|
+
transition.reject(outcome.error);
|
|
3550
|
+
else if (failure !== undefined && outcome.uncommitted)
|
|
3551
|
+
transition.reject(failure.error);
|
|
3552
|
+
else
|
|
3553
|
+
transition.resolve(outcome.value);
|
|
3554
|
+
});
|
|
3555
|
+
}
|
|
3556
|
+
}
|
|
3557
|
+
function takeBatch(queue) {
|
|
3558
|
+
let count = 1;
|
|
3559
|
+
let backend = queue[0].backend;
|
|
3560
|
+
while (count < queue.length && count < PENDING_STAGE_TRANSITION_BATCH_LIMIT) {
|
|
3561
|
+
const next = queue[count].backend;
|
|
3562
|
+
if (next !== undefined && backend !== undefined && next !== backend)
|
|
3563
|
+
break;
|
|
3564
|
+
backend ??= next;
|
|
3565
|
+
count++;
|
|
3566
|
+
}
|
|
3567
|
+
return queue.splice(0, count);
|
|
3568
|
+
}
|
|
3569
|
+
|
|
3570
|
+
class PendingStageTransitionBatch {
|
|
3571
|
+
changed = false;
|
|
3572
|
+
list;
|
|
3573
|
+
pending;
|
|
3574
|
+
context;
|
|
3575
|
+
ledgers;
|
|
3576
|
+
runId;
|
|
3577
|
+
constructor(context, ledgers, runId) {
|
|
3578
|
+
this.context = context;
|
|
3579
|
+
this.ledgers = ledgers;
|
|
3580
|
+
this.runId = runId;
|
|
3581
|
+
}
|
|
3582
|
+
messages(run) {
|
|
3583
|
+
return this.list ?? run.pendingStageMessages ?? [];
|
|
3584
|
+
}
|
|
3585
|
+
reconciled(run, messages) {
|
|
3586
|
+
if (this.changed) {
|
|
3587
|
+
this.list = messages;
|
|
3588
|
+
return;
|
|
3375
3589
|
}
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3590
|
+
if (messages !== run.pendingStageMessages) {
|
|
3591
|
+
run.pendingStageMessages = [...messages];
|
|
3592
|
+
this.context.bumpAndNotify();
|
|
3379
3593
|
}
|
|
3380
|
-
|
|
3381
|
-
|
|
3594
|
+
}
|
|
3595
|
+
async apply(backend, runId, messages) {
|
|
3596
|
+
const run = this.context.findRun(runId);
|
|
3597
|
+
try {
|
|
3598
|
+
const compacted = await compactTransition(backend, runId, messages, this.context, this.ledgers, run);
|
|
3599
|
+
this.list = compacted.messages;
|
|
3600
|
+
this.changed = true;
|
|
3601
|
+
this.pending = { backend, messages: compacted.messages, commit: compacted.commit, run };
|
|
3602
|
+
return compacted.messages;
|
|
3603
|
+
} catch (error) {
|
|
3604
|
+
if (this.changed && this.list !== undefined) {
|
|
3605
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : this.ledgers.for(run);
|
|
3606
|
+
this.list = ledger.reconcile(backend, runId, this.list);
|
|
3607
|
+
} else
|
|
3608
|
+
reconcileLive(backend, runId, this.context, this.ledgers);
|
|
3609
|
+
throw error;
|
|
3382
3610
|
}
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
|
|
3611
|
+
}
|
|
3612
|
+
async persist() {
|
|
3613
|
+
if (this.pending === undefined)
|
|
3614
|
+
return;
|
|
3615
|
+
const { backend, messages, commit, run } = this.pending;
|
|
3616
|
+
const published = this.list ?? messages;
|
|
3617
|
+
try {
|
|
3618
|
+
if (!await backend.persistPendingStageMessages(this.runId, messages)) {
|
|
3619
|
+
throw new Error(`atomic-workflows: durable workflow ${this.runId} is unavailable for pending-stage persistence`);
|
|
3392
3620
|
}
|
|
3393
|
-
|
|
3621
|
+
commit?.();
|
|
3622
|
+
} finally {
|
|
3623
|
+
reconcileLive(backend, this.runId, this.context, this.ledgers);
|
|
3624
|
+
}
|
|
3625
|
+
const live = this.context.findRun(this.runId) ?? run;
|
|
3626
|
+
if (live !== undefined)
|
|
3627
|
+
live.pendingStageMessages = [...published];
|
|
3628
|
+
this.context.bumpAndNotify();
|
|
3629
|
+
}
|
|
3630
|
+
}
|
|
3631
|
+
async function compactTransition(backend, runId, messages, context, ledgers, run) {
|
|
3632
|
+
const ledger = run === undefined ? new PendingStageTransitionLedger : ledgers.for(run);
|
|
3633
|
+
const reconciled = ledger.reconcile(backend, runId, messages);
|
|
3634
|
+
const archiveSettled = backend.archivePendingStageMessage !== undefined && backend.readSettledPendingStageMessage !== undefined ? (entry) => backend.archivePendingStageMessage(runId, entry) : undefined;
|
|
3635
|
+
if (isCompactingBackend(backend)) {
|
|
3636
|
+
return await ledger.compact(backend, runId, reconciled, context.state.runs, archiveSettled);
|
|
3637
|
+
}
|
|
3638
|
+
let retained = compactStickyStageMessageDeliveries(reconciled, context.state.runs);
|
|
3639
|
+
if (archiveSettled !== undefined)
|
|
3640
|
+
retained = await retainPendingStageMessages(retained, archiveSettled);
|
|
3641
|
+
if (run !== undefined)
|
|
3642
|
+
ledgers.reset(run);
|
|
3643
|
+
return { messages: retained };
|
|
3644
|
+
}
|
|
3645
|
+
function reconcileLive(backend, runId, context, ledgers) {
|
|
3646
|
+
const current = context.findRun(runId);
|
|
3647
|
+
if (current === undefined)
|
|
3648
|
+
return;
|
|
3649
|
+
const messages = current.pendingStageMessages ?? [];
|
|
3650
|
+
const recovered = ledgers.for(current).reconcile(backend, runId, messages);
|
|
3651
|
+
if (recovered !== messages) {
|
|
3652
|
+
current.pendingStageMessages = [...recovered];
|
|
3653
|
+
context.bumpAndNotify();
|
|
3394
3654
|
}
|
|
3395
3655
|
}
|
|
3396
|
-
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
return
|
|
3406
|
-
}
|
|
3407
|
-
|
|
3656
|
+
|
|
3657
|
+
class PendingStageLedgers {
|
|
3658
|
+
ledgers = new WeakMap;
|
|
3659
|
+
for(run) {
|
|
3660
|
+
let ledger = this.ledgers.get(run);
|
|
3661
|
+
if (ledger === undefined) {
|
|
3662
|
+
ledger = new PendingStageTransitionLedger;
|
|
3663
|
+
this.ledgers.set(run, ledger);
|
|
3664
|
+
}
|
|
3665
|
+
return ledger;
|
|
3666
|
+
}
|
|
3667
|
+
reset(run) {
|
|
3668
|
+
this.ledgers.delete(run);
|
|
3669
|
+
}
|
|
3408
3670
|
}
|
|
3671
|
+
var PENDING_STAGE_TRANSITION_BATCH_LIMIT = 256;
|
|
3409
3672
|
var init_store_pending_stage_delivery_methods = __esm(() => {
|
|
3410
3673
|
init_run_owner_backend();
|
|
3411
3674
|
init_pending_stage_delivery();
|
|
3675
|
+
init_pending_stage_transition_ledger();
|
|
3412
3676
|
});
|
|
3413
3677
|
|
|
3414
3678
|
// dist/builtin/workflows/src/shared/store-prompt-methods.ts
|
|
@@ -4998,6 +5262,204 @@ var init_resume_eligibility = () => {};
|
|
|
4998
5262
|
// dist/builtin/workflows/src/durable/types.ts
|
|
4999
5263
|
var DURABLE_TOOL_TOPOLOGY_VERSION = 1, DURABLE_STAGE_TOPOLOGY_VERSION = 1, DURABLE_BOUNDARY_TOPOLOGY_VERSION = 1;
|
|
5000
5264
|
|
|
5265
|
+
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5266
|
+
function immutableStageGroupError(stages) {
|
|
5267
|
+
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5268
|
+
if (occurrenceError !== undefined)
|
|
5269
|
+
return occurrenceError;
|
|
5270
|
+
const byReplayKey = groupByDurableStageKey(stages);
|
|
5271
|
+
for (const group of byReplayKey.values()) {
|
|
5272
|
+
const firstName = group[0]?.name;
|
|
5273
|
+
if (group.some((stage) => stage.name !== firstName))
|
|
5274
|
+
return "stage name drift within one replay-key group";
|
|
5275
|
+
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5276
|
+
const first = topologies[0];
|
|
5277
|
+
if (first === undefined)
|
|
5278
|
+
continue;
|
|
5279
|
+
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5280
|
+
return "immutable stage identity drift within one replay-key group";
|
|
5281
|
+
}
|
|
5282
|
+
}
|
|
5283
|
+
return;
|
|
5284
|
+
}
|
|
5285
|
+
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5286
|
+
return directChildTopologyErrorFromRecords({
|
|
5287
|
+
claiming: stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId),
|
|
5288
|
+
direct: (claiming) => {
|
|
5289
|
+
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5290
|
+
return stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5291
|
+
}
|
|
5292
|
+
}, replayScope, expectedRun, allowedNodeIds);
|
|
5293
|
+
}
|
|
5294
|
+
function directChildTopologyErrorFromRecords(records, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5295
|
+
const prefix = `${replayScope}:`;
|
|
5296
|
+
const claiming = records.claiming;
|
|
5297
|
+
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5298
|
+
return "child run topology exists outside its durable invocation scope";
|
|
5299
|
+
}
|
|
5300
|
+
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5301
|
+
return "child checkpoint ownership is not reciprocal";
|
|
5302
|
+
}
|
|
5303
|
+
const direct = records.direct(claiming);
|
|
5304
|
+
const immutableError = immutableStageGroupError(direct);
|
|
5305
|
+
if (immutableError !== undefined)
|
|
5306
|
+
return immutableError;
|
|
5307
|
+
const merged = [];
|
|
5308
|
+
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5309
|
+
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5310
|
+
if (topology === undefined)
|
|
5311
|
+
return "direct child stage group lacks owning-run topology";
|
|
5312
|
+
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5313
|
+
return "direct child stage topology lacks current source identity";
|
|
5314
|
+
}
|
|
5315
|
+
if (topology.stageId.length === 0)
|
|
5316
|
+
return "direct child stage id is empty";
|
|
5317
|
+
merged.push(topology);
|
|
5318
|
+
}
|
|
5319
|
+
const ids = new Set(allowedNodeIds);
|
|
5320
|
+
const stageIds = new Set;
|
|
5321
|
+
const orders = new Set;
|
|
5322
|
+
for (const topology of merged) {
|
|
5323
|
+
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5324
|
+
return "direct child stage ids are not unique";
|
|
5325
|
+
stageIds.add(topology.stageId);
|
|
5326
|
+
ids.add(topology.stageId);
|
|
5327
|
+
if (topology.sourceOrder !== undefined) {
|
|
5328
|
+
if (orders.has(topology.sourceOrder))
|
|
5329
|
+
return "direct child source orders are not unique";
|
|
5330
|
+
orders.add(topology.sourceOrder);
|
|
5331
|
+
}
|
|
5332
|
+
}
|
|
5333
|
+
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5334
|
+
return "direct child stage references an unknown parent";
|
|
5335
|
+
}
|
|
5336
|
+
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5337
|
+
}
|
|
5338
|
+
function owningBoundaryIdError(stages, owningRunId) {
|
|
5339
|
+
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5340
|
+
const seen = new Map;
|
|
5341
|
+
for (const start of starts) {
|
|
5342
|
+
const stageId = start.topology.stageId;
|
|
5343
|
+
if (stageId.length === 0)
|
|
5344
|
+
return "owning-run boundary stage id is empty";
|
|
5345
|
+
const prior = seen.get(stageId);
|
|
5346
|
+
if (prior !== undefined && prior !== start.replayKey)
|
|
5347
|
+
return "owning-run boundary stage id is aliased";
|
|
5348
|
+
seen.set(stageId, start.replayKey);
|
|
5349
|
+
}
|
|
5350
|
+
return;
|
|
5351
|
+
}
|
|
5352
|
+
function sameRun(actual, expected) {
|
|
5353
|
+
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5354
|
+
}
|
|
5355
|
+
function sameStrings(left, right) {
|
|
5356
|
+
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5357
|
+
}
|
|
5358
|
+
function sameOptionalRun(actual, expected) {
|
|
5359
|
+
if (actual === undefined || expected === undefined)
|
|
5360
|
+
return actual === expected;
|
|
5361
|
+
return sameRun(actual, expected);
|
|
5362
|
+
}
|
|
5363
|
+
function promptOccurrenceIdentityError(stages) {
|
|
5364
|
+
const replayByOccurrence = new Map;
|
|
5365
|
+
const occurrenceByReplayStage = new Map;
|
|
5366
|
+
const stageByReplayOccurrence = new Map;
|
|
5367
|
+
for (const stage of stages) {
|
|
5368
|
+
const topology = stage.topology;
|
|
5369
|
+
const occurrence = topology?.occurrenceKey;
|
|
5370
|
+
if (topology === undefined || occurrence === undefined)
|
|
5371
|
+
continue;
|
|
5372
|
+
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5373
|
+
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5374
|
+
return "prompt occurrence is bound to multiple replay keys";
|
|
5375
|
+
}
|
|
5376
|
+
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5377
|
+
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5378
|
+
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5379
|
+
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5380
|
+
return "prompt stage identity is bound to multiple occurrences";
|
|
5381
|
+
}
|
|
5382
|
+
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5383
|
+
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5384
|
+
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5385
|
+
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5386
|
+
return "prompt occurrence is bound to multiple stage identities";
|
|
5387
|
+
}
|
|
5388
|
+
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5389
|
+
}
|
|
5390
|
+
return;
|
|
5391
|
+
}
|
|
5392
|
+
function durableStageGroupKey(stage) {
|
|
5393
|
+
const occurrence = stage.topology?.occurrenceKey;
|
|
5394
|
+
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5395
|
+
}
|
|
5396
|
+
function groupByDurableStageKey(stages) {
|
|
5397
|
+
const groups = new Map;
|
|
5398
|
+
for (const stage of stages) {
|
|
5399
|
+
const key = durableStageGroupKey(stage);
|
|
5400
|
+
const group = groups.get(key) ?? [];
|
|
5401
|
+
group.push(stage);
|
|
5402
|
+
groups.set(key, group);
|
|
5403
|
+
}
|
|
5404
|
+
return groups;
|
|
5405
|
+
}
|
|
5406
|
+
function compositeKey(domain, ...parts) {
|
|
5407
|
+
return JSON.stringify([domain, ...parts]);
|
|
5408
|
+
}
|
|
5409
|
+
function hasCycle(topologies) {
|
|
5410
|
+
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5411
|
+
const visiting = new Set;
|
|
5412
|
+
const visited = new Set;
|
|
5413
|
+
const cyclic = (stageId) => {
|
|
5414
|
+
if (visiting.has(stageId))
|
|
5415
|
+
return true;
|
|
5416
|
+
if (visited.has(stageId))
|
|
5417
|
+
return false;
|
|
5418
|
+
visiting.add(stageId);
|
|
5419
|
+
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5420
|
+
return true;
|
|
5421
|
+
visiting.delete(stageId);
|
|
5422
|
+
visited.add(stageId);
|
|
5423
|
+
return false;
|
|
5424
|
+
};
|
|
5425
|
+
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5426
|
+
}
|
|
5427
|
+
var init_stage_topology_validation = () => {};
|
|
5428
|
+
|
|
5429
|
+
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5430
|
+
function parseWorkflowChildResult(value2) {
|
|
5431
|
+
if (!isOrdinaryObject(value2))
|
|
5432
|
+
return;
|
|
5433
|
+
const workflow = value2.workflow;
|
|
5434
|
+
const runId = value2.runId;
|
|
5435
|
+
const status = value2.status;
|
|
5436
|
+
const exited = value2.exited;
|
|
5437
|
+
const outputs = value2.outputs;
|
|
5438
|
+
const exitReason = value2.exitReason;
|
|
5439
|
+
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5440
|
+
return;
|
|
5441
|
+
if (exited === false) {
|
|
5442
|
+
if (status !== "completed" || exitReason !== undefined)
|
|
5443
|
+
return;
|
|
5444
|
+
return value2;
|
|
5445
|
+
}
|
|
5446
|
+
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5447
|
+
return;
|
|
5448
|
+
return value2;
|
|
5449
|
+
}
|
|
5450
|
+
function isWorkflowChildResult(value2) {
|
|
5451
|
+
return parseWorkflowChildResult(value2) !== undefined;
|
|
5452
|
+
}
|
|
5453
|
+
function isWorkflowExitStatus(value2) {
|
|
5454
|
+
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5455
|
+
}
|
|
5456
|
+
function isOrdinaryObject(value2) {
|
|
5457
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5458
|
+
return false;
|
|
5459
|
+
const prototype = Object.getPrototypeOf(value2);
|
|
5460
|
+
return prototype === Object.prototype || prototype === null;
|
|
5461
|
+
}
|
|
5462
|
+
|
|
5001
5463
|
// dist/builtin/workflows/src/durable/durable-hash.ts
|
|
5002
5464
|
import { createHash } from "node:crypto";
|
|
5003
5465
|
function durableHash(value2) {
|
|
@@ -5064,6 +5526,185 @@ function replacePendingStageMessagesForRun(existing, logicalRunId, messages) {
|
|
|
5064
5526
|
function pendingStageMessagesForDurableRun(backend, logicalRunId, rootWorkflowId = logicalRunId) {
|
|
5065
5527
|
return (backend.getWorkflow(rootWorkflowId)?.pendingStageMessages ?? []).filter((entry) => entry.runId === logicalRunId);
|
|
5066
5528
|
}
|
|
5529
|
+
|
|
5530
|
+
class StageReceipts {
|
|
5531
|
+
exact = new Map;
|
|
5532
|
+
anySession = new Map;
|
|
5533
|
+
has(logicalRunId, sessionId, admission) {
|
|
5534
|
+
return this.exact.get(logicalRunId)?.get(sessionId)?.has(admission) ?? false;
|
|
5535
|
+
}
|
|
5536
|
+
hasAnySession(logicalRunId, admission) {
|
|
5537
|
+
return this.anySession.get(logicalRunId)?.has(admission) ?? false;
|
|
5538
|
+
}
|
|
5539
|
+
add(logicalRunId, sessionId, admission) {
|
|
5540
|
+
let bySession = this.exact.get(logicalRunId);
|
|
5541
|
+
if (bySession === undefined) {
|
|
5542
|
+
bySession = new Map;
|
|
5543
|
+
this.exact.set(logicalRunId, bySession);
|
|
5544
|
+
}
|
|
5545
|
+
let admissions = bySession.get(sessionId);
|
|
5546
|
+
if (admissions === undefined) {
|
|
5547
|
+
admissions = new Set;
|
|
5548
|
+
bySession.set(sessionId, admissions);
|
|
5549
|
+
}
|
|
5550
|
+
if (admissions.has(admission))
|
|
5551
|
+
return false;
|
|
5552
|
+
admissions.add(admission);
|
|
5553
|
+
return true;
|
|
5554
|
+
}
|
|
5555
|
+
addAnySession(logicalRunId, admission) {
|
|
5556
|
+
let admissions = this.anySession.get(logicalRunId);
|
|
5557
|
+
if (admissions === undefined) {
|
|
5558
|
+
admissions = new Set;
|
|
5559
|
+
this.anySession.set(logicalRunId, admissions);
|
|
5560
|
+
}
|
|
5561
|
+
if (admissions.has(admission))
|
|
5562
|
+
return false;
|
|
5563
|
+
admissions.add(admission);
|
|
5564
|
+
return true;
|
|
5565
|
+
}
|
|
5566
|
+
}
|
|
5567
|
+
|
|
5568
|
+
class PendingStageReceiptIndex {
|
|
5569
|
+
workflows = new Map;
|
|
5570
|
+
add(workflowId, messageId, delivery, logicalRunId) {
|
|
5571
|
+
let workflow = this.workflows.get(workflowId);
|
|
5572
|
+
if (workflow === undefined) {
|
|
5573
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5574
|
+
this.workflows.set(workflowId, workflow);
|
|
5575
|
+
}
|
|
5576
|
+
let byRun = workflow.byMessage.get(messageId);
|
|
5577
|
+
if (byRun === undefined) {
|
|
5578
|
+
byRun = new Map;
|
|
5579
|
+
workflow.byMessage.set(messageId, byRun);
|
|
5580
|
+
}
|
|
5581
|
+
let byStage = byRun.get(delivery.runId);
|
|
5582
|
+
if (byStage === undefined) {
|
|
5583
|
+
byStage = new Map;
|
|
5584
|
+
byRun.set(delivery.runId, byStage);
|
|
5585
|
+
}
|
|
5586
|
+
let receipts = byStage.get(delivery.stageId);
|
|
5587
|
+
if (receipts === undefined) {
|
|
5588
|
+
receipts = new StageReceipts;
|
|
5589
|
+
byStage.set(delivery.stageId, receipts);
|
|
5590
|
+
}
|
|
5591
|
+
const added = receipts.add(logicalRunId, delivery.sessionId, delivery.admission);
|
|
5592
|
+
let changed = added;
|
|
5593
|
+
if (receipts.add(logicalRunId, undefined, undefined))
|
|
5594
|
+
changed = true;
|
|
5595
|
+
if (delivery.admission === "context" && delivery.sessionId !== undefined && receipts.addAnySession(logicalRunId, "context"))
|
|
5596
|
+
changed = true;
|
|
5597
|
+
if (changed)
|
|
5598
|
+
workflow.log.push(messageId);
|
|
5599
|
+
return added;
|
|
5600
|
+
}
|
|
5601
|
+
has(workflowId, messageId, delivery, logicalRunId, anySession = false) {
|
|
5602
|
+
const receipts = this.workflows.get(workflowId)?.byMessage.get(messageId)?.get(delivery.runId)?.get(delivery.stageId);
|
|
5603
|
+
if (receipts === undefined)
|
|
5604
|
+
return false;
|
|
5605
|
+
if (anySession && delivery.admission === "context")
|
|
5606
|
+
return receipts.hasAnySession(logicalRunId, delivery.admission) || receipts.hasAnySession(undefined, delivery.admission);
|
|
5607
|
+
return receipts.has(logicalRunId, delivery.sessionId, delivery.admission) || receipts.has(undefined, delivery.sessionId, delivery.admission);
|
|
5608
|
+
}
|
|
5609
|
+
changesSince(workflowId, cursor) {
|
|
5610
|
+
let workflow = this.workflows.get(workflowId);
|
|
5611
|
+
if (workflow === undefined) {
|
|
5612
|
+
workflow = { byMessage: new Map, log: [] };
|
|
5613
|
+
this.workflows.set(workflowId, workflow);
|
|
5614
|
+
}
|
|
5615
|
+
const next = { log: workflow.log, length: workflow.log.length };
|
|
5616
|
+
if (cursor === undefined || cursor.log !== workflow.log || cursor.length > workflow.log.length)
|
|
5617
|
+
return { cursor: next, messageIds: undefined };
|
|
5618
|
+
return { cursor: next, messageIds: workflow.log.slice(cursor.length) };
|
|
5619
|
+
}
|
|
5620
|
+
delete(workflowId) {
|
|
5621
|
+
this.workflows.delete(workflowId);
|
|
5622
|
+
}
|
|
5623
|
+
clear() {
|
|
5624
|
+
this.workflows.clear();
|
|
5625
|
+
}
|
|
5626
|
+
}
|
|
5627
|
+
function nextVersion() {
|
|
5628
|
+
lastVersion += 1;
|
|
5629
|
+
return lastVersion;
|
|
5630
|
+
}
|
|
5631
|
+
|
|
5632
|
+
class NestedSnapshotVersion {
|
|
5633
|
+
rootWorkflowId;
|
|
5634
|
+
current = nextVersion();
|
|
5635
|
+
latestCompletedAt = Number.NEGATIVE_INFINITY;
|
|
5636
|
+
replayGroups = new Map;
|
|
5637
|
+
boundaryReplayKeys = new Set;
|
|
5638
|
+
constructor(rootWorkflowId) {
|
|
5639
|
+
this.rootWorkflowId = rootWorkflowId;
|
|
5640
|
+
}
|
|
5641
|
+
get version() {
|
|
5642
|
+
return this.current;
|
|
5643
|
+
}
|
|
5644
|
+
record(checkpoint) {
|
|
5645
|
+
if (!this.isInert(checkpoint))
|
|
5646
|
+
this.current = nextVersion();
|
|
5647
|
+
this.latestCompletedAt = Math.max(this.latestCompletedAt, checkpoint.completedAt);
|
|
5648
|
+
if (checkpoint.kind !== "stage")
|
|
5649
|
+
return;
|
|
5650
|
+
if (checkpoint.topology?.boundary !== undefined)
|
|
5651
|
+
this.boundaryReplayKeys.add(checkpoint.replayKey);
|
|
5652
|
+
if (checkpoint.topology?.occurrenceKey !== undefined)
|
|
5653
|
+
return;
|
|
5654
|
+
const identity = stageIdentity(checkpoint);
|
|
5655
|
+
const exited = parseWorkflowChildResult(checkpoint.output)?.exited === true;
|
|
5656
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5657
|
+
if (summary === undefined) {
|
|
5658
|
+
this.replayGroups.set(checkpoint.replayKey, {
|
|
5659
|
+
name: checkpoint.name,
|
|
5660
|
+
identity,
|
|
5661
|
+
consistent: true,
|
|
5662
|
+
hasOutput: checkpoint.output !== undefined,
|
|
5663
|
+
hasExitedChildOutput: exited
|
|
5664
|
+
});
|
|
5665
|
+
return;
|
|
5666
|
+
}
|
|
5667
|
+
summary.consistent &&= summary.name === checkpoint.name && summary.identity === identity;
|
|
5668
|
+
summary.hasOutput ||= checkpoint.output !== undefined;
|
|
5669
|
+
summary.hasExitedChildOutput ||= exited;
|
|
5670
|
+
}
|
|
5671
|
+
isInert(checkpoint) {
|
|
5672
|
+
if (checkpoint.kind !== "stage" || !checkpoint.checkpointId.startsWith(STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX) || checkpoint.output !== undefined || checkpoint.topology === undefined || checkpoint.topology.boundary !== undefined || checkpoint.topology.occurrenceKey !== undefined || !this.isRootOwned(checkpoint) || checkpoint.completedAt < this.latestCompletedAt) {
|
|
5673
|
+
return false;
|
|
5674
|
+
}
|
|
5675
|
+
const summary = this.replayGroups.get(checkpoint.replayKey);
|
|
5676
|
+
if (summary === undefined || !summary.consistent || !summary.hasOutput || summary.hasExitedChildOutput || summary.name !== checkpoint.name || summary.identity !== stageIdentity(checkpoint)) {
|
|
5677
|
+
return false;
|
|
5678
|
+
}
|
|
5679
|
+
const id = checkpoint.checkpointId;
|
|
5680
|
+
for (let colon = id.indexOf(":");colon >= 0; colon = id.indexOf(":", colon + 1)) {
|
|
5681
|
+
if (this.boundaryReplayKeys.has(id.slice(0, colon)))
|
|
5682
|
+
return false;
|
|
5683
|
+
}
|
|
5684
|
+
return true;
|
|
5685
|
+
}
|
|
5686
|
+
isRootOwned(checkpoint) {
|
|
5687
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
5688
|
+
return runId === undefined || runId === this.rootWorkflowId;
|
|
5689
|
+
}
|
|
5690
|
+
}
|
|
5691
|
+
function stageIdentity(checkpoint) {
|
|
5692
|
+
const topology = checkpoint.topology;
|
|
5693
|
+
return JSON.stringify([
|
|
5694
|
+
durableStageGroupKey(checkpoint),
|
|
5695
|
+
topology?.version,
|
|
5696
|
+
topology?.stageId,
|
|
5697
|
+
topology?.parentIds,
|
|
5698
|
+
topology?.sourceOrder,
|
|
5699
|
+
topology?.run === undefined ? undefined : [
|
|
5700
|
+
topology.run.runId,
|
|
5701
|
+
topology.run.runName,
|
|
5702
|
+
topology.run.parentRunId,
|
|
5703
|
+
topology.run.parentStageId,
|
|
5704
|
+
topology.run.rootRunId
|
|
5705
|
+
]
|
|
5706
|
+
]);
|
|
5707
|
+
}
|
|
5067
5708
|
function checkpointKey(c) {
|
|
5068
5709
|
return `${c.kind}:${c.checkpointId}`;
|
|
5069
5710
|
}
|
|
@@ -5075,7 +5716,7 @@ class JournalProjection {
|
|
|
5075
5716
|
workflows = new Map;
|
|
5076
5717
|
deletedWorkflowIds = new Set;
|
|
5077
5718
|
pendingReceipts = new Map;
|
|
5078
|
-
stickyReceipts = new
|
|
5719
|
+
stickyReceipts = new PendingStageReceiptIndex;
|
|
5079
5720
|
registerWorkflow(handle, authoritativeOwner = false) {
|
|
5080
5721
|
this.deletedWorkflowIds.delete(handle.workflowId);
|
|
5081
5722
|
const existing = this.workflows.get(handle.workflowId);
|
|
@@ -5123,7 +5764,8 @@ class JournalProjection {
|
|
|
5123
5764
|
toolByHash: new Map,
|
|
5124
5765
|
uiByHash: new Map,
|
|
5125
5766
|
stageOutputByReplayKey: new Map,
|
|
5126
|
-
stageSessionByReplayKey: new Map
|
|
5767
|
+
stageSessionByReplayKey: new Map,
|
|
5768
|
+
nestedSnapshotVersion: new NestedSnapshotVersion(handle.workflowId)
|
|
5127
5769
|
});
|
|
5128
5770
|
}
|
|
5129
5771
|
applyWorkflowSnapshot(handle) {
|
|
@@ -5149,39 +5791,43 @@ class JournalProjection {
|
|
|
5149
5791
|
return this.pendingReceipts.get(workflowId)?.get(JSON.stringify([logicalRunId, messageId]));
|
|
5150
5792
|
}
|
|
5151
5793
|
stickyDeliveryCounts = new Map;
|
|
5794
|
+
stickyDeliveryCountVersions = new Map;
|
|
5152
5795
|
getPendingStageMessageDeliveryCount(workflowId, messageId, logicalRunId = workflowId) {
|
|
5153
|
-
return this.stickyDeliveryCounts.get(workflowId)?.get(
|
|
5796
|
+
return this.stickyDeliveryCounts.get(workflowId)?.get(logicalRunId)?.get(messageId) ?? 0;
|
|
5797
|
+
}
|
|
5798
|
+
pendingStageDeliveryCountVersion(workflowId) {
|
|
5799
|
+
return this.stickyDeliveryCountVersions.get(workflowId) ?? 0;
|
|
5800
|
+
}
|
|
5801
|
+
bumpDeliveryCountVersion(workflowId) {
|
|
5802
|
+
this.stickyDeliveryCountVersions.set(workflowId, ++deliveryCountEpoch);
|
|
5154
5803
|
}
|
|
5155
5804
|
async archivePendingStageDeliveryReceipt(workflowId, messageId, delivery, accounting) {
|
|
5156
|
-
|
|
5157
|
-
if (
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
receipts.add(pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined, admission: undefined }, accounting?.messageRunId));
|
|
5165
|
-
if (delivery.admission === "context" && delivery.sessionId !== undefined)
|
|
5166
|
-
receipts.add(`${pendingDeliveryReceiptKey(messageId, { ...delivery, sessionId: undefined }, accounting?.messageRunId)}:context-any-session`);
|
|
5167
|
-
if (accounting !== undefined && !existing) {
|
|
5168
|
-
let counts = this.stickyDeliveryCounts.get(workflowId);
|
|
5805
|
+
const added = this.stickyReceipts.add(workflowId, messageId, delivery, accounting?.messageRunId);
|
|
5806
|
+
if (accounting !== undefined && added) {
|
|
5807
|
+
let byRun = this.stickyDeliveryCounts.get(workflowId);
|
|
5808
|
+
if (byRun === undefined) {
|
|
5809
|
+
byRun = new Map;
|
|
5810
|
+
this.stickyDeliveryCounts.set(workflowId, byRun);
|
|
5811
|
+
}
|
|
5812
|
+
let counts = byRun.get(accounting.messageRunId);
|
|
5169
5813
|
if (counts === undefined) {
|
|
5170
5814
|
counts = new Map;
|
|
5171
|
-
|
|
5815
|
+
byRun.set(accounting.messageRunId, counts);
|
|
5172
5816
|
}
|
|
5173
|
-
const
|
|
5174
|
-
counts.set(
|
|
5817
|
+
const previous = counts.get(messageId) ?? 0;
|
|
5818
|
+
counts.set(messageId, Math.max(previous, accounting.deliveryCount));
|
|
5819
|
+
if (accounting.deliveryCount > previous)
|
|
5820
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5175
5821
|
}
|
|
5176
5822
|
}
|
|
5177
5823
|
async hasPendingStageDeliveryReceipt(workflowId, messageId, delivery, logicalRunId = workflowId) {
|
|
5178
|
-
return
|
|
5824
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId);
|
|
5179
5825
|
}
|
|
5180
5826
|
hasCachedPendingStageDeliveryReceipt(workflowId, messageId, delivery, anySession = false, logicalRunId = workflowId) {
|
|
5181
|
-
return
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5827
|
+
return this.stickyReceipts.has(workflowId, messageId, delivery, logicalRunId, anySession);
|
|
5828
|
+
}
|
|
5829
|
+
pendingStageDeliveryReceiptChanges(workflowId, cursor) {
|
|
5830
|
+
return this.stickyReceipts.changesSince(workflowId, cursor);
|
|
5185
5831
|
}
|
|
5186
5832
|
recordCheckpoint(recorded) {
|
|
5187
5833
|
const checkpoint = recorded.kind === "stage" ? migrateStageSessionCheckpoint(recorded) : recorded;
|
|
@@ -5200,6 +5846,7 @@ class JournalProjection {
|
|
|
5200
5846
|
if (rec.checkpoints.has(key))
|
|
5201
5847
|
return;
|
|
5202
5848
|
rec.checkpoints.set(key, currentCheckpoint);
|
|
5849
|
+
rec.nestedSnapshotVersion.record(currentCheckpoint);
|
|
5203
5850
|
if (currentCheckpoint.kind === "tool") {
|
|
5204
5851
|
const existing = rec.toolByHash.get(currentCheckpoint.argsHash);
|
|
5205
5852
|
if (currentCheckpoint.throwingFailureError === undefined && (existing === undefined || currentCheckpoint.completedAt >= existing.completedAt)) {
|
|
@@ -5257,6 +5904,9 @@ class JournalProjection {
|
|
|
5257
5904
|
getWorkflow(workflowId) {
|
|
5258
5905
|
return this.workflows.get(workflowId)?.handle;
|
|
5259
5906
|
}
|
|
5907
|
+
nestedSnapshotVersion(workflowId) {
|
|
5908
|
+
return this.workflows.get(workflowId)?.nestedSnapshotVersion.version;
|
|
5909
|
+
}
|
|
5260
5910
|
getLoadableWorkflow(workflowId) {
|
|
5261
5911
|
return this.getWorkflow(workflowId);
|
|
5262
5912
|
}
|
|
@@ -5334,6 +5984,7 @@ class JournalProjection {
|
|
|
5334
5984
|
this.pendingReceipts.delete(workflowId);
|
|
5335
5985
|
this.stickyReceipts.delete(workflowId);
|
|
5336
5986
|
this.stickyDeliveryCounts.delete(workflowId);
|
|
5987
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5337
5988
|
this.deletedWorkflowIds.add(workflowId);
|
|
5338
5989
|
}
|
|
5339
5990
|
isWorkflowLoadable(workflowId) {
|
|
@@ -5344,6 +5995,8 @@ class JournalProjection {
|
|
|
5344
5995
|
this.pendingReceipts.clear();
|
|
5345
5996
|
this.stickyReceipts.clear();
|
|
5346
5997
|
this.stickyDeliveryCounts.clear();
|
|
5998
|
+
for (const workflowId of this.stickyDeliveryCountVersions.keys())
|
|
5999
|
+
this.bumpDeliveryCountVersion(workflowId);
|
|
5347
6000
|
this.deletedWorkflowIds.clear();
|
|
5348
6001
|
}
|
|
5349
6002
|
}
|
|
@@ -5400,13 +6053,10 @@ function toResumableEntry(handle) {
|
|
|
5400
6053
|
updatedAt: handle.updatedAt
|
|
5401
6054
|
};
|
|
5402
6055
|
}
|
|
5403
|
-
|
|
5404
|
-
const identity = [messageId, delivery.runId, delivery.stageId, delivery.sessionId, delivery.admission];
|
|
5405
|
-
return JSON.stringify(logicalRunId === undefined ? identity : ["counted", logicalRunId, ...identity]);
|
|
5406
|
-
}
|
|
5407
|
-
var SESSION_HEADER_READ_LIMIT_BYTES;
|
|
6056
|
+
var SESSION_HEADER_READ_LIMIT_BYTES, STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX = "stage-replay-meta:", lastVersion = 0, deliveryCountEpoch = 0;
|
|
5408
6057
|
var init_backend = __esm(() => {
|
|
5409
6058
|
init_resume_eligibility();
|
|
6059
|
+
init_stage_topology_validation();
|
|
5410
6060
|
init_durable_hash();
|
|
5411
6061
|
SESSION_HEADER_READ_LIMIT_BYTES = 64 * 1024;
|
|
5412
6062
|
});
|
|
@@ -5490,205 +6140,24 @@ async function migrateLegacyStageSession(store, sessionId, sessionFile) {
|
|
|
5490
6140
|
}
|
|
5491
6141
|
var init_stage_session_store = () => {};
|
|
5492
6142
|
|
|
5493
|
-
// dist/builtin/workflows/src/durable/stage-topology-validation.ts
|
|
5494
|
-
function immutableStageGroupError(stages) {
|
|
5495
|
-
const occurrenceError = promptOccurrenceIdentityError(stages);
|
|
5496
|
-
if (occurrenceError !== undefined)
|
|
5497
|
-
return occurrenceError;
|
|
5498
|
-
const byReplayKey = groupByDurableStageKey(stages);
|
|
5499
|
-
for (const group of byReplayKey.values()) {
|
|
5500
|
-
const firstName = group[0]?.name;
|
|
5501
|
-
if (group.some((stage) => stage.name !== firstName))
|
|
5502
|
-
return "stage name drift within one replay-key group";
|
|
5503
|
-
const topologies = group.flatMap((stage) => stage.topology === undefined ? [] : [stage.topology]);
|
|
5504
|
-
const first = topologies[0];
|
|
5505
|
-
if (first === undefined)
|
|
5506
|
-
continue;
|
|
5507
|
-
if (topologies.some((topology) => topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.stageId !== first.stageId || !sameStrings(topology.parentIds, first.parentIds) || topology.sourceOrder !== first.sourceOrder || !sameOptionalRun(topology.run, first.run))) {
|
|
5508
|
-
return "immutable stage identity drift within one replay-key group";
|
|
5509
|
-
}
|
|
5510
|
-
}
|
|
5511
|
-
return;
|
|
5512
|
-
}
|
|
5513
|
-
function directChildTopologyError(stages, replayScope, expectedRun, allowedNodeIds = new Set) {
|
|
5514
|
-
const prefix = `${replayScope}:`;
|
|
5515
|
-
const claiming = stages.filter((stage) => stage.topology?.run?.runId === expectedRun.runId);
|
|
5516
|
-
if (claiming.some((stage) => !stage.replayKey.startsWith(prefix))) {
|
|
5517
|
-
return "child run topology exists outside its durable invocation scope";
|
|
5518
|
-
}
|
|
5519
|
-
if (claiming.some((stage) => !sameRun(stage.topology?.run, expectedRun))) {
|
|
5520
|
-
return "child checkpoint ownership is not reciprocal";
|
|
5521
|
-
}
|
|
5522
|
-
const directKeys = new Set(claiming.map((stage) => durableStageGroupKey(stage)));
|
|
5523
|
-
const direct = stages.filter((stage) => directKeys.has(durableStageGroupKey(stage)));
|
|
5524
|
-
const immutableError = immutableStageGroupError(direct);
|
|
5525
|
-
if (immutableError !== undefined)
|
|
5526
|
-
return immutableError;
|
|
5527
|
-
const merged = [];
|
|
5528
|
-
for (const group of groupByDurableStageKey(direct).values()) {
|
|
5529
|
-
const topology = group.find((stage) => stage.topology?.run?.runId === expectedRun.runId)?.topology;
|
|
5530
|
-
if (topology === undefined)
|
|
5531
|
-
return "direct child stage group lacks owning-run topology";
|
|
5532
|
-
if (topology.version !== DURABLE_STAGE_TOPOLOGY_VERSION || topology.sourceOrder === undefined) {
|
|
5533
|
-
return "direct child stage topology lacks current source identity";
|
|
5534
|
-
}
|
|
5535
|
-
if (topology.stageId.length === 0)
|
|
5536
|
-
return "direct child stage id is empty";
|
|
5537
|
-
merged.push(topology);
|
|
5538
|
-
}
|
|
5539
|
-
const ids = new Set(allowedNodeIds);
|
|
5540
|
-
const stageIds = new Set;
|
|
5541
|
-
const orders = new Set;
|
|
5542
|
-
for (const topology of merged) {
|
|
5543
|
-
if (stageIds.has(topology.stageId) || ids.has(topology.stageId))
|
|
5544
|
-
return "direct child stage ids are not unique";
|
|
5545
|
-
stageIds.add(topology.stageId);
|
|
5546
|
-
ids.add(topology.stageId);
|
|
5547
|
-
if (topology.sourceOrder !== undefined) {
|
|
5548
|
-
if (orders.has(topology.sourceOrder))
|
|
5549
|
-
return "direct child source orders are not unique";
|
|
5550
|
-
orders.add(topology.sourceOrder);
|
|
5551
|
-
}
|
|
5552
|
-
}
|
|
5553
|
-
if (merged.some((topology) => topology.parentIds.some((parentId) => !ids.has(parentId)))) {
|
|
5554
|
-
return "direct child stage references an unknown parent";
|
|
5555
|
-
}
|
|
5556
|
-
return hasCycle(merged) ? "direct child stage topology contains a cycle" : undefined;
|
|
5557
|
-
}
|
|
5558
|
-
function owningBoundaryIdError(stages, owningRunId) {
|
|
5559
|
-
const starts = stages.filter((stage) => stage.topology?.run?.runId === owningRunId && stage.topology.boundary?.event === "start");
|
|
5560
|
-
const seen = new Map;
|
|
5561
|
-
for (const start of starts) {
|
|
5562
|
-
const stageId = start.topology.stageId;
|
|
5563
|
-
if (stageId.length === 0)
|
|
5564
|
-
return "owning-run boundary stage id is empty";
|
|
5565
|
-
const prior = seen.get(stageId);
|
|
5566
|
-
if (prior !== undefined && prior !== start.replayKey)
|
|
5567
|
-
return "owning-run boundary stage id is aliased";
|
|
5568
|
-
seen.set(stageId, start.replayKey);
|
|
5569
|
-
}
|
|
5570
|
-
return;
|
|
5571
|
-
}
|
|
5572
|
-
function sameRun(actual, expected) {
|
|
5573
|
-
return actual !== undefined && actual.runId === expected.runId && actual.runName === expected.runName && actual.parentRunId === expected.parentRunId && actual.parentStageId === expected.parentStageId && actual.rootRunId === expected.rootRunId;
|
|
5574
|
-
}
|
|
5575
|
-
function sameStrings(left, right) {
|
|
5576
|
-
return left.length === right.length && left.every((value2, index) => value2 === right[index]);
|
|
5577
|
-
}
|
|
5578
|
-
function sameOptionalRun(actual, expected) {
|
|
5579
|
-
if (actual === undefined || expected === undefined)
|
|
5580
|
-
return actual === expected;
|
|
5581
|
-
return sameRun(actual, expected);
|
|
5582
|
-
}
|
|
5583
|
-
function promptOccurrenceIdentityError(stages) {
|
|
5584
|
-
const replayByOccurrence = new Map;
|
|
5585
|
-
const occurrenceByReplayStage = new Map;
|
|
5586
|
-
const stageByReplayOccurrence = new Map;
|
|
5587
|
-
for (const stage of stages) {
|
|
5588
|
-
const topology = stage.topology;
|
|
5589
|
-
const occurrence = topology?.occurrenceKey;
|
|
5590
|
-
if (topology === undefined || occurrence === undefined)
|
|
5591
|
-
continue;
|
|
5592
|
-
const priorReplay = replayByOccurrence.get(occurrence);
|
|
5593
|
-
if (priorReplay !== undefined && priorReplay !== stage.replayKey) {
|
|
5594
|
-
return "prompt occurrence is bound to multiple replay keys";
|
|
5595
|
-
}
|
|
5596
|
-
replayByOccurrence.set(occurrence, stage.replayKey);
|
|
5597
|
-
const replayStage = compositeKey("prompt-stage", stage.replayKey, topology.stageId);
|
|
5598
|
-
const priorOccurrence = occurrenceByReplayStage.get(replayStage);
|
|
5599
|
-
if (priorOccurrence !== undefined && priorOccurrence !== occurrence) {
|
|
5600
|
-
return "prompt stage identity is bound to multiple occurrences";
|
|
5601
|
-
}
|
|
5602
|
-
occurrenceByReplayStage.set(replayStage, occurrence);
|
|
5603
|
-
const replayOccurrence = compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5604
|
-
const priorStage = stageByReplayOccurrence.get(replayOccurrence);
|
|
5605
|
-
if (priorStage !== undefined && priorStage !== topology.stageId) {
|
|
5606
|
-
return "prompt occurrence is bound to multiple stage identities";
|
|
5607
|
-
}
|
|
5608
|
-
stageByReplayOccurrence.set(replayOccurrence, topology.stageId);
|
|
5609
|
-
}
|
|
5610
|
-
return;
|
|
5611
|
-
}
|
|
5612
|
-
function durableStageGroupKey(stage) {
|
|
5613
|
-
const occurrence = stage.topology?.occurrenceKey;
|
|
5614
|
-
return occurrence === undefined ? compositeKey("stage-replay", stage.replayKey) : compositeKey("prompt-occurrence", stage.replayKey, occurrence);
|
|
5615
|
-
}
|
|
5616
|
-
function groupByDurableStageKey(stages) {
|
|
5617
|
-
const groups = new Map;
|
|
5618
|
-
for (const stage of stages) {
|
|
5619
|
-
const key = durableStageGroupKey(stage);
|
|
5620
|
-
const group = groups.get(key) ?? [];
|
|
5621
|
-
group.push(stage);
|
|
5622
|
-
groups.set(key, group);
|
|
5623
|
-
}
|
|
5624
|
-
return groups;
|
|
5625
|
-
}
|
|
5626
|
-
function compositeKey(domain, ...parts) {
|
|
5627
|
-
return JSON.stringify([domain, ...parts]);
|
|
5628
|
-
}
|
|
5629
|
-
function hasCycle(topologies) {
|
|
5630
|
-
const parents = new Map(topologies.map((topology) => [topology.stageId, topology.parentIds]));
|
|
5631
|
-
const visiting = new Set;
|
|
5632
|
-
const visited = new Set;
|
|
5633
|
-
const cyclic = (stageId) => {
|
|
5634
|
-
if (visiting.has(stageId))
|
|
5635
|
-
return true;
|
|
5636
|
-
if (visited.has(stageId))
|
|
5637
|
-
return false;
|
|
5638
|
-
visiting.add(stageId);
|
|
5639
|
-
if ((parents.get(stageId) ?? []).some(cyclic))
|
|
5640
|
-
return true;
|
|
5641
|
-
visiting.delete(stageId);
|
|
5642
|
-
visited.add(stageId);
|
|
5643
|
-
return false;
|
|
5644
|
-
};
|
|
5645
|
-
return topologies.some((topology) => cyclic(topology.stageId));
|
|
5646
|
-
}
|
|
5647
|
-
var init_stage_topology_validation = () => {};
|
|
5648
|
-
|
|
5649
|
-
// dist/builtin/workflows/src/durable/workflow-child-result.ts
|
|
5650
|
-
function parseWorkflowChildResult(value2) {
|
|
5651
|
-
if (!isOrdinaryObject(value2))
|
|
5652
|
-
return;
|
|
5653
|
-
const workflow = value2.workflow;
|
|
5654
|
-
const runId = value2.runId;
|
|
5655
|
-
const status = value2.status;
|
|
5656
|
-
const exited = value2.exited;
|
|
5657
|
-
const outputs = value2.outputs;
|
|
5658
|
-
const exitReason = value2.exitReason;
|
|
5659
|
-
if (typeof workflow !== "string" || typeof runId !== "string" || !isOrdinaryObject(outputs))
|
|
5660
|
-
return;
|
|
5661
|
-
if (exited === false) {
|
|
5662
|
-
if (status !== "completed" || exitReason !== undefined)
|
|
5663
|
-
return;
|
|
5664
|
-
return value2;
|
|
5665
|
-
}
|
|
5666
|
-
if (exited !== true || !isWorkflowExitStatus(status) || exitReason !== undefined && typeof exitReason !== "string")
|
|
5667
|
-
return;
|
|
5668
|
-
return value2;
|
|
5669
|
-
}
|
|
5670
|
-
function isWorkflowChildResult(value2) {
|
|
5671
|
-
return parseWorkflowChildResult(value2) !== undefined;
|
|
5672
|
-
}
|
|
5673
|
-
function isWorkflowExitStatus(value2) {
|
|
5674
|
-
return value2 === "completed" || value2 === "skipped" || value2 === "cancelled" || value2 === "blocked" || value2 === "failed";
|
|
5675
|
-
}
|
|
5676
|
-
function isOrdinaryObject(value2) {
|
|
5677
|
-
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2))
|
|
5678
|
-
return false;
|
|
5679
|
-
const prototype = Object.getPrototypeOf(value2);
|
|
5680
|
-
return prototype === Object.prototype || prototype === null;
|
|
5681
|
-
}
|
|
5682
|
-
|
|
5683
6143
|
// dist/builtin/workflows/src/durable/boundary-lifecycle.ts
|
|
5684
6144
|
function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
5685
6145
|
const startTopology = start.topology;
|
|
5686
6146
|
const startBoundary = startTopology?.boundary;
|
|
5687
|
-
const startRun = startTopology?.run;
|
|
5688
6147
|
const related = stages.filter((stage) => {
|
|
5689
6148
|
const boundary = stage.topology?.boundary;
|
|
5690
6149
|
return boundary !== undefined && (stage.replayKey === start.replayKey || stage.topology?.stageId === startTopology?.stageId || boundary.child.runId === startBoundary?.child.runId);
|
|
5691
6150
|
});
|
|
6151
|
+
return resolveBoundaryLifecycleFromRecords(start, {
|
|
6152
|
+
related,
|
|
6153
|
+
outputRecords: () => stages.filter((stage) => stage.replayKey === start.replayKey && stage.output !== undefined)
|
|
6154
|
+
}, requireTerminal);
|
|
6155
|
+
}
|
|
6156
|
+
function resolveBoundaryLifecycleFromRecords(start, records, requireTerminal) {
|
|
6157
|
+
const startTopology = start.topology;
|
|
6158
|
+
const startBoundary = startTopology?.boundary;
|
|
6159
|
+
const startRun = startTopology?.run;
|
|
6160
|
+
const related = records.related;
|
|
5692
6161
|
const starts = related.filter((stage) => stage.topology.boundary.event === "start");
|
|
5693
6162
|
if (starts.length !== 1 || starts[0] !== start)
|
|
5694
6163
|
return failure("boundary has no unique start record");
|
|
@@ -5725,7 +6194,7 @@ function resolveBoundaryLifecycle(start, stages, requireTerminal) {
|
|
|
5725
6194
|
return failure("boundary has an illegal terminal transition");
|
|
5726
6195
|
}
|
|
5727
6196
|
const completed = terminals.find((terminal) => terminal.topology.boundary.status === "completed");
|
|
5728
|
-
const outputRecords =
|
|
6197
|
+
const outputRecords = records.outputRecords();
|
|
5729
6198
|
if (outputRecords.length !== (completed === undefined ? 0 : 1) || completed !== undefined && outputRecords[0]?.checkpointId !== completed.checkpointId) {
|
|
5730
6199
|
return failure("boundary replay key has conflicting output records");
|
|
5731
6200
|
}
|
|
@@ -5755,10 +6224,120 @@ var init_boundary_lifecycle = __esm(() => {
|
|
|
5755
6224
|
init_stage_topology_validation();
|
|
5756
6225
|
});
|
|
5757
6226
|
|
|
6227
|
+
// dist/builtin/workflows/src/durable/completed-catalog-index.ts
|
|
6228
|
+
class CheckpointCatalogIndex {
|
|
6229
|
+
checkpoints;
|
|
6230
|
+
stages;
|
|
6231
|
+
positions = new Map;
|
|
6232
|
+
groupKeys = new Map;
|
|
6233
|
+
groups = new Map;
|
|
6234
|
+
stagesByRunId = new Map;
|
|
6235
|
+
boundariesByReplayKey = new Map;
|
|
6236
|
+
boundariesByStageId = new Map;
|
|
6237
|
+
boundariesByChildRunId = new Map;
|
|
6238
|
+
outputsByReplayKey = new Map;
|
|
6239
|
+
evidence;
|
|
6240
|
+
constructor(checkpoints) {
|
|
6241
|
+
this.checkpoints = checkpoints;
|
|
6242
|
+
const stages = [];
|
|
6243
|
+
checkpoints.forEach((checkpoint, index) => {
|
|
6244
|
+
if (!this.positions.has(checkpoint))
|
|
6245
|
+
this.positions.set(checkpoint, index);
|
|
6246
|
+
if (checkpoint.kind !== "stage")
|
|
6247
|
+
return;
|
|
6248
|
+
stages.push(checkpoint);
|
|
6249
|
+
const key = durableStageGroupKey(checkpoint);
|
|
6250
|
+
this.groupKeys.set(checkpoint, key);
|
|
6251
|
+
append(this.groups, key, checkpoint);
|
|
6252
|
+
const runId = checkpoint.topology?.run?.runId;
|
|
6253
|
+
if (runId !== undefined)
|
|
6254
|
+
append(this.stagesByRunId, runId, checkpoint);
|
|
6255
|
+
if (checkpoint.output !== undefined)
|
|
6256
|
+
append(this.outputsByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6257
|
+
const boundary = checkpoint.topology?.boundary;
|
|
6258
|
+
if (boundary === undefined)
|
|
6259
|
+
return;
|
|
6260
|
+
append(this.boundariesByReplayKey, checkpoint.replayKey, checkpoint);
|
|
6261
|
+
append(this.boundariesByStageId, checkpoint.topology.stageId, checkpoint);
|
|
6262
|
+
append(this.boundariesByChildRunId, boundary.child.runId, checkpoint);
|
|
6263
|
+
});
|
|
6264
|
+
this.stages = stages;
|
|
6265
|
+
}
|
|
6266
|
+
sequence(checkpoint) {
|
|
6267
|
+
return (this.positions.get(checkpoint) ?? -1) + 1;
|
|
6268
|
+
}
|
|
6269
|
+
stageGroups() {
|
|
6270
|
+
return this.groups.values();
|
|
6271
|
+
}
|
|
6272
|
+
boundaryLifecycle(start, requireTerminal) {
|
|
6273
|
+
const topology = start.topology;
|
|
6274
|
+
const childRunId = topology?.boundary?.child.runId;
|
|
6275
|
+
const related = this.inCheckpointOrder([
|
|
6276
|
+
this.boundariesByReplayKey.get(start.replayKey),
|
|
6277
|
+
topology === undefined ? undefined : this.boundariesByStageId.get(topology.stageId),
|
|
6278
|
+
childRunId === undefined ? undefined : this.boundariesByChildRunId.get(childRunId)
|
|
6279
|
+
]);
|
|
6280
|
+
return resolveBoundaryLifecycleFromRecords(start, { related, outputRecords: () => this.outputsByReplayKey.get(start.replayKey) ?? [] }, requireTerminal);
|
|
6281
|
+
}
|
|
6282
|
+
directChildRecords(childRunId) {
|
|
6283
|
+
return {
|
|
6284
|
+
claiming: this.stagesByRunId.get(childRunId) ?? [],
|
|
6285
|
+
direct: (claiming) => this.inCheckpointOrder([...new Set(claiming.map((stage) => this.groupKeys.get(stage)))].map((key) => this.groups.get(key)))
|
|
6286
|
+
};
|
|
6287
|
+
}
|
|
6288
|
+
childScopedEvidenceExists(boundaryReplayKey, childRunId) {
|
|
6289
|
+
this.evidence ??= scopedEvidenceIndex(this.checkpoints);
|
|
6290
|
+
return this.evidence.runIds.has(childRunId) || hasPrefix(this.evidence.identities, `${boundaryReplayKey}:`);
|
|
6291
|
+
}
|
|
6292
|
+
inCheckpointOrder(lists) {
|
|
6293
|
+
const present = lists.filter((list) => list !== undefined);
|
|
6294
|
+
if (present.length === 1)
|
|
6295
|
+
return present[0];
|
|
6296
|
+
return [...new Set(present.flat())].sort((left, right) => this.sequence(left) - this.sequence(right));
|
|
6297
|
+
}
|
|
6298
|
+
}
|
|
6299
|
+
function scopedEvidenceIndex(checkpoints) {
|
|
6300
|
+
const runIds = new Set;
|
|
6301
|
+
const identities = [];
|
|
6302
|
+
for (const checkpoint of checkpoints) {
|
|
6303
|
+
identities.push(checkpoint.checkpointId, checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey);
|
|
6304
|
+
if (checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId !== undefined) {
|
|
6305
|
+
runIds.add(checkpoint.topology.run.runId);
|
|
6306
|
+
}
|
|
6307
|
+
}
|
|
6308
|
+
identities.sort(compareCodeUnits);
|
|
6309
|
+
return { runIds, identities };
|
|
6310
|
+
}
|
|
6311
|
+
function hasPrefix(sorted, prefix) {
|
|
6312
|
+
let low = 0;
|
|
6313
|
+
let high = sorted.length;
|
|
6314
|
+
while (low < high) {
|
|
6315
|
+
const middle = low + high >>> 1;
|
|
6316
|
+
if (compareCodeUnits(sorted[middle], prefix) < 0)
|
|
6317
|
+
low = middle + 1;
|
|
6318
|
+
else
|
|
6319
|
+
high = middle;
|
|
6320
|
+
}
|
|
6321
|
+
return low < sorted.length && sorted[low].startsWith(prefix);
|
|
6322
|
+
}
|
|
6323
|
+
function compareCodeUnits(left, right) {
|
|
6324
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
6325
|
+
}
|
|
6326
|
+
function append(map, key, value2) {
|
|
6327
|
+
const list = map.get(key);
|
|
6328
|
+
if (list === undefined)
|
|
6329
|
+
map.set(key, [value2]);
|
|
6330
|
+
else
|
|
6331
|
+
list.push(value2);
|
|
6332
|
+
}
|
|
6333
|
+
var init_completed_catalog_index = __esm(() => {
|
|
6334
|
+
init_boundary_lifecycle();
|
|
6335
|
+
init_stage_topology_validation();
|
|
6336
|
+
});
|
|
6337
|
+
|
|
5758
6338
|
// dist/builtin/workflows/src/durable/completed-catalog-stage-groups.ts
|
|
5759
|
-
function validBoundaryRecordSet(
|
|
5760
|
-
const
|
|
5761
|
-
const boundaries = stages.filter((item) => item.topology?.boundary !== undefined);
|
|
6339
|
+
function validBoundaryRecordSet(index, strict) {
|
|
6340
|
+
const boundaries = index.stages.filter((item) => item.topology?.boundary !== undefined);
|
|
5762
6341
|
const startsByReplay = new Map;
|
|
5763
6342
|
for (const checkpoint of boundaries) {
|
|
5764
6343
|
if (checkpoint.topology.boundary.event !== "start")
|
|
@@ -5773,7 +6352,7 @@ function validBoundaryRecordSet(checkpoints, strict) {
|
|
|
5773
6352
|
return false;
|
|
5774
6353
|
return [...startsByReplay.values()].every((starts) => {
|
|
5775
6354
|
const start = starts[0];
|
|
5776
|
-
if (
|
|
6355
|
+
if ("error" in index.boundaryLifecycle(start, strict))
|
|
5777
6356
|
return false;
|
|
5778
6357
|
return start.topology.boundary.invocationFingerprint !== undefined;
|
|
5779
6358
|
});
|
|
@@ -5837,8 +6416,8 @@ function compareDraftSourceOrder(left, right) {
|
|
|
5837
6416
|
function runTopologyFor(drafts, tools) {
|
|
5838
6417
|
return drafts.find((draft) => draft.topology?.run !== undefined)?.topology?.run ?? tools.find((checkpoint) => checkpoint.topology?.run !== undefined)?.topology?.run;
|
|
5839
6418
|
}
|
|
5840
|
-
function validateRunGroups(grouped, rootRunId,
|
|
5841
|
-
const toolsFor = (runId) =>
|
|
6419
|
+
function validateRunGroups(grouped, rootRunId, toolsByRun, index, allowPendingChildren = false) {
|
|
6420
|
+
const toolsFor = (runId) => toolsByRun.get(runId) ?? [];
|
|
5842
6421
|
const owners = new Map;
|
|
5843
6422
|
const syntheticChildIds = new Set;
|
|
5844
6423
|
for (const [parentRunId, drafts] of grouped) {
|
|
@@ -5849,12 +6428,12 @@ function validateRunGroups(grouped, rootRunId, tools, checkpoints, allowPendingC
|
|
|
5849
6428
|
if (childRunId === parentRunId || owners.has(childRunId))
|
|
5850
6429
|
return false;
|
|
5851
6430
|
if (!grouped.has(childRunId)) {
|
|
5852
|
-
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !childScopedEvidenceExists(
|
|
6431
|
+
if (allowPendingChildren && isPendingChildBoundary(draft, parentRunId, childRunId, rootRunId) && !index.childScopedEvidenceExists(draft.replayKey, childRunId)) {
|
|
5853
6432
|
syntheticChildIds.add(childRunId);
|
|
5854
6433
|
owners.set(childRunId, draft);
|
|
5855
6434
|
continue;
|
|
5856
6435
|
}
|
|
5857
|
-
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || childScopedEvidenceExists(
|
|
6436
|
+
if (!isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, childRunId))
|
|
5858
6437
|
return false;
|
|
5859
6438
|
syntheticChildIds.add(childRunId);
|
|
5860
6439
|
owners.set(childRunId, draft);
|
|
@@ -5895,13 +6474,6 @@ function isSyntheticExitedChild(draft, parentRunId, childRunId, rootRunId) {
|
|
|
5895
6474
|
const boundary = draft.topology?.boundary;
|
|
5896
6475
|
return child?.runId === childRunId && child.exited === true && boundary?.event === "terminal" && boundary.status === "completed" && boundary.child.runId === childRunId && boundary.child.parentRunId === parentRunId && boundary.child.parentStageId === draft.topology?.stageId && boundary.child.rootRunId === rootRunId;
|
|
5897
6476
|
}
|
|
5898
|
-
function childScopedEvidenceExists(checkpoints, boundaryReplayKey, childRunId) {
|
|
5899
|
-
const prefix = `${boundaryReplayKey}:`;
|
|
5900
|
-
return checkpoints.some((checkpoint) => {
|
|
5901
|
-
const lookupIdentity = checkpoint.kind === "tool" ? checkpoint.argsHash : checkpoint.kind === "ui" ? checkpoint.promptHash : checkpoint.replayKey;
|
|
5902
|
-
return checkpoint.checkpointId.startsWith(prefix) || lookupIdentity.startsWith(prefix) || checkpoint.kind !== "ui" && checkpoint.topology?.run?.runId === childRunId;
|
|
5903
|
-
});
|
|
5904
|
-
}
|
|
5905
6477
|
function retainReachableRunGroups(grouped, rootRunId) {
|
|
5906
6478
|
const reachable = new Set([rootRunId]);
|
|
5907
6479
|
const settled = new Set;
|
|
@@ -5942,14 +6514,6 @@ function childRunIdFromDraft(draft) {
|
|
|
5942
6514
|
return;
|
|
5943
6515
|
return workflowChildFromDraft(draft)?.runId ?? boundary?.child.runId;
|
|
5944
6516
|
}
|
|
5945
|
-
function boundaryOwner(grouped, childRunId) {
|
|
5946
|
-
for (const drafts of grouped.values()) {
|
|
5947
|
-
const owner = drafts.find((draft) => childRunIdFromDraft(draft) === childRunId);
|
|
5948
|
-
if (owner !== undefined)
|
|
5949
|
-
return owner;
|
|
5950
|
-
}
|
|
5951
|
-
return;
|
|
5952
|
-
}
|
|
5953
6517
|
function childRunStatus(owner) {
|
|
5954
6518
|
const child = workflowChildFromDraft(owner);
|
|
5955
6519
|
if (child !== undefined)
|
|
@@ -5957,15 +6521,15 @@ function childRunStatus(owner) {
|
|
|
5957
6521
|
const status = owner.topology?.boundary?.status;
|
|
5958
6522
|
return status === "failed" || status === "skipped" || status === "completed" ? status : "running";
|
|
5959
6523
|
}
|
|
5960
|
-
function mergeStageGroup(records,
|
|
6524
|
+
function mergeStageGroup(records, index) {
|
|
5961
6525
|
let merged;
|
|
5962
6526
|
for (const record of records)
|
|
5963
|
-
merged = mergeStageDraft(merged, record,
|
|
6527
|
+
merged = mergeStageDraft(merged, record, index.sequence(record));
|
|
5964
6528
|
const draft = merged;
|
|
5965
6529
|
const start = records.find((record) => record.topology?.boundary?.event === "start");
|
|
5966
6530
|
if (start === undefined)
|
|
5967
6531
|
return draft;
|
|
5968
|
-
const resolved =
|
|
6532
|
+
const resolved = index.boundaryLifecycle(start, false);
|
|
5969
6533
|
if ("error" in resolved)
|
|
5970
6534
|
return draft;
|
|
5971
6535
|
const terminal = resolved.lifecycle.latestTerminal;
|
|
@@ -6028,9 +6592,7 @@ function workflowChildFromDraft(draft) {
|
|
|
6028
6592
|
...typeof child.exitReason === "string" ? { exitReason: child.exitReason } : {}
|
|
6029
6593
|
};
|
|
6030
6594
|
}
|
|
6031
|
-
var init_completed_catalog_stage_groups =
|
|
6032
|
-
init_boundary_lifecycle();
|
|
6033
|
-
});
|
|
6595
|
+
var init_completed_catalog_stage_groups = () => {};
|
|
6034
6596
|
|
|
6035
6597
|
// dist/builtin/workflows/src/shared/abort.ts
|
|
6036
6598
|
function raceAbort(operation, signal) {
|
|
@@ -6564,13 +7126,19 @@ function registerRunJournalVersions(journal, versions) {
|
|
|
6564
7126
|
function runJournalVersions(journal) {
|
|
6565
7127
|
return versionReaders.get(journal);
|
|
6566
7128
|
}
|
|
7129
|
+
function registerRunJournalAppendMany(journal, appendMany) {
|
|
7130
|
+
appendManyWriters.set(journal, appendMany);
|
|
7131
|
+
}
|
|
7132
|
+
function runJournalAppendMany(journal) {
|
|
7133
|
+
return appendManyWriters.get(journal);
|
|
7134
|
+
}
|
|
6567
7135
|
function registerSqlPoolStorage(journal, storage) {
|
|
6568
7136
|
poolStorage.set(journal, storage);
|
|
6569
7137
|
}
|
|
6570
7138
|
function sqlPoolStorage(journal) {
|
|
6571
7139
|
return poolStorage.get(journal);
|
|
6572
7140
|
}
|
|
6573
|
-
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, poolStorage;
|
|
7141
|
+
var JOURNAL_READ_PAGE_SIZE = 256, NON_SNAPSHOT_EVENT_TYPES, versionReaders, appendManyWriters, poolStorage;
|
|
6574
7142
|
var init_internal = __esm(() => {
|
|
6575
7143
|
init_types();
|
|
6576
7144
|
NON_SNAPSHOT_EVENT_TYPES = [
|
|
@@ -6579,6 +7147,7 @@ var init_internal = __esm(() => {
|
|
|
6579
7147
|
RUN_EVENT_TYPES.messageDelivered
|
|
6580
7148
|
];
|
|
6581
7149
|
versionReaders = new WeakMap;
|
|
7150
|
+
appendManyWriters = new WeakMap;
|
|
6582
7151
|
poolStorage = new WeakMap;
|
|
6583
7152
|
});
|
|
6584
7153
|
|
|
@@ -6872,9 +7441,19 @@ function* invoke(action) {
|
|
|
6872
7441
|
} catch (error) {
|
|
6873
7442
|
if (error instanceof JournalError || error instanceof StorageError)
|
|
6874
7443
|
throw error;
|
|
6875
|
-
throw
|
|
7444
|
+
throw customJournalUnavailable(error);
|
|
6876
7445
|
}
|
|
6877
7446
|
}
|
|
7447
|
+
function customJournalUnavailable(failure) {
|
|
7448
|
+
const cause = failure instanceof Error ? failure : new Error(String(failure));
|
|
7449
|
+
const code = hasErrorCode(cause) ? cause.code : undefined;
|
|
7450
|
+
const detail = code === undefined ? cause.name : `${cause.name} (${String(code)})`;
|
|
7451
|
+
return new StorageUnavailableError(CUSTOM_JOURNAL, cause, detail);
|
|
7452
|
+
}
|
|
7453
|
+
function hasErrorCode(error) {
|
|
7454
|
+
const code = error.code;
|
|
7455
|
+
return typeof code === "string" || typeof code === "number";
|
|
7456
|
+
}
|
|
6878
7457
|
async function execute(operation) {
|
|
6879
7458
|
let step = operation.next();
|
|
6880
7459
|
while (!step.done) {
|
|
@@ -6902,6 +7481,8 @@ class JournalDurableBackend {
|
|
|
6902
7481
|
executions = new Map;
|
|
6903
7482
|
executionContext = new AsyncLocalStorage;
|
|
6904
7483
|
pending = new Map;
|
|
7484
|
+
reconciledPendingStages = new Map;
|
|
7485
|
+
appendBatches = new Map;
|
|
6905
7486
|
catalogQueue = Symbol("catalog");
|
|
6906
7487
|
lost = new Map;
|
|
6907
7488
|
prompts = new Map;
|
|
@@ -6934,7 +7515,7 @@ class JournalDurableBackend {
|
|
|
6934
7515
|
await backend.close();
|
|
6935
7516
|
if (error instanceof JournalError || error instanceof StorageError || error instanceof DurableNestedTopologyError)
|
|
6936
7517
|
throw error;
|
|
6937
|
-
throw
|
|
7518
|
+
throw customJournalUnavailable(error);
|
|
6938
7519
|
}
|
|
6939
7520
|
}
|
|
6940
7521
|
serialize(runId, action, requiresActiveExecution = true) {
|
|
@@ -6949,6 +7530,8 @@ class JournalDurableBackend {
|
|
|
6949
7530
|
return this.enqueue(runId, action, admitted);
|
|
6950
7531
|
}
|
|
6951
7532
|
enqueue(runId, action, admitted = this.mutationContext.getStore() ?? new Set) {
|
|
7533
|
+
if (typeof runId === "string")
|
|
7534
|
+
this.appendBatches.delete(runId);
|
|
6952
7535
|
const prior = this.pending.get(runId) ?? Promise.resolve();
|
|
6953
7536
|
const result = prior.then(() => this.mutationContext.run(admitted, () => execute(action())));
|
|
6954
7537
|
const settled = result.then(() => {}, () => {});
|
|
@@ -6963,6 +7546,7 @@ class JournalDurableBackend {
|
|
|
6963
7546
|
if (entry.seq <= (this.sequences.get(entry.runId) ?? 0))
|
|
6964
7547
|
return;
|
|
6965
7548
|
const event = entry.event;
|
|
7549
|
+
let raisedCount;
|
|
6966
7550
|
if (event.type === RUN_EVENT_TYPES.checkpointRecorded) {
|
|
6967
7551
|
const checkpoint = event.payload;
|
|
6968
7552
|
validateCheckpointInvocationFingerprint(checkpoint);
|
|
@@ -6971,7 +7555,10 @@ class JournalDurableBackend {
|
|
|
6971
7555
|
this.projection.archivePendingStageMessage(entry.runId, event.payload.message);
|
|
6972
7556
|
} else if (event.type === RUN_EVENT_TYPES.messageDelivered) {
|
|
6973
7557
|
const payload = event.payload;
|
|
7558
|
+
const countVersion = this.projection.pendingStageDeliveryCountVersion(entry.runId);
|
|
6974
7559
|
this.projection.archivePendingStageDeliveryReceipt(entry.runId, payload.messageId, payload.delivery, payload.accounting);
|
|
7560
|
+
if (payload.accounting !== undefined && this.projection.pendingStageDeliveryCountVersion(entry.runId) !== countVersion)
|
|
7561
|
+
raisedCount = { messageId: payload.messageId, messageRunId: payload.accounting.messageRunId };
|
|
6975
7562
|
} else if (event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased) {
|
|
6976
7563
|
this.projection.applyWorkflowSnapshot(decodeHandle(event.payload.handle, this.projection.getWorkflow(entry.runId)?.pendingStageMessages));
|
|
6977
7564
|
this.prompts.set(entry.runId, new PromptReservationState(event.payload.handle.pendingPrompts, event.payload.prompts));
|
|
@@ -6982,20 +7569,80 @@ class JournalDurableBackend {
|
|
|
6982
7569
|
if (event.type === RUN_EVENT_TYPES.runStarted || previous?.pendingPrompts !== handle.pendingPrompts)
|
|
6983
7570
|
this.prompts.delete(entry.runId);
|
|
6984
7571
|
}
|
|
6985
|
-
|
|
6986
|
-
if (handle?.pendingStageMessages !== undefined) {
|
|
6987
|
-
const messages = handle.pendingStageMessages.map((message) => {
|
|
6988
|
-
if (message.sticky !== true)
|
|
6989
|
-
return message;
|
|
6990
|
-
const count = this.projection.getPendingStageMessageDeliveryCount(entry.runId, message.id, message.runId);
|
|
6991
|
-
return count > (message.deliveryCount ?? message.deliveries?.length ?? 0) ? { ...message, deliveryCount: count } : message;
|
|
6992
|
-
});
|
|
6993
|
-
if (messages.some((message, index) => message !== handle.pendingStageMessages?.[index]))
|
|
6994
|
-
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: messages });
|
|
6995
|
-
}
|
|
7572
|
+
this.reconcileFoldedDeliveryCounts(entry.runId, raisedCount);
|
|
6996
7573
|
this.sequences.set(entry.runId, entry.seq);
|
|
6997
7574
|
this.foldedAt.set(entry.runId, entry.at);
|
|
6998
7575
|
}
|
|
7576
|
+
reconcileFoldedDeliveryCounts(runId, raisedCount) {
|
|
7577
|
+
const handle = this.projection.getWorkflow(runId);
|
|
7578
|
+
const list = handle?.pendingStageMessages;
|
|
7579
|
+
if (handle === undefined || list === undefined) {
|
|
7580
|
+
this.reconciledPendingStages.delete(runId);
|
|
7581
|
+
return;
|
|
7582
|
+
}
|
|
7583
|
+
let state = this.reconciledPendingStages.get(runId);
|
|
7584
|
+
let next;
|
|
7585
|
+
const raise = (index, count) => {
|
|
7586
|
+
const message = (next ?? list)[index];
|
|
7587
|
+
if (count <= (message.deliveryCount ?? message.deliveries?.length ?? 0))
|
|
7588
|
+
return;
|
|
7589
|
+
next ??= [...list];
|
|
7590
|
+
next[index] = { ...message, deliveryCount: count };
|
|
7591
|
+
};
|
|
7592
|
+
const reconcile = (index) => {
|
|
7593
|
+
const message = list[index];
|
|
7594
|
+
if (message.sticky !== true)
|
|
7595
|
+
return;
|
|
7596
|
+
raise(index, this.projection.getPendingStageMessageDeliveryCount(runId, message.id, message.runId));
|
|
7597
|
+
};
|
|
7598
|
+
if (state?.list !== list) {
|
|
7599
|
+
const previous = state?.list ?? [];
|
|
7600
|
+
let aligned = list.length >= previous.length;
|
|
7601
|
+
const changed = [];
|
|
7602
|
+
for (let index = 0;index < list.length; index++) {
|
|
7603
|
+
const message = list[index];
|
|
7604
|
+
if (previous[index] === message)
|
|
7605
|
+
continue;
|
|
7606
|
+
if (index < previous.length && previous[index].id !== message.id)
|
|
7607
|
+
aligned = false;
|
|
7608
|
+
changed.push(index);
|
|
7609
|
+
}
|
|
7610
|
+
const reconciled = aligned ? undefined : new Set(previous);
|
|
7611
|
+
for (const index of changed)
|
|
7612
|
+
if (!reconciled?.has(list[index]))
|
|
7613
|
+
reconcile(index);
|
|
7614
|
+
const positions = aligned ? state?.positions : undefined;
|
|
7615
|
+
if (positions !== undefined) {
|
|
7616
|
+
for (let index = previous.length;index < list.length; index++) {
|
|
7617
|
+
const indexes = positions.get(list[index].id);
|
|
7618
|
+
if (indexes === undefined)
|
|
7619
|
+
positions.set(list[index].id, [index]);
|
|
7620
|
+
else
|
|
7621
|
+
indexes.push(index);
|
|
7622
|
+
}
|
|
7623
|
+
}
|
|
7624
|
+
state = { list, positions };
|
|
7625
|
+
}
|
|
7626
|
+
if (raisedCount !== undefined) {
|
|
7627
|
+
const count = this.projection.getPendingStageMessageDeliveryCount(runId, raisedCount.messageId, raisedCount.messageRunId);
|
|
7628
|
+
state.positions ??= messagePositions(list);
|
|
7629
|
+
for (const index of state.positions.get(raisedCount.messageId) ?? []) {
|
|
7630
|
+
const message = list[index];
|
|
7631
|
+
if (message.sticky === true && message.runId === raisedCount.messageRunId)
|
|
7632
|
+
raise(index, count);
|
|
7633
|
+
}
|
|
7634
|
+
}
|
|
7635
|
+
if (next === undefined) {
|
|
7636
|
+
this.reconciledPendingStages.set(runId, state);
|
|
7637
|
+
return;
|
|
7638
|
+
}
|
|
7639
|
+
this.projection.applyWorkflowSnapshot({ ...handle, pendingStageMessages: next });
|
|
7640
|
+
const applied = this.projection.getWorkflow(runId)?.pendingStageMessages ?? next;
|
|
7641
|
+
this.reconciledPendingStages.set(runId, {
|
|
7642
|
+
list: applied,
|
|
7643
|
+
positions: applied.length === list.length ? state.positions : undefined
|
|
7644
|
+
});
|
|
7645
|
+
}
|
|
6999
7646
|
summarize(runId, tail) {
|
|
7000
7647
|
const event = tail.snapshot.event;
|
|
7001
7648
|
const handle = event.type === RUN_EVENT_TYPES.promptAsked || event.type === RUN_EVENT_TYPES.promptReleased ? event.payload.handle : event.payload;
|
|
@@ -7019,6 +7666,7 @@ class JournalDurableBackend {
|
|
|
7019
7666
|
*rebuild(runId) {
|
|
7020
7667
|
const entries = yield* this.entries(runId);
|
|
7021
7668
|
this.projection.deleteWorkflow(runId);
|
|
7669
|
+
this.reconciledPendingStages.delete(runId);
|
|
7022
7670
|
this.sequences.delete(runId);
|
|
7023
7671
|
this.foldedAt.delete(runId);
|
|
7024
7672
|
this.summarized.delete(runId);
|
|
@@ -7114,9 +7762,9 @@ class JournalDurableBackend {
|
|
|
7114
7762
|
}
|
|
7115
7763
|
yield* this.write(runId, lease, event, options);
|
|
7116
7764
|
}
|
|
7117
|
-
*write(runId, lease, event, options) {
|
|
7765
|
+
*write(runId, lease, event, options, context = (action) => action()) {
|
|
7118
7766
|
try {
|
|
7119
|
-
const outcome = yield* invoke(() => this.journal.append(lease, event, options));
|
|
7767
|
+
const outcome = yield* invoke(() => context(() => this.journal.append(lease, event, options)));
|
|
7120
7768
|
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
7121
7769
|
if (outcome.entry.seq > nextSeq) {
|
|
7122
7770
|
for (const entry of yield* this.entries(runId, nextSeq))
|
|
@@ -7300,13 +7948,80 @@ class JournalDurableBackend {
|
|
|
7300
7948
|
return true;
|
|
7301
7949
|
}
|
|
7302
7950
|
async archivePendingStageMessage(runId, message) {
|
|
7303
|
-
await this.
|
|
7951
|
+
await this.queueAppend(runId, { type: RUN_EVENT_TYPES.messageSettled, payload: { message } }, JSON.stringify([RUN_EVENT_TYPES.messageSettled, message.runId, message.id]));
|
|
7304
7952
|
}
|
|
7305
7953
|
async archivePendingStageDeliveryReceipt(runId, messageId, delivery, accounting) {
|
|
7306
|
-
await this.
|
|
7954
|
+
await this.queueAppend(runId, {
|
|
7307
7955
|
type: RUN_EVENT_TYPES.messageDelivered,
|
|
7308
7956
|
payload: { messageId, delivery, ...accounting === undefined ? {} : { accounting } }
|
|
7309
|
-
})
|
|
7957
|
+
});
|
|
7958
|
+
}
|
|
7959
|
+
queueAppend(runId, event, key) {
|
|
7960
|
+
try {
|
|
7961
|
+
this.ensureOpen();
|
|
7962
|
+
this.assertExecutionActive();
|
|
7963
|
+
} catch (error) {
|
|
7964
|
+
return Promise.reject(error);
|
|
7965
|
+
}
|
|
7966
|
+
const open = this.appendBatches.get(runId);
|
|
7967
|
+
const batch = open !== undefined && open.length < APPEND_BATCH_LIMIT ? open : [];
|
|
7968
|
+
const queued = new Promise((resolve, reject) => {
|
|
7969
|
+
batch.push({
|
|
7970
|
+
event,
|
|
7971
|
+
...key === undefined ? {} : { key },
|
|
7972
|
+
context: AsyncLocalStorage.snapshot(),
|
|
7973
|
+
resolve,
|
|
7974
|
+
reject
|
|
7975
|
+
});
|
|
7976
|
+
});
|
|
7977
|
+
if (batch !== open) {
|
|
7978
|
+
this.serialize(runId, () => this.appendBatch(runId, batch)).catch((error) => {
|
|
7979
|
+
for (const append of batch)
|
|
7980
|
+
append.reject(error);
|
|
7981
|
+
});
|
|
7982
|
+
this.appendBatches.set(runId, batch);
|
|
7983
|
+
}
|
|
7984
|
+
return queued;
|
|
7985
|
+
}
|
|
7986
|
+
*appendBatch(runId, batch) {
|
|
7987
|
+
if (this.appendBatches.get(runId) === batch)
|
|
7988
|
+
this.appendBatches.delete(runId);
|
|
7989
|
+
const appendMany = runJournalAppendMany(this.journal);
|
|
7990
|
+
if (appendMany === undefined || batch.length === 1) {
|
|
7991
|
+
for (const append of batch) {
|
|
7992
|
+
try {
|
|
7993
|
+
const lease = yield* this.ownership(runId);
|
|
7994
|
+
yield* this.write(runId, lease, append.event, append.key === undefined ? undefined : { key: append.key }, append.context);
|
|
7995
|
+
append.resolve();
|
|
7996
|
+
} catch (error) {
|
|
7997
|
+
append.reject(error);
|
|
7998
|
+
}
|
|
7999
|
+
}
|
|
8000
|
+
return;
|
|
8001
|
+
}
|
|
8002
|
+
let settled = 0;
|
|
8003
|
+
try {
|
|
8004
|
+
const lease = yield* this.ownership(runId);
|
|
8005
|
+
const results = yield* invoke(() => appendMany(lease, batch));
|
|
8006
|
+
for (const result of results) {
|
|
8007
|
+
const append = batch[settled];
|
|
8008
|
+
if (result.status === "rejected") {
|
|
8009
|
+
append.reject(yield* this.mapOwnership(result.reason, runId));
|
|
8010
|
+
} else {
|
|
8011
|
+
const nextSeq = (this.sequences.get(runId) ?? 0) + 1;
|
|
8012
|
+
if (result.value.entry.seq > nextSeq)
|
|
8013
|
+
for (const entry of yield* this.entries(runId, nextSeq))
|
|
8014
|
+
this.fold(entry);
|
|
8015
|
+
this.fold(result.value.entry);
|
|
8016
|
+
append.resolve();
|
|
8017
|
+
}
|
|
8018
|
+
settled++;
|
|
8019
|
+
}
|
|
8020
|
+
} catch (error) {
|
|
8021
|
+
const failure = error instanceof Error ? yield* this.mapOwnership(error, runId) : error;
|
|
8022
|
+
for (const append of batch.slice(settled))
|
|
8023
|
+
append.reject(failure);
|
|
8024
|
+
}
|
|
7310
8025
|
}
|
|
7311
8026
|
getToolOutput(...args) {
|
|
7312
8027
|
return this.projection.getToolOutput(...args);
|
|
@@ -7326,6 +8041,9 @@ class JournalDurableBackend {
|
|
|
7326
8041
|
listCheckpoints(runId) {
|
|
7327
8042
|
return this.projection.listCheckpoints(runId);
|
|
7328
8043
|
}
|
|
8044
|
+
nestedSnapshotVersion(runId) {
|
|
8045
|
+
return this.projection.nestedSnapshotVersion(runId);
|
|
8046
|
+
}
|
|
7329
8047
|
getWorkflow(runId) {
|
|
7330
8048
|
return this.projection.getWorkflow(runId);
|
|
7331
8049
|
}
|
|
@@ -7350,6 +8068,12 @@ class JournalDurableBackend {
|
|
|
7350
8068
|
getPendingStageMessageDeliveryCount(...args) {
|
|
7351
8069
|
return this.projection.getPendingStageMessageDeliveryCount(...args);
|
|
7352
8070
|
}
|
|
8071
|
+
pendingStageDeliveryReceiptChanges(...args) {
|
|
8072
|
+
return this.projection.pendingStageDeliveryReceiptChanges(...args);
|
|
8073
|
+
}
|
|
8074
|
+
pendingStageDeliveryCountVersion(runId) {
|
|
8075
|
+
return this.projection.pendingStageDeliveryCountVersion(runId);
|
|
8076
|
+
}
|
|
7353
8077
|
hasPendingStageDeliveryReceipt(...args) {
|
|
7354
8078
|
return this.projection.hasPendingStageDeliveryReceipt(...args);
|
|
7355
8079
|
}
|
|
@@ -7434,6 +8158,7 @@ class JournalDurableBackend {
|
|
|
7434
8158
|
this.foldedAt.delete(runId);
|
|
7435
8159
|
this.summarized.delete(runId);
|
|
7436
8160
|
this.prompts.delete(runId);
|
|
8161
|
+
this.reconciledPendingStages.delete(runId);
|
|
7437
8162
|
this.projection.deleteWorkflow(runId);
|
|
7438
8163
|
}
|
|
7439
8164
|
async deleteWorkflow(runId) {
|
|
@@ -7616,8 +8341,24 @@ class JournalDurableBackend {
|
|
|
7616
8341
|
}
|
|
7617
8342
|
}
|
|
7618
8343
|
}
|
|
8344
|
+
function messagePositions(list) {
|
|
8345
|
+
const positions = new Map;
|
|
8346
|
+
list.forEach((message, index) => {
|
|
8347
|
+
const indexes = positions.get(message.id);
|
|
8348
|
+
if (indexes === undefined)
|
|
8349
|
+
positions.set(message.id, [index]);
|
|
8350
|
+
else
|
|
8351
|
+
indexes.push(index);
|
|
8352
|
+
});
|
|
8353
|
+
return positions;
|
|
8354
|
+
}
|
|
7619
8355
|
function identity(message) {
|
|
7620
|
-
|
|
8356
|
+
let key = entryIdentity.get(message);
|
|
8357
|
+
if (key === undefined) {
|
|
8358
|
+
key = JSON.stringify([message.runId, message.id]);
|
|
8359
|
+
entryIdentity.set(message, key);
|
|
8360
|
+
}
|
|
8361
|
+
return key;
|
|
7621
8362
|
}
|
|
7622
8363
|
function uniqueByIdentity(messages) {
|
|
7623
8364
|
const result = new Map;
|
|
@@ -7791,7 +8532,7 @@ function decodePendingStageDeliveries(previous, messages) {
|
|
|
7791
8532
|
function hasPendingStageDeliveryDelta(messages) {
|
|
7792
8533
|
return messages?.some(isDeltaEncoded) === true;
|
|
7793
8534
|
}
|
|
7794
|
-
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, processExitCleanups, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryJson;
|
|
8535
|
+
var PROCESS_HOLDER_ID, CHANGE_APPEND_ATTEMPTS = 4, APPEND_BATCH_LIMIT = 256, processExitCleanups, CUSTOM_JOURNAL, JournalOwnershipError, JournalBackendClosedError, JournalOperationAbortedError, JournalExecutionRetiredError, JournalMessageScopeError, entryIdentity, entryJson;
|
|
7795
8536
|
var init_journal_backend = __esm(() => {
|
|
7796
8537
|
init_workflow_instance_ownership_error();
|
|
7797
8538
|
init_backend();
|
|
@@ -7803,6 +8544,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7803
8544
|
init_types();
|
|
7804
8545
|
PROCESS_HOLDER_ID = randomUUID4();
|
|
7805
8546
|
processExitCleanups = new Set;
|
|
8547
|
+
CUSTOM_JOURNAL = { adapter: "custom", location: "custom journal" };
|
|
7806
8548
|
JournalOwnershipError = class JournalOwnershipError extends WorkflowInstanceOwnershipError {
|
|
7807
8549
|
runId;
|
|
7808
8550
|
holderId;
|
|
@@ -7849,6 +8591,7 @@ var init_journal_backend = __esm(() => {
|
|
|
7849
8591
|
this.name = "JournalMessageScopeError";
|
|
7850
8592
|
}
|
|
7851
8593
|
};
|
|
8594
|
+
entryIdentity = new WeakMap;
|
|
7852
8595
|
entryJson = new WeakMap;
|
|
7853
8596
|
});
|
|
7854
8597
|
|
|
@@ -7976,6 +8719,7 @@ class SqlRunJournal {
|
|
|
7976
8719
|
runVersions: () => this.runVersions(),
|
|
7977
8720
|
snapshotTail: (runId) => this.snapshotTail(runId)
|
|
7978
8721
|
});
|
|
8722
|
+
registerRunJournalAppendMany(this, (lease, appends) => this.appendMany(lease, appends));
|
|
7979
8723
|
registerSqlPoolStorage(this, {
|
|
7980
8724
|
transaction: (body) => this.operation(() => this.database.transaction(body)),
|
|
7981
8725
|
withAppendGuard: (guard, action) => this.appendGuard.run(guard, action),
|
|
@@ -8240,38 +8984,76 @@ class SqlRunJournal {
|
|
|
8240
8984
|
this.assertOpen();
|
|
8241
8985
|
validateText("runId", lease.runId);
|
|
8242
8986
|
validateText("holderId", lease.holderId);
|
|
8243
|
-
if (options?.key !== undefined)
|
|
8244
|
-
validateText("key", options.key);
|
|
8245
8987
|
const owned = { ...lease };
|
|
8246
|
-
const
|
|
8988
|
+
const prepared = this.prepare(event, options?.key, this.appendGuard.getStore());
|
|
8247
8989
|
const expected = options?.expectSeq;
|
|
8990
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
8991
|
+
await this.checkLease(tx, owned);
|
|
8992
|
+
return this.insert(tx, owned, prepared, expected);
|
|
8993
|
+
}));
|
|
8994
|
+
}
|
|
8995
|
+
async appendMany(lease, appends) {
|
|
8996
|
+
this.assertOpen();
|
|
8997
|
+
validateText("runId", lease.runId);
|
|
8998
|
+
validateText("holderId", lease.holderId);
|
|
8999
|
+
const owned = { ...lease };
|
|
9000
|
+
const prepared = appends.map((append) => {
|
|
9001
|
+
try {
|
|
9002
|
+
return this.prepare(append.event, append.key, append.context(() => this.appendGuard.getStore()));
|
|
9003
|
+
} catch (error) {
|
|
9004
|
+
return error;
|
|
9005
|
+
}
|
|
9006
|
+
});
|
|
9007
|
+
return this.operation(() => this.database.transaction(async (tx) => {
|
|
9008
|
+
await this.checkLease(tx, owned);
|
|
9009
|
+
const results = [];
|
|
9010
|
+
for (const append of prepared) {
|
|
9011
|
+
if (append instanceof Error) {
|
|
9012
|
+
results.push({ status: "rejected", reason: append });
|
|
9013
|
+
continue;
|
|
9014
|
+
}
|
|
9015
|
+
await tx.exec(`SAVEPOINT ${APPEND_SAVEPOINT}`);
|
|
9016
|
+
try {
|
|
9017
|
+
results.push({ status: "fulfilled", value: await this.insert(tx, owned, append) });
|
|
9018
|
+
} catch (error) {
|
|
9019
|
+
await tx.exec(`ROLLBACK TO ${APPEND_SAVEPOINT}`);
|
|
9020
|
+
results.push({ status: "rejected", reason: error });
|
|
9021
|
+
}
|
|
9022
|
+
await tx.exec(`RELEASE ${APPEND_SAVEPOINT}`);
|
|
9023
|
+
}
|
|
9024
|
+
return results;
|
|
9025
|
+
}));
|
|
9026
|
+
}
|
|
9027
|
+
prepare(event, key, appendGuard) {
|
|
9028
|
+
if (key !== undefined)
|
|
9029
|
+
validateText("key", key);
|
|
8248
9030
|
const type = event.type;
|
|
8249
9031
|
const payload = this.snapshotJson(event.payload);
|
|
8250
|
-
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ?
|
|
9032
|
+
const guard = type === RUN_EVENT_TYPES.messageQueued || type === RUN_EVENT_TYPES.messageDelivered || type === RUN_EVENT_TYPES.messageSettled ? appendGuard : undefined;
|
|
8251
9033
|
const status = type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2) ? JSON.parse(payload).status : undefined;
|
|
8252
9034
|
if (type === RUN_EVENT_TYPES.statusChanged && !(payload instanceof StorageUnavailableError2))
|
|
8253
9035
|
validateText("status", status);
|
|
8254
|
-
return
|
|
8255
|
-
|
|
8256
|
-
|
|
8257
|
-
|
|
8258
|
-
|
|
8259
|
-
|
|
8260
|
-
|
|
8261
|
-
|
|
8262
|
-
|
|
8263
|
-
|
|
8264
|
-
|
|
8265
|
-
|
|
8266
|
-
|
|
8267
|
-
|
|
8268
|
-
|
|
8269
|
-
|
|
8270
|
-
|
|
8271
|
-
|
|
8272
|
-
|
|
8273
|
-
|
|
8274
|
-
}
|
|
9036
|
+
return { key, type, payload, guard, status };
|
|
9037
|
+
}
|
|
9038
|
+
async insert(tx, owned, { key, type, payload, guard, status }, expected) {
|
|
9039
|
+
await guard?.(tx, owned.runId);
|
|
9040
|
+
if (key !== undefined) {
|
|
9041
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.key} = ?`, owned.runId, key);
|
|
9042
|
+
if (row)
|
|
9043
|
+
return { status: "exists", entry: entryFromRow(row) };
|
|
9044
|
+
}
|
|
9045
|
+
const actual = await this.sequence(tx, owned.runId);
|
|
9046
|
+
if (expected !== undefined && expected !== actual)
|
|
9047
|
+
throw new JournalCasConflictError(owned.runId, expected, actual);
|
|
9048
|
+
const seq = journalCounter(owned.runId, "seq", actual + 1, "overflow");
|
|
9049
|
+
if (payload instanceof StorageUnavailableError2)
|
|
9050
|
+
throw payload;
|
|
9051
|
+
await tx.run(`INSERT INTO ${this.tables.events} VALUES (?, ?, ?, ?, ?, ?, ${STORE_NOW})`, owned.runId, seq, owned.epoch, key ?? null, type, payload);
|
|
9052
|
+
if (status !== undefined)
|
|
9053
|
+
await tx.run(`UPDATE ${this.tables.runs} SET ${JOURNAL_COLUMNS.status} = ? WHERE ${JOURNAL_COLUMNS.runId} = ?`, status, owned.runId);
|
|
9054
|
+
const row = await tx.get(`SELECT * FROM ${this.tables.events} WHERE ${JOURNAL_COLUMNS.runId} = ? AND ${JOURNAL_COLUMNS.seq} = ?`, owned.runId, seq);
|
|
9055
|
+
await guard?.(tx, owned.runId);
|
|
9056
|
+
return { status: "appended", entry: entryFromRow(row) };
|
|
8275
9057
|
}
|
|
8276
9058
|
async* read(runId, fromSeq = 1) {
|
|
8277
9059
|
this.assertOpen();
|
|
@@ -8339,7 +9121,7 @@ class SqlRunJournal {
|
|
|
8339
9121
|
return this.closePromise;
|
|
8340
9122
|
}
|
|
8341
9123
|
}
|
|
8342
|
-
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS;
|
|
9124
|
+
var STORE_NOW = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", EVENT_COLUMNS, APPEND_SAVEPOINT = "journal_append";
|
|
8343
9125
|
var init_sql_run_journal = __esm(() => {
|
|
8344
9126
|
init_internal();
|
|
8345
9127
|
init_legacy_journal_tables();
|
|
@@ -8547,8 +9329,59 @@ function firstError(schema, value2) {
|
|
|
8547
9329
|
}
|
|
8548
9330
|
return;
|
|
8549
9331
|
}
|
|
9332
|
+
function certainlySerializable(root, rootMustBeObject) {
|
|
9333
|
+
if (rootMustBeObject && (root === null || typeof root !== "object" || Array.isArray(root)))
|
|
9334
|
+
return false;
|
|
9335
|
+
const pending = [root];
|
|
9336
|
+
const ancestors = new Set;
|
|
9337
|
+
const exits = [undefined];
|
|
9338
|
+
while (pending.length > 0) {
|
|
9339
|
+
const exit = exits.pop();
|
|
9340
|
+
const value2 = pending.pop();
|
|
9341
|
+
if (exit !== undefined) {
|
|
9342
|
+
ancestors.delete(exit);
|
|
9343
|
+
continue;
|
|
9344
|
+
}
|
|
9345
|
+
if (value2 === null || typeof value2 === "string" || typeof value2 === "boolean")
|
|
9346
|
+
continue;
|
|
9347
|
+
if (typeof value2 === "number") {
|
|
9348
|
+
if (!Number.isFinite(value2))
|
|
9349
|
+
return false;
|
|
9350
|
+
continue;
|
|
9351
|
+
}
|
|
9352
|
+
if (typeof value2 !== "object" || ancestors.has(value2))
|
|
9353
|
+
return false;
|
|
9354
|
+
if (Object.getOwnPropertySymbols(value2).length > 0)
|
|
9355
|
+
return false;
|
|
9356
|
+
const names = Object.getOwnPropertyNames(value2);
|
|
9357
|
+
let children;
|
|
9358
|
+
if (Array.isArray(value2)) {
|
|
9359
|
+
if (Object.getPrototypeOf(value2) !== Array.prototype || names.length !== value2.length + 1)
|
|
9360
|
+
return false;
|
|
9361
|
+
children = [];
|
|
9362
|
+
for (let index = 0;index < value2.length; index++)
|
|
9363
|
+
children.push(value2[index]);
|
|
9364
|
+
} else {
|
|
9365
|
+
const proto = Object.getPrototypeOf(value2);
|
|
9366
|
+
if (proto !== Object.prototype && proto !== null)
|
|
9367
|
+
return false;
|
|
9368
|
+
const keys = Object.keys(value2);
|
|
9369
|
+
if (keys.length !== names.length)
|
|
9370
|
+
return false;
|
|
9371
|
+
children = keys.map((key) => value2[key]);
|
|
9372
|
+
}
|
|
9373
|
+
ancestors.add(value2);
|
|
9374
|
+
pending.push(value2);
|
|
9375
|
+
exits.push(value2);
|
|
9376
|
+
for (const child of children) {
|
|
9377
|
+
pending.push(child);
|
|
9378
|
+
exits.push(undefined);
|
|
9379
|
+
}
|
|
9380
|
+
}
|
|
9381
|
+
return true;
|
|
9382
|
+
}
|
|
8550
9383
|
function workflowSerializableValidationError(value2, label) {
|
|
8551
|
-
if (Value.Check(workflowSerializableValueSchema, value2))
|
|
9384
|
+
if (certainlySerializable(value2, false) || Value.Check(workflowSerializableValueSchema, value2))
|
|
8552
9385
|
return;
|
|
8553
9386
|
const issue = firstError(workflowSerializableValueSchema, value2);
|
|
8554
9387
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -8557,7 +9390,7 @@ function workflowSerializableValidationError(value2, label) {
|
|
|
8557
9390
|
return `${label}${location} must be ${WORKFLOW_SERIALIZABLE_DESCRIPTION}, got ${workflowSerializableTypeName(offending)}`;
|
|
8558
9391
|
}
|
|
8559
9392
|
function workflowSerializableObjectValidationError(value2, label) {
|
|
8560
|
-
if (Value.Check(workflowSerializableObjectSchema, value2))
|
|
9393
|
+
if (certainlySerializable(value2, true) || Value.Check(workflowSerializableObjectSchema, value2))
|
|
8561
9394
|
return;
|
|
8562
9395
|
const issue = firstError(workflowSerializableObjectSchema, value2);
|
|
8563
9396
|
const issuePath = issue === undefined ? "" : formatInstancePath(issue.instancePath);
|
|
@@ -60991,39 +61824,406 @@ var init_pending_stage_route_readiness = __esm(() => {
|
|
|
60991
61824
|
processState3[routeOwnersKey] = owners;
|
|
60992
61825
|
});
|
|
60993
61826
|
|
|
60994
|
-
// dist/builtin/workflows/src/shared/sticky-
|
|
60995
|
-
function
|
|
60996
|
-
|
|
60997
|
-
|
|
60998
|
-
const
|
|
60999
|
-
|
|
61000
|
-
const
|
|
61001
|
-
|
|
61002
|
-
const
|
|
61003
|
-
|
|
61827
|
+
// dist/builtin/workflows/src/shared/sticky-delivery-state-bound.ts
|
|
61828
|
+
function boundStickyDeliveryStates(states, limit = STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT) {
|
|
61829
|
+
if (states.length === 0)
|
|
61830
|
+
return { deliveryStates: states };
|
|
61831
|
+
const counts = new Map(UNRESOLVED_STATES.map((state) => [state, 0]));
|
|
61832
|
+
counts.set("delivered", 0);
|
|
61833
|
+
for (const delivery of states)
|
|
61834
|
+
counts.set(delivery.state, (counts.get(delivery.state) ?? 0) + 1);
|
|
61835
|
+
const deliveryStateCounts = Object.fromEntries([...counts].filter(([, count]) => count > 0));
|
|
61836
|
+
if (states.length <= limit)
|
|
61837
|
+
return { deliveryStates: states, deliveryStateCounts };
|
|
61838
|
+
const exemplars = [];
|
|
61839
|
+
for (const state of UNRESOLVED_STATES) {
|
|
61840
|
+
for (const delivery of states) {
|
|
61841
|
+
if (exemplars.length === limit)
|
|
61842
|
+
break;
|
|
61843
|
+
if (delivery.state === state)
|
|
61844
|
+
exemplars.push(delivery);
|
|
61845
|
+
}
|
|
61846
|
+
}
|
|
61847
|
+
for (let index = states.length - 1;index >= 0 && exemplars.length < limit; index--) {
|
|
61848
|
+
const delivery = states[index];
|
|
61849
|
+
if (delivery.state === "delivered")
|
|
61850
|
+
exemplars.push(delivery);
|
|
61851
|
+
}
|
|
61852
|
+
return {
|
|
61853
|
+
deliveryStates: exemplars,
|
|
61854
|
+
deliveryStateCounts,
|
|
61855
|
+
omittedDeliveryStates: states.length - exemplars.length
|
|
61856
|
+
};
|
|
61857
|
+
}
|
|
61858
|
+
function omittedDeliveryStatesLine(bounded) {
|
|
61859
|
+
const omitted = bounded.omittedDeliveryStates ?? 0;
|
|
61860
|
+
if (omitted === 0 || bounded.deliveryStateCounts === undefined)
|
|
61861
|
+
return;
|
|
61862
|
+
const counts = Object.entries(bounded.deliveryStateCounts).map(([state, count]) => `${count} ${state}`).join(", ");
|
|
61863
|
+
return `… ${omitted} more delivery state${omitted === 1 ? "" : "s"} omitted (all: ${counts})`;
|
|
61864
|
+
}
|
|
61865
|
+
var STICKY_DELIVERY_STATE_EXEMPLAR_LIMIT = 5, UNRESOLVED_STATES;
|
|
61866
|
+
var init_sticky_delivery_state_bound = __esm(() => {
|
|
61867
|
+
UNRESOLVED_STATES = [
|
|
61868
|
+
"queued",
|
|
61869
|
+
"receipt-unverified",
|
|
61870
|
+
"session-replaced",
|
|
61871
|
+
"delivery-unavailable",
|
|
61872
|
+
"root-unresolved",
|
|
61873
|
+
"stage-terminal"
|
|
61874
|
+
];
|
|
61875
|
+
});
|
|
61876
|
+
|
|
61877
|
+
// dist/builtin/workflows/src/shared/possible-stage-rows.ts
|
|
61878
|
+
function boundPossibleStageDeliveryStates(states) {
|
|
61879
|
+
const bounded = boundStickyDeliveryStates(states);
|
|
61880
|
+
return {
|
|
61881
|
+
...bounded,
|
|
61882
|
+
deliveryStates: bounded.deliveryStates.map(({ target: _target, ...state }) => state)
|
|
61883
|
+
};
|
|
61884
|
+
}
|
|
61885
|
+
|
|
61886
|
+
class WorkflowTopologyIndex {
|
|
61887
|
+
lineage = [];
|
|
61888
|
+
runById = new Map;
|
|
61889
|
+
roots = new Map;
|
|
61890
|
+
hops = new Map;
|
|
61891
|
+
refresh(runs) {
|
|
61892
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61893
|
+
const lineage = [];
|
|
61894
|
+
for (const run of runs) {
|
|
61895
|
+
lineage.push(run.id, run.rootRunId, run.parentRunId, run.parentStageId);
|
|
61896
|
+
if (run.parentRunId === undefined)
|
|
61897
|
+
continue;
|
|
61898
|
+
const parent = runById.get(run.parentRunId);
|
|
61899
|
+
const boundary = parent?.stages.find((stage) => stage.id === run.parentStageId);
|
|
61900
|
+
lineage.push(parent !== undefined, boundary?.name, authoritativeWorkflowChildRunId(boundary));
|
|
61901
|
+
}
|
|
61902
|
+
if (sameValues(lineage, this.lineage))
|
|
61903
|
+
return;
|
|
61904
|
+
this.lineage = lineage;
|
|
61905
|
+
this.runById = runById;
|
|
61906
|
+
this.roots.clear();
|
|
61907
|
+
this.hops.clear();
|
|
61908
|
+
}
|
|
61909
|
+
rootOf(runId) {
|
|
61910
|
+
if (this.roots.has(runId))
|
|
61911
|
+
return this.roots.get(runId);
|
|
61912
|
+
const root = reciprocalWorkflowRootRunId(this.runById, runId);
|
|
61913
|
+
this.roots.set(runId, root);
|
|
61914
|
+
return root;
|
|
61915
|
+
}
|
|
61916
|
+
hopsOf(runId) {
|
|
61917
|
+
if (this.hops.has(runId))
|
|
61918
|
+
return this.hops.get(runId);
|
|
61919
|
+
const hops = boundaryHops(this.runById, runId);
|
|
61920
|
+
this.hops.set(runId, hops);
|
|
61921
|
+
return hops;
|
|
61922
|
+
}
|
|
61923
|
+
}
|
|
61924
|
+
function boundaryHops(runById, runId) {
|
|
61925
|
+
const hops = [];
|
|
61926
|
+
let current = runById.get(runId);
|
|
61927
|
+
while (current !== undefined && current.parentRunId !== undefined) {
|
|
61928
|
+
const parent = runById.get(current.parentRunId);
|
|
61929
|
+
const boundary = parent?.stages.find((stage) => stage.id === current?.parentStageId);
|
|
61930
|
+
if (parent === undefined || boundary === undefined)
|
|
61931
|
+
return;
|
|
61932
|
+
hops.unshift({ name: boundary.name, runId: current.id });
|
|
61933
|
+
current = parent;
|
|
61934
|
+
}
|
|
61935
|
+
return current === undefined ? undefined : hops;
|
|
61936
|
+
}
|
|
61937
|
+
function sameValues(left, right) {
|
|
61938
|
+
if (left.length !== right.length)
|
|
61939
|
+
return false;
|
|
61940
|
+
for (let index = 0;index < left.length; index++)
|
|
61941
|
+
if (left[index] !== right[index])
|
|
61942
|
+
return false;
|
|
61943
|
+
return true;
|
|
61944
|
+
}
|
|
61945
|
+
|
|
61946
|
+
class PossibleStageRowsIndex {
|
|
61947
|
+
rootRunId;
|
|
61948
|
+
topology;
|
|
61949
|
+
targets = new Map;
|
|
61950
|
+
patterns = [];
|
|
61951
|
+
targetOf = new Map;
|
|
61952
|
+
targetsByMessage = new Map;
|
|
61953
|
+
list = [];
|
|
61954
|
+
stageFields = new WeakMap;
|
|
61955
|
+
backend;
|
|
61956
|
+
receiptCursor;
|
|
61957
|
+
scanMatches = new Map;
|
|
61958
|
+
rowCache = new Map;
|
|
61959
|
+
boundedStates = new WeakMap;
|
|
61960
|
+
deliveriesByEntry = new WeakMap;
|
|
61961
|
+
statusCache = new WeakMap;
|
|
61962
|
+
receiptGeneration = 0;
|
|
61963
|
+
receiptEpochs = new Map;
|
|
61964
|
+
constructor(rootRunId, topology) {
|
|
61965
|
+
this.rootRunId = rootRunId;
|
|
61966
|
+
this.topology = topology;
|
|
61967
|
+
}
|
|
61968
|
+
rows(runs, rootRun, availableBackend) {
|
|
61969
|
+
if (isTerminalRunStatus(rootRun.status))
|
|
61970
|
+
return [];
|
|
61971
|
+
this.refreshEntries(rootRun.pendingStageMessages ?? []);
|
|
61972
|
+
const resolvedBackend = availableRunDurableBackend(runs, this.rootRunId) ?? availableBackend;
|
|
61973
|
+
const backend = resolvedBackend === undefined ? undefined : durableBackendForRun(resolvedBackend, runs, this.rootRunId);
|
|
61974
|
+
this.refreshReceipts(backend);
|
|
61975
|
+
this.refreshStages(runs);
|
|
61976
|
+
for (const state of this.patterns)
|
|
61977
|
+
if (state.dirty)
|
|
61978
|
+
this.recompute(state, backend);
|
|
61979
|
+
return this.assemble(runs, rootRun);
|
|
61980
|
+
}
|
|
61981
|
+
targetFor(entry) {
|
|
61004
61982
|
const target = entry.targetPath ?? entry.stageKey;
|
|
61005
|
-
let
|
|
61006
|
-
if (
|
|
61983
|
+
let state = this.targets.get(target);
|
|
61984
|
+
if (state === undefined) {
|
|
61007
61985
|
const parsed = parseWorkflowStageTarget(target);
|
|
61008
|
-
|
|
61009
|
-
|
|
61010
|
-
|
|
61011
|
-
|
|
61986
|
+
const owned = parsed?.rootRunId === this.rootRunId;
|
|
61987
|
+
state = {
|
|
61988
|
+
target,
|
|
61989
|
+
segments: owned ? parsed.segments : undefined,
|
|
61990
|
+
pattern: owned ? this.patterns.length : -1,
|
|
61991
|
+
entries: [],
|
|
61992
|
+
positions: [],
|
|
61993
|
+
queued: 0,
|
|
61994
|
+
stages: [],
|
|
61995
|
+
statuses: [],
|
|
61996
|
+
dirty: true
|
|
61997
|
+
};
|
|
61998
|
+
this.targets.set(target, state);
|
|
61999
|
+
if (owned)
|
|
62000
|
+
this.patterns.push(state);
|
|
61012
62001
|
}
|
|
61013
|
-
|
|
62002
|
+
return state;
|
|
62003
|
+
}
|
|
62004
|
+
refreshEntries(list) {
|
|
62005
|
+
const previous = this.list;
|
|
62006
|
+
if (list === previous)
|
|
61014
62007
|
return;
|
|
61015
|
-
|
|
61016
|
-
|
|
61017
|
-
|
|
61018
|
-
|
|
61019
|
-
|
|
61020
|
-
|
|
61021
|
-
|
|
61022
|
-
|
|
61023
|
-
|
|
62008
|
+
this.list = list;
|
|
62009
|
+
let incremental = list.length >= previous.length;
|
|
62010
|
+
const replaced = [];
|
|
62011
|
+
for (let index = 0;incremental && index < previous.length; index++) {
|
|
62012
|
+
if (list[index] === previous[index])
|
|
62013
|
+
continue;
|
|
62014
|
+
const before = previous[index];
|
|
62015
|
+
const after = list[index];
|
|
62016
|
+
if (before.sticky !== true && after.sticky !== true)
|
|
62017
|
+
continue;
|
|
62018
|
+
if (before.sticky !== true || after.sticky !== true || (before.targetPath ?? before.stageKey) !== (after.targetPath ?? after.stageKey)) {
|
|
62019
|
+
incremental = false;
|
|
62020
|
+
break;
|
|
62021
|
+
}
|
|
62022
|
+
replaced.push(index);
|
|
62023
|
+
}
|
|
62024
|
+
if (!incremental) {
|
|
62025
|
+
this.rebuildEntries(list);
|
|
62026
|
+
return;
|
|
62027
|
+
}
|
|
62028
|
+
for (const index of replaced) {
|
|
62029
|
+
const before = previous[index];
|
|
62030
|
+
const after = list[index];
|
|
62031
|
+
const state = this.targetOf.get(before);
|
|
62032
|
+
const slot = state.entries.indexOf(before);
|
|
62033
|
+
state.entries[slot] = after;
|
|
62034
|
+
this.targetOf.delete(before);
|
|
62035
|
+
this.targetOf.set(after, state);
|
|
62036
|
+
state.queued += (after.status === "queued" ? 1 : 0) - (before.status === "queued" ? 1 : 0);
|
|
62037
|
+
if (before.id !== after.id)
|
|
62038
|
+
this.rebuildMessageIndex();
|
|
62039
|
+
state.dirty = true;
|
|
62040
|
+
}
|
|
62041
|
+
for (let index = previous.length;index < list.length; index++)
|
|
62042
|
+
this.addEntry(list[index], index);
|
|
62043
|
+
}
|
|
62044
|
+
addEntry(entry, position) {
|
|
62045
|
+
if (entry.sticky !== true)
|
|
62046
|
+
return;
|
|
62047
|
+
const state = this.targetFor(entry);
|
|
62048
|
+
state.entries.push(entry);
|
|
62049
|
+
state.positions.push(position);
|
|
62050
|
+
if (entry.status === "queued")
|
|
62051
|
+
state.queued++;
|
|
62052
|
+
state.dirty = true;
|
|
62053
|
+
this.targetOf.set(entry, state);
|
|
62054
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62055
|
+
if (targets === undefined) {
|
|
62056
|
+
targets = new Set;
|
|
62057
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62058
|
+
}
|
|
62059
|
+
targets.add(state);
|
|
62060
|
+
}
|
|
62061
|
+
rebuildEntries(list) {
|
|
62062
|
+
const before = new Map;
|
|
62063
|
+
for (const state of this.targets.values()) {
|
|
62064
|
+
before.set(state, state.entries);
|
|
62065
|
+
state.entries = [];
|
|
62066
|
+
state.positions = [];
|
|
62067
|
+
state.queued = 0;
|
|
62068
|
+
}
|
|
62069
|
+
this.targetOf.clear();
|
|
62070
|
+
this.targetsByMessage.clear();
|
|
62071
|
+
list.forEach((entry, position) => {
|
|
62072
|
+
this.addEntry(entry, position);
|
|
62073
|
+
});
|
|
62074
|
+
for (const state of this.targets.values()) {
|
|
62075
|
+
const previous = before.get(state);
|
|
62076
|
+
state.dirty ||= previous === undefined || !sameValues(previous, state.entries);
|
|
62077
|
+
}
|
|
62078
|
+
}
|
|
62079
|
+
rebuildMessageIndex() {
|
|
62080
|
+
this.targetsByMessage.clear();
|
|
62081
|
+
for (const [entry, state] of this.targetOf) {
|
|
62082
|
+
let targets = this.targetsByMessage.get(entry.id);
|
|
62083
|
+
if (targets === undefined) {
|
|
62084
|
+
targets = new Set;
|
|
62085
|
+
this.targetsByMessage.set(entry.id, targets);
|
|
62086
|
+
}
|
|
62087
|
+
targets.add(state);
|
|
62088
|
+
}
|
|
62089
|
+
}
|
|
62090
|
+
refreshReceipts(backend) {
|
|
62091
|
+
if (backend !== this.backend) {
|
|
62092
|
+
this.backend = backend;
|
|
62093
|
+
this.receiptCursor = undefined;
|
|
62094
|
+
this.invalidateReceipts();
|
|
62095
|
+
}
|
|
62096
|
+
if (backend?.hasCachedPendingStageDeliveryReceipt === undefined)
|
|
62097
|
+
return;
|
|
62098
|
+
const changes = backend.pendingStageDeliveryReceiptChanges?.(this.rootRunId, this.receiptCursor);
|
|
62099
|
+
if (changes === undefined) {
|
|
62100
|
+
this.invalidateReceipts();
|
|
62101
|
+
return;
|
|
62102
|
+
}
|
|
62103
|
+
const firstRead = this.receiptCursor === undefined;
|
|
62104
|
+
this.receiptCursor = changes.cursor;
|
|
62105
|
+
if (changes.messageIds === undefined) {
|
|
62106
|
+
if (!firstRead)
|
|
62107
|
+
this.invalidateReceipts();
|
|
62108
|
+
return;
|
|
62109
|
+
}
|
|
62110
|
+
for (const messageId of changes.messageIds) {
|
|
62111
|
+
this.receiptEpochs.set(messageId, (this.receiptEpochs.get(messageId) ?? 0) + 1);
|
|
62112
|
+
for (const state of this.targetsByMessage.get(messageId) ?? [])
|
|
62113
|
+
state.dirty = true;
|
|
62114
|
+
}
|
|
62115
|
+
}
|
|
62116
|
+
invalidateReceipts() {
|
|
62117
|
+
this.receiptGeneration++;
|
|
62118
|
+
this.receiptEpochs.clear();
|
|
62119
|
+
for (const state of this.patterns)
|
|
62120
|
+
state.dirty = true;
|
|
62121
|
+
}
|
|
62122
|
+
refreshStages(runs) {
|
|
62123
|
+
const current = new Map;
|
|
62124
|
+
let ordinal = 0;
|
|
62125
|
+
for (const run of runs) {
|
|
62126
|
+
const root = this.topology.rootOf(run.id);
|
|
62127
|
+
if (run.id !== this.rootRunId && run.rootRunId !== this.rootRunId && root !== this.rootRunId)
|
|
62128
|
+
continue;
|
|
62129
|
+
const resolved = root === this.rootRunId;
|
|
62130
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62131
|
+
for (const stage of run.stages) {
|
|
62132
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
62133
|
+
continue;
|
|
62134
|
+
const fields = this.fieldsOf(run, stage, resolved, hops);
|
|
62135
|
+
const status = { run, stage, resolved, hops, ordinal: ordinal++ };
|
|
62136
|
+
for (const state of fields.matched) {
|
|
62137
|
+
const stages = current.get(state);
|
|
62138
|
+
if (stages === undefined)
|
|
62139
|
+
current.set(state, [status]);
|
|
62140
|
+
else
|
|
62141
|
+
stages.push(status);
|
|
62142
|
+
}
|
|
62143
|
+
}
|
|
62144
|
+
}
|
|
62145
|
+
for (const state of this.patterns) {
|
|
62146
|
+
const stages = current.get(state) ?? [];
|
|
62147
|
+
if (stages.length !== state.stages.length || stages.some((status, index) => {
|
|
62148
|
+
const previous = state.stages[index];
|
|
62149
|
+
return previous.stage !== status.stage || previous.run !== status.run;
|
|
62150
|
+
}))
|
|
62151
|
+
state.dirty = true;
|
|
62152
|
+
state.stages = stages;
|
|
62153
|
+
}
|
|
62154
|
+
}
|
|
62155
|
+
fieldsOf(run, stage, resolved, hops) {
|
|
62156
|
+
let fields = this.stageFields.get(stage);
|
|
62157
|
+
const matchable = resolved && hops !== undefined;
|
|
62158
|
+
let changed = false;
|
|
62159
|
+
if (fields === undefined || fields.run !== run || fields.resolved !== resolved || fields.hops !== hops || fields.id !== stage.id || fields.name !== stage.name || fields.status !== stage.status || fields.error !== stage.error || fields.sessionId !== stage.sessionId || fields.pendingStageDeliveryAvailable !== stage.pendingStageDeliveryAvailable) {
|
|
62160
|
+
const previous = fields;
|
|
62161
|
+
const sameMatching = previous !== undefined && previous.run === run && previous.resolved === resolved && previous.hops === hops && previous.id === stage.id && previous.name === stage.name;
|
|
62162
|
+
fields = {
|
|
62163
|
+
run,
|
|
62164
|
+
status: stage.status,
|
|
62165
|
+
error: stage.error,
|
|
62166
|
+
sessionId: stage.sessionId,
|
|
62167
|
+
pendingStageDeliveryAvailable: stage.pendingStageDeliveryAvailable,
|
|
62168
|
+
resolved,
|
|
62169
|
+
hops,
|
|
62170
|
+
id: stage.id,
|
|
62171
|
+
name: stage.name,
|
|
62172
|
+
tested: sameMatching ? previous.tested : 0,
|
|
62173
|
+
matched: sameMatching ? previous.matched : []
|
|
62174
|
+
};
|
|
62175
|
+
this.stageFields.set(stage, fields);
|
|
62176
|
+
changed = true;
|
|
62177
|
+
}
|
|
62178
|
+
if (fields.tested < this.patterns.length) {
|
|
62179
|
+
const added = this.patterns.slice(fields.tested);
|
|
62180
|
+
if (!matchable)
|
|
62181
|
+
fields.matched.push(...added);
|
|
62182
|
+
else {
|
|
62183
|
+
const index = createStagePathPatternIndex(added.map((state) => state.segments));
|
|
62184
|
+
for (const position of index.matching(hops, [stage.id, stage.name]))
|
|
62185
|
+
fields.matched.push(added[position]);
|
|
62186
|
+
}
|
|
62187
|
+
fields.tested = this.patterns.length;
|
|
62188
|
+
}
|
|
62189
|
+
if (changed)
|
|
62190
|
+
for (const state of fields.matched)
|
|
62191
|
+
state.dirty = true;
|
|
62192
|
+
return fields;
|
|
62193
|
+
}
|
|
62194
|
+
recompute(state, backend) {
|
|
62195
|
+
state.dirty = false;
|
|
62196
|
+
const statuses = [];
|
|
62197
|
+
for (const { run, stage, resolved, hops } of state.stages) {
|
|
62198
|
+
const fields = this.stageFields.get(stage);
|
|
62199
|
+
for (const entry of state.entries) {
|
|
62200
|
+
let byStage = this.statusCache.get(entry);
|
|
62201
|
+
if (byStage === undefined) {
|
|
62202
|
+
byStage = new Map;
|
|
62203
|
+
this.statusCache.set(entry, byStage);
|
|
62204
|
+
}
|
|
62205
|
+
const epoch = this.receiptEpochs.get(entry.id) ?? 0;
|
|
62206
|
+
let cached = byStage.get(stage);
|
|
62207
|
+
if (cached === undefined || cached.fields !== fields || cached.generation !== this.receiptGeneration || cached.epoch !== epoch) {
|
|
62208
|
+
cached = {
|
|
62209
|
+
fields,
|
|
62210
|
+
generation: this.receiptGeneration,
|
|
62211
|
+
epoch,
|
|
62212
|
+
status: this.status(entry, run, stage, resolved && hops !== undefined, backend)
|
|
62213
|
+
};
|
|
62214
|
+
byStage.set(stage, cached);
|
|
62215
|
+
}
|
|
62216
|
+
statuses.push(cached.status);
|
|
62217
|
+
}
|
|
62218
|
+
}
|
|
62219
|
+
if (!sameStatuses(statuses, state.statuses))
|
|
62220
|
+
state.statuses = statuses;
|
|
62221
|
+
}
|
|
62222
|
+
receiptsOf(entry, runId, stageId) {
|
|
62223
|
+
let byStage = this.deliveriesByEntry.get(entry);
|
|
61024
62224
|
if (byStage === undefined) {
|
|
61025
62225
|
byStage = new Map;
|
|
61026
|
-
for (const delivery of
|
|
62226
|
+
for (const delivery of entry.deliveries ?? []) {
|
|
61027
62227
|
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
61028
62228
|
const list = byStage.get(key);
|
|
61029
62229
|
if (list === undefined)
|
|
@@ -61031,75 +62231,185 @@ function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
|
61031
62231
|
else
|
|
61032
62232
|
list.push(delivery);
|
|
61033
62233
|
}
|
|
61034
|
-
deliveriesByEntry.set(
|
|
62234
|
+
this.deliveriesByEntry.set(entry, byStage);
|
|
61035
62235
|
}
|
|
61036
62236
|
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
61037
|
-
}
|
|
61038
|
-
|
|
61039
|
-
|
|
61040
|
-
|
|
61041
|
-
|
|
61042
|
-
|
|
61043
|
-
|
|
61044
|
-
if (
|
|
61045
|
-
|
|
61046
|
-
|
|
61047
|
-
|
|
61048
|
-
|
|
61049
|
-
|
|
62237
|
+
}
|
|
62238
|
+
status(entry, run, stage, matchable, backend) {
|
|
62239
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
62240
|
+
const receipts = this.receiptsOf(entry, run.id, stage.id);
|
|
62241
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(this.rootRunId, entry.id, { runId: run.id, stageId: stage.id, sessionId: stage.sessionId, admission: "context" }, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
62242
|
+
let state;
|
|
62243
|
+
let reason;
|
|
62244
|
+
if (!matchable) {
|
|
62245
|
+
state = "root-unresolved";
|
|
62246
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
62247
|
+
} else if (delivered) {
|
|
62248
|
+
state = "delivered";
|
|
62249
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
62250
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
62251
|
+
state = "stage-terminal";
|
|
62252
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
62253
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
62254
|
+
state = "receipt-unverified";
|
|
62255
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
62256
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
62257
|
+
state = "delivery-unavailable";
|
|
62258
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
62259
|
+
} else if (receipts.length > 0) {
|
|
62260
|
+
state = "session-replaced";
|
|
62261
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
62262
|
+
} else {
|
|
62263
|
+
state = "queued";
|
|
62264
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
62265
|
+
}
|
|
62266
|
+
return {
|
|
62267
|
+
messageId: entry.message.id,
|
|
62268
|
+
target,
|
|
62269
|
+
runId: run.id,
|
|
62270
|
+
stageId: stage.id,
|
|
62271
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
62272
|
+
state,
|
|
62273
|
+
reason
|
|
62274
|
+
};
|
|
62275
|
+
}
|
|
62276
|
+
assemble(runs, rootRun) {
|
|
62277
|
+
const rows = [];
|
|
62278
|
+
const scanTargets = [...this.targets.values()].filter((state) => state.queued > 0 && state.segments !== undefined && !(state.segments.length === 1 && state.segments[0] === "**"));
|
|
62279
|
+
let liveStageMatches;
|
|
62280
|
+
const rowTargets = new Set;
|
|
62281
|
+
for (const [position, scanEntry] of (rootRun.possibleStages ?? []).entries()) {
|
|
62282
|
+
const rowSegments = splitStagePathSegments(scanEntry);
|
|
62283
|
+
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61050
62284
|
continue;
|
|
61051
|
-
|
|
61052
|
-
|
|
61053
|
-
|
|
61054
|
-
|
|
61055
|
-
const receipts = entryReceipts(index, run.id, stage.id);
|
|
61056
|
-
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
61057
|
-
runId: run.id,
|
|
61058
|
-
stageId: stage.id,
|
|
61059
|
-
sessionId: stage.sessionId,
|
|
61060
|
-
admission: "context"
|
|
61061
|
-
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
61062
|
-
let state;
|
|
61063
|
-
let reason;
|
|
61064
|
-
if (!resolved || !hops) {
|
|
61065
|
-
state = "root-unresolved";
|
|
61066
|
-
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
61067
|
-
} else if (delivered) {
|
|
61068
|
-
state = "delivered";
|
|
61069
|
-
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
61070
|
-
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
61071
|
-
state = "stage-terminal";
|
|
61072
|
-
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
61073
|
-
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
61074
|
-
state = "receipt-unverified";
|
|
61075
|
-
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
61076
|
-
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
61077
|
-
state = "delivery-unavailable";
|
|
61078
|
-
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
61079
|
-
} else if (receipts.length > 0) {
|
|
61080
|
-
state = "session-replaced";
|
|
61081
|
-
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
61082
|
-
} else {
|
|
61083
|
-
state = "queued";
|
|
61084
|
-
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
61085
|
-
}
|
|
61086
|
-
result.push({
|
|
61087
|
-
messageId: entry.message.id,
|
|
61088
|
-
target,
|
|
61089
|
-
runId: run.id,
|
|
61090
|
-
stageId: stage.id,
|
|
61091
|
-
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
61092
|
-
state,
|
|
61093
|
-
reason
|
|
61094
|
-
});
|
|
62285
|
+
if (!rowSegments.some((segment) => segment.includes("*"))) {
|
|
62286
|
+
liveStageMatches ??= this.materializedStageMatcher(runs);
|
|
62287
|
+
if (liveStageMatches(rowSegments))
|
|
62288
|
+
continue;
|
|
61095
62289
|
}
|
|
62290
|
+
let matches = this.scanMatches.get(scanEntry);
|
|
62291
|
+
if (matches === undefined) {
|
|
62292
|
+
matches = new Map;
|
|
62293
|
+
this.scanMatches.set(scanEntry, matches);
|
|
62294
|
+
}
|
|
62295
|
+
let queuedCount = 0;
|
|
62296
|
+
for (const state of scanTargets) {
|
|
62297
|
+
let match = matches.get(state);
|
|
62298
|
+
if (match === undefined) {
|
|
62299
|
+
const segments = state.segments;
|
|
62300
|
+
match = matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
62301
|
+
matches.set(state, match);
|
|
62302
|
+
}
|
|
62303
|
+
if (match)
|
|
62304
|
+
queuedCount += state.queued;
|
|
62305
|
+
}
|
|
62306
|
+
const target = formatWorkflowStageTarget(this.rootRunId, ...rowSegments);
|
|
62307
|
+
rows.push({ slot: `s${position}`, target, queuedCount });
|
|
62308
|
+
rowTargets.add(target);
|
|
62309
|
+
}
|
|
62310
|
+
let broadcastCount = 0;
|
|
62311
|
+
for (const state of this.targets.values()) {
|
|
62312
|
+
if (state.segments?.length === 1 && state.segments[0] === "**")
|
|
62313
|
+
broadcastCount += state.queued;
|
|
62314
|
+
}
|
|
62315
|
+
const broadcast = formatWorkflowStageTarget(this.rootRunId, "**");
|
|
62316
|
+
rows.push({ slot: "b", target: broadcast, queuedCount: broadcastCount });
|
|
62317
|
+
rowTargets.add(broadcast);
|
|
62318
|
+
const statusTargets = this.patterns.filter((state) => state.statuses.length > 0).sort((left, right) => left.stages[0].ordinal - right.stages[0].ordinal || left.positions[0] - right.positions[0]);
|
|
62319
|
+
const statusesByTarget = new Map(statusTargets.map((state) => [state.target, state.statuses]));
|
|
62320
|
+
for (const state of statusTargets) {
|
|
62321
|
+
if (rowTargets.has(state.target))
|
|
62322
|
+
continue;
|
|
62323
|
+
rowTargets.add(state.target);
|
|
62324
|
+
rows.push({ slot: `d${state.target}`, target: state.target, queuedCount: state.queued, deliveryOnly: true });
|
|
61096
62325
|
}
|
|
62326
|
+
const result = rows.map((row) => this.row(row.slot, row.target, row.queuedCount, statusesByTarget.get(row.target), row.deliveryOnly));
|
|
62327
|
+
this.pruneRows(result);
|
|
62328
|
+
return result;
|
|
62329
|
+
}
|
|
62330
|
+
row(slot, target, queuedCount, deliveryStates, deliveryOnly) {
|
|
62331
|
+
const cached = this.rowCache.get(slot);
|
|
62332
|
+
const same = cached !== undefined && cached.row.target === target && cached.row.queuedCount === queuedCount && cached.row.deliveryOnly === deliveryOnly;
|
|
62333
|
+
if (same && cached.statuses === deliveryStates)
|
|
62334
|
+
return cached.row;
|
|
62335
|
+
const bounded = deliveryStates === undefined ? undefined : this.bounded(deliveryStates);
|
|
62336
|
+
if (same && sameBoundedRow(cached.row, bounded)) {
|
|
62337
|
+
this.rowCache.set(slot, { row: cached.row, statuses: deliveryStates });
|
|
62338
|
+
return cached.row;
|
|
62339
|
+
}
|
|
62340
|
+
const row = {
|
|
62341
|
+
target,
|
|
62342
|
+
queuedCount,
|
|
62343
|
+
...deliveryOnly === true ? { deliveryOnly } : {},
|
|
62344
|
+
...bounded
|
|
62345
|
+
};
|
|
62346
|
+
this.rowCache.set(slot, { row, statuses: deliveryStates });
|
|
62347
|
+
return row;
|
|
62348
|
+
}
|
|
62349
|
+
bounded(states) {
|
|
62350
|
+
let bounded = this.boundedStates.get(states);
|
|
62351
|
+
if (bounded === undefined) {
|
|
62352
|
+
bounded = boundPossibleStageDeliveryStates(states);
|
|
62353
|
+
this.boundedStates.set(states, bounded);
|
|
62354
|
+
}
|
|
62355
|
+
return bounded;
|
|
62356
|
+
}
|
|
62357
|
+
pruneRows(rows) {
|
|
62358
|
+
if (this.rowCache.size <= rows.length * 4 + 16)
|
|
62359
|
+
return;
|
|
62360
|
+
const kept = new Set(rows);
|
|
62361
|
+
for (const [key, cached] of this.rowCache)
|
|
62362
|
+
if (!kept.has(cached.row))
|
|
62363
|
+
this.rowCache.delete(key);
|
|
62364
|
+
}
|
|
62365
|
+
materializedStageMatcher(runs) {
|
|
62366
|
+
const invocationRuns = runs.filter((run) => this.topology.rootOf(run.id) === this.rootRunId);
|
|
62367
|
+
return (entrySegments) => {
|
|
62368
|
+
for (const run of invocationRuns) {
|
|
62369
|
+
const hops = this.topology.hopsOf(run.id);
|
|
62370
|
+
if (hops === undefined)
|
|
62371
|
+
continue;
|
|
62372
|
+
for (const stage of run.stages) {
|
|
62373
|
+
if (stage.pendingStageDeliveryAvailable !== true)
|
|
62374
|
+
continue;
|
|
62375
|
+
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
62376
|
+
continue;
|
|
62377
|
+
}
|
|
62378
|
+
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
62379
|
+
return true;
|
|
62380
|
+
}
|
|
62381
|
+
}
|
|
62382
|
+
return false;
|
|
62383
|
+
};
|
|
61097
62384
|
}
|
|
61098
|
-
return result;
|
|
61099
62385
|
}
|
|
61100
|
-
|
|
62386
|
+
function sameBoundedRow(row, bounded) {
|
|
62387
|
+
if (bounded === undefined)
|
|
62388
|
+
return row.deliveryStates === undefined;
|
|
62389
|
+
if (row.deliveryStates === undefined || row.omittedDeliveryStates !== bounded.omittedDeliveryStates)
|
|
62390
|
+
return false;
|
|
62391
|
+
const counts = Object.entries(row.deliveryStateCounts ?? {});
|
|
62392
|
+
const boundedCounts = Object.entries(bounded.deliveryStateCounts ?? {});
|
|
62393
|
+
if (counts.length !== boundedCounts.length || counts.some(([state, count], index) => boundedCounts[index][0] !== state || boundedCounts[index][1] !== count))
|
|
62394
|
+
return false;
|
|
62395
|
+
return sameStatuses(row.deliveryStates, bounded.deliveryStates);
|
|
62396
|
+
}
|
|
62397
|
+
function sameStatuses(left, right) {
|
|
62398
|
+
if (left.length !== right.length)
|
|
62399
|
+
return false;
|
|
62400
|
+
for (let index = 0;index < left.length; index++) {
|
|
62401
|
+
const a = left[index];
|
|
62402
|
+
const b = right[index];
|
|
62403
|
+
if (a.messageId !== b.messageId || a.target !== b.target || a.runId !== b.runId || a.stageId !== b.stageId || a.sessionId !== b.sessionId || a.state !== b.state || a.reason !== b.reason)
|
|
62404
|
+
return false;
|
|
62405
|
+
}
|
|
62406
|
+
return true;
|
|
62407
|
+
}
|
|
62408
|
+
var init_possible_stage_rows = __esm(() => {
|
|
61101
62409
|
init_run_owner_backend();
|
|
61102
62410
|
init_pending_stage_status();
|
|
62411
|
+
init_sticky_delivery_state_bound();
|
|
62412
|
+
init_store_internal();
|
|
61103
62413
|
init_workflow_stage_target();
|
|
61104
62414
|
});
|
|
61105
62415
|
|
|
@@ -61164,11 +62474,32 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61164
62474
|
announceIfPending();
|
|
61165
62475
|
return routeCompletions.get(runId);
|
|
61166
62476
|
});
|
|
62477
|
+
const topology = new WorkflowTopologyIndex;
|
|
62478
|
+
const possibleStageRowIndexes = new Map;
|
|
62479
|
+
const rowIdentities = new WeakMap;
|
|
62480
|
+
let nextRowIdentity = 0;
|
|
62481
|
+
const rowIdentity = (row) => {
|
|
62482
|
+
let identity = rowIdentities.get(row);
|
|
62483
|
+
if (identity === undefined) {
|
|
62484
|
+
identity = nextRowIdentity++;
|
|
62485
|
+
rowIdentities.set(row, identity);
|
|
62486
|
+
}
|
|
62487
|
+
return identity;
|
|
62488
|
+
};
|
|
62489
|
+
const possibleStageRowsIndex = (rootRunId) => {
|
|
62490
|
+
let index = possibleStageRowIndexes.get(rootRunId);
|
|
62491
|
+
if (index === undefined) {
|
|
62492
|
+
index = new PossibleStageRowsIndex(rootRunId, topology);
|
|
62493
|
+
possibleStageRowIndexes.set(rootRunId, index);
|
|
62494
|
+
}
|
|
62495
|
+
return index;
|
|
62496
|
+
};
|
|
61167
62497
|
const announceRoutes = () => {
|
|
61168
62498
|
if (disposed)
|
|
61169
62499
|
return;
|
|
61170
62500
|
const ownedRunIds = new Set;
|
|
61171
62501
|
const runs = activeStore.runs();
|
|
62502
|
+
topology.refresh(runs);
|
|
61172
62503
|
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61173
62504
|
const childrenByBoundary = new Map;
|
|
61174
62505
|
for (const run of runs) {
|
|
@@ -61183,7 +62514,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61183
62514
|
for (const run of runs) {
|
|
61184
62515
|
if (externalRunRouteOwner(activeStore, run.id) !== undefined || !ownsRun(run.id))
|
|
61185
62516
|
continue;
|
|
61186
|
-
const rootRunId =
|
|
62517
|
+
const rootRunId = topology.rootOf(run.id);
|
|
61187
62518
|
if (rootRunId === undefined)
|
|
61188
62519
|
continue;
|
|
61189
62520
|
ownedRunIds.add(run.id);
|
|
@@ -61193,6 +62524,13 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61193
62524
|
runId: parentRun.id,
|
|
61194
62525
|
stageKeys: [boundary.id, boundary.name].filter((key) => resolveIndexedChildRun(childrenByBoundary, parentRun, key)?.id === run.id)
|
|
61195
62526
|
};
|
|
62527
|
+
let possibleStages;
|
|
62528
|
+
if (run.id === rootRunId && isTerminalRunStatus(run.status)) {
|
|
62529
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62530
|
+
possibleStages = [];
|
|
62531
|
+
} else if (run.id === rootRunId) {
|
|
62532
|
+
possibleStages = possibleStageRowsIndex(rootRunId).rows(runs, run, availableStoreDurableBackend(activeStore, rootRunId));
|
|
62533
|
+
}
|
|
61196
62534
|
const announcement = {
|
|
61197
62535
|
runId: run.id,
|
|
61198
62536
|
group: workflowInvocationIntercomGroup(rootRunId),
|
|
@@ -61204,7 +62542,7 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61204
62542
|
stageName: stage.name,
|
|
61205
62543
|
target: stageRouteTarget(runs, rootRunId, run.id, stage.id, resolveBoundarySegments),
|
|
61206
62544
|
lifecycle: stage.sessionId === undefined && stage.sessionFile === undefined ? "pending" : "running",
|
|
61207
|
-
routeEligible: stage.pendingStageDeliveryAvailable === true && (stage.status
|
|
62545
|
+
routeEligible: stage.pendingStageDeliveryAvailable === true && isAddressableStageStatus(stage.status),
|
|
61208
62546
|
group: stage.intercomGroup ?? workflowInvocationIntercomGroup(rootRunId)
|
|
61209
62547
|
})),
|
|
61210
62548
|
...nonAgentNodes(run).map((node) => ({
|
|
@@ -61217,11 +62555,9 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61217
62555
|
group: workflowInvocationIntercomGroup(rootRunId)
|
|
61218
62556
|
}))
|
|
61219
62557
|
],
|
|
61220
|
-
...
|
|
61221
|
-
possibleStages: possibleStageRows(runs, rootRunId, run, availableStoreDurableBackend(activeStore, rootRunId))
|
|
61222
|
-
} : {}
|
|
62558
|
+
...possibleStages === undefined ? {} : { possibleStages }
|
|
61223
62559
|
};
|
|
61224
|
-
const signature = JSON.stringify(announcement)
|
|
62560
|
+
const signature = possibleStages === undefined ? JSON.stringify(announcement) : `${JSON.stringify({ ...announcement, possibleStages: undefined })}\x00${possibleStages.map(rowIdentity).join(",")}`;
|
|
61225
62561
|
if (announcedRoutes.get(run.id)?.signature === signature)
|
|
61226
62562
|
continue;
|
|
61227
62563
|
const entry = { signature };
|
|
@@ -61256,6 +62592,10 @@ function registerPendingStageIntercomBridge(pi, activeStore, ownsRun = () => tru
|
|
|
61256
62592
|
if (!ownedRunIds.has(runId))
|
|
61257
62593
|
routeCompletions.delete(runId);
|
|
61258
62594
|
}
|
|
62595
|
+
for (const rootRunId of possibleStageRowIndexes.keys()) {
|
|
62596
|
+
if (!ownedRunIds.has(rootRunId))
|
|
62597
|
+
possibleStageRowIndexes.delete(rootRunId);
|
|
62598
|
+
}
|
|
61259
62599
|
sweepPromise = sweepPromise.then(() => outsideWorkflowExecutions(() => settleUndeliverablePendingStageMessages(activeStore, notifyUndeliverable, ownsRun))).then(() => {
|
|
61260
62600
|
return;
|
|
61261
62601
|
}).catch((error) => reportWarning(`atomic-workflows: pending stage delivery sweep failed: ${error.message}`));
|
|
@@ -61416,102 +62756,6 @@ function stageRouteTarget(runs, rootRunId, runId, stageId, resolveBoundarySegmen
|
|
|
61416
62756
|
const boundarySegments = runId === rootRunId ? [] : resolveBoundarySegments(runId);
|
|
61417
62757
|
return formatWorkflowStageTarget(rootRunId, ...boundarySegments ?? [runId], stageId);
|
|
61418
62758
|
}
|
|
61419
|
-
function possibleStageRows(runs, rootRunId, rootRun, backend) {
|
|
61420
|
-
if (isTerminalRunStatus(rootRun.status))
|
|
61421
|
-
return [];
|
|
61422
|
-
const stickyQueued = (rootRun.pendingStageMessages ?? []).filter((entry) => entry.sticky === true && entry.status === "queued");
|
|
61423
|
-
const segmentsByTarget = new Map;
|
|
61424
|
-
const stickySegments = (entry) => {
|
|
61425
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61426
|
-
if (segmentsByTarget.has(target))
|
|
61427
|
-
return segmentsByTarget.get(target);
|
|
61428
|
-
const parsed = parseWorkflowStageTarget(target);
|
|
61429
|
-
const segments = parsed?.rootRunId === rootRunId ? parsed.segments : undefined;
|
|
61430
|
-
segmentsByTarget.set(target, segments);
|
|
61431
|
-
return segments;
|
|
61432
|
-
};
|
|
61433
|
-
const scanEntries = stickyQueued.filter((entry) => {
|
|
61434
|
-
const segments = stickySegments(entry);
|
|
61435
|
-
return !(segments !== undefined && segments.length === 1 && segments[0] === "**");
|
|
61436
|
-
});
|
|
61437
|
-
const liveStageMatches = materializedStageMatcher(runs, rootRunId);
|
|
61438
|
-
const rows = [];
|
|
61439
|
-
for (const scanEntry of rootRun.possibleStages ?? []) {
|
|
61440
|
-
const rowSegments = splitStagePathSegments(scanEntry);
|
|
61441
|
-
if (rowSegments.length === 0 || rowSegments.some((segment) => segment.length === 0))
|
|
61442
|
-
continue;
|
|
61443
|
-
if (!rowSegments.some((segment) => segment.includes("*")) && liveStageMatches(rowSegments)) {
|
|
61444
|
-
continue;
|
|
61445
|
-
}
|
|
61446
|
-
rows.push({
|
|
61447
|
-
target: formatWorkflowStageTarget(rootRunId, ...rowSegments),
|
|
61448
|
-
queuedCount: scanEntries.filter((entry) => {
|
|
61449
|
-
const segments = stickySegments(entry);
|
|
61450
|
-
if (segments === undefined)
|
|
61451
|
-
return false;
|
|
61452
|
-
return matchStagePathSegments(rowSegments, segments) || matchStagePathSegments(segments, rowSegments);
|
|
61453
|
-
}).length
|
|
61454
|
-
});
|
|
61455
|
-
}
|
|
61456
|
-
rows.push({
|
|
61457
|
-
target: formatWorkflowStageTarget(rootRunId, "**"),
|
|
61458
|
-
queuedCount: stickyQueued.filter((entry) => {
|
|
61459
|
-
const segments = stickySegments(entry);
|
|
61460
|
-
return segments !== undefined && segments.length === 1 && segments[0] === "**";
|
|
61461
|
-
}).length
|
|
61462
|
-
});
|
|
61463
|
-
const statuses = stickyStageDeliveryStatuses(runs, rootRunId, backend);
|
|
61464
|
-
const statusesByTarget = new Map;
|
|
61465
|
-
for (const status of statuses) {
|
|
61466
|
-
const forTarget = statusesByTarget.get(status.target);
|
|
61467
|
-
if (forTarget === undefined)
|
|
61468
|
-
statusesByTarget.set(status.target, [status]);
|
|
61469
|
-
else
|
|
61470
|
-
forTarget.push(status);
|
|
61471
|
-
}
|
|
61472
|
-
const queuedByTarget = new Map;
|
|
61473
|
-
for (const entry of stickyQueued) {
|
|
61474
|
-
const target = entry.targetPath ?? entry.stageKey;
|
|
61475
|
-
queuedByTarget.set(target, (queuedByTarget.get(target) ?? 0) + 1);
|
|
61476
|
-
}
|
|
61477
|
-
const rowTargets = new Set(rows.map((row) => row.target));
|
|
61478
|
-
for (const target of statusesByTarget.keys()) {
|
|
61479
|
-
if (rowTargets.has(target))
|
|
61480
|
-
continue;
|
|
61481
|
-
rowTargets.add(target);
|
|
61482
|
-
rows.push({ target, queuedCount: queuedByTarget.get(target) ?? 0, deliveryOnly: true });
|
|
61483
|
-
}
|
|
61484
|
-
for (let index = 0;index < rows.length; index++) {
|
|
61485
|
-
const row = rows[index];
|
|
61486
|
-
const deliveryStates = statusesByTarget.get(row.target);
|
|
61487
|
-
if (deliveryStates !== undefined)
|
|
61488
|
-
rows[index] = { ...row, deliveryStates };
|
|
61489
|
-
}
|
|
61490
|
-
return rows;
|
|
61491
|
-
}
|
|
61492
|
-
function materializedStageMatcher(runs, rootRunId) {
|
|
61493
|
-
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
61494
|
-
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
61495
|
-
const invocationRuns = runs.filter((run) => reciprocalWorkflowRootRunId(runById, run.id) === rootRunId);
|
|
61496
|
-
return (entrySegments) => materializedStageMatches(invocationRuns, hopsFor, entrySegments);
|
|
61497
|
-
}
|
|
61498
|
-
function materializedStageMatches(invocationRuns, hopsFor, entrySegments) {
|
|
61499
|
-
for (const run of invocationRuns) {
|
|
61500
|
-
const hops = hopsFor(run.id);
|
|
61501
|
-
if (hops === undefined)
|
|
61502
|
-
continue;
|
|
61503
|
-
for (const stage of run.stages) {
|
|
61504
|
-
if (stage.pendingStageDeliveryAvailable !== true)
|
|
61505
|
-
continue;
|
|
61506
|
-
if (stage.status !== "pending" && stage.status !== "running" && stage.status !== "awaiting_input" && stage.status !== "paused" && stage.status !== "blocked") {
|
|
61507
|
-
continue;
|
|
61508
|
-
}
|
|
61509
|
-
if (stageMatchesPathPattern(entrySegments, hops, [stage.id, stage.name]))
|
|
61510
|
-
return true;
|
|
61511
|
-
}
|
|
61512
|
-
}
|
|
61513
|
-
return false;
|
|
61514
|
-
}
|
|
61515
62759
|
function isPendingStageMessageEvent(value2) {
|
|
61516
62760
|
if (typeof value2 !== "object" || value2 === null)
|
|
61517
62761
|
return false;
|
|
@@ -61801,7 +63045,7 @@ function knownUninitializedStage(run, stageKey) {
|
|
|
61801
63045
|
if (candidates.length !== 1)
|
|
61802
63046
|
return;
|
|
61803
63047
|
const stage = candidates[0];
|
|
61804
|
-
return (stage.status
|
|
63048
|
+
return isAddressableStageStatus(stage.status) && stage.sessionId === undefined && stage.sessionFile === undefined ? stage : undefined;
|
|
61805
63049
|
}
|
|
61806
63050
|
function pendingStageDestination(run, entry) {
|
|
61807
63051
|
if (entry.stageId !== undefined) {
|
|
@@ -61895,7 +63139,7 @@ var init_pending_stage_intercom = __esm(() => {
|
|
|
61895
63139
|
init_pending_stage_route_capability();
|
|
61896
63140
|
init_pending_stage_route_readiness();
|
|
61897
63141
|
init_pending_stage_status();
|
|
61898
|
-
|
|
63142
|
+
init_possible_stage_rows();
|
|
61899
63143
|
init_store_internal();
|
|
61900
63144
|
init_workflow_stage_target();
|
|
61901
63145
|
});
|
|
@@ -64766,9 +66010,146 @@ var init_definitions = __esm(() => {
|
|
|
64766
66010
|
workerDefinitionSources = new WeakMap;
|
|
64767
66011
|
});
|
|
64768
66012
|
|
|
66013
|
+
// dist/builtin/workflows/src/durable/journal/pool-checkpoint-index.ts
|
|
66014
|
+
function push(map, key, value2) {
|
|
66015
|
+
if (key === undefined)
|
|
66016
|
+
return;
|
|
66017
|
+
const list = map.get(key);
|
|
66018
|
+
if (list)
|
|
66019
|
+
list.push(value2);
|
|
66020
|
+
else
|
|
66021
|
+
map.set(key, [value2]);
|
|
66022
|
+
}
|
|
66023
|
+
|
|
66024
|
+
class Partitions {
|
|
66025
|
+
byRun = new Map;
|
|
66026
|
+
startsByChild = new Map;
|
|
66027
|
+
stagesByGroup = new Map;
|
|
66028
|
+
stagesByReplayKey = new Map;
|
|
66029
|
+
boundariesByStageId = new Map;
|
|
66030
|
+
boundariesByChild = new Map;
|
|
66031
|
+
add(entry) {
|
|
66032
|
+
const checkpoint = entry.checkpoint;
|
|
66033
|
+
push(this.byRun, checkpoint.topology?.run?.runId, entry);
|
|
66034
|
+
if (checkpoint.kind !== "stage")
|
|
66035
|
+
return;
|
|
66036
|
+
const stage = entry;
|
|
66037
|
+
push(this.stagesByGroup, durableStageGroupKey(checkpoint), stage);
|
|
66038
|
+
push(this.stagesByReplayKey, checkpoint.replayKey, stage);
|
|
66039
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66040
|
+
if (!boundary)
|
|
66041
|
+
return;
|
|
66042
|
+
push(this.boundariesByStageId, checkpoint.topology?.stageId, stage);
|
|
66043
|
+
push(this.boundariesByChild, boundary.child?.runId, stage);
|
|
66044
|
+
if (boundary.event === "start")
|
|
66045
|
+
push(this.startsByChild, boundary.child?.runId, stage);
|
|
66046
|
+
}
|
|
66047
|
+
}
|
|
66048
|
+
function inKeyOrder(entries) {
|
|
66049
|
+
return [...new Set(entries)].sort((left, right) => Buffer.compare(left.order, right.order) || left.seq - right.seq).map((entry) => entry.checkpoint);
|
|
66050
|
+
}
|
|
66051
|
+
|
|
66052
|
+
class PoolCheckpointView {
|
|
66053
|
+
base;
|
|
66054
|
+
from;
|
|
66055
|
+
delta;
|
|
66056
|
+
constructor(base, from, delta) {
|
|
66057
|
+
this.base = base;
|
|
66058
|
+
this.from = from;
|
|
66059
|
+
this.delta = delta;
|
|
66060
|
+
}
|
|
66061
|
+
entries(partition, key) {
|
|
66062
|
+
const committed = (this.base?.partitions[partition].get(key) ?? []).filter((entry) => entry.seq <= this.from);
|
|
66063
|
+
return [...committed, ...this.delta[partition].get(key) ?? []];
|
|
66064
|
+
}
|
|
66065
|
+
}
|
|
66066
|
+
|
|
66067
|
+
class PoolCheckpointRead {
|
|
66068
|
+
index;
|
|
66069
|
+
root;
|
|
66070
|
+
pending;
|
|
66071
|
+
constructor(index, root) {
|
|
66072
|
+
this.index = index;
|
|
66073
|
+
this.root = root;
|
|
66074
|
+
}
|
|
66075
|
+
read(tx, events) {
|
|
66076
|
+
return this.index.read(tx, events, this.root, (apply) => {
|
|
66077
|
+
this.pending = apply;
|
|
66078
|
+
});
|
|
66079
|
+
}
|
|
66080
|
+
commit() {
|
|
66081
|
+
this.pending?.();
|
|
66082
|
+
this.pending = undefined;
|
|
66083
|
+
}
|
|
66084
|
+
}
|
|
66085
|
+
|
|
66086
|
+
class PoolCheckpointIndex {
|
|
66087
|
+
roots = new Map;
|
|
66088
|
+
begin(root) {
|
|
66089
|
+
return new PoolCheckpointRead(this, root);
|
|
66090
|
+
}
|
|
66091
|
+
async read(tx, events, root, onCommit) {
|
|
66092
|
+
const current = this.roots.get(root);
|
|
66093
|
+
let base = current;
|
|
66094
|
+
const head = await tx.get(`SELECT seq, key, at FROM ${events} WHERE run_id = ? ORDER BY seq DESC LIMIT 1`, root);
|
|
66095
|
+
if (base && base.seq > 0) {
|
|
66096
|
+
const mark = head?.seq === base.seq ? head : await tx.get(`SELECT key, at FROM ${events} WHERE run_id = ? AND seq = ?`, root, base.seq);
|
|
66097
|
+
if (!mark || mark.key !== base.key || Number(mark.at) !== base.at)
|
|
66098
|
+
base = undefined;
|
|
66099
|
+
}
|
|
66100
|
+
const from = base?.seq ?? 0;
|
|
66101
|
+
const to = head ? Number(head.seq) : 0;
|
|
66102
|
+
const delta = new Partitions;
|
|
66103
|
+
const added = [];
|
|
66104
|
+
if (to > from) {
|
|
66105
|
+
const rows = await tx.all(`SELECT seq, key, payload FROM ${events} WHERE run_id = ? AND seq > ? AND seq <= ? AND type = ? ORDER BY seq`, root, from, to, RUN_EVENT_TYPES.checkpointRecorded);
|
|
66106
|
+
for (const row of rows) {
|
|
66107
|
+
const checkpoint = JSON.parse(String(row.payload));
|
|
66108
|
+
if (checkpoint.kind === "ui")
|
|
66109
|
+
continue;
|
|
66110
|
+
const entry = {
|
|
66111
|
+
seq: Number(row.seq),
|
|
66112
|
+
order: row.key === null ? Buffer.alloc(0) : Buffer.from(row.key, "utf8"),
|
|
66113
|
+
checkpoint
|
|
66114
|
+
};
|
|
66115
|
+
delta.add(entry);
|
|
66116
|
+
added.push(entry);
|
|
66117
|
+
}
|
|
66118
|
+
}
|
|
66119
|
+
onCommit(() => {
|
|
66120
|
+
const latest = this.roots.get(root);
|
|
66121
|
+
if (!base) {
|
|
66122
|
+
if (latest !== current)
|
|
66123
|
+
return;
|
|
66124
|
+
const fresh = {
|
|
66125
|
+
seq: to,
|
|
66126
|
+
key: head?.key ?? null,
|
|
66127
|
+
at: Number(head?.at ?? 0),
|
|
66128
|
+
partitions: delta
|
|
66129
|
+
};
|
|
66130
|
+
this.roots.set(root, fresh);
|
|
66131
|
+
return;
|
|
66132
|
+
}
|
|
66133
|
+
if (latest !== base || to <= base.seq)
|
|
66134
|
+
return;
|
|
66135
|
+
for (const entry of added)
|
|
66136
|
+
if (entry.seq > base.seq)
|
|
66137
|
+
base.partitions.add(entry);
|
|
66138
|
+
base.seq = to;
|
|
66139
|
+
base.key = head?.key ?? null;
|
|
66140
|
+
base.at = Number(head?.at ?? 0);
|
|
66141
|
+
});
|
|
66142
|
+
return new PoolCheckpointView(base, from, delta);
|
|
66143
|
+
}
|
|
66144
|
+
}
|
|
66145
|
+
var init_pool_checkpoint_index = __esm(() => {
|
|
66146
|
+
init_stage_topology_validation();
|
|
66147
|
+
init_types();
|
|
66148
|
+
});
|
|
66149
|
+
|
|
64769
66150
|
// dist/builtin/workflows/src/durable/journal/pool-mutations.ts
|
|
64770
66151
|
import { StorageProvider as StorageProvider2 } from "@bastani/atomic/storage";
|
|
64771
|
-
async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
66152
|
+
async function validatePoolMutation(tx, journal, root, work, mutation, checkpoints) {
|
|
64772
66153
|
if (!["checkpoint", "session-create", "session-replace"].includes(mutation.kind))
|
|
64773
66154
|
throw new Error("Invalid pool mutation");
|
|
64774
66155
|
const checkpoint = mutation.checkpoint;
|
|
@@ -64788,42 +66169,59 @@ async function validatePoolMutation(tx, journal, root, work, mutation) {
|
|
|
64788
66169
|
}
|
|
64789
66170
|
if (checkpoint.kind === "ui")
|
|
64790
66171
|
return;
|
|
64791
|
-
const
|
|
64792
|
-
const records = rows.map((row) => JSON.parse(String(row.payload)));
|
|
64793
|
-
const stages = records.filter((entry) => entry.kind === "stage");
|
|
66172
|
+
const view = await checkpoints.read(tx, journal.events);
|
|
64794
66173
|
const topology = checkpoint.topology;
|
|
64795
66174
|
const run = topology?.run;
|
|
64796
66175
|
if (!topology || !run || run.rootRunId !== root || run.runId === root)
|
|
64797
66176
|
throw new Error("Pool topology scope mismatch");
|
|
64798
|
-
const owner =
|
|
66177
|
+
const owner = inKeyOrder(view.entries("startsByChild", run.runId))[0]?.topology?.boundary;
|
|
64799
66178
|
if (!owner || !sameRun(run, owner.child) || !key.startsWith(`${owner.replayScope}:`) || !checkpoint.checkpointId.startsWith(`${owner.replayScope}:`) || !(owner.replayScope === work.scope || owner.replayScope.startsWith(prefix)))
|
|
64800
66179
|
throw new Error("Pool topology scope mismatch");
|
|
64801
66180
|
if (mutation.kind !== "checkpoint" && "header" in mutation && mutation.header.workflow && (mutation.header.workflow.runId !== run.runId || mutation.header.workflow.stageId !== ("stageId" in topology ? topology.stageId : undefined)))
|
|
64802
66181
|
throw new Error("Pool session scope mismatch");
|
|
66182
|
+
const runEntries = view.entries("byRun", run.runId);
|
|
64803
66183
|
if (checkpoint.kind === "tool") {
|
|
64804
66184
|
const tool = checkpoint.topology;
|
|
64805
|
-
const peers =
|
|
66185
|
+
const peers = runEntries.map((entry) => entry.checkpoint);
|
|
64806
66186
|
const ids = new Set(peers.map((entry) => entry.kind === "tool" ? entry.topology.nodeId : entry.topology.stageId));
|
|
64807
66187
|
if (tool.version !== DURABLE_TOOL_TOPOLOGY_VERSION || !tool.nodeId || tool.parentIds.some((id) => id === tool.nodeId || !ids.has(id)) || peers.some((entry) => entry.kind === "tool" ? entry.argsHash === checkpoint.argsHash ? entry.topology.nodeId !== tool.nodeId || !sameStrings(entry.topology.parentIds, tool.parentIds) : entry.topology.nodeId === tool.nodeId : entry.topology.stageId === tool.nodeId))
|
|
64808
66188
|
throw new Error("Pool tool topology scope mismatch");
|
|
64809
66189
|
return;
|
|
64810
66190
|
}
|
|
64811
|
-
const
|
|
64812
|
-
const
|
|
66191
|
+
const runStages = runEntries.filter((entry) => entry.checkpoint.kind === "stage");
|
|
66192
|
+
const relevant = [...runStages];
|
|
66193
|
+
const groups = new Set([...runStages.map((entry) => entry.checkpoint), checkpoint].filter((entry) => entry === checkpoint || entry.checkpointId !== checkpoint.checkpointId).map(durableStageGroupKey));
|
|
66194
|
+
for (const group of groups)
|
|
66195
|
+
relevant.push(...view.entries("stagesByGroup", group));
|
|
66196
|
+
const boundary = checkpoint.topology?.boundary;
|
|
66197
|
+
let start;
|
|
66198
|
+
if (boundary) {
|
|
66199
|
+
const replayed = view.entries("stagesByReplayKey", checkpoint.replayKey);
|
|
66200
|
+
start = [...stagesExcept(replayed, checkpoint), checkpoint].find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
66201
|
+
relevant.push(...replayed);
|
|
66202
|
+
if (start) {
|
|
66203
|
+
const startTopology = start.topology;
|
|
66204
|
+
relevant.push(...view.entries("boundariesByStageId", startTopology.stageId));
|
|
66205
|
+
relevant.push(...view.entries("boundariesByChild", startTopology.boundary.child.runId));
|
|
66206
|
+
}
|
|
66207
|
+
}
|
|
66208
|
+
const next = [...stagesExcept(relevant, checkpoint), checkpoint];
|
|
66209
|
+
const tools = new Set(runEntries.flatMap((entry) => entry.checkpoint.kind === "tool" ? [entry.checkpoint.topology.nodeId] : []));
|
|
64813
66210
|
const stageError = directChildTopologyError(next, owner.replayScope, run, tools);
|
|
64814
66211
|
if (stageError)
|
|
64815
66212
|
throw new Error(`Pool topology scope: ${stageError}`);
|
|
64816
|
-
const boundary = checkpoint.topology?.boundary;
|
|
64817
66213
|
if (!boundary) {
|
|
64818
66214
|
if (parseWorkflowChildResult(checkpoint.output))
|
|
64819
66215
|
throw new Error("Pool boundary output without ownership");
|
|
64820
66216
|
return;
|
|
64821
66217
|
}
|
|
64822
|
-
const start = next.find((entry) => entry.replayKey === checkpoint.replayKey && entry.topology?.boundary?.event === "start");
|
|
64823
66218
|
const boundaryError = start ? boundaryTransitionError(start, next, false) : "missing start";
|
|
64824
66219
|
if (!boundary.replayScope.startsWith(`${owner.replayScope}:`) || boundary.child.runId === root || boundary.child.runId === work.childRunId || boundaryError)
|
|
64825
66220
|
throw new Error(`Pool boundary scope: ${boundaryError ?? "invalid child"}`);
|
|
64826
66221
|
}
|
|
66222
|
+
function stagesExcept(entries, checkpoint) {
|
|
66223
|
+
return inKeyOrder(entries).filter((entry) => entry.checkpointId !== checkpoint.checkpointId);
|
|
66224
|
+
}
|
|
64827
66225
|
async function sessionStore(tx, prefix) {
|
|
64828
66226
|
const database = {
|
|
64829
66227
|
descriptor: { adapter: "pool-transaction", location: prefix },
|
|
@@ -64872,20 +66270,52 @@ var init_pool_mutations = __esm(() => {
|
|
|
64872
66270
|
init_boundary_lifecycle();
|
|
64873
66271
|
init_boundary_topology();
|
|
64874
66272
|
init_stage_topology_validation();
|
|
64875
|
-
|
|
66273
|
+
init_pool_checkpoint_index();
|
|
64876
66274
|
});
|
|
64877
66275
|
|
|
64878
66276
|
// dist/builtin/workflows/src/durable/journal/pool-stage-usage.ts
|
|
64879
66277
|
function poolUsageTable(runs) {
|
|
64880
66278
|
return `${runs}_pool_usage`;
|
|
64881
66279
|
}
|
|
66280
|
+
function poolUsageTotalsTable(runs) {
|
|
66281
|
+
return `${runs}_pool_usage_totals`;
|
|
66282
|
+
}
|
|
66283
|
+
function addToTotals(totals, runs, work, sign, row) {
|
|
66284
|
+
const assignments = POOL_USAGE_FIELDS.map((field) => {
|
|
66285
|
+
const total = field.toLowerCase();
|
|
66286
|
+
const value2 = `(${sign}${row}.${total})`;
|
|
66287
|
+
return `${total} = ${total} + ${value2}, ${total}_err = ${total}_err + CASE WHEN abs(${total}) > abs(${value2}) THEN (${total} - (${total} + ${value2})) + ${value2} ELSE (${value2} - (${total} + ${value2})) + ${total} END`;
|
|
66288
|
+
});
|
|
66289
|
+
return `UPDATE ${totals} SET ${assignments.join(", ")} WHERE root = (SELECT root FROM ${runs}_work WHERE id = ${work});`;
|
|
66290
|
+
}
|
|
64882
66291
|
function poolUsageSchema(runs) {
|
|
64883
|
-
|
|
66292
|
+
const usage = poolUsageTable(runs);
|
|
66293
|
+
const totals = poolUsageTotalsTable(runs);
|
|
66294
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66295
|
+
return `CREATE TABLE IF NOT EXISTS ${usage}(
|
|
64884
66296
|
work TEXT NOT NULL REFERENCES ${runs}_work(id) ON DELETE CASCADE,
|
|
64885
66297
|
generation INTEGER NOT NULL, key TEXT NOT NULL,
|
|
64886
66298
|
input REAL NOT NULL, output REAL NOT NULL, cacheread REAL NOT NULL, cachewrite REAL NOT NULL, cost REAL NOT NULL,
|
|
64887
66299
|
PRIMARY KEY(work, generation, key)
|
|
64888
|
-
)
|
|
66300
|
+
);
|
|
66301
|
+
CREATE TABLE IF NOT EXISTS ${totals}(
|
|
66302
|
+
root TEXT PRIMARY KEY REFERENCES ${runs}_pool(root) ON DELETE CASCADE,
|
|
66303
|
+
${columns.map((column) => `${column} REAL NOT NULL, ${column}_err REAL NOT NULL`).join(", ")}
|
|
66304
|
+
);
|
|
66305
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_insert AFTER INSERT ON ${usage} BEGIN
|
|
66306
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66307
|
+
END;
|
|
66308
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_update AFTER UPDATE ON ${usage}
|
|
66309
|
+
WHEN ${columns.map((column) => `OLD.${column} IS NOT NEW.${column}`).join(" OR ")} OR OLD.work IS NOT NEW.work BEGIN
|
|
66310
|
+
${addToTotals(totals, runs, "OLD.work", "-", "OLD")}
|
|
66311
|
+
${addToTotals(totals, runs, "NEW.work", "", "NEW")}
|
|
66312
|
+
END;
|
|
66313
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_delete AFTER DELETE ON ${usage} BEGIN
|
|
66314
|
+
DELETE FROM ${totals} WHERE root = (SELECT root FROM ${runs}_work WHERE id = OLD.work);
|
|
66315
|
+
END;
|
|
66316
|
+
CREATE TRIGGER IF NOT EXISTS ${totals}_work_delete BEFORE DELETE ON ${runs}_work BEGIN
|
|
66317
|
+
DELETE FROM ${totals} WHERE root = OLD.root;
|
|
66318
|
+
END;`;
|
|
64889
66319
|
}
|
|
64890
66320
|
function poolStageUsageKey(runId, stageId) {
|
|
64891
66321
|
return `stage:${runId}:${stageId}`;
|
|
@@ -64912,7 +66342,7 @@ var init_pool_stage_usage = __esm(() => {
|
|
|
64912
66342
|
|
|
64913
66343
|
// dist/builtin/workflows/src/durable/journal/sql-pool-queue.ts
|
|
64914
66344
|
import { createHash as createHash4, randomUUID as randomUUID16 } from "node:crypto";
|
|
64915
|
-
import { assertStorageSchemaVersion as assertStorageSchemaVersion2 } from "@bastani/atomic/storage";
|
|
66345
|
+
import { assertStorageSchemaVersion as assertStorageSchemaVersion2, StorageError as StorageError2 } from "@bastani/atomic/storage";
|
|
64916
66346
|
function requestIdentity(request) {
|
|
64917
66347
|
return createHash4("sha256").update(request).digest("hex");
|
|
64918
66348
|
}
|
|
@@ -64951,6 +66381,7 @@ class SqlPoolQueue {
|
|
|
64951
66381
|
operations;
|
|
64952
66382
|
transaction;
|
|
64953
66383
|
journal;
|
|
66384
|
+
checkpoints = new PoolCheckpointIndex;
|
|
64954
66385
|
constructor(storage) {
|
|
64955
66386
|
this.storage = storage;
|
|
64956
66387
|
const { transaction, tables: journal } = storage;
|
|
@@ -65062,17 +66493,23 @@ class SqlPoolQueue {
|
|
|
65062
66493
|
lease = snapshotLease(lease);
|
|
65063
66494
|
token = { ...token };
|
|
65064
66495
|
return this.storage.withAppendGuard(async (tx, runId) => {
|
|
65065
|
-
|
|
65066
|
-
|
|
65067
|
-
|
|
65068
|
-
|
|
65069
|
-
|
|
65070
|
-
|
|
65071
|
-
|
|
65072
|
-
|
|
65073
|
-
|
|
65074
|
-
|
|
65075
|
-
|
|
66496
|
+
try {
|
|
66497
|
+
if (runId !== lease.runId)
|
|
66498
|
+
throw new Error("Pool delivery cannot write another root");
|
|
66499
|
+
await this.authority(tx, lease).catch((error) => {
|
|
66500
|
+
if (error instanceof StaleLeaseError)
|
|
66501
|
+
throw new Error("Stale pool coordinator authority", { cause: error });
|
|
66502
|
+
throw error;
|
|
66503
|
+
});
|
|
66504
|
+
const row = await this.current(tx, lease, token);
|
|
66505
|
+
if (row.epoch !== lease.epoch)
|
|
66506
|
+
throw new Error(STALE_WORK_LEASE);
|
|
66507
|
+
assertSession();
|
|
66508
|
+
} catch (error) {
|
|
66509
|
+
if (error instanceof JournalError || error instanceof StorageError2 || !(error instanceof Error))
|
|
66510
|
+
throw error;
|
|
66511
|
+
throw new PoolAppendRefusedError(error.message, { cause: error });
|
|
66512
|
+
}
|
|
65076
66513
|
}, action);
|
|
65077
66514
|
}
|
|
65078
66515
|
async assertCurrent(lease, token) {
|
|
@@ -65185,7 +66622,8 @@ class SqlPoolQueue {
|
|
|
65185
66622
|
const request = serializeJson({ epoch: token.epoch, worker: token.worker, mutation });
|
|
65186
66623
|
const identity = requestIdentity(request);
|
|
65187
66624
|
mutation = JSON.parse(request).mutation;
|
|
65188
|
-
|
|
66625
|
+
const checkpoints = this.checkpoints.begin(lease.runId);
|
|
66626
|
+
const receipt = await this.transaction(async (tx) => {
|
|
65189
66627
|
const { now, status } = await this.authority(tx, lease);
|
|
65190
66628
|
const row = await this.current(tx, lease, token);
|
|
65191
66629
|
if (row.epoch !== lease.epoch)
|
|
@@ -65200,24 +66638,26 @@ class SqlPoolQueue {
|
|
|
65200
66638
|
throw new Error(STALE_WORK_LEASE);
|
|
65201
66639
|
if (!await this.admitsAttemptWrites(tx, lease, status))
|
|
65202
66640
|
throw new Error("Pool root is not running");
|
|
65203
|
-
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now);
|
|
66641
|
+
const receipt = mutation.kind === "session-append" ? await this.appendSession(tx, row, mutation, now) : await this.recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints);
|
|
65204
66642
|
await tx.run(`INSERT INTO ${this.operations} VALUES (?, ?, ?, ?, ?)`, token.id, token.generation, operationId, identity, JSON.stringify(receipt));
|
|
65205
66643
|
return receipt;
|
|
65206
66644
|
});
|
|
66645
|
+
checkpoints.commit();
|
|
66646
|
+
return receipt;
|
|
65207
66647
|
}
|
|
65208
66648
|
async appendSession(tx, row, mutation, now) {
|
|
65209
66649
|
return { at: now, lastSeq: await applyPoolSession(tx, `${this.journal.runs}_pool_`, row.id, mutation) };
|
|
65210
66650
|
}
|
|
65211
|
-
async recordCheckpoint(tx, lease, token, row, mutation, now) {
|
|
66651
|
+
async recordCheckpoint(tx, lease, token, row, mutation, now, checkpoints) {
|
|
65212
66652
|
const checkpoint = mutation.checkpoint;
|
|
65213
|
-
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation);
|
|
66653
|
+
await validatePoolMutation(tx, this.journal, lease.runId, fromRow(row), mutation, checkpoints);
|
|
65214
66654
|
if (checkpoint.completedAt > Math.max(now, Date.now()) + MAX_WORKER_CLOCK_SKEW_MS)
|
|
65215
66655
|
throw new Error("Pool checkpoint timestamp is in the future");
|
|
65216
66656
|
assertWorkflowSerializableObject(checkpoint, "Pool checkpoint");
|
|
65217
66657
|
const key = JSON.stringify([RUN_EVENT_TYPES.checkpointRecorded, checkpoint.kind, checkpoint.checkpointId]);
|
|
65218
66658
|
const payload = serializeJson(checkpoint);
|
|
65219
66659
|
const existing = await tx.get(`SELECT seq, at, payload FROM ${this.journal.events} WHERE run_id = ? AND key = ?`, lease.runId, key);
|
|
65220
|
-
if (existing && existing.payload
|
|
66660
|
+
if (existing && !isRecordedCheckpoint(existing.payload, payload))
|
|
65221
66661
|
throw new Error("Pool checkpoint identity conflict");
|
|
65222
66662
|
if (existing && mutation.kind !== "checkpoint")
|
|
65223
66663
|
throw new Error("Pool session checkpoint identity conflict");
|
|
@@ -65315,22 +66755,37 @@ class SqlPoolQueue {
|
|
|
65315
66755
|
return this.transaction(async (tx) => (await tx.all(`SELECT * FROM ${this.work} WHERE root = ? ORDER BY ready_at, rowid`, root)).map(fromRow));
|
|
65316
66756
|
}
|
|
65317
66757
|
}
|
|
66758
|
+
function isRecordedCheckpoint(stored, payload) {
|
|
66759
|
+
if (stored === payload)
|
|
66760
|
+
return true;
|
|
66761
|
+
const { completedAt } = JSON.parse(stored);
|
|
66762
|
+
if (typeof completedAt !== "number")
|
|
66763
|
+
return false;
|
|
66764
|
+
return serializeJson({ ...JSON.parse(payload), completedAt }) === stored;
|
|
66765
|
+
}
|
|
65318
66766
|
function openSqlPoolQueue(journal) {
|
|
65319
66767
|
const storage = sqlPoolStorage(journal);
|
|
65320
66768
|
if (!storage)
|
|
65321
66769
|
return Promise.reject(new Error("Worker pool requires a SQL run journal"));
|
|
65322
66770
|
return SqlPoolQueue.open(storage);
|
|
65323
66771
|
}
|
|
65324
|
-
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease";
|
|
66772
|
+
var STORE_NOW2 = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", MAX_WORKER_CLOCK_SKEW_MS = 1000, POOL_SCHEMA_VERSION = 2, STALE_WORK_LEASE = "Stale work lease", PoolAppendRefusedError;
|
|
65325
66773
|
var init_sql_pool_queue = __esm(() => {
|
|
65326
66774
|
init_serializable();
|
|
65327
66775
|
init_boundary_topology();
|
|
65328
66776
|
init_durable_hash();
|
|
65329
66777
|
init_internal();
|
|
66778
|
+
init_pool_checkpoint_index();
|
|
65330
66779
|
init_pool_mutations();
|
|
65331
66780
|
init_pool_stage_usage();
|
|
65332
66781
|
init_types();
|
|
65333
66782
|
init_values();
|
|
66783
|
+
PoolAppendRefusedError = class PoolAppendRefusedError extends JournalError {
|
|
66784
|
+
constructor(message, options) {
|
|
66785
|
+
super(message, options);
|
|
66786
|
+
this.name = "PoolAppendRefusedError";
|
|
66787
|
+
}
|
|
66788
|
+
};
|
|
65334
66789
|
});
|
|
65335
66790
|
|
|
65336
66791
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
@@ -65353,15 +66808,22 @@ function validate(report) {
|
|
|
65353
66808
|
throw new Error(`Pool usage ${field} must be a finite non-negative number`);
|
|
65354
66809
|
}
|
|
65355
66810
|
}
|
|
66811
|
+
function addCompensated(total, value2) {
|
|
66812
|
+
const sum = total.sum + value2;
|
|
66813
|
+
total.error += Math.abs(total.sum) > Math.abs(value2) ? total.sum - sum + value2 : value2 - sum + total.sum;
|
|
66814
|
+
total.sum = sum;
|
|
66815
|
+
}
|
|
65356
66816
|
|
|
65357
66817
|
class SqlPoolUsage {
|
|
65358
66818
|
transaction;
|
|
65359
66819
|
journal;
|
|
65360
66820
|
table;
|
|
66821
|
+
totals;
|
|
65361
66822
|
constructor(transaction, journal) {
|
|
65362
66823
|
this.transaction = transaction;
|
|
65363
66824
|
this.journal = journal;
|
|
65364
66825
|
this.table = poolUsageTable(journal.runs);
|
|
66826
|
+
this.totals = poolUsageTotalsTable(journal.runs);
|
|
65365
66827
|
}
|
|
65366
66828
|
static async open(transaction, journal) {
|
|
65367
66829
|
const usage = new SqlPoolUsage(transaction, journal);
|
|
@@ -65369,9 +66831,27 @@ class SqlPoolUsage {
|
|
|
65369
66831
|
return usage;
|
|
65370
66832
|
}
|
|
65371
66833
|
async totalIn(tx, root) {
|
|
66834
|
+
const columns = POOL_USAGE_FIELDS.map((field) => field.toLowerCase());
|
|
66835
|
+
const kept = await tx.get(`SELECT * FROM ${this.totals} WHERE root = ?`, root);
|
|
66836
|
+
if (kept)
|
|
66837
|
+
return Object.fromEntries(POOL_USAGE_FIELDS.map((field, index) => [
|
|
66838
|
+
field,
|
|
66839
|
+
Number(kept[columns[index]]) + Number(kept[`${columns[index]}_err`])
|
|
66840
|
+
]));
|
|
65372
66841
|
const row = await tx.get(`SELECT ${POOL_USAGE_FIELDS.map((field) => `COALESCE(SUM(u.${field.toLowerCase()}), 0) AS ${field}`).join(", ")}
|
|
65373
66842
|
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
65374
|
-
|
|
66843
|
+
const total = Object.fromEntries(POOL_USAGE_FIELDS.map((field) => [field, Number(row?.[field] ?? 0)]));
|
|
66844
|
+
if (!await tx.get(`SELECT 1 FROM ${this.journal.runs}_pool WHERE root = ?`, root))
|
|
66845
|
+
return total;
|
|
66846
|
+
const sums = columns.map(() => ({ sum: 0, error: 0 }));
|
|
66847
|
+
const rows = await tx.all(`SELECT ${columns.map((column) => `u.${column}`).join(", ")}
|
|
66848
|
+
FROM ${this.table} u JOIN ${this.journal.runs}_work w ON w.id = u.work WHERE w.root = ?`, root);
|
|
66849
|
+
for (const usage of rows)
|
|
66850
|
+
for (const [index, column] of columns.entries())
|
|
66851
|
+
addCompensated(sums[index], Number(usage[column]));
|
|
66852
|
+
await tx.run(`INSERT INTO ${this.totals}(root, ${columns.map((column) => `${column}, ${column}_err`).join(", ")})
|
|
66853
|
+
VALUES (?, ${columns.map(() => "?, ?").join(", ")})`, root, ...sums.flatMap(({ sum, error }) => [sum, error]));
|
|
66854
|
+
return total;
|
|
65375
66855
|
}
|
|
65376
66856
|
async report(lease, token, reports) {
|
|
65377
66857
|
validateText("runId", lease.runId);
|
|
@@ -66261,15 +67741,74 @@ function isWorkflowPendingStageDeliveryFailure(error) {
|
|
|
66261
67741
|
return true;
|
|
66262
67742
|
return error?.code === "pending_stage_delivery_failed";
|
|
66263
67743
|
}
|
|
66264
|
-
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action()) {
|
|
67744
|
+
function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageName, commit = (action) => action(), options = {}) {
|
|
66265
67745
|
let resolveReady;
|
|
66266
67746
|
let rejectReady;
|
|
66267
67747
|
let readyPromise;
|
|
66268
67748
|
let drainError;
|
|
66269
67749
|
let terminalError;
|
|
66270
67750
|
let drainPromise;
|
|
67751
|
+
let drainedSessionId;
|
|
66271
67752
|
let currentRecipient;
|
|
66272
|
-
|
|
67753
|
+
let lastProgressAt = Date.now();
|
|
67754
|
+
let stallReported = false;
|
|
67755
|
+
const progress = () => {
|
|
67756
|
+
lastProgressAt = Date.now();
|
|
67757
|
+
};
|
|
67758
|
+
const progressingCommit = async (action) => {
|
|
67759
|
+
try {
|
|
67760
|
+
return await commit(action);
|
|
67761
|
+
} finally {
|
|
67762
|
+
progress();
|
|
67763
|
+
}
|
|
67764
|
+
};
|
|
67765
|
+
const reportStall = (sessionId) => {
|
|
67766
|
+
if (stallReported)
|
|
67767
|
+
return;
|
|
67768
|
+
stallReported = true;
|
|
67769
|
+
const undelivered = [
|
|
67770
|
+
...activeStore.pendingStageMessagesFor(runId, stageId),
|
|
67771
|
+
...stageId === stageName ? [] : activeStore.pendingStageMessagesFor(runId, stageName)
|
|
67772
|
+
].map((entry) => `${entry.message.id} (direct → ${entry.stageKey})`).concat(stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).map((entry) => `${entry.message.id} (sticky → ${entry.targetPath ?? entry.stageKey})`));
|
|
67773
|
+
const listed = undelivered.slice(0, 10).join(", ");
|
|
67774
|
+
const more = undelivered.length > 10 ? `, and ${undelivered.length - 10} more` : "";
|
|
67775
|
+
const message = `atomic-workflows: stage "${stageName}" (${stageId}) is starting without its queued Intercom messages: ` + `no delivery progress for ${PENDING_STAGE_READY_STALL_MS / 1000}s; ` + `drain ${drainPromise === undefined ? "not running" : "running but hung"}; ` + `undelivered: ${listed.length === 0 ? "none" : `${listed}${more}`}. ` + "They stay queued and arrive live or on the stage's next drain.";
|
|
67776
|
+
try {
|
|
67777
|
+
(options.reportStall ?? console.warn)(message);
|
|
67778
|
+
} catch {}
|
|
67779
|
+
};
|
|
67780
|
+
const watchStall = (pending, sessionId, onStall) => {
|
|
67781
|
+
let timer;
|
|
67782
|
+
let settled = false;
|
|
67783
|
+
const check = () => {
|
|
67784
|
+
if (settled)
|
|
67785
|
+
return;
|
|
67786
|
+
const remaining = lastProgressAt + PENDING_STAGE_READY_STALL_MS - Date.now();
|
|
67787
|
+
if (remaining > 0) {
|
|
67788
|
+
timer = setTimeout(check, remaining);
|
|
67789
|
+
timer.unref?.();
|
|
67790
|
+
return;
|
|
67791
|
+
}
|
|
67792
|
+
reportStall(sessionId);
|
|
67793
|
+
onStall();
|
|
67794
|
+
};
|
|
67795
|
+
const stop = () => {
|
|
67796
|
+
settled = true;
|
|
67797
|
+
clearTimeout(timer);
|
|
67798
|
+
};
|
|
67799
|
+
pending.then(stop, stop);
|
|
67800
|
+
check();
|
|
67801
|
+
};
|
|
67802
|
+
const boundedDrain = (drain, sessionId) => {
|
|
67803
|
+
const waiting = new Promise((resolve, reject) => {
|
|
67804
|
+
drain.then(resolve, reject);
|
|
67805
|
+
watchStall(drain, sessionId, resolve);
|
|
67806
|
+
});
|
|
67807
|
+
waiting.catch(() => {});
|
|
67808
|
+
return waiting;
|
|
67809
|
+
};
|
|
67810
|
+
const hasUndeliveredExact = () => activeStore.pendingStageMessagesFor(runId, stageId).length > 0 || stageId !== stageName && activeStore.pendingStageMessagesFor(runId, stageName).length > 0;
|
|
67811
|
+
const hasUndelivered = (sessionId) => hasUndeliveredExact() || stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, sessionId === undefined ? undefined : { sessionId }).length > 0;
|
|
66273
67812
|
const pendingReady = () => {
|
|
66274
67813
|
if (readyPromise === undefined) {
|
|
66275
67814
|
readyPromise = new Promise((resolve, reject) => {
|
|
@@ -66279,35 +67818,64 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
66279
67818
|
}
|
|
66280
67819
|
return readyPromise;
|
|
66281
67820
|
};
|
|
67821
|
+
let followUp;
|
|
67822
|
+
const startDrain = (deliver, recipient) => {
|
|
67823
|
+
if (terminalError !== undefined)
|
|
67824
|
+
return Promise.resolve();
|
|
67825
|
+
drainError = undefined;
|
|
67826
|
+
progress();
|
|
67827
|
+
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, async (from, message) => {
|
|
67828
|
+
progress();
|
|
67829
|
+
await deliver(from, message);
|
|
67830
|
+
progress();
|
|
67831
|
+
}, progressingCommit, recipient);
|
|
67832
|
+
const drain = attempt.then(() => {
|
|
67833
|
+
progress();
|
|
67834
|
+
drainedSessionId = recipient?.sessionId;
|
|
67835
|
+
resolveReady?.();
|
|
67836
|
+
readyPromise = undefined;
|
|
67837
|
+
resolveReady = undefined;
|
|
67838
|
+
rejectReady = undefined;
|
|
67839
|
+
if (drainPromise === drain)
|
|
67840
|
+
drainPromise = undefined;
|
|
67841
|
+
}, (error) => {
|
|
67842
|
+
progress();
|
|
67843
|
+
drainError = error;
|
|
67844
|
+
rejectReady?.(error);
|
|
67845
|
+
readyPromise = undefined;
|
|
67846
|
+
resolveReady = undefined;
|
|
67847
|
+
rejectReady = undefined;
|
|
67848
|
+
if (drainPromise === drain)
|
|
67849
|
+
drainPromise = undefined;
|
|
67850
|
+
throw error;
|
|
67851
|
+
});
|
|
67852
|
+
drainPromise = drain;
|
|
67853
|
+
return boundedDrain(drain, recipient?.sessionId);
|
|
67854
|
+
};
|
|
67855
|
+
const requestDrain = (deliver, recipient) => {
|
|
67856
|
+
const running = drainPromise;
|
|
67857
|
+
if (running === undefined)
|
|
67858
|
+
return startDrain(deliver, recipient);
|
|
67859
|
+
if (followUp !== undefined) {
|
|
67860
|
+
followUp.deliver = deliver;
|
|
67861
|
+
followUp.recipient = recipient;
|
|
67862
|
+
return boundedDrain(followUp.settled.promise, recipient?.sessionId);
|
|
67863
|
+
}
|
|
67864
|
+
const next = { deliver, recipient, settled: Promise.withResolvers() };
|
|
67865
|
+
next.settled.promise.catch(() => {});
|
|
67866
|
+
followUp = next;
|
|
67867
|
+
running.catch(() => {}).then(() => {
|
|
67868
|
+
if (followUp === next)
|
|
67869
|
+
followUp = undefined;
|
|
67870
|
+
return requestDrain(next.deliver, next.recipient);
|
|
67871
|
+
}).then(next.settled.resolve, next.settled.reject);
|
|
67872
|
+
return boundedDrain(next.settled.promise, recipient?.sessionId);
|
|
67873
|
+
};
|
|
66282
67874
|
return {
|
|
66283
67875
|
routeCapability: workflowPendingStageRouteCapability(activeStore, runId),
|
|
66284
67876
|
deliverPending(deliver, recipient) {
|
|
66285
67877
|
currentRecipient = recipient;
|
|
66286
|
-
|
|
66287
|
-
return drainPromise;
|
|
66288
|
-
if (terminalError !== undefined)
|
|
66289
|
-
return Promise.resolve();
|
|
66290
|
-
drainError = undefined;
|
|
66291
|
-
const attempt = deliverPendingStageMessages(activeStore, runId, stageId, stageName, deliver, commit, recipient);
|
|
66292
|
-
const drain = attempt.then(() => {
|
|
66293
|
-
resolveReady?.();
|
|
66294
|
-
readyPromise = undefined;
|
|
66295
|
-
resolveReady = undefined;
|
|
66296
|
-
rejectReady = undefined;
|
|
66297
|
-
if (drainPromise === drain)
|
|
66298
|
-
drainPromise = undefined;
|
|
66299
|
-
}, (error) => {
|
|
66300
|
-
drainError = error;
|
|
66301
|
-
rejectReady?.(error);
|
|
66302
|
-
readyPromise = undefined;
|
|
66303
|
-
resolveReady = undefined;
|
|
66304
|
-
rejectReady = undefined;
|
|
66305
|
-
if (drainPromise === drain)
|
|
66306
|
-
drainPromise = undefined;
|
|
66307
|
-
throw error;
|
|
66308
|
-
});
|
|
66309
|
-
drainPromise = drain;
|
|
66310
|
-
return drainPromise;
|
|
67878
|
+
return requestDrain(deliver, recipient);
|
|
66311
67879
|
},
|
|
66312
67880
|
ready(sessionId) {
|
|
66313
67881
|
if (!hasUndelivered(sessionId)) {
|
|
@@ -66317,6 +67885,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
66317
67885
|
return Promise.reject(terminalError);
|
|
66318
67886
|
if (drainError !== undefined)
|
|
66319
67887
|
return Promise.reject(drainError);
|
|
67888
|
+
if (drainPromise === undefined && sessionId !== undefined && drainedSessionId === sessionId && !hasUndeliveredExact())
|
|
67889
|
+
return;
|
|
66320
67890
|
const ready = pendingReady();
|
|
66321
67891
|
if (sessionId !== undefined && activeStore.pendingStageMessagesFor(runId, stageId).length === 0 && activeStore.pendingStageMessagesFor(runId, stageName).length === 0) {
|
|
66322
67892
|
const entries = stickyPendingStageEntriesForStage(activeStore, runId, stageId, stageName, { sessionId });
|
|
@@ -66340,6 +67910,14 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
66340
67910
|
rejectReady?.(error);
|
|
66341
67911
|
});
|
|
66342
67912
|
}
|
|
67913
|
+
watchStall(ready, sessionId, () => {
|
|
67914
|
+
if (readyPromise !== ready)
|
|
67915
|
+
return;
|
|
67916
|
+
resolveReady?.();
|
|
67917
|
+
readyPromise = undefined;
|
|
67918
|
+
resolveReady = undefined;
|
|
67919
|
+
rejectReady = undefined;
|
|
67920
|
+
});
|
|
66343
67921
|
return ready;
|
|
66344
67922
|
},
|
|
66345
67923
|
fail(reason) {
|
|
@@ -66467,7 +68045,7 @@ function toPendingStageSender(entry) {
|
|
|
66467
68045
|
lastActivity: entry.from.lastActivity ?? entry.message.timestamp
|
|
66468
68046
|
};
|
|
66469
68047
|
}
|
|
66470
|
-
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError;
|
|
68048
|
+
var pendingDeliveryClaims, WorkflowPendingStageDeliveryFailedError, PENDING_STAGE_READY_STALL_MS = 60000;
|
|
66471
68049
|
var init_pending_stage_delivery2 = __esm(() => {
|
|
66472
68050
|
init_run_owner_backend();
|
|
66473
68051
|
init_store_backend();
|
|
@@ -68724,29 +70302,23 @@ var init_run_durable_stage_session = __esm(() => {
|
|
|
68724
70302
|
|
|
68725
70303
|
// dist/builtin/workflows/src/durable/completed-subtree.ts
|
|
68726
70304
|
function durableCompletedNestedRunSubtree(backend, rootWorkflowId, childRunId) {
|
|
68727
|
-
const
|
|
68728
|
-
if (!
|
|
70305
|
+
const catalog = durableNestedRunCatalog(backend, rootWorkflowId);
|
|
70306
|
+
if (!catalog.positionsById.has(childRunId))
|
|
68729
70307
|
return;
|
|
68730
|
-
const intentionalExitStatuses = new Map;
|
|
68731
|
-
for (const checkpoint of backend.listCheckpoints(rootWorkflowId)) {
|
|
68732
|
-
if (checkpoint.kind !== "stage")
|
|
68733
|
-
continue;
|
|
68734
|
-
const child = parseWorkflowChildResult(checkpoint.output);
|
|
68735
|
-
if (child?.exited === true)
|
|
68736
|
-
intentionalExitStatuses.set(child.runId, child.status);
|
|
68737
|
-
}
|
|
68738
70308
|
const included = new Set([childRunId]);
|
|
68739
|
-
|
|
68740
|
-
|
|
68741
|
-
|
|
68742
|
-
|
|
68743
|
-
|
|
68744
|
-
|
|
70309
|
+
const pending = [childRunId];
|
|
70310
|
+
for (let runId = pending.pop();runId !== undefined; runId = pending.pop()) {
|
|
70311
|
+
for (const childId of catalog.childIdsByParent.get(runId) ?? []) {
|
|
70312
|
+
if (included.has(childId))
|
|
70313
|
+
continue;
|
|
70314
|
+
included.add(childId);
|
|
70315
|
+
pending.push(childId);
|
|
68745
70316
|
}
|
|
68746
70317
|
}
|
|
68747
|
-
const
|
|
68748
|
-
const
|
|
68749
|
-
|
|
70318
|
+
const positions = [...included].flatMap((runId) => catalog.positionsById.get(runId) ?? []).sort((left, right) => left - right);
|
|
70319
|
+
const subtree = positions.map((position) => catalog.runs[position]);
|
|
70320
|
+
const complete = subtree.length === included.size && subtree.every((run) => isCompleteCachedRun(run, catalog.intentionalExitStatuses));
|
|
70321
|
+
return complete ? subtree.map(cloneRunSnapshot) : undefined;
|
|
68750
70322
|
}
|
|
68751
70323
|
function isCompleteCachedRun(run, intentionalExitStatuses) {
|
|
68752
70324
|
if (run.endedAt === undefined || !run.stages.every((stage) => isTerminalStage2(stage.status) && stage.endedAt !== undefined)) {
|
|
@@ -68867,7 +70439,7 @@ function createDurableCachedStageRecorder(input) {
|
|
|
68867
70439
|
const persistence = stage === undefined ? undefined : input.backend.commitCheckpoint({
|
|
68868
70440
|
kind: "stage",
|
|
68869
70441
|
workflowId: input.run.id,
|
|
68870
|
-
checkpointId:
|
|
70442
|
+
checkpointId: `${STAGE_REPLAY_METADATA_CHECKPOINT_PREFIX}${durableHash({ replayKey, stageId: stage.id, parentIds: stage.parentIds })}`,
|
|
68871
70443
|
name,
|
|
68872
70444
|
replayKey,
|
|
68873
70445
|
completedAt: Date.now(),
|
|
@@ -77288,11 +78860,68 @@ function runResultFields(backend, handle) {
|
|
|
77288
78860
|
const result = durableRunResult(backend, handle.workflowId);
|
|
77289
78861
|
return result === undefined ? {} : { result };
|
|
77290
78862
|
}
|
|
77291
|
-
function
|
|
78863
|
+
function durableNestedRunCatalog(backend, rootWorkflowId) {
|
|
77292
78864
|
const handle = backend.getWorkflow(rootWorkflowId);
|
|
77293
78865
|
if (handle === undefined)
|
|
77294
|
-
return
|
|
77295
|
-
|
|
78866
|
+
return EMPTY_NESTED_RUN_CATALOG;
|
|
78867
|
+
const version = backend.nestedSnapshotVersion?.(rootWorkflowId);
|
|
78868
|
+
const cached = version === undefined ? undefined : nestedRunCatalogs.get(backend)?.get(rootWorkflowId);
|
|
78869
|
+
if (cached !== undefined && cached.version === version && cached.name === handle.name && (cached.updatedAt === undefined || cached.updatedAt === handle.updatedAt)) {
|
|
78870
|
+
return cached.catalog;
|
|
78871
|
+
}
|
|
78872
|
+
const checkpoints = backend.listCheckpoints(rootWorkflowId);
|
|
78873
|
+
const runs = runSnapshotsFromCheckpoints(checkpoints, rootWorkflowId, handle.name, handle.updatedAt, ACTIVE_INSPECTION_RECONSTRUCTION).filter((run) => run.id !== rootWorkflowId);
|
|
78874
|
+
const catalog = nestedRunCatalog(runs, checkpoints);
|
|
78875
|
+
if (version !== undefined) {
|
|
78876
|
+
const byRoot = nestedRunCatalogs.get(backend) ?? new Map;
|
|
78877
|
+
nestedRunCatalogs.set(backend, byRoot);
|
|
78878
|
+
byRoot.delete(rootWorkflowId);
|
|
78879
|
+
byRoot.set(rootWorkflowId, {
|
|
78880
|
+
version,
|
|
78881
|
+
name: handle.name,
|
|
78882
|
+
updatedAt: runs.some(readsFallbackCompletedAt) ? handle.updatedAt : undefined,
|
|
78883
|
+
catalog
|
|
78884
|
+
});
|
|
78885
|
+
if (byRoot.size > MAX_CACHED_ROOTS_PER_BACKEND)
|
|
78886
|
+
byRoot.delete(byRoot.keys().next().value);
|
|
78887
|
+
}
|
|
78888
|
+
return catalog;
|
|
78889
|
+
}
|
|
78890
|
+
function nestedRunCatalog(runs, checkpoints) {
|
|
78891
|
+
const positionsById = new Map;
|
|
78892
|
+
const childIdsByParent = new Map;
|
|
78893
|
+
runs.forEach((run, position) => {
|
|
78894
|
+
const positions = positionsById.get(run.id);
|
|
78895
|
+
if (positions === undefined)
|
|
78896
|
+
positionsById.set(run.id, [position]);
|
|
78897
|
+
else
|
|
78898
|
+
positions.push(position);
|
|
78899
|
+
if (run.parentRunId === undefined)
|
|
78900
|
+
return;
|
|
78901
|
+
const children = childIdsByParent.get(run.parentRunId);
|
|
78902
|
+
if (children === undefined)
|
|
78903
|
+
childIdsByParent.set(run.parentRunId, [run.id]);
|
|
78904
|
+
else
|
|
78905
|
+
children.push(run.id);
|
|
78906
|
+
});
|
|
78907
|
+
const intentionalExitStatuses = new Map;
|
|
78908
|
+
for (const checkpoint of checkpoints) {
|
|
78909
|
+
if (checkpoint.kind !== "stage")
|
|
78910
|
+
continue;
|
|
78911
|
+
const child = parseWorkflowChildResult(checkpoint.output);
|
|
78912
|
+
if (child?.exited === true)
|
|
78913
|
+
intentionalExitStatuses.set(child.runId, child.status);
|
|
78914
|
+
}
|
|
78915
|
+
return { runs, positionsById, childIdsByParent, intentionalExitStatuses };
|
|
78916
|
+
}
|
|
78917
|
+
function readsFallbackCompletedAt(run) {
|
|
78918
|
+
return run.endedAt !== undefined && run.stages.some((stage) => stage.endedAt === undefined);
|
|
78919
|
+
}
|
|
78920
|
+
function cloneRunSnapshot(run) {
|
|
78921
|
+
const copy = structuredClone(run);
|
|
78922
|
+
for (const tool of copy.toolNodes)
|
|
78923
|
+
Object.freeze(tool.parentIds);
|
|
78924
|
+
return copy;
|
|
77296
78925
|
}
|
|
77297
78926
|
async function validatedStageTranscript(stage, store) {
|
|
77298
78927
|
if (stage.sessionId !== undefined && await isReopenableSession(store, stage.sessionId))
|
|
@@ -77322,11 +78951,8 @@ function checkpointDrafts(checkpoints) {
|
|
|
77322
78951
|
tools: [...toolByHash.values()]
|
|
77323
78952
|
};
|
|
77324
78953
|
}
|
|
77325
|
-
function completedToolNodes(
|
|
77326
|
-
return
|
|
77327
|
-
const topologyRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
77328
|
-
if (topologyRunId !== runId)
|
|
77329
|
-
return [];
|
|
78954
|
+
function completedToolNodes(ownedCheckpoints, sourceIds) {
|
|
78955
|
+
return ownedCheckpoints.flatMap((checkpoint) => {
|
|
77330
78956
|
const topology = checkpoint.topology;
|
|
77331
78957
|
if (topology === undefined)
|
|
77332
78958
|
return [];
|
|
@@ -77363,18 +78989,34 @@ function summarizeCompletedToolResult(value2) {
|
|
|
77363
78989
|
return serialized.length <= 240 ? serialized : `${flattenTruncatedString(serialized.slice(0, 237))}...`;
|
|
77364
78990
|
}
|
|
77365
78991
|
function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallbackCompletedAt, policy = COMPLETED_RECONSTRUCTION) {
|
|
77366
|
-
|
|
78992
|
+
const index = new CheckpointCatalogIndex(checkpoints);
|
|
78993
|
+
if (!validBoundaryRecordSet(index, policy.requireTerminal))
|
|
77367
78994
|
return [];
|
|
77368
|
-
const stageRecords =
|
|
78995
|
+
const stageRecords = index.stages;
|
|
77369
78996
|
const identityRecords = stageRecords.filter((stage) => stage.topology?.sourceOrder !== undefined || stage.topology?.status !== undefined || stage.topology?.occurrenceKey !== undefined || stage.topology?.boundary !== undefined);
|
|
77370
78997
|
if (immutableStageGroupError(identityRecords) !== undefined)
|
|
77371
78998
|
return [];
|
|
77372
|
-
const
|
|
77373
|
-
const candidates = [...recordsByGroup.values()].map((records) => mergeStageGroup(records, stageRecords, checkpoints)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
78999
|
+
const candidates = [...index.stageGroups()].map((records) => mergeStageGroup(records, index)).filter((draft) => policy.retainActive || draft.topology?.run === undefined || draft.endedAt !== undefined || draft.output !== undefined).sort(compareDraftSourceOrder);
|
|
77374
79000
|
const drafts = checkpointDrafts(checkpoints);
|
|
77375
79001
|
const toolCheckpoints = drafts.tools;
|
|
77376
79002
|
if (toolCheckpoints.some((checkpoint) => checkpoint.topology === undefined))
|
|
77377
79003
|
return [];
|
|
79004
|
+
const toolNodeIdsByClaimedRun = new Map;
|
|
79005
|
+
const toolsByRun = new Map;
|
|
79006
|
+
for (const checkpoint of toolCheckpoints) {
|
|
79007
|
+
const claimedRunId = checkpoint.topology?.run?.runId;
|
|
79008
|
+
if (claimedRunId !== undefined) {
|
|
79009
|
+
const ids = toolNodeIdsByClaimedRun.get(claimedRunId) ?? new Set;
|
|
79010
|
+
ids.add(checkpoint.topology.nodeId);
|
|
79011
|
+
toolNodeIdsByClaimedRun.set(claimedRunId, ids);
|
|
79012
|
+
}
|
|
79013
|
+
const ownerRunId = claimedRunId ?? rootRunId;
|
|
79014
|
+
const owned = toolsByRun.get(ownerRunId);
|
|
79015
|
+
if (owned === undefined)
|
|
79016
|
+
toolsByRun.set(ownerRunId, [checkpoint]);
|
|
79017
|
+
else
|
|
79018
|
+
owned.push(checkpoint);
|
|
79019
|
+
}
|
|
77378
79020
|
for (const draft of candidates) {
|
|
77379
79021
|
const boundary = draft.topology?.boundary;
|
|
77380
79022
|
if (boundary === undefined || childRunIdFromDraft(draft) === undefined)
|
|
@@ -77388,8 +79030,8 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77388
79030
|
parentStageId: boundary.child.parentStageId,
|
|
77389
79031
|
rootRunId: boundary.child.rootRunId
|
|
77390
79032
|
};
|
|
77391
|
-
const childToolIds =
|
|
77392
|
-
if (
|
|
79033
|
+
const childToolIds = toolNodeIdsByClaimedRun.get(expectedChildRun.runId) ?? new Set;
|
|
79034
|
+
if (directChildTopologyErrorFromRecords(index.directChildRecords(expectedChildRun.runId), boundary.replayScope, expectedChildRun, childToolIds) !== undefined)
|
|
77393
79035
|
return [];
|
|
77394
79036
|
}
|
|
77395
79037
|
if (candidates.length === 0 && toolCheckpoints.length === 0) {
|
|
@@ -77417,7 +79059,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77417
79059
|
group.sort(compareDraftSourceOrder);
|
|
77418
79060
|
if (!retainReachableRunGroups(grouped, rootRunId) && policy.failOnUnreachable)
|
|
77419
79061
|
return [];
|
|
77420
|
-
if (!validateRunGroups(grouped, rootRunId,
|
|
79062
|
+
if (!validateRunGroups(grouped, rootRunId, toolsByRun, index, policy.retainActive))
|
|
77421
79063
|
return [];
|
|
77422
79064
|
const syntheticChildren = [];
|
|
77423
79065
|
const pendingChildren = [];
|
|
@@ -77431,7 +79073,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77431
79073
|
const child = workflowChildFromDraft(draft);
|
|
77432
79074
|
if (child?.exited !== true || grouped.has(child.runId))
|
|
77433
79075
|
continue;
|
|
77434
|
-
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || childScopedEvidenceExists(
|
|
79076
|
+
if (!isSyntheticExitedChild(draft, parentRunId, child.runId, rootRunId) || index.childScopedEvidenceExists(draft.replayKey, child.runId)) {
|
|
77435
79077
|
return [];
|
|
77436
79078
|
}
|
|
77437
79079
|
syntheticChildren.push({ parentRunId, owner: draft, child });
|
|
@@ -77439,7 +79081,7 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77439
79081
|
}
|
|
77440
79082
|
const idMaps = new Map;
|
|
77441
79083
|
for (const [runId, runDrafts] of grouped) {
|
|
77442
|
-
const ownedTools =
|
|
79084
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77443
79085
|
if (!validStageGroup(runDrafts, runId, rootRunId, ownedTools))
|
|
77444
79086
|
return [];
|
|
77445
79087
|
const ids = new Map;
|
|
@@ -77457,32 +79099,45 @@ function runSnapshotsFromCheckpoints(checkpoints, rootRunId, rootRunName, fallba
|
|
|
77457
79099
|
});
|
|
77458
79100
|
if (hasIdentityConflict)
|
|
77459
79101
|
return [];
|
|
77460
|
-
for (const checkpoint of
|
|
77461
|
-
const ownerRunId = checkpoint.topology?.run?.runId ?? rootRunId;
|
|
79102
|
+
for (const checkpoint of ownedTools) {
|
|
77462
79103
|
const sourceId = checkpoint.topology?.nodeId ?? checkpoint.checkpointId;
|
|
77463
|
-
|
|
77464
|
-
ids.set(sourceId, sourceId);
|
|
79104
|
+
ids.set(sourceId, sourceId);
|
|
77465
79105
|
}
|
|
77466
79106
|
idMaps.set(runId, ids);
|
|
77467
79107
|
}
|
|
79108
|
+
const boundaryOwners = new Map;
|
|
79109
|
+
const boundarySourcesByParent = new Map;
|
|
79110
|
+
for (const [parentRunId, runDrafts] of grouped) {
|
|
79111
|
+
const sources = new Map;
|
|
79112
|
+
for (const draft of runDrafts) {
|
|
79113
|
+
const childRunId = childRunIdFromDraft(draft);
|
|
79114
|
+
if (childRunId === undefined)
|
|
79115
|
+
continue;
|
|
79116
|
+
if (!boundaryOwners.has(childRunId))
|
|
79117
|
+
boundaryOwners.set(childRunId, draft);
|
|
79118
|
+
if (!sources.has(childRunId))
|
|
79119
|
+
sources.set(childRunId, draft);
|
|
79120
|
+
}
|
|
79121
|
+
boundarySourcesByParent.set(parentRunId, sources);
|
|
79122
|
+
}
|
|
77468
79123
|
const runs = [];
|
|
77469
79124
|
const emittedRunIds = new Set;
|
|
77470
79125
|
for (const [runId, runDrafts] of grouped) {
|
|
77471
79126
|
const ids = idMaps.get(runId);
|
|
77472
|
-
const ownedTools =
|
|
79127
|
+
const ownedTools = toolsByRun.get(runId) ?? [];
|
|
77473
79128
|
const hasUnknownParents = runDrafts.some((draft) => draft.topology.parentIds.some((parentId) => !ids.has(parentId))) || ownedTools.some((checkpoint) => checkpoint.topology?.parentIds.some((parentId) => !ids.has(parentId)) === true);
|
|
77474
79129
|
if (hasUnknownParents)
|
|
77475
79130
|
return [];
|
|
77476
79131
|
const stages = runDrafts.map((draft) => stageSnapshotFromDraft(draft, ids.get(draft.topology.stageId), draft.topology.parentIds.map((parentId) => ids.get(parentId))));
|
|
77477
|
-
const toolNodes = completedToolNodes(
|
|
79132
|
+
const toolNodes = completedToolNodes(ownedTools, ids);
|
|
77478
79133
|
const run = runTopologyFor(runDrafts, ownedTools);
|
|
77479
79134
|
const startedAt = Math.min(...stages.map((stage) => stage.startedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.startedAt ?? tool.endedAt ?? fallbackCompletedAt));
|
|
77480
79135
|
const endedAt = Math.max(...stages.map((stage) => stage.endedAt ?? fallbackCompletedAt), ...toolNodes.map((tool) => tool.endedAt ?? fallbackCompletedAt));
|
|
77481
|
-
const owner = runId === rootRunId ? undefined :
|
|
79136
|
+
const owner = runId === rootRunId ? undefined : boundaryOwners.get(runId);
|
|
77482
79137
|
const status = owner === undefined ? "completed" : childRunStatus(owner);
|
|
77483
79138
|
const terminal = status !== "running" && status !== "pending" && status !== "paused" && status !== "blocked";
|
|
77484
79139
|
const parentRunId = run?.parentRunId ?? rootRunId;
|
|
77485
|
-
const boundarySourceId =
|
|
79140
|
+
const boundarySourceId = boundarySourcesByParent.get(parentRunId)?.get(runId)?.topology?.stageId;
|
|
77486
79141
|
const declaredParentStageId = run?.parentStageId === undefined ? undefined : idMaps.get(parentRunId)?.get(run.parentStageId);
|
|
77487
79142
|
const parentStageId = declaredParentStageId ?? (boundarySourceId === undefined ? undefined : idMaps.get(parentRunId)?.get(boundarySourceId));
|
|
77488
79143
|
const ownerChild = owner === undefined ? undefined : workflowChildFromDraft(owner);
|
|
@@ -77618,10 +79273,11 @@ function emptyGraphRun(runId, runName, completedAt) {
|
|
|
77618
79273
|
resumable: false
|
|
77619
79274
|
};
|
|
77620
79275
|
}
|
|
77621
|
-
var COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
79276
|
+
var MAX_CACHED_ROOTS_PER_BACKEND = 8, nestedRunCatalogs, EMPTY_NESTED_RUN_CATALOG, COMPLETED_RECONSTRUCTION, ACTIVE_INSPECTION_RECONSTRUCTION;
|
|
77622
79277
|
var init_completed_catalog = __esm(() => {
|
|
77623
79278
|
init_stage_session_store();
|
|
77624
79279
|
init_tool_payload_bounds();
|
|
79280
|
+
init_completed_catalog_index();
|
|
77625
79281
|
init_completed_catalog_stage_groups();
|
|
77626
79282
|
init_resume_eligibility();
|
|
77627
79283
|
init_resume_runtime();
|
|
@@ -77630,6 +79286,13 @@ var init_completed_catalog = __esm(() => {
|
|
|
77630
79286
|
init_workflow_heartbeat_anchor();
|
|
77631
79287
|
init_stage_topology_validation();
|
|
77632
79288
|
init_tool_outcome();
|
|
79289
|
+
nestedRunCatalogs = new WeakMap;
|
|
79290
|
+
EMPTY_NESTED_RUN_CATALOG = {
|
|
79291
|
+
runs: [],
|
|
79292
|
+
positionsById: new Map,
|
|
79293
|
+
childIdsByParent: new Map,
|
|
79294
|
+
intentionalExitStatuses: new Map
|
|
79295
|
+
};
|
|
77633
79296
|
COMPLETED_RECONSTRUCTION = {
|
|
77634
79297
|
requireTerminal: true,
|
|
77635
79298
|
retainActive: false,
|
|
@@ -88913,6 +90576,7 @@ init_sdk_recovery_scope();
|
|
|
88913
90576
|
init_intercom_group();
|
|
88914
90577
|
init_pending_stage_delivery();
|
|
88915
90578
|
init_pending_stage_route_capability();
|
|
90579
|
+
init_pending_stage_status();
|
|
88916
90580
|
|
|
88917
90581
|
// dist/builtin/workflows/src/shared/persistence-restore.ts
|
|
88918
90582
|
init_persistence_restore_helpers();
|
|
@@ -89033,13 +90697,6 @@ var PENDING_STAGE_MESSAGE_EVENT3 = "atomic:workflow-pending-stage-message";
|
|
|
89033
90697
|
var PENDING_STAGE_ROUTE_REVOKED_EVENT = "atomic:workflow-pending-stage-route-revoked";
|
|
89034
90698
|
var DORMANT_RUN_SUBAGENT_HOLD_EVENT = "atomic:workflow-dormant-run-subagent-hold";
|
|
89035
90699
|
var DORMANT_RUN_RECHECK_MS = 15000;
|
|
89036
|
-
var ADDRESSABLE_STAGE_STATUSES = new Set([
|
|
89037
|
-
"pending",
|
|
89038
|
-
"running",
|
|
89039
|
-
"awaiting_input",
|
|
89040
|
-
"paused",
|
|
89041
|
-
"blocked"
|
|
89042
|
-
]);
|
|
89043
90700
|
var DORMANT_ASK_REFUSAL = "Cannot ask a stage of an interrupted workflow run. Use send; Atomic will queue the message until the run is resumed.";
|
|
89044
90701
|
function registerDormantRunIntercomRoutes(pi, activeStore, deps) {
|
|
89045
90702
|
const reportWarning = createWorkflowBackgroundWarningReporter(pi);
|
|
@@ -89328,7 +90985,7 @@ function queueRefusal(result, target) {
|
|
|
89328
90985
|
return { outcome: "refused", reason: "Target workflow run is in a different intercom group" };
|
|
89329
90986
|
}
|
|
89330
90987
|
function isAddressableStage(stage) {
|
|
89331
|
-
return
|
|
90988
|
+
return isAddressableStageStatus(stage.status) && stage.nodeKind !== "tool" && stage.replayKey?.startsWith("prompt:") !== true;
|
|
89332
90989
|
}
|
|
89333
90990
|
function isDormantMessageEvent(value2) {
|
|
89334
90991
|
if (typeof value2 !== "object" || value2 === null)
|
|
@@ -89860,7 +91517,119 @@ function observeRunExecution(run, now = Date.now()) {
|
|
|
89860
91517
|
}
|
|
89861
91518
|
|
|
89862
91519
|
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89863
|
-
|
|
91520
|
+
init_sticky_delivery_state_bound();
|
|
91521
|
+
|
|
91522
|
+
// dist/builtin/workflows/src/shared/sticky-stage-delivery-status.ts
|
|
91523
|
+
init_run_owner_backend();
|
|
91524
|
+
init_pending_stage_status();
|
|
91525
|
+
init_workflow_stage_target();
|
|
91526
|
+
function stickyStageDeliveryStatuses(runs, rootRunId, availableBackend) {
|
|
91527
|
+
const root = runs.find((run) => run.id === rootRunId);
|
|
91528
|
+
const entries = (root?.pendingStageMessages ?? []).filter((entry) => entry.sticky === true);
|
|
91529
|
+
const result = [];
|
|
91530
|
+
const patternByTarget = new Map;
|
|
91531
|
+
const patterns = [];
|
|
91532
|
+
const entriesByPattern = [];
|
|
91533
|
+
const ownEntries = [];
|
|
91534
|
+
entries.forEach((entry, index) => {
|
|
91535
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91536
|
+
let pattern = patternByTarget.get(target);
|
|
91537
|
+
if (pattern === undefined) {
|
|
91538
|
+
const parsed = parseWorkflowStageTarget(target);
|
|
91539
|
+
pattern = parsed?.rootRunId === rootRunId ? patterns.push(parsed.segments) - 1 : -1;
|
|
91540
|
+
if (pattern >= 0)
|
|
91541
|
+
entriesByPattern.push([]);
|
|
91542
|
+
patternByTarget.set(target, pattern);
|
|
91543
|
+
}
|
|
91544
|
+
if (pattern < 0)
|
|
91545
|
+
return;
|
|
91546
|
+
entriesByPattern[pattern].push(index);
|
|
91547
|
+
ownEntries.push(index);
|
|
91548
|
+
});
|
|
91549
|
+
if (ownEntries.length === 0)
|
|
91550
|
+
return result;
|
|
91551
|
+
const patternIndex = createStagePathPatternIndex(patterns);
|
|
91552
|
+
const deliveriesByEntry = new Map;
|
|
91553
|
+
const entryReceipts = (index, runId, stageId) => {
|
|
91554
|
+
let byStage = deliveriesByEntry.get(index);
|
|
91555
|
+
if (byStage === undefined) {
|
|
91556
|
+
byStage = new Map;
|
|
91557
|
+
for (const delivery of entries[index].deliveries ?? []) {
|
|
91558
|
+
const key = `${delivery.runId}\x00${delivery.stageId}`;
|
|
91559
|
+
const list = byStage.get(key);
|
|
91560
|
+
if (list === undefined)
|
|
91561
|
+
byStage.set(key, [delivery]);
|
|
91562
|
+
else
|
|
91563
|
+
list.push(delivery);
|
|
91564
|
+
}
|
|
91565
|
+
deliveriesByEntry.set(index, byStage);
|
|
91566
|
+
}
|
|
91567
|
+
return byStage.get(`${runId}\x00${stageId}`) ?? [];
|
|
91568
|
+
};
|
|
91569
|
+
availableBackend = availableRunDurableBackend(runs, rootRunId) ?? availableBackend;
|
|
91570
|
+
const backend = availableBackend === undefined ? undefined : durableBackendForRun(availableBackend, runs, rootRunId);
|
|
91571
|
+
const runById = new Map(runs.map((run) => [run.id, run]));
|
|
91572
|
+
const hopsFor = createWorkflowBoundaryHopsResolver(runs);
|
|
91573
|
+
for (const run of runs) {
|
|
91574
|
+
const durableRootRunId = reciprocalWorkflowRootRunId(runById, run.id);
|
|
91575
|
+
if (run.id !== rootRunId && run.rootRunId !== rootRunId && durableRootRunId !== rootRunId)
|
|
91576
|
+
continue;
|
|
91577
|
+
const resolved = durableRootRunId === rootRunId;
|
|
91578
|
+
const hops = hopsFor(run.id);
|
|
91579
|
+
for (const stage of run.stages) {
|
|
91580
|
+
if (stage.nodeKind === "tool" || stage.replayKey?.startsWith("prompt:") === true || stage.replayKey?.startsWith("workflow:") === true || stage.workflowChildRun !== undefined || stage.workflowChild !== undefined)
|
|
91581
|
+
continue;
|
|
91582
|
+
const matchedEntries = resolved && hops ? patternIndex.matching(hops, [stage.id, stage.name]).flatMap((pattern) => entriesByPattern[pattern]).sort((left, right) => left - right) : ownEntries;
|
|
91583
|
+
for (const index of matchedEntries) {
|
|
91584
|
+
const entry = entries[index];
|
|
91585
|
+
const target = entry.targetPath ?? entry.stageKey;
|
|
91586
|
+
const receipts = entryReceipts(index, run.id, stage.id);
|
|
91587
|
+
const delivered = backend?.hasCachedPendingStageDeliveryReceipt?.(rootRunId, entry.id, {
|
|
91588
|
+
runId: run.id,
|
|
91589
|
+
stageId: stage.id,
|
|
91590
|
+
sessionId: stage.sessionId,
|
|
91591
|
+
admission: "context"
|
|
91592
|
+
}, stage.sessionId === undefined) === true || receipts.some((delivery) => delivery.admission === "context" && delivery.sessionId !== undefined && (stage.sessionId === undefined || delivery.sessionId === stage.sessionId));
|
|
91593
|
+
let state;
|
|
91594
|
+
let reason;
|
|
91595
|
+
if (!resolved || !hops) {
|
|
91596
|
+
state = "root-unresolved";
|
|
91597
|
+
reason = "Repair reciprocal parent/boundary ownership before startup";
|
|
91598
|
+
} else if (delivered) {
|
|
91599
|
+
state = "delivered";
|
|
91600
|
+
reason = "Startup context admission confirmed; sticky subscription remains for future stages";
|
|
91601
|
+
} else if (["completed", "failed", "skipped"].includes(stage.status)) {
|
|
91602
|
+
state = "stage-terminal";
|
|
91603
|
+
reason = `${stage.error ? `${stage.error}; ` : ""}Stage ended without confirmed context admission; steer a live stage or start a new stage`;
|
|
91604
|
+
} else if (receipts.some((delivery) => delivery.admission !== "context" && (delivery.sessionId === undefined || stage.sessionId === undefined || delivery.sessionId === stage.sessionId))) {
|
|
91605
|
+
state = "receipt-unverified";
|
|
91606
|
+
reason = "Live delivery receipt only; inspect the session transcript to confirm context admission";
|
|
91607
|
+
} else if (stage.pendingStageDeliveryAvailable !== true) {
|
|
91608
|
+
state = "delivery-unavailable";
|
|
91609
|
+
reason = "Stage has no pre-start Intercom helper; enable Intercom in the invocation group";
|
|
91610
|
+
} else if (receipts.length > 0) {
|
|
91611
|
+
state = "session-replaced";
|
|
91612
|
+
reason = "Only a previous session has a receipt; replacement must drain before its first turn";
|
|
91613
|
+
} else {
|
|
91614
|
+
state = "queued";
|
|
91615
|
+
reason = "No delivery receipt; stage startup must drain before its first turn";
|
|
91616
|
+
}
|
|
91617
|
+
result.push({
|
|
91618
|
+
messageId: entry.message.id,
|
|
91619
|
+
target,
|
|
91620
|
+
runId: run.id,
|
|
91621
|
+
stageId: stage.id,
|
|
91622
|
+
...stage.sessionId === undefined ? {} : { sessionId: stage.sessionId },
|
|
91623
|
+
state,
|
|
91624
|
+
reason
|
|
91625
|
+
});
|
|
91626
|
+
}
|
|
91627
|
+
}
|
|
91628
|
+
}
|
|
91629
|
+
return result;
|
|
91630
|
+
}
|
|
91631
|
+
|
|
91632
|
+
// dist/builtin/workflows/src/runs/background/run-inspect.ts
|
|
89864
91633
|
init_store();
|
|
89865
91634
|
|
|
89866
91635
|
// dist/builtin/workflows/src/shared/workflow-identity.ts
|
|
@@ -89903,7 +91672,7 @@ function inspectRun(runId, opts) {
|
|
|
89903
91672
|
inputs: copy.inputs,
|
|
89904
91673
|
stages: expandedStages.map((stage) => structuredClone(stage)),
|
|
89905
91674
|
tools: expandedGraph.tools.map((tool) => structuredClone(tool)),
|
|
89906
|
-
|
|
91675
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(activeStore.runs(), copy.rootRunId ?? copy.id, availableStoreDurableBackend(activeStore, copy.id)).filter((delivery) => copy.id === (copy.rootRunId ?? copy.id) || delivery.runId === copy.id)),
|
|
89907
91676
|
result: copy.result,
|
|
89908
91677
|
error: copy.error ?? (strandedRoot ? IMPOSSIBLE_ROOT_LIVENESS_MESSAGE : effectiveRunStatus(copy) === copy.status ? undefined : structuredRecoverableWorkflowFailureText(copy) ?? actionableReturnedStatusText(copy.result)),
|
|
89909
91678
|
exited: copy.exited,
|
|
@@ -97628,7 +99397,7 @@ function buildWorkflowRunStatusDigest(detail) {
|
|
|
97628
99397
|
// dist/builtin/workflows/src/extension/workflow-status-summary.ts
|
|
97629
99398
|
init_pending_stage_status();
|
|
97630
99399
|
init_returned_run_status();
|
|
97631
|
-
|
|
99400
|
+
init_sticky_delivery_state_bound();
|
|
97632
99401
|
var workflowStatusRenderRuns = new WeakMap;
|
|
97633
99402
|
function setWorkflowStatusRenderRuns(result, runs) {
|
|
97634
99403
|
workflowStatusRenderRuns.set(result, runs);
|
|
@@ -97735,7 +99504,7 @@ function summarizeRunSnapshot(run, now = Date.now(), options) {
|
|
|
97735
99504
|
...stage.rateLimits !== undefined ? { rateLimits: stage.rateLimits } : {}
|
|
97736
99505
|
})),
|
|
97737
99506
|
pendingStages: pendingWorkflowStageStatuses(run, options?.owningRunStatus, options?.resolveBoundarySegments),
|
|
97738
|
-
|
|
99507
|
+
...boundStickyDeliveryStates(stickyStageDeliveryStatuses(options?.allRuns ?? [run], run.id, options?.durableBackend)),
|
|
97739
99508
|
tools: run.toolNodes.map((tool) => {
|
|
97740
99509
|
const owner = tool;
|
|
97741
99510
|
return {
|
|
@@ -97937,7 +99706,7 @@ function formatStageStartup(startup, now = Date.now()) {
|
|
|
97937
99706
|
}
|
|
97938
99707
|
|
|
97939
99708
|
// dist/builtin/workflows/src/extension/workflow-tool-content.ts
|
|
97940
|
-
|
|
99709
|
+
init_sticky_delivery_state_bound();
|
|
97941
99710
|
function stringifyWorkflowToolResult(result) {
|
|
97942
99711
|
return JSON.stringify(result, null, 2);
|
|
97943
99712
|
}
|
|
@@ -98036,9 +99805,13 @@ function renderStatusToolContent(result) {
|
|
|
98036
99805
|
}
|
|
98037
99806
|
for (const entry of run.awaitingInput)
|
|
98038
99807
|
lines.push(statusAwaitingInputLine(entry));
|
|
98039
|
-
|
|
99808
|
+
const deliveries = run.deliveryStates === undefined ? boundStickyDeliveryStates(stickyStageDeliveryStatuses(allRuns, run.runId, undefined)) : run;
|
|
99809
|
+
for (const delivery of deliveries.deliveryStates ?? []) {
|
|
98040
99810
|
lines.push(` Intercom delivery ${delivery.messageId} target=${delivery.target} stage=${delivery.runId}/${delivery.stageId}${delivery.sessionId === undefined ? "" : ` session=${delivery.sessionId}`} state=${delivery.state} — ${delivery.reason}`);
|
|
98041
99811
|
}
|
|
99812
|
+
const omittedDeliveries = omittedDeliveryStatesLine(deliveries);
|
|
99813
|
+
if (omittedDeliveries !== undefined)
|
|
99814
|
+
lines.push(` ${omittedDeliveries}`);
|
|
98042
99815
|
for (const stage of run.pendingStages.slice(0, STATUS_PENDING_STAGE_LIMIT)) {
|
|
98043
99816
|
lines.push(statusPendingStageLine(stage));
|
|
98044
99817
|
}
|