@mastra/factory 0.15.0 → 0.16.0-alpha.5

package/dist/rules/dispatcher.js

@@ -3,11 +3,11 @@ import { boardForWorkItem, resolvePhaseSemantics, workItemPhaseSemantics } from
 import { FACTORY_RULE_STAGES, externallyAuthoredWorkItem } from "./types.js";
 import { createBoardRegistry } from "../boards/registry.js";
 import "../boards/index.js";
-import { FACTORY_RULE_MATERIALIZATION_KEY } from "../storage/domains/work-items/base.js";
+import { FACTORY_RULE_MATERIALIZATION_KEY, WorkItemClaimConflictError } from "../storage/domains/work-items/base.js";
 import { FactoryDispatchError, factoryDispatchFailureCode, factoryDispatchFailureMetadata } from "./dispatch-errors.js";
 import { isHumanActorId } from "../storage/domains/audit/actors.js";
-import { resolvePromptInvocation, resolveSkillInvocation } from "../skills/service.js";
-import { recordSessionRunStart } from "../session/run-audit.js";
+import { resolvePromptInvocation, resolveSkillInvocation, resolveSkillResumeInvocation } from "../skills/service.js";
+import { FACTORY_OPEN_RUN_STALE_MS, heartbeatSessionOpenRun, isOpenRunLiveElsewhere, listSessionOpenRuns, recordSessionRunStart, waitForSessionRunAudit } from "../session/run-audit.js";
 import { withWorkItemFeed } from "../storage/domains/comments/feed-context.js";
 import { RequestContext } from "@mastra/core/request-context";
 import { randomUUID } from "crypto";
@@ -103,7 +103,7 @@ function watchRun(session, { timeoutMs, approvePlans, onAgentEnd, label, runStil
 				});
 				observed = await wait();
 			}
-			if (!observed) throw new Error(`${label} terminal event was not observed before timeout.`);
+			if (!observed) throw new FactoryDispatchError("run_terminal_event_missing", `${label} terminal event was not observed before timeout.`);
 			if (endReason === "error") throw new Error(`${label} ended in error.`);
 			if (endReason === "aborted") {
 				if (omFailure) {
@@ -243,6 +243,7 @@ var FactoryDecisionDispatcher = class {
 	#timer;
 	#activeClaim;
 	#inFlight = /* @__PURE__ */ new Set();
+	#bindingSkillRuns = /* @__PURE__ */ new Map();
 	constructor(options) {
 		this.#audit = options.audit;
 		this.#controller = options.controller;
@@ -521,93 +522,120 @@ var FactoryDecisionDispatcher = class {
 			case "invokeSkill": {
 				if (await this.#roleSuperseded(record, decision.role)) return;
 				const binding = await this.#requireOrPrepareBinding(record, decision.role);
-				const item = record.workItemId ? await this.#storage.get({
-					orgId: record.orgId,
-					id: record.workItemId
-				}) : null;
-				const startedBy = item?.sessions[binding.role]?.startedBy;
-				if (!startedBy) throw new Error(`Factory binding ${binding.id} has no authenticated session owner.`);
-				await this.#primeCredentials?.({
-					orgId: record.orgId,
-					userId: startedBy
-				});
-				const session = await this.#requireSession(binding);
-				const requestContext = factoryRequestContext({
-					session,
-					binding,
-					userId: startedBy,
-					orgId: record.orgId
-				});
-				const resolved = decision.skillName === void 0 ? await resolvePromptInvocation(this.#controller, {
-					resourceId: binding.resourceId,
-					prompt: decision.prompt
-				}) : await resolveSkillInvocation(this.#controller, {
-					resourceId: binding.resourceId,
-					name: decision.skillName,
-					arguments: decision.arguments
-				});
-				await this.#switchThread(session, binding);
-				const deliveryId = record.deliveryGeneration === 0 ? record.id : `${record.id}:retry:${record.deliveryGeneration}`;
-				if ((await session.thread.listActiveMessages()).some((message) => message.id === deliveryId)) return;
-				const kickoffContents = await withWorkItemFeed(this.#feedReader, {
-					orgId: record.orgId,
-					factoryProjectId: record.factoryProjectId,
-					workItemId: record.workItemId
-				}, resolved.message);
-				if (decision.cancelInFlight && session.stream.isActive()) session.abort();
-				const precedingMessage = decision.precedingMessage;
-				if (precedingMessage) await awaitNotification(() => session.sendNotificationSignal({
-					source: "factory",
-					kind: "stage-transition",
-					summary: precedingMessage,
-					priority: "medium",
-					payload: { message: precedingMessage },
-					sourceId: `${record.id}:stage-transition`,
-					dedupeKey: `${record.idempotencyKey}:stage-transition`
-				}, {
-					ifActive: { behavior: "deliver" },
-					ifIdle: { behavior: "persist" },
-					requestContext
-				}));
-				const run = watchRun(session, {
-					timeoutMs: this.#skillCompletionObservationTimeoutMs,
-					approvePlans: await this.#plansAreAutoApproved(record, item),
-					onAgentEnd: () => this.#roleSuperseded(record, decision.role),
-					label: "Factory skill run",
-					runStillActive: () => this.#controller.listActiveThreadRuns().some((active) => active.threadId === binding.threadId)
-				});
-				const sendKickoff = async () => {
-					const settled = await session.sendSignal({
-						id: deliveryId,
-						type: "user",
-						tagName: "user",
-						contents: kickoffContents
-					}, {
-						requestContext,
-						requireDelivery: true
-					}).accepted;
-					if (settled.action !== "wake" && settled.action !== "deliver") throw new Error(`Factory skill invocation signal did not reach the agent (${String(settled.action)}).`);
-					return settled;
-				};
-				try {
-					let settled = await sendKickoff();
-					if (settled.action === "deliver") {
-						if (!(await session.thread.listActiveMessages()).some((message) => message.id === deliveryId)) {
-							if (!await run.wait()) throw new Error("Factory skill invocation is waiting on a run that has not ended.");
-							run.arm();
-							settled = await sendKickoff();
-							if (settled.action !== "wake") throw new Error("Factory skill invocation was queued onto an ending run and never reached the agent.");
+				await this.#withBindingSkillRun(binding.id, async () => {
+					const item = record.workItemId ? await this.#storage.get({
+						orgId: record.orgId,
+						id: record.workItemId
+					}) : null;
+					const startedBy = item?.sessions[binding.role]?.startedBy;
+					if (!startedBy) throw new Error(`Factory binding ${binding.id} has no authenticated session owner.`);
+					await this.#primeCredentials?.({
+						orgId: record.orgId,
+						userId: startedBy
+					});
+					const session = await this.#requireSession(binding);
+					const requestContext = factoryRequestContext({
+						session,
+						binding,
+						userId: startedBy,
+						orgId: record.orgId
+					});
+					const resolved = decision.skillName === void 0 ? await resolvePromptInvocation(this.#controller, {
+						resourceId: binding.resourceId,
+						prompt: decision.prompt
+					}) : decision.resume === true ? await resolveSkillResumeInvocation(this.#controller, {
+						resourceId: binding.resourceId,
+						name: decision.skillName,
+						arguments: decision.arguments
+					}) : await resolveSkillInvocation(this.#controller, {
+						resourceId: binding.resourceId,
+						name: decision.skillName,
+						arguments: decision.arguments
+					});
+					await this.#switchThread(session, binding);
+					const deliveryId = record.deliveryGeneration === 0 ? record.id : `${record.id}:retry:${record.deliveryGeneration}`;
+					const runStillActive = () => this.#controller.listActiveThreadRuns().some((active) => active.threadId === binding.threadId);
+					while (true) {
+						const openRun = watchRun(session, {
+							timeoutMs: this.#skillCompletionObservationTimeoutMs,
+							approvePlans: false,
+							label: "Factory skill run",
+							runStillActive
+						});
+						try {
+							const open = (await listSessionOpenRuns(session)).filter((entry) => entry.bindingId === binding.id);
+							if (open.length === 0) break;
+							if (open.some((entry) => isOpenRunLiveElsewhere(entry, this.#ownerId))) throw new FactoryDispatchError("session_unavailable", `Factory binding ${binding.id} has a run in flight on another dispatcher.`);
+							if (!runStillActive()) break;
+							if (!await openRun.wait()) break;
+						} finally {
+							openRun.close();
 						}
+						await waitForSessionRunAudit(session);
 					}
-					await this.#recordRunStart(session, binding, deliveryId, record.approvedBy ?? void 0, run.endReason, item?.title);
+					if (await this.#roleSuperseded(record, decision.role)) return;
+					if ((await this.#requireBinding(record, decision.role)).id !== binding.id) throw new FactoryDispatchError("session_unavailable", `Factory binding ${binding.id} changed while waiting for its open run to end.`);
+					if ((await session.thread.listActiveMessages()).some((message) => message.id === deliveryId)) return;
+					const kickoffContents = await withWorkItemFeed(this.#feedReader, {
+						orgId: record.orgId,
+						factoryProjectId: record.factoryProjectId,
+						workItemId: record.workItemId
+					}, resolved.message);
+					if (decision.cancelInFlight && session.stream.isActive()) session.abort();
+					const precedingMessage = decision.precedingMessage;
+					if (precedingMessage) await awaitNotification(() => session.sendNotificationSignal({
+						source: "factory",
+						kind: "stage-transition",
+						summary: precedingMessage,
+						priority: "medium",
+						payload: { message: precedingMessage },
+						sourceId: `${record.id}:stage-transition`,
+						dedupeKey: `${record.idempotencyKey}:stage-transition`
+					}, {
+						ifActive: { behavior: "deliver" },
+						ifIdle: { behavior: "persist" },
+						requestContext
+					}));
+					const run = watchRun(session, {
+						timeoutMs: this.#skillCompletionObservationTimeoutMs,
+						approvePlans: await this.#plansAreAutoApproved(record, item),
+						onAgentEnd: () => this.#roleSuperseded(record, decision.role),
+						label: "Factory skill run",
+						runStillActive
+					});
+					const sendKickoff = async () => {
+						const settled = await session.sendSignal({
+							id: deliveryId,
+							type: "user",
+							tagName: "user",
+							contents: kickoffContents
+						}, {
+							requestContext,
+							requireDelivery: true
+						}).accepted;
+						if (settled.action !== "wake" && settled.action !== "deliver") throw new FactoryDispatchError(settled.action ? "notification_delivery_failed" : "skill_delivery_ambiguous", `Factory skill invocation signal did not reach the agent (${String(settled.action)}).`);
+						return settled;
+					};
 					try {
-						await run.settle();
-					} catch (error) {
-						if (!(await run.supersededAtEnd() ?? await this.#roleSuperseded(record, decision.role))) throw error;
+						let settled = await sendKickoff();
+						if (settled.action === "deliver") {
+							if (!(await session.thread.listActiveMessages()).some((message) => message.id === deliveryId)) {
+								if (!await run.wait()) throw new FactoryDispatchError("run_terminal_event_missing", "Factory skill invocation is waiting on a run whose terminal event was not observed.");
+								run.arm();
+								settled = await sendKickoff();
+								if (settled.action !== "wake") throw new FactoryDispatchError("skill_delivery_ambiguous", "Factory skill invocation was queued onto an ending run and never reached the agent.");
+							}
+						}
+						await this.#recordRunStart(session, binding, deliveryId, record.approvedBy ?? void 0, run.endReason, item?.title);
+						try {
+							await this.#withOpenRunHeartbeat(session, deliveryId, () => run.settle());
+						} catch (error) {
+							if (!(await run.supersededAtEnd() ?? await this.#roleSuperseded(record, decision.role))) throw error;
+						}
+					} finally {
+						run.close();
 					}
-				} finally {
-					run.close();
-				}
+				});
 				return;
 			}
 			case "sendMessage": {
@@ -680,24 +708,42 @@ var FactoryDecisionDispatcher = class {
 			factoryProjectId: record.factoryProjectId,
 			decision
 		}) ?? null;
-		let result = await this.#storage.upsert({
-			orgId: record.orgId,
-			userId: "factory-rule-dispatcher",
-			factoryProjectId: record.factoryProjectId,
-			input: {
-				externalSource: externalSourceForDecision(decision),
-				parentWorkItemId,
-				title: decision.title,
-				board: decision.board,
-				stages: [initialPhase],
-				sessions: {},
-				metadata: {
-					...decision.metadata,
-					[FACTORY_RULE_MATERIALIZATION_KEY]: record.idempotencyKey
-				}
-			},
-			reuseMode: "preserve"
-		});
+		let result;
+		try {
+			result = await this.#storage.upsert({
+				orgId: record.orgId,
+				userId: "factory-rule-dispatcher",
+				factoryProjectId: record.factoryProjectId,
+				input: {
+					externalSource: externalSourceForDecision(decision),
+					...decision.claimKey ? { claimKey: decision.claimKey } : {},
+					parentWorkItemId,
+					title: decision.title,
+					board: decision.board,
+					stages: [initialPhase],
+					sessions: {},
+					metadata: {
+						...decision.metadata,
+						[FACTORY_RULE_MATERIALIZATION_KEY]: record.idempotencyKey
+					}
+				},
+				reuseMode: "preserve"
+			});
+		} catch (error) {
+			if (error instanceof WorkItemClaimConflictError) return;
+			throw error;
+		}
+		if (!result.created && decision.claimKey && !result.item.claimKey) {
+			const claimed = await this.#storage.claimWorkItem({
+				orgId: record.orgId,
+				id: result.item.id,
+				claimKey: decision.claimKey
+			});
+			if (claimed) result = {
+				...result,
+				item: claimed
+			};
+		}
 		const itemBoard = boardForWorkItem(result.item);
 		if (itemBoard !== decision.board) throw new Error(`The work item belongs to board "${itemBoard}", not "${decision.board}".`);
 		if (!result.item.parentWorkItemId && parentWorkItemId && parentWorkItemId !== result.item.id) {
@@ -821,6 +867,11 @@ var FactoryDecisionDispatcher = class {
 		const bindings = await this.#storage.listRunBindings(record.orgId, record.factoryProjectId, record.workItemId);
 		const own = bindings.filter((candidate) => candidate.role === role);
 		if (own.some((candidate) => candidate.status === "active")) return false;
+		const item = await this.#storage.get({
+			orgId: record.orgId,
+			id: record.workItemId
+		}).catch(() => null);
+		if (item && workItemPhaseSemantics(this.#boards, item)?.kind === "terminal") return true;
 		return own.some((revoked) => revoked.revokedAt !== null && revoked.revokedAt.getTime() >= record.createdAt.getTime() && bindings.some((successor) => successor.role !== role && successor.status === "active" && successor.resourceId === revoked.resourceId && successor.sessionId === revoked.sessionId && successor.threadId === revoked.threadId && successor.createdAt.getTime() >= revoked.revokedAt.getTime()));
 	}
 	async #requireOrPrepareBinding(record, role) {
@@ -864,6 +915,15 @@ var FactoryDecisionDispatcher = class {
 		if (session.thread.requireId() === binding.threadId) return;
 		await session.thread.switch({ threadId: binding.threadId });
 	}
+	async #withBindingSkillRun(bindingId, effect) {
+		const run = (this.#bindingSkillRuns.get(bindingId) ?? Promise.resolve()).catch(() => {}).then(effect);
+		this.#bindingSkillRuns.set(bindingId, run);
+		try {
+			await run;
+		} finally {
+			if (this.#bindingSkillRuns.get(bindingId) === run) this.#bindingSkillRuns.delete(bindingId);
+		}
+	}
 	async #withLease(renew, effect) {
 		let renewalFailure;
 		let renewal = Promise.resolve();
@@ -886,6 +946,17 @@ var FactoryDecisionDispatcher = class {
 			await renewal;
 		}
 	}
+	/** Keep this process's claim on an open run fresh for as long as it is watching the run. */
+	async #withOpenRunHeartbeat(session, kickoffId, effect) {
+		if (!this.#audit) return effect();
+		const timer = setInterval(() => void heartbeatSessionOpenRun(session, kickoffId), Math.floor(FACTORY_OPEN_RUN_STALE_MS / 3));
+		timer.unref?.();
+		try {
+			return await effect();
+		} finally {
+			clearInterval(timer);
+		}
+	}
 	async #recordRunStart(session, binding, kickoffId, approvedBy, observedEnd, workItemName) {
 		if (!this.#audit) return;
 		const humanApproved = isHumanActorId(approvedBy);
@@ -895,6 +966,8 @@ var FactoryDecisionDispatcher = class {
 			observedEnd,
 			run: {
 				kickoffId,
+				ownerId: this.#ownerId,
+				heartbeatAt: Date.now(),
 				bindingId: binding.id,
 				role: binding.role,
 				startedBy: humanApproved ? approvedBy : "factory-rule-dispatcher",
@@ -987,6 +1060,7 @@ const FACTORY_DISPATCH_CONSTANTS = {
 	maxBackoffMs: MAX_BACKOFF_MS,
 	skillCompletionObservationTimeoutMs: SKILL_COMPLETION_OBSERVATION_TIMEOUT_MS,
 	runRegistryHeartbeatMs: RUN_REGISTRY_HEARTBEAT_MS,
+	openRunStaleMs: FACTORY_OPEN_RUN_STALE_MS,
 	maxInFlight: MAX_IN_FLIGHT,
 	stages: FACTORY_RULE_STAGES
 };