@mastra/factory 0.24.0-alpha.0 → 0.24.0-alpha.10

package/dist/rules/dispatcher.js

@@ -6,10 +6,10 @@ import "../boards/index.js";
 import { FactoryDispatchError, factoryDispatchFailureCode, factoryDispatchFailureMetadata } from "./dispatch-errors.js";
 import { FACTORY_RULE_MATERIALIZATION_KEY, WorkItemClaimConflictError } from "../storage/domains/work-items/base.js";
 import { moveCardToBoard } from "../boards/relocate.js";
-import { isHumanActorId } from "../storage/domains/audit/actors.js";
-import { resolvePromptInvocation, resolveSkillInvocation, resolveSkillResumeInvocation } from "../skills/service.js";
 import { decisionOvertaken } from "./decision-applicability.js";
 import { FACTORY_OPEN_RUN_STALE_MS, heartbeatSessionOpenRun, isOpenRunLiveElsewhere, listSessionOpenRuns, recordSessionRunStart, waitForSessionRunAudit } from "../session/run-audit.js";
+import { isHumanActorId } from "../storage/domains/audit/actors.js";
+import { resolvePromptInvocation, resolveSkillInvocation, resolveSkillResumeInvocation } from "../skills/service.js";
 import { withWorkItemFeed } from "../storage/domains/comments/feed-context.js";
 import { RequestContext } from "@mastra/core/request-context";
 //#region src/rules/dispatcher.ts
@@ -36,6 +36,29 @@ const STALE_BINDING_TTL_MS = 1440 * 6e4;
 const RECONCILE_INTERVAL_MS = 3e4;
 /** The card moved on before this run could act; it settles as superseded, not succeeded. */
 var FactoryDecisionSuperseded = class extends Error {};
+/** A run that ran and ended in error; the session's own listener reports it to followers. */
+var RunEndedInError = class extends FactoryDispatchError {};
+const CREDENTIAL_RESOLUTION_FAILURE = "tenant model credential policy could not be resolved";
+const NO_AUTHENTICATED_USER = "the request has no authenticated user";
+function httpStatus(error) {
+	if (!error || typeof error !== "object") return void 0;
+	if ("statusCode" in error && typeof error.statusCode === "number") return error.statusCode;
+	return "status" in error && typeof error.status === "number" ? error.status : void 0;
+}
+function causeOf(error) {
+	return error && typeof error === "object" && "cause" in error ? error.cause : void 0;
+}
+function runErrorCode(error, message) {
+	const lower = message.toLowerCase();
+	if (lower.includes(CREDENTIAL_RESOLUTION_FAILURE)) return lower.includes(NO_AUTHENTICATED_USER) ? "run_user_unauthenticated" : "run_credentials_unavailable";
+	const status = httpStatus(error) ?? httpStatus(causeOf(error));
+	return status === 401 || status === 403 ? "run_credentials_unavailable" : "run_failed";
+}
+function runErrorFailure(label, error) {
+	const cause = (error instanceof Error ? error.message : String(error ?? "")).trim();
+	if (!cause) return new RunEndedInError("run_failed", `${label} ended in error.`);
+	return new RunEndedInError(runErrorCode(error, cause), `${label} ended in error: ${cause}`);
+}
 function isTerminalFailure(attempts, failureCode) {
 	return attempts >= MAX_ATTEMPTS || !factoryDispatchFailureMetadata(failureCode).canRetry;
 }
@@ -57,10 +80,12 @@ function watchRun(session, { timeoutMs, approvePlans, onAgentEnd, label, runStil
 	let supersededAtEnd;
 	let parked;
 	let omFailure;
+	let runError;
 	const arm = () => {
 		endReason = void 0;
 		supersededAtEnd = void 0;
 		omFailure = void 0;
+		runError = void 0;
 		agentEnd = new Promise((resolve) => {
 			resolveAgentEnd = resolve;
 		});
@@ -73,6 +98,10 @@ function watchRun(session, { timeoutMs, approvePlans, onAgentEnd, label, runStil
 			resolveAgentEnd();
 			return;
 		}
+		if (event.type === "error") {
+			runError = event.error;
+			return;
+		}
 		if (event.type === "om_observation_failed" || event.type === "om_buffering_failed") {
 			if (event.error) omFailure = event.error;
 			return;
@@ -100,6 +129,8 @@ function watchRun(session, { timeoutMs, approvePlans, onAgentEnd, label, runStil
 		wait,
 		supersededAtEnd: () => supersededAtEnd,
 		endReason: () => endReason,
+		/** The tool the run is parked on, when it ended suspended. */
+		parked: () => parked,
 		close: unsubscribe,
 		/** The run's own verdict, thrown as what the dispatcher should record. */
 		async settle() {
@@ -115,7 +146,7 @@ function watchRun(session, { timeoutMs, approvePlans, onAgentEnd, label, runStil
 				observed = await wait();
 			}
 			if (!observed) throw new FactoryDispatchError("run_terminal_event_missing", `${label} terminal event was not observed before timeout.`);
-			if (endReason === "error") throw new Error(`${label} ended in error.`);
+			if (endReason === "error") throw runErrorFailure(label, runError);
 			if (endReason === "aborted") {
 				if (omFailure) {
 					if (isPermanentProviderRejection(omFailure)) throw new FactoryDispatchError("run_configuration_invalid", `${label} was aborted by an observational-memory failure that will not succeed on retry: ${omFailure}`);
@@ -160,8 +191,20 @@ function factoryRequestContext(input) {
 function positiveMs(value, fallback) {
 	return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
 }
+const KEY_SHAPED_TOKEN = /\b(?:sk-[A-Za-z0-9_-]{3,}|gh[pousr]_[A-Za-z0-9]{10,}|github_pat_[A-Za-z0-9_]{10,}|xox[abprs]-[A-Za-z0-9-]{10,})/g;
+function looksLikeSecret(token) {
+	return /[A-Z]/.test(token) && /[a-z]/.test(token) && /\d/.test(token) && /[A-Za-z0-9]{16,}/.test(token);
+}
+function redactSecretRun(token) {
+	if (looksLikeSecret(token) && !token.split("/").some((piece) => /^[A-Za-z][a-z]+$/.test(piece))) return "[redacted]";
+	return token.replace(/[A-Za-z0-9_+=-]{24,}/g, (piece) => looksLikeSecret(piece) || isGroupedKey(piece) ? "[redacted]" : piece);
+}
+function isGroupedKey(token) {
+	const groups = token.split(/[-_]/);
+	return groups.length >= 4 && groups.every((group) => /[A-Z]/.test(group) && /[a-z]/.test(group) && /\d/.test(group));
+}
 function sanitizeDispatchError(error) {
-	return (error instanceof Error ? error.message : String(error)).replace(/\b(?:bearer|token|api[-_ ]?key|authorization)\s*[:=]?\s*[^\s,;]+/gi, "[redacted]").slice(0, MAX_ERROR_LENGTH);
+	return (error instanceof Error ? error.message : String(error)).replace(KEY_SHAPED_TOKEN, "[redacted]").replace(/\b[A-Za-z0-9_+=/-]{24,}/g, redactSecretRun).replace(/\b(?:bearer|token|api[-_ ]?key|authorization)\s*[:=]?\s*(?:bearer\s+)?[^\s,;]+/gi, "[redacted]").slice(0, MAX_ERROR_LENGTH);
 }
 function retryAt(now, attempts) {
 	return new Date(now.getTime() + Math.min(1e3 * 2 ** Math.max(0, attempts - 1), MAX_BACKOFF_MS));
@@ -243,6 +286,7 @@ var FactoryDecisionDispatcher = class {
 	#reconcileToolResults;
 	#prepareBinding;
 	#primeCredentials;
+	#reportRunEnd;
 	#feedReader;
 	#dismissStaleReviews;
 	#resolveLinkedWorkItemParentId;
@@ -275,6 +319,7 @@ var FactoryDecisionDispatcher = class {
 		this.#reconcileToolResults = options.reconcileToolResults;
 		this.#prepareBinding = options.prepareBinding;
 		this.#primeCredentials = options.primeCredentials;
+		this.#reportRunEnd = options.reportRunEnd;
 		this.#feedReader = options.feedReader;
 		this.#dismissStaleReviews = options.dismissStaleReviews;
 		this.#resolveLinkedWorkItemParentId = options.resolveLinkedWorkItemParentId;
@@ -376,7 +421,16 @@ var FactoryDecisionDispatcher = class {
 		try {
 			const revoked = await this.#storage.revokeStaleRunBindings({
 				olderThan: new Date(now.getTime() - this.#staleBindingTtlMs),
-				now
+				now,
+				keep: async (binding) => {
+					try {
+						const session = await this.#controller.getSession(binding.sessionId);
+						if (session?.thread.getId() !== binding.threadId) return false;
+						return session.suspensions.hasPending() || session.run.isRunning();
+					} catch {
+						return false;
+					}
+				}
 			});
 			if (revoked > 0) console.info(`Factory stale-binding sweep revoked ${revoked} binding(s)`);
 		} catch (error) {
@@ -591,14 +645,14 @@ var FactoryDecisionDispatcher = class {
 						orgId: record.orgId
 					});
 					const resolved = decision.skillName === void 0 ? await resolvePromptInvocation(this.#controller, {
-						resourceId: binding.resourceId,
+						sessionId: binding.sessionId,
 						prompt: decision.prompt
 					}) : decision.resume === true ? await resolveSkillResumeInvocation(this.#controller, {
-						resourceId: binding.resourceId,
+						sessionId: binding.sessionId,
 						name: decision.skillName,
 						arguments: decision.arguments
 					}) : await resolveSkillInvocation(this.#controller, {
-						resourceId: binding.resourceId,
+						sessionId: binding.sessionId,
 						name: decision.skillName,
 						arguments: decision.arguments
 					});
@@ -654,6 +708,7 @@ var FactoryDecisionDispatcher = class {
 						}));
 						if (await kickoffStale()) throw new FactoryDecisionSuperseded();
 					}
+					const since = /* @__PURE__ */ new Date();
 					const run = watchRun(session, {
 						timeoutMs: this.#skillCompletionObservationTimeoutMs,
 						approvePlans: await this.#plansAreAutoApproved(record),
@@ -686,7 +741,13 @@ var FactoryDecisionDispatcher = class {
 						}
 						await this.#recordRunStart(session, binding, deliveryId, record.approvedBy ?? void 0, run.endReason, item?.title);
 						try {
-							await this.#withOpenRunHeartbeat(session, deliveryId, () => run.settle());
+							await this.#reportingRunEnd({
+								binding,
+								startedBy,
+								since,
+								attempts: record.attempts,
+								superseded: async () => await run.supersededAtEnd() ?? await this.#roleSuperseded(record, decision.role)
+							}, () => this.#withOpenRunHeartbeat(session, deliveryId, () => run.settle()));
 						} catch (error) {
 							if (!(await run.supersededAtEnd() ?? await this.#roleSuperseded(record, decision.role))) throw error;
 						}
@@ -970,7 +1031,7 @@ var FactoryDecisionDispatcher = class {
 	async #requireOrPrepareBinding(record, role) {
 		const binding = await this.#findBinding(record, role);
 		if (binding) {
-			if (await this.#controller.getSessionByResource(binding.resourceId)) return binding;
+			if (await this.#controller.getSession(binding.sessionId)) return binding;
 		}
 		if (!this.#prepareBinding) throw new FactoryDispatchError("session_unavailable", binding ? "Bound Factory session not found." : `No active Factory binding for role ${role}.`);
 		const item = await this.#requireItem(record);
@@ -982,7 +1043,7 @@ var FactoryDecisionDispatcher = class {
 		return this.#requireBinding(record, role);
 	}
 	async #findSession(binding) {
-		const session = await this.#controller.getSessionByResource(binding.resourceId);
+		const session = await this.#controller.getSession(binding.sessionId);
 		if (!session) return void 0;
 		await this.#switchThread(session, binding);
 		return session;
@@ -1045,6 +1106,44 @@ var FactoryDecisionDispatcher = class {
 			clearInterval(timer);
 		}
 	}
+	/**
+	* Settle a watched run; when it fails terminally, tell the item's followers.
+	* A retryable failure says nothing, since the retry reports. How a run that
+	* ran ends (done, blocked, an error) is reported by the session's own
+	* listener; this covers failures no run end shows. The reporter skips
+	* recipients that already have a final report from this run, and never
+	* throws, so reporting can't change the decision's outcome.
+	*/
+	async #reportingRunEnd({ binding, startedBy, since, attempts, superseded }, settle) {
+		try {
+			await settle();
+		} catch (error) {
+			const reportRunEnd = this.#reportRunEnd;
+			if (reportRunEnd && !(error instanceof RunEndedInError) && !(error instanceof FactoryDecisionSuperseded) && !await superseded?.()) {
+				const failureCode = factoryDispatchFailureCode(error);
+				if (isTerminalFailure(attempts, failureCode)) reportRunEnd({
+					orgId: binding.orgId,
+					factoryProjectId: binding.factoryProjectId,
+					workItemId: binding.workItemId,
+					role: binding.role,
+					sender: {
+						sessionId: binding.sessionId,
+						threadId: binding.threadId,
+						userId: startedBy
+					},
+					since,
+					outcome: {
+						status: "failed",
+						failureCode,
+						reason: sanitizeDispatchError(error)
+					}
+				}).catch((reportError) => {
+					console.warn(`[factory] run-end report failed for binding ${binding.id}:`, reportError);
+				});
+			}
+			throw error;
+		}
+	}
 	async #recordRunStart(session, binding, kickoffId, approvedBy, observedEnd, workItemName) {
 		if (!this.#audit) return;
 		const humanApproved = isHumanActorId(approvedBy);
@@ -1092,6 +1191,7 @@ var FactoryDecisionDispatcher = class {
 					userId: startedBy,
 					orgId: record.orgId
 				});
+				const since = /* @__PURE__ */ new Date();
 				const run = watchRun(session, {
 					timeoutMs: this.#skillCompletionObservationTimeoutMs,
 					approvePlans: await this.#plansAreAutoApproved(record),
@@ -1121,7 +1221,12 @@ var FactoryDecisionDispatcher = class {
 						settled = await sendKickoff(`factory-kickoff:${record.kickoffKey}:retry:${record.attempts}:${deliveryGeneration}`);
 					}
 					await this.#recordRunStart(session, binding, `factory-kickoff:${record.kickoffKey}:${record.attempts}`, startedBy, run.endReason, item?.title);
-					await run.settle();
+					await this.#reportingRunEnd({
+						binding,
+						startedBy,
+						since,
+						attempts: record.attempts
+					}, () => run.settle());
 				} finally {
 					run.close();
 				}