@notionhq/apps 0.0.40 → 0.0.41

package/src/workflow.ts

@@ -19,7 +19,7 @@ import { join } from "node:path";
 
 import type { CapabilityContext } from "./context.js";
 import { createCapabilityContext } from "./context.js";
-import { ExecutionError } from "./error.js";
+import { ExecutionError, FatalError, RateLimitError, RetryableError } from "./error.js";
 import { writeOutput } from "./output.js";
 import { resolveRuntimeInput } from "./runtime-input.js";
 import { readRunMetadata, type RunMetadata } from "./runtime-metadata.js";
@@ -186,6 +186,7 @@ type WorkflowWaitResult = WorkflowWaitUntilResult | WorkflowWaitForInputResult;
 type WorkflowHandlerResult = { status: "success" } | WorkflowWaitResult;
 
 const MAX_WORKFLOW_WAIT_MS = 7 * 24 * 60 * 60 * 1_000;
+export { FatalError, RetryableError } from "./error.js";
 
 export type {
 	WorkflowAccess,
@@ -228,6 +229,58 @@ export {
 	type OAuthConnection,
 } from "./connections.js";
 
+export type WorkflowDeadline = {
+	/** Maximum wall-clock lifetime of one logical workflow run. */
+	afterMs: number;
+};
+
+/**
+ * An error class, or a predicate, that marks an error as retryable.
+ * Classes match with `instanceof`; any other function is called with the error.
+ */
+export type WorkflowRetryableErrorMatcher =
+	| (abstract new (...args: never[]) => Error)
+	| ((error: unknown) => boolean);
+
+export type WorkflowRetryPolicy = {
+	/** Total attempts, including the initial attempt. At most 4. */
+	maxAttempts: number;
+	/** Delay before the first retry. Defaults to 1 second. */
+	initialDelayMs?: number;
+	/** Maximum delay between attempts. Defaults to 5 minutes. */
+	maxDelayMs?: number;
+	/** Exponential backoff multiplier. Defaults to 2. */
+	backoffMultiplier?: number;
+	/**
+	 * Errors that may be retried, as one matcher or an array of matchers. When
+	 * set, only matching errors and `RetryableError` are retried; every other
+	 * error fails immediately. When omitted, every error except `FatalError` is
+	 * retried.
+	 */
+	retryOn?: WorkflowRetryableErrorMatcher | readonly WorkflowRetryableErrorMatcher[];
+};
+
+/** Platform limit on `maxAttempts`. */
+const MAX_WORKFLOW_RETRY_ATTEMPTS = 3;
+
+/** Platform limit on deadlines and retry delays: seven days. */
+const MAX_WORKFLOW_DURATION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * One workflow invocation runs in a sandbox that stops after five minutes, and
+ * step retries wait inside that invocation. A step's worst-case retry delays
+ * and timeouts must fit in this budget.
+ */
+const WORKFLOW_STEP_RETRY_BUDGET_MS = 5 * 60 * 1000;
+
+/** Retry policy used when neither the workflow nor the step sets one. */
+const DEFAULT_WORKFLOW_RETRY_POLICY = {
+	maxAttempts: 3,
+	initialDelayMs: 1_000,
+	maxDelayMs: 5 * 60 * 1000,
+	backoffMultiplier: 2,
+} as const satisfies WorkflowRetryPolicy;
+
 /**
  * The inbound HTTP request passed to a webhook verify handler.
  */
@@ -332,6 +385,15 @@ export type WorkflowConfiguration<
 	 */
 	connections?: TConnections;
 
+	/** Maximum wall-clock lifetime of one logical workflow run. */
+	deadline?: WorkflowDeadline;
+
+	/**
+	 * Default retry policy for every step, and the retry policy for failures
+	 * that happen outside a step. A step's own `retry` replaces it.
+	 */
+	retry?: WorkflowRetryPolicy;
+
 	/**
 	 * Optional synchronous verification handler, run at Notion's webhook
 	 * ingress before the HTTP response is sent. Use it for providers that
@@ -400,6 +462,8 @@ export type Workflow<
 		triggers: WorkflowManifestTrigger[];
 		connections?: readonly WorkflowConnectionRequirement[];
 		access?: WorkflowAccessRequirements;
+		deadline?: WorkflowDeadline;
+		retry?: WorkflowRetryPolicy;
 	};
 	handler: (
 		input: WorkflowEventForTriggers<TTriggers> | WebhookVerifyInvocation,
@@ -468,6 +532,16 @@ export function workflow<
 			? configuration.triggers({ events: createWorkflowEvents<TConnections>() })
 			: configuration.triggers;
 	validateTriggerConnections(triggers, requirements);
+	validateWorkflowDeadline(configuration.deadline);
+	validateWorkflowRetryPolicy(configuration.retry, "Workflow retry policy");
+	if (configuration.retry !== undefined) {
+		validateStepRetryBudget(configuration.retry, undefined, "Workflow retry policy");
+	}
+	const deadline =
+		configuration.deadline === undefined ? undefined : structuredClone(configuration.deadline);
+	const retry =
+		configuration.retry === undefined ? undefined : manifestRetryPolicy(configuration.retry);
+
 	const manualTriggers = triggers.filter(
 		(trigger): trigger is ManualWorkflowTrigger => trigger.type === "workflow.manual",
 	);
@@ -493,11 +567,17 @@ export function workflow<
 			triggers: manifestTriggers(triggers, configuration.verify !== undefined),
 			...(configuration.connections === undefined ? {} : { connections: requirements }),
 			...(configuration.access === undefined ? {} : { access: accessRequirements }),
+			...(deadline === undefined ? {} : { deadline }),
+			...(retry === undefined ? {} : { retry }),
 		},
 		async handler(
 			input: WorkflowEventForTriggers<TTriggers> | WebhookVerifyInvocation,
 			options?: HandlerOptions,
 		): Promise<WorkflowHandlerResult | WebhookVerifyResponse | undefined> {
+			const invocationStartedAt = Date.now();
+			// Errors from steps that already used their retry policy. The run is not
+			// retried for these, so they are reported as terminal.
+			const exhaustedStepErrors = new WeakSet<object>();
 			try {
 				input = await resolveRuntimeInput(input);
 
@@ -525,7 +605,11 @@ export function workflow<
 				const event = input;
 				const runMetadata = readRunMetadata();
 				const baseContext = createCapabilityContext();
-				const step = createStep(runMetadata);
+				const step = createStep(runMetadata, {
+					defaultRetry: configuration.retry,
+					exhaustedErrors: exhaustedStepErrors,
+					invocationStartedAt,
+				});
 				const capabilityContext: WorkflowContext<TConnections, TAccess> = {
 					...baseContext,
 					access: hydrateWorkflowAccess<TAccess>(accessRequirements),
@@ -567,15 +651,51 @@ export function workflow<
 				}
 
 				const error = new ExecutionError(err);
+				const terminal =
+					err instanceof FatalError ||
+					(isObject(err) && exhaustedStepErrors.has(err)) ||
+					(!(err instanceof RetryableError) &&
+						!isRetryableError(err, configuration.retry));
 
 				if (!options?.concreteOutput) {
 					writeOutput({
 						_tag: "error",
-						error: {
-							name: error.name,
-							message: error.message,
-							trace: error.stack,
-						},
+						error:
+							!terminal && err instanceof RateLimitError
+								? {
+										_tag: "rate_limit",
+										name: err.name,
+										message: err.message,
+										...(err.retryAfter === undefined
+											? {}
+											: { retryAfter: err.retryAfter }),
+									}
+								: !terminal && err instanceof RetryableError
+									? {
+											_tag: "retryable",
+											name: err.name,
+											message: err.message,
+											trace: err.stack,
+											...(err.retryAfterMs === undefined
+												? {}
+												: { retryAfterMs: err.retryAfterMs }),
+										}
+									: terminal
+										? {
+												_tag: "terminal",
+												name: err instanceof Error ? err.name : error.name,
+												message:
+													err instanceof Error
+														? err.message
+														: error.message,
+												trace:
+													err instanceof Error ? err.stack : error.stack,
+											}
+										: {
+												name: error.name,
+												message: error.message,
+												trace: error.stack,
+											},
 					});
 				}
 
@@ -635,7 +755,11 @@ export type WorkflowStepOptions = {
 	 * Pass a string for a single key segment or an array of strings for a composite
 	 * key. Defaults to the step name. Changing the key creates a new logical step.
 	 */
-	key: string | string[];
+	key?: string | string[];
+	/** Maximum duration of one invocation of the step callback. */
+	timeoutMs?: number;
+	/** Retry policy for failures from this step callback. */
+	retry?: WorkflowRetryPolicy;
 };
 
 type WorkflowStep = {
@@ -733,8 +857,15 @@ function writeStepEvent(
 
 function createStep(
 	metadata: Pick<RunMetadata, "runGroupId" | "workerId">,
+	options: {
+		defaultRetry?: WorkflowRetryPolicy | undefined;
+		exhaustedErrors?: WeakSet<object>;
+		/** When the handler started. Step retries share one invocation's budget. */
+		invocationStartedAt: number;
+	},
 ): WorkflowContext["step"] {
 	const { runGroupId } = metadata;
+	const { defaultRetry, exhaustedErrors, invocationStartedAt } = options;
 	const stepNameByKey = new Map<string, string>();
 
 	async function step<T>(
@@ -742,13 +873,17 @@ function createStep(
 		optionsOrFn: WorkflowStepOptions | ((context: StepContext) => T | Promise<T>),
 		maybeFn?: (context: StepContext) => T | Promise<T>,
 	): Promise<WorkflowStepResult<T>> {
-		const options = typeof optionsOrFn === "function" ? undefined : optionsOrFn;
+		const stepOptions = typeof optionsOrFn === "function" ? undefined : optionsOrFn;
 		const fn = typeof optionsOrFn === "function" ? optionsOrFn : maybeFn;
 		if (!fn) {
 			throw new Error(`Workflow step "${name}" is missing its callback.`);
 		}
 
-		const keyInput = options?.key ?? name;
+		validateWorkflowStepOptions(stepOptions);
+		const retry = stepOptions?.retry ?? defaultRetry ?? DEFAULT_WORKFLOW_RETRY_POLICY;
+		validateStepRetryBudget(retry, stepOptions?.timeoutMs, `Workflow step "${name}"`);
+
+		const keyInput = stepOptions?.key ?? name;
 		const key = createWorkflowStepKey(typeof keyInput === "string" ? [keyInput] : keyInput);
 		const existingStepName = stepNameByKey.get(key);
 		if (existingStepName !== undefined) {
@@ -759,9 +894,7 @@ function createStep(
 		stepNameByKey.set(key, name);
 
 		const id = `${runGroupId}:${key}`;
-		const stepState = createWorkflowStepState(metadata, id);
-		const context: StepContext = { id, state: stepState.state };
-		const event = { id: context.id, name, key };
+		const event = { id, name, key };
 		const startedAt = performance.now();
 
 		writeStepEvent("started", event, startedAt, startedAt);
@@ -789,7 +922,16 @@ function createStep(
 				return checkpoint.value as WorkflowStepResult<T>;
 			}
 
-			const value = normalizeWorkflowStepResult(await stepState.run(() => fn(context)));
+			const value = normalizeWorkflowStepResult(
+				await runWorkflowStepCallback({
+					metadata,
+					id,
+					fn,
+					timeoutMs: stepOptions?.timeoutMs,
+					retry,
+					invocationStartedAt,
+				}),
+			);
 			writeStepEvent("success", { ...event, value }, startedAt);
 			writeStepEvent(
 				"completed",
@@ -800,7 +942,11 @@ function createStep(
 				startedAt,
 			);
 			return value;
-		} catch (err) {
+		} catch (caught) {
+			// A retry that no longer fits in this invocation is left to the run's
+			// retry policy, so the step is not reported as having used its retries.
+			const deferred = caught instanceof DeferredStepRetry;
+			const err = deferred ? caught.error : caught;
 			const error = new ExecutionError(err);
 			writeStepEvent(
 				"failure",
@@ -822,6 +968,9 @@ function createStep(
 				},
 				startedAt,
 			);
+			if (!deferred && isObject(err)) {
+				exhaustedErrors?.add(err);
+			}
 			throw err;
 		}
 	}
@@ -829,6 +978,11 @@ function createStep(
 	return step;
 }
 
+/** Signals that a step retry must be deferred to a later workflow invocation. */
+class DeferredStepRetry {
+	constructor(readonly error: unknown) {}
+}
+
 class WorkflowWaitInterrupt extends Error {
 	constructor(readonly result: WorkflowWaitResult) {
 		super(workflowWaitMessage(result));
@@ -1035,6 +1189,235 @@ function waitDurationUnitMultiplier(unit: string): number {
 	}
 }
 
+async function runWorkflowStepCallback<T>(args: {
+	metadata: Pick<RunMetadata, "workerId">;
+	id: string;
+	fn: (context: StepContext) => T | Promise<T>;
+	timeoutMs: number | undefined;
+	retry: WorkflowRetryPolicy;
+	invocationStartedAt: number;
+}): Promise<T> {
+	let attempt = 1;
+
+	while (true) {
+		// Each attempt gets its own state scope so a failed or timed-out attempt
+		// never commits, and its pending mutations are not visible to the retry.
+		const stepState = createWorkflowStepState(args.metadata, args.id);
+		const context: StepContext = { id: args.id, state: stepState.state };
+		try {
+			return await stepState.run(() => runWithTimeout(args.fn(context), args.timeoutMs));
+		} catch (error) {
+			if (attempt >= args.retry.maxAttempts || !isRetryableError(error, args.retry)) {
+				throw error;
+			}
+
+			const delayMs =
+				explicitRetryDelay(error) ?? calculateRetryDelay(args.retry, attempt - 1);
+			// Earlier work in this invocation shares the sandbox, so measure from
+			// the handler's start rather than the step's.
+			const nextAttemptBudgetMs = delayMs + (args.timeoutMs ?? 0);
+			if (
+				Date.now() - args.invocationStartedAt + nextAttemptBudgetMs >
+				WORKFLOW_STEP_RETRY_BUDGET_MS
+			) {
+				throw new DeferredStepRetry(error);
+			}
+			if (delayMs > 0) {
+				await new Promise((resolve) => setTimeout(resolve, delayMs));
+			}
+			attempt += 1;
+		}
+	}
+}
+
+/**
+ * Whether a policy permits retrying an error. `FatalError` never retries and
+ * `RetryableError` always does; `retryOn`, when set, limits everything else.
+ */
+function isRetryableError(error: unknown, policy: WorkflowRetryPolicy | undefined): boolean {
+	if (error instanceof FatalError) {
+		return false;
+	}
+	if (error instanceof RetryableError || policy?.retryOn === undefined) {
+		return true;
+	}
+	const matchers: readonly WorkflowRetryableErrorMatcher[] =
+		typeof policy.retryOn === "function" ? [policy.retryOn] : policy.retryOn;
+	return matchers.some((matcher) =>
+		isErrorClass(matcher)
+			? error instanceof matcher
+			: (matcher as (error: unknown) => boolean)(error),
+	);
+}
+
+function isErrorClass(
+	matcher: WorkflowRetryableErrorMatcher,
+): matcher is abstract new (...args: never[]) => Error {
+	return matcher === Error || matcher.prototype instanceof Error;
+}
+
+/** A delay requested by the error itself, which takes precedence over backoff. */
+function explicitRetryDelay(error: unknown): number | undefined {
+	if (error instanceof RetryableError) {
+		return error.retryAfterMs;
+	}
+	if (error instanceof RateLimitError && error.retryAfter !== undefined) {
+		return error.retryAfter * 1_000;
+	}
+	return undefined;
+}
+
+async function runWithTimeout<T>(value: T | Promise<T>, timeoutMs: number | undefined): Promise<T> {
+	if (timeoutMs === undefined) {
+		return await value;
+	}
+
+	let timer: NodeJS.Timeout | undefined;
+	try {
+		return await Promise.race([
+			value,
+			new Promise<never>((_resolve, reject) => {
+				timer = setTimeout(() => {
+					const error = new Error(`Workflow step timed out after ${timeoutMs}ms.`);
+					error.name = "StepTimeoutError";
+					reject(error);
+				}, timeoutMs);
+			}),
+		]);
+	} finally {
+		if (timer !== undefined) {
+			clearTimeout(timer);
+		}
+	}
+}
+
+/**
+ * Exponential backoff with "equal jitter": half the delay is fixed and half is
+ * random, so retries from many runs spread out without retrying too early.
+ */
+function calculateRetryDelay(policy: WorkflowRetryPolicy, retryIndex: number): number {
+	const delayMs = maxRetryDelay(policy, retryIndex);
+	return delayMs / 2 + Math.random() * (delayMs / 2);
+}
+
+/** The backoff delay before jitter, which is also its upper bound. */
+function maxRetryDelay(policy: WorkflowRetryPolicy, retryIndex: number): number {
+	const initialDelayMs = policy.initialDelayMs ?? DEFAULT_WORKFLOW_RETRY_POLICY.initialDelayMs;
+	const maxDelayMs = policy.maxDelayMs ?? DEFAULT_WORKFLOW_RETRY_POLICY.maxDelayMs;
+	const backoffMultiplier =
+		policy.backoffMultiplier ?? DEFAULT_WORKFLOW_RETRY_POLICY.backoffMultiplier;
+	return Math.min(initialDelayMs * backoffMultiplier ** retryIndex, maxDelayMs);
+}
+
+/**
+ * Rejects retry policies whose longest possible retry delays plus step timeouts
+ * cannot fit in one workflow invocation.
+ */
+function validateStepRetryBudget(
+	policy: WorkflowRetryPolicy,
+	timeoutMs: number | undefined,
+	label: string,
+): void {
+	let worstCaseMs = (timeoutMs ?? 0) * policy.maxAttempts;
+	for (let retryIndex = 0; retryIndex < policy.maxAttempts - 1; retryIndex++) {
+		worstCaseMs += maxRetryDelay(policy, retryIndex);
+	}
+	if (worstCaseMs > WORKFLOW_STEP_RETRY_BUDGET_MS) {
+		throw new Error(
+			`${label} can spend up to ${worstCaseMs}ms on retry delays and timeouts, which exceeds the ${WORKFLOW_STEP_RETRY_BUDGET_MS}ms limit for one workflow invocation. Lower maxAttempts, the retry delays, or timeoutMs, or use context.wait.until() for longer waits.`,
+		);
+	}
+}
+
+/** The manifest carries only serializable policy fields; `retryOn` runs in the SDK. */
+function manifestRetryPolicy(policy: WorkflowRetryPolicy): Omit<WorkflowRetryPolicy, "retryOn"> {
+	const { retryOn: _retryOn, ...rest } = policy;
+	return structuredClone(rest);
+}
+
+function validateWorkflowDeadline(deadline: WorkflowDeadline | undefined): void {
+	if (deadline === undefined) {
+		return;
+	}
+	assertPositiveFiniteNumber(deadline.afterMs, "Workflow deadline afterMs");
+	assertAtMost(deadline.afterMs, MAX_WORKFLOW_DURATION_MS, "Workflow deadline afterMs");
+}
+
+function validateWorkflowRetryPolicy(policy: WorkflowRetryPolicy | undefined, label: string): void {
+	if (policy === undefined) {
+		return;
+	}
+	if (
+		!Number.isInteger(policy.maxAttempts) ||
+		policy.maxAttempts < 1 ||
+		policy.maxAttempts > MAX_WORKFLOW_RETRY_ATTEMPTS
+	) {
+		throw new Error(
+			`${label} maxAttempts must be an integer between 1 and ${MAX_WORKFLOW_RETRY_ATTEMPTS}.`,
+		);
+	}
+	if (policy.initialDelayMs !== undefined) {
+		assertNonNegativeFiniteNumber(policy.initialDelayMs, `${label} initialDelayMs`);
+		assertAtMost(policy.initialDelayMs, MAX_WORKFLOW_DURATION_MS, `${label} initialDelayMs`);
+	}
+	if (policy.maxDelayMs !== undefined) {
+		assertNonNegativeFiniteNumber(policy.maxDelayMs, `${label} maxDelayMs`);
+		assertAtMost(policy.maxDelayMs, MAX_WORKFLOW_DURATION_MS, `${label} maxDelayMs`);
+	}
+	if (
+		policy.retryOn !== undefined &&
+		typeof policy.retryOn !== "function" &&
+		(!Array.isArray(policy.retryOn) ||
+			policy.retryOn.some((matcher) => typeof matcher !== "function"))
+	) {
+		throw new Error(
+			`${label} retryOn must be an error class, a predicate, or an array of them.`,
+		);
+	}
+	if (
+		policy.initialDelayMs !== undefined &&
+		policy.maxDelayMs !== undefined &&
+		policy.maxDelayMs < policy.initialDelayMs
+	) {
+		throw new Error(`${label} maxDelayMs must be at least initialDelayMs.`);
+	}
+	if (
+		policy.backoffMultiplier !== undefined &&
+		(!Number.isFinite(policy.backoffMultiplier) || policy.backoffMultiplier < 1)
+	) {
+		throw new Error(`${label} backoffMultiplier must be at least one.`);
+	}
+}
+
+function validateWorkflowStepOptions(options: WorkflowStepOptions | undefined): void {
+	if (options?.timeoutMs !== undefined) {
+		assertPositiveFiniteNumber(options.timeoutMs, "Workflow step timeoutMs");
+	}
+	validateWorkflowRetryPolicy(options?.retry, "Workflow step retry policy");
+}
+
+function assertAtMost(value: number, max: number, label: string): void {
+	if (value > max) {
+		throw new Error(`${label} must be at most ${max}.`);
+	}
+}
+
+function isObject(value: unknown): value is object {
+	return (typeof value === "object" && value !== null) || typeof value === "function";
+}
+
+function assertPositiveFiniteNumber(value: number, label: string): void {
+	if (!Number.isFinite(value) || value <= 0) {
+		throw new Error(`${label} must be greater than zero.`);
+	}
+}
+
+function assertNonNegativeFiniteNumber(value: number, label: string): void {
+	if (!Number.isFinite(value) || value < 0) {
+		throw new Error(`${label} must be zero or greater.`);
+	}
+}
+
 function createWorkflowStepKey(segments: readonly unknown[]): string {
 	if (segments.length === 0) {
 		throw new Error("Workflow step keys must contain at least one segment.");