@oomol-lab/open-flow 0.1.0-beta.67 → 0.1.0-beta.69

package/dist/common/scheduler.js

@@ -5066,7 +5066,7 @@ var ZodString = /*@__PURE__*/ $constructor("ZodString", (inst, def) => {
 		return this.check(/* @__PURE__ */ _isoDuration(ZodISODuration, params));
 	}
 });
-function string(params) {
+function string$1(params) {
 	return /* @__PURE__ */ _string(ZodString, params);
 }
 var ZodStringFormat = /*@__PURE__*/ $constructor("ZodStringFormat", (inst, def) => {
@@ -5417,7 +5417,7 @@ var ZodRecord = /*@__PURE__*/ $constructor("ZodRecord", (inst, def) => {
 function record$1(keyType, valueType, params) {
 	if (!valueType || !valueType._zod) return new ZodRecord({
 		type: "record",
-		keyType: string(),
+		keyType: string$1(),
 		valueType: keyType,
 		...normalizeParams(valueType)
 	});
@@ -5665,12 +5665,12 @@ function superRefine(fn, params) {
 function json$1(params) {
 	const jsonSchema = lazy(() => {
 		return union([
-			string(params),
+			string$1(params),
 			number(),
 			boolean(),
 			_null(),
 			array(jsonSchema),
-			record$1(string(), jsonSchema)
+			record$1(string$1(), jsonSchema)
 		]);
 	});
 	return jsonSchema;
@@ -9467,11 +9467,33 @@ function answerSchema(question) {
 	};
 }
 //#endregion
+//#region src/flow/common/retry.ts
+var retryNodeTypes = [
+	"code",
+	"connector",
+	"openapi",
+	"llm",
+	"decision"
+];
+var nodeRetryLimits = {
+	maxRetries: 5,
+	minIntervalMs: 1e3,
+	maxIntervalMs: 6e4,
+	intervalStepMs: 1e3
+};
+var retryConfigurationSchema = strictObject({
+	maxRetries: number().int().min(0).max(nodeRetryLimits.maxRetries),
+	intervalMs: number().int().min(nodeRetryLimits.minIntervalMs).max(nodeRetryLimits.maxIntervalMs).multipleOf(nodeRetryLimits.intervalStepMs)
+});
+function supportsNodeRetry(node) {
+	return node.kind == "task" && retryNodeTypes.some((type) => type == ("moduleId" in node.task ? "code" : node.task.executor.kind));
+}
+//#endregion
 //#region src/flow/common/triggerScheduleSchema.ts
 var triggerScheduleSchema = array(union([object({
 	type: literal("cron"),
-	expression: string(),
-	timezone: string()
+	expression: string$1(),
+	timezone: string$1()
 }), object({
 	type: literal("every"),
 	unit: _enum([
@@ -9495,7 +9517,7 @@ var webhookMethods = [
 function webhookSupportsBody(method) {
 	return method !== "GET";
 }
-var text = string();
+var text = string$1();
 var actionConnection = text.describe("Account selection: omit while unconfigured; use an account ID to select it; use \"\" to explicitly defer configuration. Deferred Actions fail if called; deferred Agent tools are excluded from the callable tool set. Not supported by Triggers.");
 var json = json$1();
 var strings = array(text);
@@ -9515,6 +9537,7 @@ var source = union([object({
 	kind: literal("node"),
 	nodeId: text,
 	output: text,
+	system: literal(true).optional(),
 	field: text.optional()
 }), object({
 	kind: literal("binding"),
@@ -9773,7 +9796,9 @@ var base = {
 	name: text.optional(),
 	description: text.optional(),
 	icon: text.optional(),
-	timeoutMs: number().optional()
+	timeoutMs: number().optional(),
+	retry: retryConfigurationSchema.optional(),
+	onError: _enum(["stop", "error"]).optional()
 };
 var node = union([
 	strictObject({
@@ -9894,7 +9919,8 @@ object({
 		literal(5),
 		literal(6),
 		literal(7),
-		literal(8)
+		literal(8),
+		literal(9)
 	]),
 	document,
 	modules: record$1(text, module)
@@ -9955,6 +9981,13 @@ var shapes = {
 	},
 	"graph.node.delete": at,
 	"graph.node.field.set": union([
+		object({
+			...at,
+			kind: literal("graph.node.field.set"),
+			field: literal("retry"),
+			before: retryConfigurationSchema.optional(),
+			value: retryConfigurationSchema.optional()
+		}),
 		object({
 			...at,
 			kind: literal("graph.node.field.set"),
@@ -9968,7 +10001,8 @@ var shapes = {
 			field: _enum([
 				"description",
 				"icon",
-				"name"
+				"name",
+				"onError"
 			]),
 			before: text.optional(),
 			value: text.optional()
@@ -10266,6 +10300,37 @@ function selectConditionBranches(node, values) {
 	return selected.length == 0 ? [otherwiseOutput] : selected;
 }
 //#endregion
+//#region src/flow/common/errorHandling.ts
+var nodeErrorSchema = {
+	type: "object",
+	additionalProperties: false,
+	required: [
+		"code",
+		"message",
+		"nodeId",
+		"nodeName",
+		"runId",
+		"executionId"
+	],
+	properties: {
+		code: { type: "string" },
+		message: { type: "string" },
+		nodeId: { type: "string" },
+		nodeName: { type: "string" },
+		runId: { type: "string" },
+		executionId: { type: "string" }
+	}
+};
+function supportsNodeErrorHandling(node) {
+	return supportsNodeRetry(node);
+}
+function nodeSystemOutputPorts(node) {
+	return supportsNodeErrorHandling(node) && node.onError == "error" ? { error: {
+		jsonSchema: nodeErrorSchema,
+		nullable: true
+	} } : {};
+}
+//#endregion
 //#region src/flow/common/change.ts
 function portsByHandle(ports) {
 	return Object.fromEntries(ports.flatMap((port) => "handle" in port ? [[port.handle, port]] : []));
@@ -10309,23 +10374,23 @@ var schema = lazy(() => union([boolean(), strictObject({
 	if: schema.optional(),
 	then: schema.optional(),
 	else: schema.optional(),
-	$ref: string().optional(),
+	$ref: string$1().optional(),
 	$schema: _enum([
 		"http://json-schema.org/draft-07/schema#",
 		"https://json-schema.org/draft-07/schema",
 		"https://json-schema.org/draft/2019-09/schema",
 		"https://json-schema.org/draft/2020-12/schema"
 	]).optional(),
-	$defs: record$1(string(), schema).optional(),
-	definitions: record$1(string(), schema).optional(),
-	pattern: string().refine((value) => {
+	$defs: record$1(string$1(), schema).optional(),
+	definitions: record$1(string$1(), schema).optional(),
+	pattern: string$1().refine((value) => {
 		try {
 			return new RegExp(value, "u").source.length > 0;
 		} catch {
 			return false;
 		}
 	}, "Invalid regular expression.").optional(),
-	format: string().refine((value) => Object.hasOwn(format, value), "Unsupported string format.").optional(),
+	format: string$1().refine((value) => Object.hasOwn(format, value), "Unsupported string format.").optional(),
 	multipleOf: number().positive().optional(),
 	uniqueItems: boolean().optional(),
 	contains: schema.optional(),
@@ -10334,31 +10399,31 @@ var schema = lazy(() => union([boolean(), strictObject({
 	minProperties: number().int().nonnegative().optional(),
 	maxProperties: number().int().nonnegative().optional(),
 	propertyNames: schema.optional(),
-	patternProperties: record$1(string().refine((value) => {
+	patternProperties: record$1(string$1().refine((value) => {
 		try {
 			return new RegExp(value, "u").source.length > 0;
 		} catch {
 			return false;
 		}
 	}, "Invalid regular expression."), schema).optional(),
-	dependentRequired: record$1(string(), array(string())).optional(),
-	dependentSchemas: record$1(string(), schema).optional(),
-	dependencies: record$1(string(), union([schema, array(string())])).optional(),
+	dependentRequired: record$1(string$1(), array(string$1())).optional(),
+	dependentSchemas: record$1(string$1(), schema).optional(),
+	dependencies: record$1(string$1(), union([schema, array(string$1())])).optional(),
 	prefixItems: array(schema).optional(),
 	additionalItems: schema.optional(),
 	unevaluatedItems: schema.optional(),
 	unevaluatedProperties: schema.optional(),
-	description: string().optional(),
-	title: string().optional(),
+	description: string$1().optional(),
+	title: string$1().optional(),
 	default: json$1().optional(),
 	examples: array(json$1()).optional(),
 	readOnly: boolean().optional(),
 	writeOnly: boolean().optional(),
 	deprecated: boolean().optional(),
-	$comment: string().optional(),
+	$comment: string$1().optional(),
 	enum: array(json$1()).min(1).optional(),
-	properties: record$1(string(), schema).optional(),
-	required: array(string()).optional(),
+	properties: record$1(string$1(), schema).optional(),
+	required: array(string$1()).optional(),
 	additionalProperties: schema.optional(),
 	items: union([schema, array(schema)]).optional(),
 	minItems: number().int().nonnegative().optional(),
@@ -10664,10 +10729,11 @@ function nodeOutputPorts(node) {
 }
 //#endregion
 //#region src/execution/common/dependencies.ts
-var outputsSchema = record$1(string(), json$1());
-var inputSchema = record$1(string(), strictObject({
+var outputsSchema = record$1(string$1(), json$1());
+var inputSchema = record$1(string$1(), strictObject({
 	execution: number().int().nonnegative(),
-	outputs: outputsSchema
+	outputs: outputsSchema,
+	systemOutputs: outputsSchema.optional()
 }));
 function dependencyValues(input) {
 	return Object.fromEntries(Object.entries(input).map(([id, result]) => [id, result.outputs]));
@@ -10682,15 +10748,16 @@ var stateSchema = strictObject({
 	]),
 	input: inputSchema,
 	output: outputsSchema.optional(),
-	previous: array(string()),
-	released: array(string()),
-	jobId: string().optional(),
+	systemOutputs: outputsSchema.optional(),
+	previous: array(string$1()),
+	released: array(string$1()),
+	jobId: string$1().optional(),
 	seeded: boolean()
 });
 var dependenciesSchema = strictObject({
 	nextId: number().int().nonnegative(),
-	iterations: record$1(string(), number().int().nonnegative()),
-	nodes: record$1(string(), stateSchema)
+	iterations: record$1(string$1(), number().int().nonnegative()),
+	nodes: record$1(string$1(), stateSchema)
 });
 /** Dependency completion belongs to an invocation, never to a Run-wide latest-result table. */
 var Dependencies = class {
@@ -10786,7 +10853,7 @@ var Dependencies = class {
 			if (this.graph.nodes[id] == null || state.id >= nextId || ids.has(state.id)) throw new Error("Checkpoint dependency identity is invalid.");
 			ids.add(state.id);
 			if (state.status == "running" && state.jobId == null) throw new Error("Checkpoint running dependency has no job.");
-			if ((state.status == "pending" || state.status == "skipped") && (state.jobId != null || state.output != null || state.released.length != 0)) throw new Error("Checkpoint inactive dependency contains an execution.");
+			if ((state.status == "pending" || state.status == "skipped") && (state.jobId != null || state.output != null || state.systemOutputs != null || state.released.length != 0)) throw new Error("Checkpoint inactive dependency contains an execution.");
 			if (state.status == "skipped" && (state.seeded || state.previous.length != 0 || Object.keys(state.input).length != 0)) throw new Error("Checkpoint skipped dependency contains values.");
 			if (state.seeded && (!this.loops.has(id) || state.previous.some((source) => !this.loops.get(id).feedback.some((edge) => edge.source == source)))) throw new Error("Checkpoint recurrence seed is invalid.");
 			if (state.released.length != new Set(state.released).size) throw new Error("Checkpoint released handles are duplicated.");
@@ -10826,7 +10893,11 @@ var Dependencies = class {
 				outputs: {
 					...previous.outputs,
 					...result.outputs
-				}
+				},
+				...previous.systemOutputs || result.systemOutputs ? { systemOutputs: {
+					...previous.systemOutputs,
+					...result.systemOutputs
+				} } : {}
 			};
 		};
 		for (const edge of edges) {
@@ -10835,12 +10906,16 @@ var Dependencies = class {
 			const outputs = edge.sourceHandle != null && this.graph.nodes[edge.source].kind != "condition" ? Object.fromEntries(Object.entries(source.output).filter(([handle]) => handle == edge.sourceHandle)) : source.output;
 			merge(edge.source, {
 				execution: source.id,
-				outputs
+				outputs,
+				...source.systemOutputs == null ? {} : { systemOutputs: source.systemOutputs }
 			});
 		}
 		for (const id of Object.keys(input)) if (this.state.nodes[id].status == "skipped") delete input[id];
 		return input;
 	}
+	systemInputValues(nodeId) {
+		return Object.fromEntries(Object.entries(this.state.nodes[nodeId].input).flatMap(([id, value]) => value.systemOutputs == null ? [] : [[id, value.systemOutputs]]));
+	}
 	inputValues(nodeId) {
 		return dependencyValues(this.state.nodes[nodeId].input);
 	}
@@ -10931,9 +11006,10 @@ var Dependencies = class {
 		state.status = "running";
 		state.jobId = jobId;
 	}
-	release(nodeId, outputs, handles, complete) {
+	release(nodeId, outputs, handles, complete, systemOutputs) {
 		const state = this.state.nodes[nodeId];
 		state.output = outputs;
+		state.systemOutputs = systemOutputs;
 		state.released = [.../* @__PURE__ */ new Set([...state.released, ...handles])];
 		if (complete) state.status = "completed";
 	}
@@ -10942,9 +11018,33 @@ var Dependencies = class {
 	}
 };
 //#endregion
+//#region src/execution/common/events.ts
+var encoder = new TextEncoder();
+function string(value, description) {
+	if (typeof value != "string" || value.length == 0) throw new TypeError(`${description} must be a non-empty string.`);
+	return value;
+}
+/** Stable public identity shared by events and durable workflow-call provenance. */
+function projectExecutionId(platformRunId, rawRunId, jobId) {
+	return opaqueId("execution", platformRunId, rawRunId, jobId);
+}
+async function opaqueId(prefix, ...parts) {
+	const bytes = encoder.encode(JSON.stringify([
+		1,
+		prefix,
+		...parts
+	]));
+	const input = new Uint8Array(bytes.byteLength);
+	input.set(bytes);
+	return `${prefix}_${[...new Uint8Array(await crypto.subtle.digest("SHA-256", input))].slice(0, 16).map((byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+}
+function message(value) {
+	return string(value, "Runtime event message").replace(/Bearer\s+\S+/gi, "Bearer [REDACTED]").replace(/\b(api[-_ ]?key|authorization|password|secret|token)\s*[:=]\s*\S+/gi, "$1=[REDACTED]");
+}
+//#endregion
 //#region src/execution/common/wait.ts
 /** Shared by decision entry points, the editor, and durable decision storage. */
-var waitCommentSchema = string().trim().refine((value) => [...value].length <= 2e3, "Wait comment must contain at most 2,000 Unicode code points.").nullable().optional();
+var waitCommentSchema = string$1().trim().refine((value) => [...value].length <= 2e3, "Wait comment must contain at most 2,000 Unicode code points.").nullable().optional();
 function normalizeWaitComment$1(value) {
 	return waitCommentSchema.parse(value) || null;
 }
@@ -10972,12 +11072,46 @@ function usesPending(graph, nodeId) {
 	return graph.edges.some((edge) => edge.source == nodeId && edge.sourceHandle == "pending") || Object.values(graph.nodes).some((node) => !isTriggerNode(node) && Object.values(nodeInputMappings(node)).some((mapping) => mapping.kind == "sources" && mapping.sources.some((source) => source.kind == "node" && source.nodeId == nodeId && source.output == "pending")));
 }
 var waitRetentionMs = 12e4;
+var NodeValidationError = class extends PortValidationError {
+	constructor(code, error) {
+		super(error instanceof Error ? error.message : String(error));
+		this.code = code;
+	}
+};
 function nodeFailure(error, project) {
+	if (error instanceof NodeValidationError) return {
+		code: error.code,
+		message: error.message,
+		catchable: true
+	};
 	return project?.(error) ?? {
 		code: "node.failed",
 		message: error instanceof Error ? error.message : String(error)
 	};
 }
+function invokeWithRetry(context, node, invocation) {
+	const retry = supportsNodeRetry(node) ? node.retry : void 0;
+	const attempt = (index) => Effect.suspend(() => context.invokeTask(index == 0 ? invocation : {
+		...invocation,
+		invocationId: context.createId()
+	}).pipe(Effect.catch((error) => {
+		const failure = nodeFailure(error, context.projectFailure);
+		if (retry == null || index >= retry.maxRetries || failure.retryable !== true) return Effect.fail(error);
+		return Effect.gen(function* () {
+			yield* context.emit({
+				type: "node.log",
+				level: "warn",
+				runId: invocation.runId,
+				nodeId: invocation.nodeId,
+				jobId: invocation.jobId,
+				message: `Attempt ${index + 1}/${retry.maxRetries + 1} failed (${failure.code}): ${failure.message}. Retrying attempt ${index + 2}/${retry.maxRetries + 1} in ${retry.intervalMs} ms.`
+			});
+			yield* Effect.sleep(retry.intervalMs);
+			return yield* attempt(index + 1);
+		});
+	})));
+	return attempt(0);
+}
 function nodeTitle(node) {
 	if (node.name != null) return node.name;
 	switch (node.kind) {
@@ -11015,9 +11149,9 @@ function checkpointJson(value, description, depth = 0) {
 }
 var agentCheckpointSchema = strictObject({
 	state: json$1(),
-	callId: string().min(1),
-	toolId: string().min(1),
-	input: record$1(string(), json$1()),
+	callId: string$1().min(1),
+	toolId: string$1().min(1),
+	input: record$1(string$1(), json$1()),
 	rounds: number().int().nonnegative(),
 	version: literal(1)
 });
@@ -11041,7 +11175,7 @@ function decodeFlowRunCheckpoint(input) {
 		"waits",
 		...source.calls === void 0 ? [] : ["calls"]
 	], "Flow Run checkpoint");
-	if (source.version != 5) throw new Error("Flow Run checkpoint version is unsupported.");
+	if (source.version != 6) throw new Error("Flow Run checkpoint version is unsupported.");
 	const bindingValues = Object.fromEntries(Object.entries(checkpointRecord(source.bindingValues, "Checkpoint bindings")).map(([id, value]) => {
 		if (typeof value != "string") throw new Error("Checkpoint binding must be a string.");
 		return [id, value];
@@ -11049,31 +11183,39 @@ function decodeFlowRunCheckpoint(input) {
 	const inputs = Object.fromEntries(Object.entries(checkpointRecord(source.inputs, "Checkpoint inputs")).map(([id, value]) => [id, Object.fromEntries(Object.entries(checkpointRecord(value, "Checkpoint node inputs")).map(([handle, item]) => [handle, checkpointJson(item, "Checkpoint input")]))]));
 	const results = Object.fromEntries(Object.entries(checkpointRecord(source.results, "Checkpoint results")).map(([id, value]) => {
 		const result = checkpointRecord(value, "Checkpoint result");
-		checkpointExact(result, ["jobId", "outputs"], "Checkpoint result");
+		if (result.handled !== void 0 && result.handled !== true) throw new Error("Checkpoint handled flag is invalid.");
+		checkpointExact(result, [
+			"jobId",
+			"outputs",
+			...result.systemOutputs === void 0 ? [] : ["systemOutputs"],
+			...result.handled === void 0 ? [] : ["handled"]
+		], "Checkpoint result");
 		return [id, {
 			jobId: checkpointString(result.jobId, "Checkpoint job ID"),
+			...result.systemOutputs === void 0 ? {} : { systemOutputs: outputRecord(checkpointJson(result.systemOutputs, "Checkpoint system outputs"), id) },
+			...result.handled === true ? { handled: true } : {},
 			outputs: Object.fromEntries(Object.entries(checkpointRecord(result.outputs, "Checkpoint outputs")).map(([handle, item]) => [handle, checkpointJson(item, "Checkpoint output")]))
 		}];
 	}));
-	const counts = record$1(string(), record$1(string(), number().int().positive().max(Number.MAX_SAFE_INTEGER))).parse(source.counts);
+	const counts = record$1(string$1(), record$1(string$1(), number().int().positive().max(Number.MAX_SAFE_INTEGER))).parse(source.counts);
 	const dependencies = dependenciesSchema.parse(source.dependencies);
 	const pendingWaits = array(strictObject({
-		jobId: string().min(1),
-		nodeId: string().min(1),
+		jobId: string$1().min(1),
+		nodeId: string$1().min(1),
 		value: json$1(),
-		waitId: string().min(1),
+		waitId: string$1().min(1),
 		pending: json$1().optional()
 	})).parse(source.waits);
 	const calls = array(strictObject({
-		jobId: string().min(1),
-		nodeId: string().min(1),
-		childRunId: string().min(1),
-		input: record$1(string(), json$1())
+		jobId: string$1().min(1),
+		nodeId: string$1().min(1),
+		childRunId: string$1().min(1),
+		input: record$1(string$1(), json$1())
 	})).parse(source.calls ?? []);
 	if (pendingWaits.length + calls.length == 0) throw new Error("Checkpoint has no pending execution.");
-	const agents = record$1(string(), strictObject({
-		invocationId: string().min(1),
-		input: record$1(string(), json$1()),
+	const agents = record$1(string$1(), strictObject({
+		invocationId: string$1().min(1),
+		input: record$1(string$1(), json$1()),
 		remainingMs: number().positive().optional(),
 		checkpoint: agentCheckpointSchema
 	})).parse(source.agents);
@@ -11097,7 +11239,7 @@ function decodeFlowRunCheckpoint(input) {
 		dependencies,
 		inputs,
 		results,
-		version: 5,
+		version: 6,
 		waits: pendingWaits,
 		...source.calls === void 0 ? {} : { calls }
 	};
@@ -11182,14 +11324,27 @@ function validateCheckpoint(target, checkpoint, resolveInputs) {
 			if (node == null || isTriggerNode(node) || count > (node.maxExecutions ?? 1e3)) throw new Error("Checkpoint execution count is invalid.");
 		}
 	}
-	const validateResults = (results, frame = false) => {
+	const validateResults = (results, frame = false, metadata = checkpoint.results) => {
 		for (const [id, outputs] of Object.entries(results)) {
+			const result = metadata[id];
 			const node = target.graph.nodes[id];
 			if (node == null) throw new Error("Checkpoint result node is missing.");
+			const systemEnabled = nodeSystemOutputPorts(node).error != null;
+			if (systemEnabled && result?.systemOutputs == null) throw new Error("Checkpoint system output is missing.");
+			if (!isTriggerNode(node) && checkpoint.counts[""]?.[id] == null) throw new Error("Checkpoint execution count is missing.");
+			if (result?.systemOutputs != null) {
+				const error = result.systemOutputs.error;
+				if (!systemEnabled || Object.keys(result.systemOutputs).length != 1 || !Object.hasOwn(result.systemOutputs, "error") || error !== null && (!matchesSchema(error, nodeErrorSchema) || typeof error != "object" || !("nodeId" in error) || error.nodeId != id)) throw new Error("Checkpoint system output is invalid.");
+			}
+			const handled = frame ? result?.systemOutputs?.error != null : result?.handled === true;
+			if (handled !== (result?.systemOutputs?.error != null)) throw new Error("Checkpoint failure status is inconsistent.");
+			if (handled) {
+				if (Object.keys(outputs).length || result.systemOutputs?.error == null) throw new Error("Checkpoint handled failure is invalid.");
+				continue;
+			}
 			if (isTriggerNode(node)) {
 				if (!matchesTriggerOutputs(node, outputs)) throw new Error("Checkpoint Trigger output is invalid.");
 			} else {
-				if (checkpoint.counts[""]?.[id] == null) throw new Error("Checkpoint execution count is missing.");
 				if (!jsonEqual(validateOutputs(id, node, outputs), outputs)) throw new Error("Checkpoint node outputs are incomplete.");
 				const count = Object.keys(outputs).length;
 				if (node.kind == "condition" && count != 0 || isResolutionNode(node) && (resolutionActions(node).filter((action) => Object.hasOwn(outputs, action)).length > 1 || !resolutionActions(node).some((action) => Object.hasOwn(outputs, action)) && !(frame && Object.hasOwn(outputs, "pending")))) throw new Error("Checkpoint branch result is invalid.");
@@ -11208,7 +11363,7 @@ function validateCheckpoint(target, checkpoint, resolveInputs) {
 	}
 	for (const [nodeId, state] of Object.entries(checkpoint.dependencies.nodes)) {
 		const frame = dependencyValues(state.input);
-		validateResults(frame, true);
+		validateResults(frame, true, state.input);
 		if (!isTriggerNode(target.graph.nodes[nodeId]) && (state.status == "running" || state.status == "completed")) {
 			const triggers = Object.keys(frame).filter((id) => isTriggerNode(target.graph.nodes[id]));
 			if (triggers.length != 1 || triggers[0] != triggerIds[0] || !jsonEqual(frame[triggers[0]], checkpoint.results[triggers[0]].outputs)) throw new Error("Checkpoint input Trigger does not match the Run.");
@@ -11216,7 +11371,7 @@ function validateCheckpoint(target, checkpoint, resolveInputs) {
 		if (state.output != null) validateResults({ [nodeId]: state.output }, state.status == "running");
 		if (state.status == "completed") {
 			const result = checkpoint.results[nodeId];
-			if (result == null || state.jobId != null && state.jobId != result.jobId || !jsonEqual(state.output, result.outputs)) throw new Error("Checkpoint completed dependency conflicts with its result.");
+			if (result == null || state.jobId != null && state.jobId != result.jobId || !jsonEqual(state.output, result.outputs) || !jsonEqual(state.systemOutputs ?? {}, result.systemOutputs ?? {})) throw new Error("Checkpoint completed dependency conflicts with its result.");
 		}
 	}
 	for (const call of checkpoint.calls ?? []) {
@@ -11290,15 +11445,15 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 		let measuredAt = yield* Clock.currentTimeMillis;
 		let idleSince;
 		const agents = { ...resume?.checkpoint.agents };
-		const resolveInput = (mapping, port, supplied, description, valuesByNode, required = false) => {
+		const resolveInput = (mapping, port, supplied, description, valuesByNode, required = false, systemByNode = {}) => {
 			let value = inputValue(mapping, supplied === void 0 ? port.value : supplied);
 			if (mapping?.kind == "sources") {
 				const values = mapping.sources.flatMap((source) => {
 					if (source.kind == "binding") return Object.hasOwn(context.bindingValues, source.bindingId) ? [context.bindingValues[source.bindingId]] : [];
-					const outputs = valuesByNode[source.nodeId];
+					const outputs = (source.system ? systemByNode : valuesByNode)[source.nodeId];
 					if (outputs == null || !Object.hasOwn(outputs, source.output)) return [];
 					const output = outputs[source.output];
-					if (source.field === void 0) return [output];
+					if (source.field === void 0 || output === null) return [output];
 					return output != null && typeof output === "object" && !Array.isArray(output) && Object.hasOwn(output, source.field) ? [output[source.field]] : [];
 				});
 				if (values.length > 1) throw new Error(`${description} has multiple available sources.`);
@@ -11313,21 +11468,44 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 		const resolveInputs = (nodeId, node) => {
 			const mappings = nodeInputMappings(node);
 			const frame = dependencies.inputValues(nodeId);
-			return Object.fromEntries(Object.entries(nodeInputPorts(node)).map(([handle, port]) => [handle, resolveInput(mappings[handle], port, launch[nodeId]?.[handle], `Node "${nodeId}" input "${handle}"`, frame, node.kind == "condition")]));
+			const systemFrame = dependencies.systemInputValues(nodeId);
+			return Object.fromEntries(Object.entries(nodeInputPorts(node)).map(([handle, port]) => [handle, resolveInput(mappings[handle], port, launch[nodeId]?.[handle], `Node "${nodeId}" input "${handle}"`, frame, node.kind == "condition", systemFrame)]));
 		};
-		const commit = (nodeId, jobId, outputs, releasedHandles = Object.keys(outputs)) => Effect.uninterruptible(Effect.gen(function* () {
-			yield* context.emit({
+		const commit = (nodeId, jobId, outputs, releasedHandles = Object.keys(outputs), failure) => Effect.uninterruptible(Effect.gen(function* () {
+			const node = target.graph.nodes[nodeId];
+			const systemOutputs = nodeSystemOutputPorts(node).error == null ? void 0 : { error: failure == null ? null : {
+				code: failure.code,
+				message: message(failure.message),
+				nodeId,
+				nodeName: nodeTitle(node) ?? nodeId,
+				runId,
+				executionId: yield* Effect.tryPromise(() => projectExecutionId(runId, runId, jobId))
+			} };
+			const system = systemOutputs == null ? {} : { systemOutputs };
+			const handled = failure == null ? {} : { handled: true };
+			yield* context.emit(failure == null ? {
 				jobId,
 				nodeId,
 				outputs,
 				runId,
+				...system,
 				type: "node.completed"
+			} : {
+				...failure,
+				jobId,
+				nodeId,
+				runId,
+				...system,
+				handled: true,
+				type: "node.failed"
 			});
 			completed.set(nodeId, {
 				jobId,
-				outputs
+				outputs,
+				...system,
+				...handled
 			});
-			dependencies.release(nodeId, outputs, releasedHandles, true);
+			dependencies.release(nodeId, outputs, releasedHandles, true, systemOutputs);
 		}));
 		const executeNode = (nodeId, node, jobId, nodeInputs, resolution) => {
 			const config = agentConfig(node);
@@ -11366,9 +11544,10 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 						break;
 					case "task": {
 						const frame = dependencies.inputValues(nodeId);
+						const systemFrame = dependencies.systemInputValues(nodeId);
 						const previousIds = dependencies.state.nodes[nodeId].previous;
 						const additional = new Set((node.additionalInputs ?? []).map((port) => port.handle));
-						const result = yield* context.invokeTask({
+						const result = yield* invokeWithRetry(context, node, {
 							...pendingCalls.has(jobId) ? { workflowRunId: pendingCalls.get(jobId).childRunId } : {},
 							additionalInputs: Object.fromEntries(Object.entries(nodeInputs).filter(([handle]) => additional.has(handle))),
 							blockId: "moduleId" in node.task ? node.task.moduleId : nodeId,
@@ -11394,16 +11573,17 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 								getPrevious: () => {
 									return previousIds.map((previousId) => {
 										const previous = target.graph.nodes[previousId];
-										return {
+										const systemOutputs = systemFrame[previousId];
+										return Object.assign({
 											id: previousId,
-											name: previous.name ?? (!isTriggerNode(previous) ? nodeTitle(previous) : void 0) ?? previousId,
+											name: nodeTitle(previous) ?? previousId,
 											outputs: frame[previousId],
 											outputDefs: Object.entries(nodeOutputPorts(previous)).map(([handle, definition]) => Object.assign({
 												handle,
 												jsonSchema: definition.jsonSchema,
 												nullable: definition.nullable
 											}, definition.description == null ? {} : { description: definition.description }))
-										};
+										}, systemOutputs == null ? {} : { systemOutputs });
 									});
 								}
 							} : { kind: "managed" }
@@ -11411,10 +11591,10 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 						if ("executor" in node.task && node.task.executor.kind == "workflow") {
 							const response = discriminatedUnion("kind", [strictObject({
 								kind: literal("calling"),
-								childRunId: string().min(1)
+								childRunId: string$1().min(1)
 							}), strictObject({
 								kind: literal("completed"),
-								outputs: record$1(string(), json$1())
+								outputs: record$1(string$1(), json$1())
 							})]).parse(result);
 							if (response.kind == "calling") return {
 								kind: "calling",
@@ -11442,7 +11622,10 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 							}
 							outputs = { output: response.output };
 						} else if ("executor" in node.task && node.task.executor.kind == "connector" && node.task.executor.outputMode == "value") outputs = { output: checkpointJson(result, `Node "${nodeId}" output`) };
-						else outputs = outputRecord(result, nodeId);
+						else outputs = yield* Effect.try({
+							try: () => outputRecord(result, nodeId),
+							catch: (error) => new NodeValidationError("node.output-invalid", error)
+						});
 						break;
 					}
 					case "approval":
@@ -11453,7 +11636,7 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 						kind: "completed",
 						outputs: validateOutputs(nodeId, node, outputs)
 					}),
-					catch: (error) => error instanceof Error ? error : new Error(String(error))
+					catch: (error) => new NodeValidationError("node.output-invalid", error)
 				});
 			});
 			return nodeRemainingMs == null ? execution : execution.pipe(Effect.timeoutOrElse({
@@ -11539,28 +11722,34 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 			counts[nodeId] = count + 1;
 			dependencies.begin(nodeId, jobId);
 			runNode(observe(nodeId, jobId, Effect.gen(function* () {
-				const nodeInputs = yield* Effect.try({
-					try: () => resolveInputs(nodeId, node),
-					catch: (error) => error instanceof Error ? error : new Error(String(error))
-				});
-				if (isResolutionNode(node)) {
-					yield* context.emit({
-						inputs: Object.fromEntries(Object.entries(nodeInputs).filter(([handle]) => {
-							const mapping = nodeInputMappings(node)[handle];
-							return mapping?.kind != "sources" || mapping.sources.every((source) => source.kind != "binding");
-						})),
-						jobId,
-						nodeId,
-						nodeKind: node.kind,
-						...node.name == null ? {} : { nodeTitle: node.name },
-						runId,
-						type: "node.started"
+				const result = yield* Effect.gen(function* () {
+					const nodeInputs = yield* Effect.try({
+						try: () => resolveInputs(nodeId, node),
+						catch: (error) => new NodeValidationError("node.input-invalid", error)
 					});
-					yield* createWait(nodeId, jobId, nodeInputs);
-					return;
-				}
-				const result = yield* executeNode(nodeId, node, jobId, nodeInputs);
-				yield* settleNode(nodeId, jobId, result);
+					if (isResolutionNode(node)) {
+						yield* context.emit({
+							inputs: Object.fromEntries(Object.entries(nodeInputs).filter(([handle]) => {
+								const mapping = nodeInputMappings(node)[handle];
+								return mapping?.kind != "sources" || mapping.sources.every((source) => source.kind != "binding");
+							})),
+							jobId,
+							nodeId,
+							nodeKind: node.kind,
+							...node.name == null ? {} : { nodeTitle: node.name },
+							runId,
+							type: "node.started"
+						});
+						yield* createWait(nodeId, jobId, nodeInputs);
+						return;
+					}
+					return yield* executeNode(nodeId, node, jobId, nodeInputs);
+				}).pipe(Effect.catch((error) => {
+					const failure = nodeFailure(error, context.projectFailure);
+					if (!supportsNodeErrorHandling(node) || node.onError != "error" || failure.catchable !== true) return Effect.fail(error);
+					return commit(nodeId, jobId, {}, [], failure);
+				}));
+				if (result != null) yield* settleNode(nodeId, jobId, result);
 			})).pipe(Effect.ensuring(Queue.offer(changes, void 0))));
 		};
 		if (resume != null) yield* Effect.try({
@@ -11638,7 +11827,7 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 					results: Object.fromEntries(completed),
 					counts: context.counts,
 					dependencies: dependencies.state,
-					version: 5,
+					version: 6,
 					agents,
 					waits: [...pendingWaits.values()],
 					...pendingCalls.size == 0 ? {} : { calls: [...pendingCalls.values()] }
@@ -11672,9 +11861,10 @@ function runGraph(context, target, runId, launchInputs = {}, trigger, resume) {
 				const saved = completed.get(nodeId);
 				return saved == null ? [] : [{
 					nodeId,
-					status: "completed",
+					status: saved.handled ? "error-handled" : "completed",
 					jobId: saved.jobId,
-					outputs: saved.outputs
+					outputs: saved.outputs,
+					...saved.systemOutputs == null ? {} : { systemOutputs: saved.systemOutputs }
 				}];
 			})
 		};