llmz 0.6.3 → 0.6.5

package/DOCS.md

@@ -1582,6 +1582,31 @@ await execute({
 
 ---
 
+### Buffered Delivery (midStreamFallback)
+
+By default, LLMz streams responses: send blocks are dispatched as soon as they are parsed, and the ■run block starts executing while the model is still generating the rest of the response. This gives the lowest time-to-first-token but runs code before the response is complete.
+
+Set `options.midStreamFallback: true` to allow Cognitive to restart a failed stream on another model safely. LLMz buffers delivery and discards all content from abandoned attempts:
+
+```typescript
+await execute({
+  // ...
+  options: { midStreamFallback: true },
+})
+```
+
+When enabled, the runtime buffers the full generated response and only dispatches it (sends and code execution) once the stream has finished. Trade-offs:
+
+- **Latency**: higher time-to-first-send/token, since nothing is delivered until the response completes.
+- **No early code**: the VM is not run mid-stream; code executes only after the full response is parsed.
+- **Cancellation**: cancellation still aborts the original generation request as usual.
+- **No extra retry loop**: Cognitive owns fallback, bounded by the model chain. LLMz does not retry abandoned attempts itself.
+- **Not exactly-once**: abandoned attempts deliver nothing, but successful-attempt callbacks cannot be rolled back if delivery fails or cancellation arrives during delivery. Caller retries can duplicate effects.
+
+This option defaults to false and applies only to streaming clients. Non-streaming Cognitive requests already fall back transparently. It can be combined with `maxTimeToFirstToken` and `transcriptionModel`; do not enable it globally underneath LLMz without also enabling LLMz's buffering option.
+
+Restarts emit `llm_call_restarted` traces with the attempt, source/destination models, and reason. Token timings remain relative to the original request and describe the surviving attempt (including time spent on prior attempts). Usage and cost retain Cognitive's final reported metadata; LLMz does not infer or sum abandoned-attempt billing. Cancellation deadlines are not reset, and the existing stream inactivity guard remains active during handoffs.
+
 ## API Reference
 
 ### Core Functions

package/dist/context.d.ts

@@ -564,6 +564,11 @@ export declare class Context implements Serializable<Context.JSON> {
      * before the cognitive service falls back to the next model/provider.
      */
     maxTimeToFirstToken?: number;
+    /**
+     * Allow Cognitive mid-stream fallback, buffering delivery until generation
+     * succeeds and discarding abandoned attempts. Streaming-only; defaults to false.
+     */
+    midStreamFallback?: boolean;
     /**
      * STT model used by the cognitive service to transcribe audio attachments
      * when the target LLM does not support audio natively. Defaults to 'fast'.
@@ -600,6 +605,7 @@ export declare class Context implements Serializable<Context.JSON> {
         timeout?: number;
         maxTokens?: number;
         maxTimeToFirstToken?: number;
+        midStreamFallback?: boolean;
         transcriptionModel?: SttModels;
     });
     toJSON(): {

package/dist/{custom-client-BPmEoWKu.cjs → custom-client-BX8gx0tI.cjs}

@@ -2254,6 +2254,11 @@ var Context = class {
 	*/
 	maxTimeToFirstToken;
 	/**
+	* Allow Cognitive mid-stream fallback, buffering delivery until generation
+	* succeeds and discarding abandoned attempts. Streaming-only; defaults to false.
+	*/
+	midStreamFallback;
+	/**
 	* STT model used by the cognitive service to transcribe audio attachments
 	* when the target LLM does not support audio natively. Defaults to 'fast'.
 	*/
@@ -2509,10 +2514,12 @@ var Context = class {
 		this.snapshot = props.snapshot;
 		this.maxTokens = props.maxTokens;
 		this.maxTimeToFirstToken = props.maxTimeToFirstToken;
+		this.midStreamFallback = props.midStreamFallback;
 		this.transcriptionModel = props.transcriptionModel;
 		if (this.loop < 1 || this.loop > 100) throw new Error("Invalid loop. Expected a number between 1 and 100.");
 		if (this.maxTokens !== void 0 && (!Number.isFinite(this.maxTokens) || this.maxTokens < 1)) throw new Error("Invalid maxTokens. Expected a positive number.");
 		if (this.maxTimeToFirstToken !== void 0 && (!Number.isFinite(this.maxTimeToFirstToken) || this.maxTimeToFirstToken < 1)) throw new Error("Invalid maxTimeToFirstToken. Expected a positive number of milliseconds.");
+		if (this.midStreamFallback !== void 0 && typeof this.midStreamFallback !== "boolean") throw new Error("Invalid midStreamFallback. Expected a boolean.");
 	}
 	toJSON() {
 		var _this$snapshot4;

package/dist/{custom-client-NUoxrFQh.js → custom-client-BnlDpjdI.js}

@@ -2254,6 +2254,11 @@ var Context = class {
 	*/
 	maxTimeToFirstToken;
 	/**
+	* Allow Cognitive mid-stream fallback, buffering delivery until generation
+	* succeeds and discarding abandoned attempts. Streaming-only; defaults to false.
+	*/
+	midStreamFallback;
+	/**
 	* STT model used by the cognitive service to transcribe audio attachments
 	* when the target LLM does not support audio natively. Defaults to 'fast'.
 	*/
@@ -2509,10 +2514,12 @@ var Context = class {
 		this.snapshot = props.snapshot;
 		this.maxTokens = props.maxTokens;
 		this.maxTimeToFirstToken = props.maxTimeToFirstToken;
+		this.midStreamFallback = props.midStreamFallback;
 		this.transcriptionModel = props.transcriptionModel;
 		if (this.loop < 1 || this.loop > 100) throw new Error("Invalid loop. Expected a number between 1 and 100.");
 		if (this.maxTokens !== void 0 && (!Number.isFinite(this.maxTokens) || this.maxTokens < 1)) throw new Error("Invalid maxTokens. Expected a positive number.");
 		if (this.maxTimeToFirstToken !== void 0 && (!Number.isFinite(this.maxTimeToFirstToken) || this.maxTimeToFirstToken < 1)) throw new Error("Invalid maxTimeToFirstToken. Expected a positive number of milliseconds.");
+		if (this.midStreamFallback !== void 0 && typeof this.midStreamFallback !== "boolean") throw new Error("Invalid midStreamFallback. Expected a boolean.");
 	}
 	toJSON() {
 		var _this$snapshot4;

package/dist/{execute-BEaq3jkV.cjs → execute-BJF4Txh2.cjs}

@@ -1,6 +1,6 @@
 const require__baseForOwn = require("./_baseForOwn-buOZDFbK.cjs");
 const require_dual_modes = require("./dual-modes-wAOAlaBS.cjs");
-const require_custom_client = require("./custom-client-BPmEoWKu.cjs");
+const require_custom_client = require("./custom-client-BX8gx0tI.cjs");
 const require_utils = require("./utils-ZpJm00ed.cjs");
 const require_jsx = require("./jsx-BWvT6ukg.cjs");
 const require_stack_traces = require("./stack-traces-ntw_qhoy.cjs");
@@ -121,7 +121,7 @@ const RESPONSE_LENGTH_BUFFER = {
 	PERCENTAGE: .1
 };
 /** Maximum time to wait between two stream chunks before considering the stream stalled. */
-const STREAM_INACTIVITY_TIMEOUT = 6e4;
+const STREAM_INACTIVITY_TIMEOUT = 18e4;
 const getModelOutputLimit = (inputLength) => clamp(RESPONSE_LENGTH_BUFFER.PERCENTAGE * inputLength, RESPONSE_LENGTH_BUFFER.MIN_TOKENS, RESPONSE_LENGTH_BUFFER.MAX_TOKENS);
 /**
 * Models sometimes wrap their whole response in a code fence. The fence has to
@@ -197,6 +197,12 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 	/** Milliseconds between the stream request and the first/last streamed tokens. */
 	let timeToFirstToken;
 	let timeToLastToken;
+	const bufferedDelivery = ctx.midStreamFallback === true;
+	let pendingDeliveries = [];
+	const deliver = async (callback) => {
+		if (bufferedDelivery) pendingDeliveries.push(callback);
+		else await callback();
+	};
 	const liveItems = /* @__PURE__ */ new Map();
 	const liveContent = /* @__PURE__ */ new Map();
 	let codeGenerationTraced = false;
@@ -206,48 +212,58 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 			liveItems.set(event.item.id, event.item);
 			if (event.item.kind === "run" && !codeGenerationTraced) {
 				codeGenerationTraced = true;
-				traces.push({
-					type: "code_generation_started",
-					started_at: Date.now()
+				const started_at = Date.now();
+				await deliver(() => {
+					traces.push({
+						type: "code_generation_started",
+						started_at
+					});
+					onRunStart === null || onRunStart === void 0 || onRunStart();
 				});
-				onRunStart === null || onRunStart === void 0 || onRunStart();
 			}
 		} else if (event.type === "body-delta" && onSendDelta) {
 			const item = liveItems.get(event.itemId);
 			if ((item === null || item === void 0 ? void 0 : item.kind) !== "send") continue;
 			const content = (liveContent.get(item.id) ?? "") + event.delta;
 			liveContent.set(item.id, content);
-			try {
-				await onSendDelta({
-					id: `${iteration.id}:${item.id}`,
-					component: item.name,
-					props: item.props,
-					delta: event.delta,
-					content
-				});
-			} catch (err) {}
-		} else if (event.type === "item-complete") {
-			if (event.item.kind === "send" && onSend) await onSend({
-				name: event.item.name,
-				props: event.item.props,
-				body: event.item.body
+			const delta = {
+				id: `${iteration.id}:${item.id}`,
+				component: item.name,
+				props: bufferedDelivery ? structuredClone(item.props) : item.props,
+				delta: event.delta,
+				content
+			};
+			await deliver(async () => {
+				try {
+					await onSendDelta(delta);
+				} catch (err) {}
 			});
-			else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
+		} else if (event.type === "item-complete") {
+			if (event.item.kind === "send" && onSend) {
+				const send = {
+					name: event.item.name,
+					props: bufferedDelivery ? structuredClone(event.item.props) : event.item.props,
+					body: event.item.body
+				};
+				await deliver(() => onSend(send));
+			} else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
 				runCompleted = true;
-				onRunComplete === null || onRunComplete === void 0 || onRunComplete((event.item.body ?? "").trim());
+				const code = (event.item.body ?? "").trim();
+				await deliver(() => onRunComplete === null || onRunComplete === void 0 ? void 0 : onRunComplete(code));
 			}
 		}
 	};
 	if (typeof cognitive.generateTextStream === "function") {
-		const parser = new require_dual_modes.StreamingMessageParser();
-		const fence = new LeadingFenceFilter();
+		let parser = new require_dual_modes.StreamingMessageParser();
+		let fence = new LeadingFenceFilter();
 		const streamController = createJoinedAbortController([controller.signal]);
 		const requestedAt = Date.now();
 		const stream = cognitive.generateTextStream({
 			...input,
-			...ctx.maxTimeToFirstToken ? { options: {
+			...ctx.maxTimeToFirstToken || bufferedDelivery ? { options: {
 				...input.options,
-				maxTimeToFirstToken: ctx.maxTimeToFirstToken
+				...ctx.maxTimeToFirstToken ? { maxTimeToFirstToken: ctx.maxTimeToFirstToken } : {},
+				...bufferedDelivery ? { midStreamFallback: true } : {}
 			} } : {}
 		}, { signal: streamController.signal });
 		const inactivityTimeout = Math.max(STREAM_INACTIVITY_TIMEOUT, ctx.maxTimeToFirstToken ?? 0);
@@ -266,28 +282,68 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 			}
 		};
 		raw = "";
-		while (true) {
-			var _chunk$value, _chunk$value2;
-			let chunk;
-			try {
-				chunk = await nextChunk();
-			} catch (thrown) {
-				throw new require_errors.CognitiveError(`LLM generation failed: ${require_utils.getErrorMessage(thrown)}`);
+		let streamCompleted = false;
+		try {
+			while (true) {
+				var _chunk$value, _chunk$value2, _chunk$value3, _chunk$value4;
+				let chunk;
+				try {
+					chunk = await nextChunk();
+				} catch (thrown) {
+					throw new require_errors.CognitiveError(`LLM generation failed: ${require_utils.getErrorMessage(thrown)}`);
+				}
+				if (chunk.done) {
+					streamCompleted = true;
+					break;
+				}
+				if (((_chunk$value = chunk.value) === null || _chunk$value === void 0 ? void 0 : _chunk$value.restart) && !bufferedDelivery) {
+					streamController.abort("Unexpected LLM stream restart");
+					throw new require_errors.CognitiveError("LLM stream restarted without options.midStreamFallback enabled");
+				}
+				if ((_chunk$value2 = chunk.value) === null || _chunk$value2 === void 0 ? void 0 : _chunk$value2.restart) {
+					traces.push({
+						type: "llm_call_restarted",
+						started_at: Date.now(),
+						...chunk.value.restart
+					});
+					raw = "";
+					parser = new require_dual_modes.StreamingMessageParser();
+					fence = new LeadingFenceFilter();
+					liveItems.clear();
+					liveContent.clear();
+					pendingDeliveries = [];
+					codeGenerationTraced = false;
+					runCompleted = false;
+					responseMetadata = void 0;
+					timeToFirstToken = void 0;
+					timeToLastToken = void 0;
+					continue;
+				}
+				if ((_chunk$value3 = chunk.value) === null || _chunk$value3 === void 0 ? void 0 : _chunk$value3.metadata) responseMetadata = chunk.value.metadata;
+				const delta = (_chunk$value4 = chunk.value) === null || _chunk$value4 === void 0 ? void 0 : _chunk$value4.output;
+				if (!delta) continue;
+				timeToLastToken = Date.now() - requestedAt;
+				timeToFirstToken ??= timeToLastToken;
+				raw += delta;
+				await dispatchSends(parser.push(fence.push(delta)));
+			}
+			if (!responseMetadata) throw new require_errors.CognitiveError("LLM streaming completed without metadata");
+			const remaining = fence.flush();
+			if (remaining) await dispatchSends(parser.push(remaining));
+			await dispatchSends(parser.finish());
+			assistantResponse = require_dual_modes.toParsedAssistantResponse(parser.items, raw);
+		} finally {
+			streamController.abort("LLM stream closed");
+			if (!streamCompleted) stream.return(void 0).catch(() => {});
+		}
+		if (bufferedDelivery) {
+			controller.signal.throwIfAborted();
+			for (const callback of pendingDeliveries) {
+				controller.signal.throwIfAborted();
+				await callback();
 			}
-			if (chunk.done) break;
-			if ((_chunk$value = chunk.value) === null || _chunk$value === void 0 ? void 0 : _chunk$value.metadata) responseMetadata = chunk.value.metadata;
-			const delta = (_chunk$value2 = chunk.value) === null || _chunk$value2 === void 0 ? void 0 : _chunk$value2.output;
-			if (!delta) continue;
-			timeToLastToken = Date.now() - requestedAt;
-			timeToFirstToken ??= timeToLastToken;
-			raw += delta;
-			await dispatchSends(parser.push(fence.push(delta)));
+			pendingDeliveries = [];
 		}
-		if (!responseMetadata) throw new require_errors.CognitiveError("LLM streaming completed without metadata");
-		const remaining = fence.flush();
-		if (remaining) await dispatchSends(parser.push(remaining));
-		await dispatchSends(parser.finish());
-		assistantResponse = require_dual_modes.toParsedAssistantResponse(parser.items, raw);
 	} else {
 		const response = await cognitive.generateText(input, { signal: controller.signal }).catch((thrown) => {
 			throw new require_errors.CognitiveError(`LLM generation failed: ${require_utils.getErrorMessage(thrown)}`);
@@ -329,7 +385,7 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 		type: "llm_call_success",
 		started_at: startedAt,
 		ended_at: iteration.llm.ended_at,
-		model: model.id,
+		model: iteration.llm.model,
 		code: iteration.code ?? ""
 	});
 };
@@ -677,7 +733,7 @@ const executeContext = async (props) => {
 	return result;
 };
 const executeContextInternal = async (props) => {
-	var _props$options, _props$options2, _props$options3, _props$options4, _props$options5;
+	var _props$options, _props$options2, _props$options3, _props$options4, _props$options5, _props$options6;
 	const controller = createJoinedAbortController([props.signal]);
 	const { onIterationStart, onIterationEnd, onTrace, onExit, onBeforeExecution, onAfterTool, onBeforeTool } = props;
 	const client = props.client ?? new _botpress_client.Client();
@@ -691,7 +747,8 @@ const executeContextInternal = async (props) => {
 		timeout: (_props$options2 = props.options) === null || _props$options2 === void 0 ? void 0 : _props$options2.timeout,
 		maxTokens: (_props$options3 = props.options) === null || _props$options3 === void 0 ? void 0 : _props$options3.maxTokens,
 		maxTimeToFirstToken: (_props$options4 = props.options) === null || _props$options4 === void 0 ? void 0 : _props$options4.maxTimeToFirstToken,
-		transcriptionModel: (_props$options5 = props.options) === null || _props$options5 === void 0 ? void 0 : _props$options5.transcriptionModel,
+		midStreamFallback: (_props$options5 = props.options) === null || _props$options5 === void 0 ? void 0 : _props$options5.midStreamFallback,
+		transcriptionModel: (_props$options6 = props.options) === null || _props$options6 === void 0 ? void 0 : _props$options6.transcriptionModel,
 		exits: props.exits,
 		snapshot: props.snapshot,
 		model: props.model,
@@ -884,7 +941,7 @@ const executeIteration = async ({ iteration, ctx, cognitive, controller, onExit,
 			};
 		});
 	};
-	const canExecuteEarly = typeof onBeforeExecution !== "function";
+	const canExecuteEarly = typeof onBeforeExecution !== "function" && !ctx.midStreamFallback;
 	let earlyExecution;
 	await generateCode({
 		iteration,

package/dist/{execute-DYvFRs54.js → execute-BjBBTshA.js}

@@ -1,5 +1,5 @@
 import { a as StreamingMessageParser, d as isEqual, h as toNumber, i as toParsedAssistantResponse } from "./dual-modes-D_p-jedw.js";
-import { d as Snapshot, f as parseExit, l as PartialExecutionResult, n as Context, s as ErrorExecutionResult, t as _CustomModelClient, u as SuccessExecutionResult } from "./custom-client-NUoxrFQh.js";
+import { d as Snapshot, f as parseExit, l as PartialExecutionResult, n as Context, s as ErrorExecutionResult, t as _CustomModelClient, u as SuccessExecutionResult } from "./custom-client-BnlDpjdI.js";
 import { d as stripInvalidIdentifiers, i as getErrorMessage, s as init } from "./utils-UBeAFeEI.js";
 import { t as createJsxComponent } from "./jsx-CIgdGGVs.js";
 import { t as cleanStackTrace } from "./stack-traces-BIE_5Xwf.js";
@@ -119,7 +119,7 @@ const RESPONSE_LENGTH_BUFFER = {
 	PERCENTAGE: .1
 };
 /** Maximum time to wait between two stream chunks before considering the stream stalled. */
-const STREAM_INACTIVITY_TIMEOUT = 6e4;
+const STREAM_INACTIVITY_TIMEOUT = 18e4;
 const getModelOutputLimit = (inputLength) => clamp(RESPONSE_LENGTH_BUFFER.PERCENTAGE * inputLength, RESPONSE_LENGTH_BUFFER.MIN_TOKENS, RESPONSE_LENGTH_BUFFER.MAX_TOKENS);
 /**
 * Models sometimes wrap their whole response in a code fence. The fence has to
@@ -195,6 +195,12 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 	/** Milliseconds between the stream request and the first/last streamed tokens. */
 	let timeToFirstToken;
 	let timeToLastToken;
+	const bufferedDelivery = ctx.midStreamFallback === true;
+	let pendingDeliveries = [];
+	const deliver = async (callback) => {
+		if (bufferedDelivery) pendingDeliveries.push(callback);
+		else await callback();
+	};
 	const liveItems = /* @__PURE__ */ new Map();
 	const liveContent = /* @__PURE__ */ new Map();
 	let codeGenerationTraced = false;
@@ -204,48 +210,58 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 			liveItems.set(event.item.id, event.item);
 			if (event.item.kind === "run" && !codeGenerationTraced) {
 				codeGenerationTraced = true;
-				traces.push({
-					type: "code_generation_started",
-					started_at: Date.now()
+				const started_at = Date.now();
+				await deliver(() => {
+					traces.push({
+						type: "code_generation_started",
+						started_at
+					});
+					onRunStart === null || onRunStart === void 0 || onRunStart();
 				});
-				onRunStart === null || onRunStart === void 0 || onRunStart();
 			}
 		} else if (event.type === "body-delta" && onSendDelta) {
 			const item = liveItems.get(event.itemId);
 			if ((item === null || item === void 0 ? void 0 : item.kind) !== "send") continue;
 			const content = (liveContent.get(item.id) ?? "") + event.delta;
 			liveContent.set(item.id, content);
-			try {
-				await onSendDelta({
-					id: `${iteration.id}:${item.id}`,
-					component: item.name,
-					props: item.props,
-					delta: event.delta,
-					content
-				});
-			} catch (err) {}
-		} else if (event.type === "item-complete") {
-			if (event.item.kind === "send" && onSend) await onSend({
-				name: event.item.name,
-				props: event.item.props,
-				body: event.item.body
+			const delta = {
+				id: `${iteration.id}:${item.id}`,
+				component: item.name,
+				props: bufferedDelivery ? structuredClone(item.props) : item.props,
+				delta: event.delta,
+				content
+			};
+			await deliver(async () => {
+				try {
+					await onSendDelta(delta);
+				} catch (err) {}
 			});
-			else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
+		} else if (event.type === "item-complete") {
+			if (event.item.kind === "send" && onSend) {
+				const send = {
+					name: event.item.name,
+					props: bufferedDelivery ? structuredClone(event.item.props) : event.item.props,
+					body: event.item.body
+				};
+				await deliver(() => onSend(send));
+			} else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
 				runCompleted = true;
-				onRunComplete === null || onRunComplete === void 0 || onRunComplete((event.item.body ?? "").trim());
+				const code = (event.item.body ?? "").trim();
+				await deliver(() => onRunComplete === null || onRunComplete === void 0 ? void 0 : onRunComplete(code));
 			}
 		}
 	};
 	if (typeof cognitive.generateTextStream === "function") {
-		const parser = new StreamingMessageParser();
-		const fence = new LeadingFenceFilter();
+		let parser = new StreamingMessageParser();
+		let fence = new LeadingFenceFilter();
 		const streamController = createJoinedAbortController([controller.signal]);
 		const requestedAt = Date.now();
 		const stream = cognitive.generateTextStream({
 			...input,
-			...ctx.maxTimeToFirstToken ? { options: {
+			...ctx.maxTimeToFirstToken || bufferedDelivery ? { options: {
 				...input.options,
-				maxTimeToFirstToken: ctx.maxTimeToFirstToken
+				...ctx.maxTimeToFirstToken ? { maxTimeToFirstToken: ctx.maxTimeToFirstToken } : {},
+				...bufferedDelivery ? { midStreamFallback: true } : {}
 			} } : {}
 		}, { signal: streamController.signal });
 		const inactivityTimeout = Math.max(STREAM_INACTIVITY_TIMEOUT, ctx.maxTimeToFirstToken ?? 0);
@@ -264,28 +280,68 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 			}
 		};
 		raw = "";
-		while (true) {
-			var _chunk$value, _chunk$value2;
-			let chunk;
-			try {
-				chunk = await nextChunk();
-			} catch (thrown) {
-				throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
+		let streamCompleted = false;
+		try {
+			while (true) {
+				var _chunk$value, _chunk$value2, _chunk$value3, _chunk$value4;
+				let chunk;
+				try {
+					chunk = await nextChunk();
+				} catch (thrown) {
+					throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
+				}
+				if (chunk.done) {
+					streamCompleted = true;
+					break;
+				}
+				if (((_chunk$value = chunk.value) === null || _chunk$value === void 0 ? void 0 : _chunk$value.restart) && !bufferedDelivery) {
+					streamController.abort("Unexpected LLM stream restart");
+					throw new CognitiveError("LLM stream restarted without options.midStreamFallback enabled");
+				}
+				if ((_chunk$value2 = chunk.value) === null || _chunk$value2 === void 0 ? void 0 : _chunk$value2.restart) {
+					traces.push({
+						type: "llm_call_restarted",
+						started_at: Date.now(),
+						...chunk.value.restart
+					});
+					raw = "";
+					parser = new StreamingMessageParser();
+					fence = new LeadingFenceFilter();
+					liveItems.clear();
+					liveContent.clear();
+					pendingDeliveries = [];
+					codeGenerationTraced = false;
+					runCompleted = false;
+					responseMetadata = void 0;
+					timeToFirstToken = void 0;
+					timeToLastToken = void 0;
+					continue;
+				}
+				if ((_chunk$value3 = chunk.value) === null || _chunk$value3 === void 0 ? void 0 : _chunk$value3.metadata) responseMetadata = chunk.value.metadata;
+				const delta = (_chunk$value4 = chunk.value) === null || _chunk$value4 === void 0 ? void 0 : _chunk$value4.output;
+				if (!delta) continue;
+				timeToLastToken = Date.now() - requestedAt;
+				timeToFirstToken ??= timeToLastToken;
+				raw += delta;
+				await dispatchSends(parser.push(fence.push(delta)));
+			}
+			if (!responseMetadata) throw new CognitiveError("LLM streaming completed without metadata");
+			const remaining = fence.flush();
+			if (remaining) await dispatchSends(parser.push(remaining));
+			await dispatchSends(parser.finish());
+			assistantResponse = toParsedAssistantResponse(parser.items, raw);
+		} finally {
+			streamController.abort("LLM stream closed");
+			if (!streamCompleted) stream.return(void 0).catch(() => {});
+		}
+		if (bufferedDelivery) {
+			controller.signal.throwIfAborted();
+			for (const callback of pendingDeliveries) {
+				controller.signal.throwIfAborted();
+				await callback();
 			}
-			if (chunk.done) break;
-			if ((_chunk$value = chunk.value) === null || _chunk$value === void 0 ? void 0 : _chunk$value.metadata) responseMetadata = chunk.value.metadata;
-			const delta = (_chunk$value2 = chunk.value) === null || _chunk$value2 === void 0 ? void 0 : _chunk$value2.output;
-			if (!delta) continue;
-			timeToLastToken = Date.now() - requestedAt;
-			timeToFirstToken ??= timeToLastToken;
-			raw += delta;
-			await dispatchSends(parser.push(fence.push(delta)));
+			pendingDeliveries = [];
 		}
-		if (!responseMetadata) throw new CognitiveError("LLM streaming completed without metadata");
-		const remaining = fence.flush();
-		if (remaining) await dispatchSends(parser.push(remaining));
-		await dispatchSends(parser.finish());
-		assistantResponse = toParsedAssistantResponse(parser.items, raw);
 	} else {
 		const response = await cognitive.generateText(input, { signal: controller.signal }).catch((thrown) => {
 			throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
@@ -327,7 +383,7 @@ const generateCode = async ({ iteration, ctx, cognitive, controller, metadata, o
 		type: "llm_call_success",
 		started_at: startedAt,
 		ended_at: iteration.llm.ended_at,
-		model: model.id,
+		model: iteration.llm.model,
 		code: iteration.code ?? ""
 	});
 };
@@ -675,7 +731,7 @@ const executeContext = async (props) => {
 	return result;
 };
 const executeContextInternal = async (props) => {
-	var _props$options, _props$options2, _props$options3, _props$options4, _props$options5;
+	var _props$options, _props$options2, _props$options3, _props$options4, _props$options5, _props$options6;
 	const controller = createJoinedAbortController([props.signal]);
 	const { onIterationStart, onIterationEnd, onTrace, onExit, onBeforeExecution, onAfterTool, onBeforeTool } = props;
 	const client = props.client ?? new Client();
@@ -689,7 +745,8 @@ const executeContextInternal = async (props) => {
 		timeout: (_props$options2 = props.options) === null || _props$options2 === void 0 ? void 0 : _props$options2.timeout,
 		maxTokens: (_props$options3 = props.options) === null || _props$options3 === void 0 ? void 0 : _props$options3.maxTokens,
 		maxTimeToFirstToken: (_props$options4 = props.options) === null || _props$options4 === void 0 ? void 0 : _props$options4.maxTimeToFirstToken,
-		transcriptionModel: (_props$options5 = props.options) === null || _props$options5 === void 0 ? void 0 : _props$options5.transcriptionModel,
+		midStreamFallback: (_props$options5 = props.options) === null || _props$options5 === void 0 ? void 0 : _props$options5.midStreamFallback,
+		transcriptionModel: (_props$options6 = props.options) === null || _props$options6 === void 0 ? void 0 : _props$options6.transcriptionModel,
 		exits: props.exits,
 		snapshot: props.snapshot,
 		model: props.model,
@@ -882,7 +939,7 @@ const executeIteration = async ({ iteration, ctx, cognitive, controller, onExit,
 			};
 		});
 	};
-	const canExecuteEarly = typeof onBeforeExecution !== "function";
+	const canExecuteEarly = typeof onBeforeExecution !== "function" && !ctx.midStreamFallback;
 	let earlyExecution;
 	await generateCode({
 		iteration,

package/dist/index.cjs

@@ -1,6 +1,6 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
 const require_dual_modes = require("./dual-modes-wAOAlaBS.cjs");
-const require_custom_client = require("./custom-client-BPmEoWKu.cjs");
+const require_custom_client = require("./custom-client-BX8gx0tI.cjs");
 const require_utils = require("./utils-ZpJm00ed.cjs");
 const require_tool = require("./tool-DdPZb0JW.cjs");
 const require_errors = require("./errors-C962uJdS.cjs");
@@ -11,7 +11,7 @@ const require_truncator = require("./truncator-B6aaJaN1.cjs");
 const require_quickjs_variant = require("./quickjs-variant-CsIwTVFC.cjs");
 let _bpinternal_zui = require("@bpinternal/zui");
 //#region package.json
-var version = "0.6.3";
+var version = "0.6.5";
 //#endregion
 //#region src/hoist.ts
 /**
@@ -1774,7 +1774,7 @@ const utils = {
 * })
 */
 const execute = async (props) => {
-	const { executeContext } = await Promise.resolve().then(() => require("./execute-BEaq3jkV.cjs"));
+	const { executeContext } = await Promise.resolve().then(() => require("./execute-BJF4Txh2.cjs"));
 	return executeContext(props);
 };
 /**
@@ -1783,7 +1783,7 @@ const execute = async (props) => {
 * It's recommended to call this function at the beginning of your application without awaiting it (void init())
 */
 const init = async () => {
-	await Promise.resolve().then(() => require("./execute-BEaq3jkV.cjs"));
+	await Promise.resolve().then(() => require("./execute-BJF4Txh2.cjs"));
 	await Promise.resolve().then(() => require("./component-BwixNVH8.cjs")).then((n) => n.component_exports);
 	await Promise.resolve().then(() => require("./tool-DdPZb0JW.cjs")).then((n) => n.tool_exports);
 	await Promise.resolve().then(() => require("./exit-BGi0Kk0y.cjs")).then((n) => n.exit_exports);

package/dist/index.js

@@ -1,5 +1,5 @@
 import { a as StreamingMessageParser, c as NAME_REGEX, o as tryParseJson, r as generateInstructions, s as MARKER } from "./dual-modes-D_p-jedw.js";
-import { a as ThinkExit, c as ExecutionResult, d as Snapshot, f as parseExit, i as ListenExit, l as PartialExecutionResult, o as getValue, r as DefaultExit, s as ErrorExecutionResult, t as _CustomModelClient, u as SuccessExecutionResult } from "./custom-client-NUoxrFQh.js";
+import { a as ThinkExit, c as ExecutionResult, d as Snapshot, f as parseExit, i as ListenExit, l as PartialExecutionResult, o as getValue, r as DefaultExit, s as ErrorExecutionResult, t as _CustomModelClient, u as SuccessExecutionResult } from "./custom-client-BnlDpjdI.js";
 import { a as getMultilineComment, c as isJsonSchema, l as isValidIdentifier, m as toValidObjectName, n as escapeString, p as toValidFunctionName, t as configureTokenizer, u as isZuiSchema, v as upperFirst } from "./utils-UBeAFeEI.js";
 import { t as Tool } from "./tool-CAo-BQHs.js";
 import { c as SnapshotSignal, l as ThinkSignal, o as LoopExceededError, r as CodeFormattingError } from "./errors-Cg1G0Awu.js";
@@ -10,7 +10,7 @@ import { i as wrapContent, n as truncateWrappedContent, t as stripTruncationTags
 import { t as configureQuickJS } from "./quickjs-variant-BYhMBpwU.js";
 import { transforms, z } from "@bpinternal/zui";
 //#region package.json
-var version = "0.6.3";
+var version = "0.6.5";
 //#endregion
 //#region src/hoist.ts
 /**
@@ -1773,7 +1773,7 @@ const utils = {
 * })
 */
 const execute = async (props) => {
-	const { executeContext } = await import("./execute-DYvFRs54.js");
+	const { executeContext } = await import("./execute-BjBBTshA.js");
 	return executeContext(props);
 };
 /**
@@ -1782,7 +1782,7 @@ const execute = async (props) => {
 * It's recommended to call this function at the beginning of your application without awaiting it (void init())
 */
 const init = async () => {
-	await import("./execute-DYvFRs54.js");
+	await import("./execute-BjBBTshA.js");
 	await import("./component-DGG7d_Mz.js").then((n) => n.r);
 	await import("./tool-CAo-BQHs.js").then((n) => n.n);
 	await import("./exit-CDdxDBRT.js").then((n) => n.n);

package/dist/index.workerd.js

@@ -52,7 +52,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 }) : target, mod));
 //#endregion
 //#region package.json
-var version = "0.6.3";
+var version = "0.6.5";
 //#endregion
 //#region ../../node_modules/.pnpm/lodash-es@4.17.21/node_modules/lodash-es/_freeGlobal.js
 var freeGlobal;
@@ -12215,6 +12215,11 @@ var init_context = __esmMin((() => {
 		*/
 		maxTimeToFirstToken;
 		/**
+		* Allow Cognitive mid-stream fallback, buffering delivery until generation
+		* succeeds and discarding abandoned attempts. Streaming-only; defaults to false.
+		*/
+		midStreamFallback;
+		/**
 		* STT model used by the cognitive service to transcribe audio attachments
 		* when the target LLM does not support audio natively. Defaults to 'fast'.
 		*/
@@ -12449,10 +12454,12 @@ var init_context = __esmMin((() => {
 			this.snapshot = props.snapshot;
 			this.maxTokens = props.maxTokens;
 			this.maxTimeToFirstToken = props.maxTimeToFirstToken;
+			this.midStreamFallback = props.midStreamFallback;
 			this.transcriptionModel = props.transcriptionModel;
 			if (this.loop < 1 || this.loop > 100) throw new Error("Invalid loop. Expected a number between 1 and 100.");
 			if (this.maxTokens !== void 0 && (!Number.isFinite(this.maxTokens) || this.maxTokens < 1)) throw new Error("Invalid maxTokens. Expected a positive number.");
 			if (this.maxTimeToFirstToken !== void 0 && (!Number.isFinite(this.maxTimeToFirstToken) || this.maxTimeToFirstToken < 1)) throw new Error("Invalid maxTimeToFirstToken. Expected a positive number of milliseconds.");
+			if (this.midStreamFallback !== void 0 && typeof this.midStreamFallback !== "boolean") throw new Error("Invalid midStreamFallback. Expected a boolean.");
 		}
 		toJSON() {
 			return {
@@ -17023,7 +17030,7 @@ var init_generate = __esmMin((() => {
 		MAX_TOKENS: 16e3,
 		PERCENTAGE: .1
 	};
-	STREAM_INACTIVITY_TIMEOUT = 6e4;
+	STREAM_INACTIVITY_TIMEOUT = 18e4;
 	getModelOutputLimit = (inputLength) => clamp(RESPONSE_LENGTH_BUFFER.PERCENTAGE * inputLength, RESPONSE_LENGTH_BUFFER.MIN_TOKENS, RESPONSE_LENGTH_BUFFER.MAX_TOKENS);
 	LeadingFenceFilter = class {
 		_buffer = "";
@@ -17093,6 +17100,12 @@ var init_generate = __esmMin((() => {
 		/** Milliseconds between the stream request and the first/last streamed tokens. */
 		let timeToFirstToken;
 		let timeToLastToken;
+		const bufferedDelivery = ctx.midStreamFallback === true;
+		let pendingDeliveries = [];
+		const deliver = async (callback) => {
+			if (bufferedDelivery) pendingDeliveries.push(callback);
+			else await callback();
+		};
 		const liveItems = /* @__PURE__ */ new Map();
 		const liveContent = /* @__PURE__ */ new Map();
 		let codeGenerationTraced = false;
@@ -17102,48 +17115,58 @@ var init_generate = __esmMin((() => {
 				liveItems.set(event.item.id, event.item);
 				if (event.item.kind === "run" && !codeGenerationTraced) {
 					codeGenerationTraced = true;
-					traces.push({
-						type: "code_generation_started",
-						started_at: Date.now()
+					const started_at = Date.now();
+					await deliver(() => {
+						traces.push({
+							type: "code_generation_started",
+							started_at
+						});
+						onRunStart?.();
 					});
-					onRunStart?.();
 				}
 			} else if (event.type === "body-delta" && onSendDelta) {
 				const item = liveItems.get(event.itemId);
 				if (item?.kind !== "send") continue;
 				const content = (liveContent.get(item.id) ?? "") + event.delta;
 				liveContent.set(item.id, content);
-				try {
-					await onSendDelta({
-						id: `${iteration.id}:${item.id}`,
-						component: item.name,
-						props: item.props,
-						delta: event.delta,
-						content
-					});
-				} catch (err) {}
-			} else if (event.type === "item-complete") {
-				if (event.item.kind === "send" && onSend) await onSend({
-					name: event.item.name,
-					props: event.item.props,
-					body: event.item.body
+				const delta = {
+					id: `${iteration.id}:${item.id}`,
+					component: item.name,
+					props: bufferedDelivery ? structuredClone(item.props) : item.props,
+					delta: event.delta,
+					content
+				};
+				await deliver(async () => {
+					try {
+						await onSendDelta(delta);
+					} catch (err) {}
 				});
-				else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
+			} else if (event.type === "item-complete") {
+				if (event.item.kind === "send" && onSend) {
+					const send = {
+						name: event.item.name,
+						props: bufferedDelivery ? structuredClone(event.item.props) : event.item.props,
+						body: event.item.body
+					};
+					await deliver(() => onSend(send));
+				} else if (event.item.kind === "run" && event.item.status === "complete" && !runCompleted) {
 					runCompleted = true;
-					onRunComplete?.((event.item.body ?? "").trim());
+					const code = (event.item.body ?? "").trim();
+					await deliver(() => onRunComplete?.(code));
 				}
 			}
 		};
 		if (typeof cognitive.generateTextStream === "function") {
-			const parser = new StreamingMessageParser();
-			const fence = new LeadingFenceFilter();
+			let parser = new StreamingMessageParser();
+			let fence = new LeadingFenceFilter();
 			const streamController = createJoinedAbortController([controller.signal]);
 			const requestedAt = Date.now();
 			const stream = cognitive.generateTextStream({
 				...input,
-				...ctx.maxTimeToFirstToken ? { options: {
+				...ctx.maxTimeToFirstToken || bufferedDelivery ? { options: {
 					...input.options,
-					maxTimeToFirstToken: ctx.maxTimeToFirstToken
+					...ctx.maxTimeToFirstToken ? { maxTimeToFirstToken: ctx.maxTimeToFirstToken } : {},
+					...bufferedDelivery ? { midStreamFallback: true } : {}
 				} } : {}
 			}, { signal: streamController.signal });
 			const inactivityTimeout = Math.max(STREAM_INACTIVITY_TIMEOUT, ctx.maxTimeToFirstToken ?? 0);
@@ -17162,27 +17185,67 @@ var init_generate = __esmMin((() => {
 				}
 			};
 			raw = "";
-			while (true) {
-				let chunk;
-				try {
-					chunk = await nextChunk();
-				} catch (thrown) {
-					throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
+			let streamCompleted = false;
+			try {
+				while (true) {
+					let chunk;
+					try {
+						chunk = await nextChunk();
+					} catch (thrown) {
+						throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
+					}
+					if (chunk.done) {
+						streamCompleted = true;
+						break;
+					}
+					if (chunk.value?.restart && !bufferedDelivery) {
+						streamController.abort("Unexpected LLM stream restart");
+						throw new CognitiveError("LLM stream restarted without options.midStreamFallback enabled");
+					}
+					if (chunk.value?.restart) {
+						traces.push({
+							type: "llm_call_restarted",
+							started_at: Date.now(),
+							...chunk.value.restart
+						});
+						raw = "";
+						parser = new StreamingMessageParser();
+						fence = new LeadingFenceFilter();
+						liveItems.clear();
+						liveContent.clear();
+						pendingDeliveries = [];
+						codeGenerationTraced = false;
+						runCompleted = false;
+						responseMetadata = void 0;
+						timeToFirstToken = void 0;
+						timeToLastToken = void 0;
+						continue;
+					}
+					if (chunk.value?.metadata) responseMetadata = chunk.value.metadata;
+					const delta = chunk.value?.output;
+					if (!delta) continue;
+					timeToLastToken = Date.now() - requestedAt;
+					timeToFirstToken ??= timeToLastToken;
+					raw += delta;
+					await dispatchSends(parser.push(fence.push(delta)));
+				}
+				if (!responseMetadata) throw new CognitiveError("LLM streaming completed without metadata");
+				const remaining = fence.flush();
+				if (remaining) await dispatchSends(parser.push(remaining));
+				await dispatchSends(parser.finish());
+				assistantResponse = toParsedAssistantResponse(parser.items, raw);
+			} finally {
+				streamController.abort("LLM stream closed");
+				if (!streamCompleted) stream.return(void 0).catch(() => {});
+			}
+			if (bufferedDelivery) {
+				controller.signal.throwIfAborted();
+				for (const callback of pendingDeliveries) {
+					controller.signal.throwIfAborted();
+					await callback();
 				}
-				if (chunk.done) break;
-				if (chunk.value?.metadata) responseMetadata = chunk.value.metadata;
-				const delta = chunk.value?.output;
-				if (!delta) continue;
-				timeToLastToken = Date.now() - requestedAt;
-				timeToFirstToken ??= timeToLastToken;
-				raw += delta;
-				await dispatchSends(parser.push(fence.push(delta)));
+				pendingDeliveries = [];
 			}
-			if (!responseMetadata) throw new CognitiveError("LLM streaming completed without metadata");
-			const remaining = fence.flush();
-			if (remaining) await dispatchSends(parser.push(remaining));
-			await dispatchSends(parser.finish());
-			assistantResponse = toParsedAssistantResponse(parser.items, raw);
 		} else {
 			const response = await cognitive.generateText(input, { signal: controller.signal }).catch((thrown) => {
 				throw new CognitiveError(`LLM generation failed: ${getErrorMessage(thrown)}`);
@@ -17224,7 +17287,7 @@ var init_generate = __esmMin((() => {
 			type: "llm_call_success",
 			started_at: startedAt,
 			ended_at: iteration.llm.ended_at,
-			model: model.id,
+			model: iteration.llm.model,
 			code: iteration.code ?? ""
 		});
 	};
@@ -17614,6 +17677,7 @@ var init_execute = __esmMin((() => {
 			timeout: props.options?.timeout,
 			maxTokens: props.options?.maxTokens,
 			maxTimeToFirstToken: props.options?.maxTimeToFirstToken,
+			midStreamFallback: props.options?.midStreamFallback,
 			transcriptionModel: props.options?.transcriptionModel,
 			exits: props.exits,
 			snapshot: props.snapshot,
@@ -17805,7 +17869,7 @@ var init_execute = __esmMin((() => {
 				};
 			});
 		};
-		const canExecuteEarly = typeof onBeforeExecution !== "function";
+		const canExecuteEarly = typeof onBeforeExecution !== "function" && !ctx.midStreamFallback;
 		let earlyExecution;
 		await generateCode({
 			iteration,

package/dist/runtime/types.d.ts

@@ -122,6 +122,13 @@ type Options = Partial<Pick<Context, 'loop' | 'timeout'>> & {
      * `model` is an array of fallback models.
      */
     maxTimeToFirstToken?: number;
+    /**
+     * Allow Cognitive to restart failed streams on another model. Buffers message
+     * previews, sends, and code execution until generation succeeds; abandoned
+     * attempts deliver nothing. Streaming-only; defaults to false, preserving
+     * progressive sends and early code execution.
+     */
+    midStreamFallback?: boolean;
     /**
      * STT model used to transcribe audio attachments (voice messages) when the
      * target LLM does not support audio natively. Audio-capable models receive

package/dist/types.d.ts

@@ -55,6 +55,13 @@ export declare namespace Traces {
         model: string;
         code: string;
     }>;
+    /** Diagnostic only: content and effects from the preceding attempt are discarded. */
+    type LLMCallRestart = TraceTemplate<'llm_call_restarted', {
+        attempt: number;
+        fromModel: string;
+        toModel: string;
+        reason: string;
+    }>;
     /**
      * Emitted on streaming clients the moment the model starts writing a `■run`
      * block — before the code is fully generated. Useful to show a
@@ -77,7 +84,7 @@ export declare namespace Traces {
         started_at: number;
         ended_at?: number;
     } & Content;
-    type Trace = Comment | Log | ToolCall | ToolSlow | PropertyMutation | YieldTrace | LLMCallStart | LLMCallSuccess | CodeGenerationStart | ThinkSignal | CodeExecution | CodeExecutionException | InvalidCodeExceptionTrace | AbortTrace;
+    type Trace = Comment | Log | ToolCall | ToolSlow | PropertyMutation | YieldTrace | LLMCallStart | LLMCallSuccess | LLMCallRestart | CodeGenerationStart | ThinkSignal | CodeExecution | CodeExecutionException | InvalidCodeExceptionTrace | AbortTrace;
 }
 export type Trace = Traces.Trace;
 export type VMExecutionResult = {

package/package.json

@@ -2,7 +2,7 @@
   "name": "llmz",
   "type": "module",
   "description": "LLMz - An LLM-native Typescript VM built on top of Zui",
-  "version": "0.6.3",
+  "version": "0.6.5",
   "files": [
     "dist",
     "DOCS.md"
@@ -69,7 +69,7 @@
   },
   "peerDependencies": {
     "@botpress/client": "2.4.0",
-    "@botpress/cognitive": "1.1.0",
+    "@botpress/cognitive": "1.2.0",
     "@bpinternal/thicktoken": "^3.1.0",
     "@bpinternal/zui": "^2.4.1"
   },