libfx 0.0.11 → 0.0.12

package/README.md

@@ -39,25 +39,39 @@ await agent.close();
 
 `apiKey` is required. `model` is optional and defaults to fx's built-in model.
 Agent configuration uses named options; `env` is reserved for
-`createFxTerminal()`.
-
-`effort` sets the reasoning effort for models that advertise effort levels. It
-uses the same vocabulary as the fx CLI's `--effort` flag: `"default"` leaves
-the choice to the model, and named levels such as `"low"`, `"medium"`,
-`"high"`, or `"xhigh"` request a specific level. A named level is validated
-against the selected model's advertised levels at creation; an unsupported
-level rejects with an error (code `LIBFX_UNSUPPORTED_EFFORT`) naming the
-supported set. When `effort` is omitted or `"default"`, the model default
-applies.
-
-`fast` enables the fast lane for models that advertise one, matching the fx
-CLI's `--fast` flag. Enabling it is validated against the selected model at
-creation; a model without a fast path rejects with an error (code
-`LIBFX_UNSUPPORTED_FAST`). When `fast` is omitted or `false`, the model
-default applies.
+`createFxTerminal()`. The canonical model configuration groups the model ID
+and model-specific options:
+
+```js
+const agent = await createFxAgent({
+  apiKey,
+  model: { id: "anthropic/claude-opus-5.5-fast", effort: "low", fast: true },
+});
+```
+
+A string `model` remains supported as shorthand. Top-level `effort` and
+`fast` are deprecated but remain supported with a string model or no model;
+they cannot be mixed with a model object. New code should use the model object.
+
+`model.effort` sets the reasoning effort for models that advertise effort
+levels. It uses the same vocabulary as the fx CLI's `--effort` flag:
+`"default"` leaves the choice to the model; named levels such as `"low"`,
+`"medium"`, `"high"`, or `"xhigh"` request a specific level. A named level is
+validated at creation; an unsupported level rejects with an Error carrying
+`code: "LIBFX_MODEL_UNSUPPORTED_EFFORT"`, `model`, and
+`capability: "effort"`. Its message names the supported set when available.
+Omitting effort or using `"default"` leaves the model default in place.
+
+`model.fast` enables the fast lane for models that advertise one, matching the
+fx CLI's `--fast` flag. A model without a fast path rejects at creation with
+`code: "LIBFX_MODEL_UNSUPPORTED_FAST"`, `model`, and `capability: "fast"`.
+Omitting fast or setting it to `false` leaves the model default in place.
+The new codes replace `LIBFX_UNSUPPORTED_EFFORT` and `LIBFX_UNSUPPORTED_FAST`
+for both nested and legacy top-level settings. Callers that check the old codes
+must update their error handling.
 
 The host selects the model. Agent creation does not fetch the Gateway model
-catalog unless `effort` requests a named level or `fast` is enabled. Prompting
+catalog unless effort requests a named level or fast is enabled. Prompting
 can resolve model capabilities and context capacity through the supplied
 `fetch`; fx caches that metadata for the agent.
 
@@ -96,9 +110,11 @@ const result = await turn.result;
 ```
 
 A turn has one event consumer. Breaking out of its iterator cancels the turn;
-`turn.cancel()` and `agent.close()` also release blocked output. Transport or
-message-decoding failures reject the result instead of returning success with
-missing text.
+`turn.cancel()` and `agent.close()` also release blocked output. Embedded agents
+have no implicit model-step cap, so hosts should cancel turns that exceed their
+own budgets. The CLI's `max_agent_steps` setting does not apply to
+`createFxAgent()`. Transport or message-decoding failures reject the result
+instead of returning success with missing text.
 
 Native transport buffers at most 8 MiB of output bytes. Unread SDK events apply
 backpressure at 1 MiB of encoded messages or 256 events. One message can exceed
@@ -106,22 +122,32 @@ that threshold when the queue is empty; an individual encoded ACP message is
 limited to 64 MiB on both backends. These are transport bounds, not a total
 answer-size limit or a bound on retained conversation history.
 
-Image blocks use ACP's content shape and carry canonical base64 (no line
-wrapping) of a PNG, JPEG, GIF, or WebP payload:
+Image blocks accept a `Blob` or `File` with a non-empty `type`, or the
+existing canonical base64 (no line wrapping) and explicit `mimeType` of a PNG,
+JPEG, GIF, or WebP payload:
 
 ```js
 const turn = agent.prompt([
   { type: "text", text: "What does this screenshot show?" },
-  { type: "image", data: base64Png, mimeType: "image/png" },
+  { type: "image", data: file }, // File or Blob, with file.type
+  // Or: { type: "image", data: base64Png, mimeType: "image/png" }
 ]);
 ```
 
 A prompt may contain up to 8 images, each with up to 5 MiB of base64 data,
-with at most 8 MiB of image data per prompt; the SDK rejects larger input with
-typed `RangeError`s before any request. The kernel then validates the decoded
-bytes against the declared `mimeType` and fails the turn with
-`Invalid image prompt block` on a mismatch. Images are routed only to models
-that advertise image input; for any other model the turn fails with
+with at most 8 MiB of image data per prompt. The SDK checks Blob size before
+reading it, encodes it for the same ACP wire format, and rejects larger input
+with typed `RangeError`s. The total frame size is checked before reading a
+Blob, and the actual byte count is checked before encoding it. Blob reads are
+asynchronous: `prompt()` returns a turn, and read failures reject
+`turn.result`. Cancelling or closing while a Blob is being read settles the
+turn without sending its prompt. For base64 input, size errors still throw
+synchronously from `prompt()`.
+
+The kernel sniffs decoded bytes and compares them with the claimed MIME type
+for both input forms; a mismatch fails the turn with
+`Invalid image prompt block`. Images are routed only to models that advertise
+image input; for any other model the turn fails with
 `Image prompts are unavailable for the selected model` and no image bytes
 leave the process. Prompt images are retained in checkpoints within the
 existing 4 MiB checkpoint bound, so a restored agent can refer to earlier
@@ -133,8 +159,10 @@ without discarding the in-flight response or completed tool work. Steering also
 accepts an array of text blocks; image and resource steering blocks are rejected.
 Each message is limited to 64 KiB, with at most 64 queued messages and 1 MiB of
 queued steering text. Accepted steering appears as a `user_message` event before
-the model's continued output. Calling `steer()` after the turn settles rejects
-with `no prompt is running`.
+the model's continued output. For a Blob prompt, steering during the read
+waits for the prompt to be sent; cancelling before then rejects the pending
+steering. Calling `steer()` after the turn settles rejects with
+`no prompt is running`.
 
 ```js
 const turn = agent.prompt("Build the feature.");
@@ -162,8 +190,8 @@ The checkpoint contains conversation history and usage only. The host owns
 durable storage and must resupply models, credentials, instructions, tools,
 MCP clients, and skill records. Reasoning effort and fast mode are
 agent-creation options and are not stored in a checkpoint: recreate the agent
-with new `effort` or `fast` values to change them, the same path as switching
-models.
+with new `model.effort` or `model.fast` values to change them, the same path as
+switching models.
 
 ## Models
 

package/fx-sdk.js

@@ -84,7 +84,7 @@ function normalizeFast(value) {
   return value;
 }
 
-function normalizeAgentOptions(value) {
+export function normalizeAgentOptions(value) {
   if (!value || typeof value !== "object" || Array.isArray(value)) {
     throw new TypeError("createFxAgent() options must be an object");
   }
@@ -93,9 +93,24 @@ function normalizeAgentOptions(value) {
     throw new TypeError("createFxAgent() does not accept env; pass apiKey and model directly");
   }
   options.apiKey = boundedString(options.apiKey, "apiKey", maxApiKeyBytes, true);
-  options.model = boundedString(options.model, "model", maxModelBytes, false);
-  options.effort = normalizeEffort(options.effort);
-  options.fast = normalizeFast(options.fast);
+  if (options.model !== null && typeof options.model === "object" && !Array.isArray(options.model)) {
+    if (Object.hasOwn(options, "effort") || Object.hasOwn(options, "fast")) {
+      throw new TypeError("model options cannot be mixed with top-level effort or fast");
+    }
+    const model = options.model;
+    for (const name of Object.keys(model)) {
+      if (name !== "id" && name !== "effort" && name !== "fast") {
+        throw new TypeError(`unsupported model option: ${name}`);
+      }
+    }
+    options.model = boundedString(model.id, "model.id", maxModelBytes, true);
+    options.effort = normalizeEffort(model.effort);
+    options.fast = normalizeFast(model.fast);
+  } else {
+    options.model = boundedString(options.model, "model", maxModelBytes, false);
+    options.effort = normalizeEffort(options.effort);
+    options.fast = normalizeFast(options.fast);
+  }
   validateGatewayChatUrl(options.gatewayChatUrl);
   return options;
 }
@@ -110,16 +125,15 @@ function agentEnvironment(options) {
   };
 }
 
-// The kernel rejects an unsupported effort or fast lane during initialize;
-// these messages originate only from that validation, so the rejection is
-// safe to retype.
-function agentBootstrapError(error) {
-  if (error instanceof Error &&
-    (error.message === "Invalid reasoning effort" || error.message.startsWith("Reasoning effort"))) {
-    error.code ??= "LIBFX_UNSUPPORTED_EFFORT";
-  }
-  if (error instanceof Error && error.message.startsWith("Fast mode")) {
-    error.code ??= "LIBFX_UNSUPPORTED_FAST";
+function agentRpcError(response) {
+  const error = new Error(response.message);
+  const data = response.data;
+  if (data && ["LIBFX_MODEL_UNSUPPORTED_EFFORT", "LIBFX_MODEL_UNSUPPORTED_FAST"].includes(data.code) &&
+    typeof data.model === "string" &&
+    data.capability === (data.code === "LIBFX_MODEL_UNSUPPORTED_FAST" ? "fast" : "effort")) {
+    error.code = data.code;
+    error.model = data.model;
+    error.capability = data.capability;
   }
   return error;
 }
@@ -1276,6 +1290,15 @@ export async function createFxTerminal(options) {
   };
 }
 
+function blobByteLength(value) {
+  if (typeof Blob === "undefined" || value == null) return null;
+  try {
+    return Object.getOwnPropertyDescriptor(Blob.prototype, "size").get.call(value);
+  } catch {
+    return null;
+  }
+}
+
 function normalizePromptInput(input) {
   if (typeof input === "string") return [{ type: "text", text: input }];
   if (!Array.isArray(input)) throw new TypeError("prompt input must be a string or an array of prompt blocks");
@@ -1284,24 +1307,34 @@ function normalizePromptInput(input) {
   return input.map((block, index) => {
     if (!block || typeof block !== "object") throw new TypeError(`prompt block ${index} must be an object`);
     if (block.type === "image") {
-      if (typeof block.data !== "string" || block.data.length === 0) {
-        throw new TypeError(`image prompt block ${index} requires base64 data`);
+      const size = blobByteLength(block.data);
+      const blob = size !== null;
+      if (!blob && (typeof block.data !== "string" || block.data.length === 0)) {
+        throw new TypeError(`image prompt block ${index} requires base64 data or a Blob`);
+      }
+      const mimeType = blob ? block.data.type : block.mimeType;
+      if (blob && block.mimeType !== undefined && block.mimeType !== mimeType) {
+        throw new TypeError(`image prompt block ${index} mimeType disagrees with Blob.type`);
       }
-      if (typeof block.mimeType !== "string" || block.mimeType.length === 0 || block.mimeType.length > 128) {
+      if (typeof mimeType !== "string" || mimeType.length === 0 || mimeType.length > 128) {
         throw new TypeError(`image prompt block ${index} requires a mimeType`);
       }
-      if (block.data.length > maxPromptImageDataBytes) {
+      if (blob && (!Number.isSafeInteger(size) || size <= 0)) {
+        throw new TypeError(`image prompt block ${index} requires a non-empty Blob with a valid size`);
+      }
+      const encodedLength = blob ? Math.ceil(size / 3) * 4 : block.data.length;
+      if (encodedLength > maxPromptImageDataBytes) {
         throw new RangeError(`image prompt block ${index} exceeds the ${maxPromptImageDataBytes} byte per-image libfx limit`);
       }
       imageCount += 1;
       if (imageCount > maxPromptImages) {
         throw new RangeError(`prompt cannot contain more than ${maxPromptImages} images`);
       }
-      imageBytes += block.data.length;
+      imageBytes += encodedLength;
       if (imageBytes > maxPromptImagesBytes) {
         throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
       }
-      return { type: "image", data: block.data, mimeType: block.mimeType };
+      return { type: "image", data: block.data, mimeType };
     }
     if (block.type === "text") {
       if (typeof block.text !== "string") throw new TypeError(`text prompt block ${index} requires text`);
@@ -1317,6 +1350,50 @@ function normalizePromptInput(input) {
   });
 }
 
+function promptFrameSize(prompt) {
+  let blobDataBytes = 0;
+  const projected = prompt.map((block) => {
+    if (block.type !== "image" || typeof block.data === "string") return block;
+    const size = blobByteLength(block.data);
+    if (!Number.isSafeInteger(size)) throw new TypeError("image prompt requires a valid Blob size");
+    blobDataBytes += Math.ceil(size / 3) * 4;
+    return { type: "image", data: "", mimeType: block.mimeType };
+  });
+  return encoder.encode(JSON.stringify({ sessionId: "", prompt: projected })).length + blobDataBytes + promptFrameEnvelopeBytes;
+}
+
+async function materializePromptBlobs(blocks, isCancelled) {
+  const encoded = [];
+  let imageBytes = 0;
+  for (const [index, block] of blocks.entries()) {
+    if (isCancelled()) return null;
+    if (block.type !== "image") {
+      encoded.push(block);
+      continue;
+    }
+    if (typeof block.data === "string") {
+      imageBytes += block.data.length;
+      if (imageBytes > maxPromptImagesBytes) {
+        throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
+      }
+      encoded.push(block);
+      continue;
+    }
+    const bytes = new Uint8Array(await block.data.arrayBuffer());
+    if (isCancelled()) return null;
+    const encodedLength = Math.ceil(bytes.length / 3) * 4;
+    if (encodedLength > maxPromptImageDataBytes) {
+      throw new RangeError(`image prompt block ${index} exceeds the ${maxPromptImageDataBytes} byte per-image libfx limit`);
+    }
+    imageBytes += encodedLength;
+    if (imageBytes > maxPromptImagesBytes) {
+      throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
+    }
+    encoded.push({ type: "image", data: bytesToBase64(bytes), mimeType: block.mimeType });
+  }
+  return normalizePromptInput(encoded);
+}
+
 function normalizeSteeringInput(input) {
   const blocks = normalizePromptInput(input);
   if (blocks.some((block) => block.type !== "text")) {
@@ -1443,6 +1520,7 @@ export async function createFxAgent(options = {}) {
   let sessionId = null;
   let activeTurn = null;
   let closing = false;
+  let coreExitError = null;
   const isCurrentTurn = (turn) => turn && activeTurn === turn && !turn.cancelled && !closing;
   const emit = (type, detail = {}) => {
     try { options.onEvent?.({ type, timestamp: performance.now(), ...detail }); } catch {}
@@ -1572,7 +1650,9 @@ export async function createFxAgent(options = {}) {
   const send = (message) => {
     if (closing) throw new Error("fx agent is closing");
     emit("acp.send", { message });
+    if (message.method === "session/prompt" && activeTurn?.cancelled) throw new Error("Cancelled");
     runtime.write(`${JSON.stringify(message)}\n`);
+    if (message.method === "session/prompt") activeTurn?.promptWritten();
   };
   const request = (method, params = {}) => new Promise((resolve, reject) => {
     const id = nextId++;
@@ -1580,11 +1660,13 @@ export async function createFxAgent(options = {}) {
     try { send({ jsonrpc: "2.0", id, method, params }); } catch (error) { pending.delete(id); reject(error); }
   });
   runtime.exited.then((code) => {
-    emit("runtime.exit", { code });
     closing = true;
     const error = runtime.error ?? new Error(`fx-core exited with code ${code} before completing the ACP request`);
+    coreExitError = error;
+    activeTurn?.failPendingBlob(error);
     for (const waiter of pending.values()) waiter.reject(error);
     pending.clear();
+    emit("runtime.exit", { code });
   });
   runtime.setLineHandler((message, size) => {
     emit("acp.receive", { message });
@@ -1623,7 +1705,7 @@ export async function createFxAgent(options = {}) {
       return;
     }
     const waiter = pending.get(message.id); if (!waiter) return; pending.delete(message.id);
-    if (message.error) waiter.reject(new Error(message.error.message)); else waiter.resolve(message.result);
+    if (message.error) waiter.reject(agentRpcError(message.error)); else waiter.resolve(message.result);
   }
   try {
     await request("initialize", {
@@ -1648,7 +1730,7 @@ export async function createFxAgent(options = {}) {
     try { runtime.abortHostEffects(); } catch {}
     try { runtime.closeStdin(); } catch {}
     try { await runtime.exited; } catch {}
-    throw agentBootstrapError(error);
+    throw error;
   }
 
   const agent = {
@@ -1765,7 +1847,8 @@ export async function createFxAgent(options = {}) {
 
   function startTurn(input, promptOptions) {
     const prompt = normalizePromptInput(input);
-    if (encoder.encode(JSON.stringify({ sessionId: "", prompt })).length + promptFrameEnvelopeBytes > maxPromptFrameBytes) {
+    const hasBlobs = prompt.some((block) => block.type === "image" && typeof block.data !== "string");
+    if (promptFrameSize(prompt) > maxPromptFrameBytes) {
       throw new RangeError(`prompt exceeds the ${maxPromptFrameBytes} byte libfx frame limit`);
     }
     const signal = promptOptions.signal;
@@ -1778,6 +1861,12 @@ export async function createFxAgent(options = {}) {
     let terminalError;
     let reportedPressure = false;
     let discardedBytes = 0;
+    let cancelBlobRead = null;
+    let rejectBlobRead = null;
+    let resolvePromptStart = null;
+    const promptStarted = hasBlobs ? new Promise((resolve) => { resolvePromptStart = resolve; }) : null;
+    let pendingSteeringCount = 0;
+    let pendingSteeringBytes = 0;
     const toolControllers = new Set();
     let finished = false;
     let cancelled = false;
@@ -1802,34 +1891,63 @@ export async function createFxAgent(options = {}) {
       transportBytes: 0,
       lastTransportActivityAt: null,
       get cancelled() { return cancelled; },
+      failPendingBlob(error) {
+        rejectBlobRead?.(error);
+        rejectBlobRead = null;
+        cancelBlobRead = null;
+      },
+      promptWritten() {
+        resolvePromptStart?.(true);
+        resolvePromptStart = null;
+      },
       steer(text) {
         if (finished || cancelled || activeTurn !== turn) {
           return Promise.reject(new Error("no prompt is running"));
         }
-        let accepted;
-        try {
-          if (typeof runtime.steer === "function") {
-            runtime.steer(text);
-            void turn.push({
-              sessionUpdate: "user_message_chunk",
-              content: { type: "text", text },
-            });
-            accepted = Promise.resolve();
-          } else {
-            accepted = request("libfx/steer", { sessionId, text });
+        if (closing) return Promise.reject(coreExitError ?? new Error("fx agent is closing"));
+        const apply = () => {
+          if (finished || cancelled || activeTurn !== turn) {
+            return Promise.reject(new Error("no prompt is running"));
+          }
+          if (closing) return Promise.reject(coreExitError ?? new Error("fx agent is closing"));
+          try {
+            if (typeof runtime.steer === "function") {
+              runtime.steer(text);
+              void turn.push({
+                sessionUpdate: "user_message_chunk",
+                content: { type: "text", text },
+              });
+              return Promise.resolve();
+            }
+            return request("libfx/steer", { sessionId, text });
+          } catch (error) {
+            return Promise.reject(error);
           }
-        } catch (error) {
-          return Promise.reject(error);
+        };
+        if (!resolvePromptStart) return apply();
+        const bytes = encoder.encode(text).length;
+        if (pendingSteeringCount >= maxSteeringMessages || bytes > maxSteeringQueueBytes - pendingSteeringBytes) {
+          return Promise.reject(new Error("steering queue is full"));
         }
-        return accepted;
+        pendingSteeringCount++;
+        pendingSteeringBytes += bytes;
+        return promptStarted.then((started) => {
+          if (coreExitError && !cancelled) throw coreExitError;
+          return started === true ? apply() : Promise.reject(started instanceof Error ? started : new Error("no prompt is running"));
+        }).finally(() => { pendingSteeringCount--; pendingSteeringBytes -= bytes; });
       },
       cancel() {
         if (finished || cancelled) return;
         cancelled = true;
+        cancelBlobRead?.();
+        cancelBlobRead = null;
+        rejectBlobRead = null;
+        resolvePromptStart?.(false);
+        resolvePromptStart = null;
         runtime.closeSteering?.();
         resumeOutput?.();
         resumeOutput = null;
-        send({ jsonrpc: "2.0", method: "session/cancel", params: { sessionId } });
+        if (!closing) send({ jsonrpc: "2.0", method: "session/cancel", params: { sessionId } });
         for (const controller of toolControllers) controller.abort();
         runtime.abortHostEffects();
       },
@@ -1854,6 +1972,8 @@ export async function createFxAgent(options = {}) {
       },
     };
     if (signal?.aborted) {
+      resolvePromptStart?.(false);
+      resolvePromptStart = null;
       finished = true;
       turn.result = Promise.resolve({ stopReason: "cancelled" });
       return turn;
@@ -1862,10 +1982,37 @@ export async function createFxAgent(options = {}) {
     runtime.openSteering?.();
     const abort = () => turn.cancel();
     signal?.addEventListener("abort", abort, { once: true });
-    turn.result = request("session/prompt", { sessionId, prompt })
-      .then((response) => ({ stopReason: cancelled ? "cancelled" : response.stopReason, usage: response.usage }))
+    const blobReadCancelled = hasBlobs ? new Promise((resolve, reject) => {
+      cancelBlobRead = () => resolve(null);
+      rejectBlobRead = reject;
+    }) : null;
+    const response = hasBlobs
+      ? Promise.race([
+        Promise.resolve().then(() => materializePromptBlobs(prompt, () => cancelled || closing)),
+        blobReadCancelled,
+      ]).then((encodedPrompt) => {
+        cancelBlobRead = null;
+        rejectBlobRead = null;
+        if (coreExitError && !cancelled) throw coreExitError;
+        if (encodedPrompt === null || cancelled || closing) {
+          resolvePromptStart?.(false);
+          resolvePromptStart = null;
+          return { stopReason: "cancelled" };
+        }
+        if (promptFrameSize(encodedPrompt) > maxPromptFrameBytes) {
+          throw new RangeError(`prompt exceeds the ${maxPromptFrameBytes} byte libfx frame limit`);
+        }
+        return request("session/prompt", { sessionId, prompt: encodedPrompt });
+      })
+      : request("session/prompt", { sessionId, prompt });
+    turn.result = response
+      .then((value) => ({ stopReason: cancelled ? "cancelled" : value.stopReason, usage: value.usage }))
       .catch((error) => {
-        if (error.message === "Cancelled") return { stopReason: "cancelled" };
+        cancelBlobRead = null;
+        rejectBlobRead = null;
+        resolvePromptStart?.(error);
+        resolvePromptStart = null;
+        if (cancelled && error.message === "Cancelled") return { stopReason: "cancelled" };
         terminalError = error;
         throw error;
       })

package/node.cjs

@@ -264,9 +264,24 @@ function normalizeAgentOptions(value) {
     throw new TypeError("createFxAgent() does not accept env; pass apiKey and model directly");
   }
   options.apiKey = boundedString(options.apiKey, "apiKey", maxApiKeyBytes, true);
-  options.model = boundedString(options.model, "model", maxModelBytes, false);
-  options.effort = normalizeEffort(options.effort);
-  options.fast = normalizeFast(options.fast);
+  if (options.model !== null && typeof options.model === "object" && !Array.isArray(options.model)) {
+    if (Object.hasOwn(options, "effort") || Object.hasOwn(options, "fast")) {
+      throw new TypeError("model options cannot be mixed with top-level effort or fast");
+    }
+    const model = options.model;
+    for (const name of Object.keys(model)) {
+      if (name !== "id" && name !== "effort" && name !== "fast") {
+        throw new TypeError(`unsupported model option: ${name}`);
+      }
+    }
+    options.model = boundedString(model.id, "model.id", maxModelBytes, true);
+    options.effort = normalizeEffort(model.effort);
+    options.fast = normalizeFast(model.fast);
+  } else {
+    options.model = boundedString(options.model, "model", maxModelBytes, false);
+    options.effort = normalizeEffort(options.effort);
+    options.fast = normalizeFast(options.fast);
+  }
   validateGatewayChatUrl(options.gatewayChatUrl);
   return options;
 }
@@ -279,12 +294,13 @@ function agentEnvironment(options) {
     ...options.gatewayChatUrl === undefined ? {} : { FX_GATEWAY_CHAT_URL: options.gatewayChatUrl }
   };
 }
-function agentBootstrapError(error) {
-  if (error instanceof Error && (error.message === "Invalid reasoning effort" || error.message.startsWith("Reasoning effort"))) {
-    error.code ??= "LIBFX_UNSUPPORTED_EFFORT";
-  }
-  if (error instanceof Error && error.message.startsWith("Fast mode")) {
-    error.code ??= "LIBFX_UNSUPPORTED_FAST";
+function agentRpcError(response) {
+  const error = new Error(response.message);
+  const data = response.data;
+  if (data && ["LIBFX_MODEL_UNSUPPORTED_EFFORT", "LIBFX_MODEL_UNSUPPORTED_FAST"].includes(data.code) && typeof data.model === "string" && data.capability === (data.code === "LIBFX_MODEL_UNSUPPORTED_FAST" ? "fast" : "effort")) {
+    error.code = data.code;
+    error.model = data.model;
+    error.capability = data.capability;
   }
   return error;
 }
@@ -1634,6 +1650,15 @@ async function createFxTerminal(options) {
     }
   };
 }
+function blobByteLength(value) {
+  if (typeof Blob === "undefined" || value == null)
+    return null;
+  try {
+    return Object.getOwnPropertyDescriptor(Blob.prototype, "size").get.call(value);
+  } catch {
+    return null;
+  }
+}
 function normalizePromptInput(input) {
   if (typeof input === "string")
     return [{ type: "text", text: input }];
@@ -1645,24 +1670,34 @@ function normalizePromptInput(input) {
     if (!block || typeof block !== "object")
       throw new TypeError(`prompt block ${index} must be an object`);
     if (block.type === "image") {
-      if (typeof block.data !== "string" || block.data.length === 0) {
-        throw new TypeError(`image prompt block ${index} requires base64 data`);
+      const size = blobByteLength(block.data);
+      const blob = size !== null;
+      if (!blob && (typeof block.data !== "string" || block.data.length === 0)) {
+        throw new TypeError(`image prompt block ${index} requires base64 data or a Blob`);
+      }
+      const mimeType = blob ? block.data.type : block.mimeType;
+      if (blob && block.mimeType !== undefined && block.mimeType !== mimeType) {
+        throw new TypeError(`image prompt block ${index} mimeType disagrees with Blob.type`);
       }
-      if (typeof block.mimeType !== "string" || block.mimeType.length === 0 || block.mimeType.length > 128) {
+      if (typeof mimeType !== "string" || mimeType.length === 0 || mimeType.length > 128) {
         throw new TypeError(`image prompt block ${index} requires a mimeType`);
       }
-      if (block.data.length > maxPromptImageDataBytes) {
+      if (blob && (!Number.isSafeInteger(size) || size <= 0)) {
+        throw new TypeError(`image prompt block ${index} requires a non-empty Blob with a valid size`);
+      }
+      const encodedLength = blob ? Math.ceil(size / 3) * 4 : block.data.length;
+      if (encodedLength > maxPromptImageDataBytes) {
         throw new RangeError(`image prompt block ${index} exceeds the ${maxPromptImageDataBytes} byte per-image libfx limit`);
       }
       imageCount += 1;
       if (imageCount > maxPromptImages) {
         throw new RangeError(`prompt cannot contain more than ${maxPromptImages} images`);
       }
-      imageBytes += block.data.length;
+      imageBytes += encodedLength;
       if (imageBytes > maxPromptImagesBytes) {
         throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
       }
-      return { type: "image", data: block.data, mimeType: block.mimeType };
+      return { type: "image", data: block.data, mimeType };
     }
     if (block.type === "text") {
       if (typeof block.text !== "string")
@@ -1680,6 +1715,52 @@ function normalizePromptInput(input) {
     throw new TypeError(`unsupported prompt block type: ${String(block.type)}`);
   });
 }
+function promptFrameSize(prompt) {
+  let blobDataBytes = 0;
+  const projected = prompt.map((block) => {
+    if (block.type !== "image" || typeof block.data === "string")
+      return block;
+    const size = blobByteLength(block.data);
+    if (!Number.isSafeInteger(size))
+      throw new TypeError("image prompt requires a valid Blob size");
+    blobDataBytes += Math.ceil(size / 3) * 4;
+    return { type: "image", data: "", mimeType: block.mimeType };
+  });
+  return encoder.encode(JSON.stringify({ sessionId: "", prompt: projected })).length + blobDataBytes + promptFrameEnvelopeBytes;
+}
+async function materializePromptBlobs(blocks, isCancelled) {
+  const encoded = [];
+  let imageBytes = 0;
+  for (const [index, block] of blocks.entries()) {
+    if (isCancelled())
+      return null;
+    if (block.type !== "image") {
+      encoded.push(block);
+      continue;
+    }
+    if (typeof block.data === "string") {
+      imageBytes += block.data.length;
+      if (imageBytes > maxPromptImagesBytes) {
+        throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
+      }
+      encoded.push(block);
+      continue;
+    }
+    const bytes = new Uint8Array(await block.data.arrayBuffer());
+    if (isCancelled())
+      return null;
+    const encodedLength = Math.ceil(bytes.length / 3) * 4;
+    if (encodedLength > maxPromptImageDataBytes) {
+      throw new RangeError(`image prompt block ${index} exceeds the ${maxPromptImageDataBytes} byte per-image libfx limit`);
+    }
+    imageBytes += encodedLength;
+    if (imageBytes > maxPromptImagesBytes) {
+      throw new RangeError(`prompt images exceed the ${maxPromptImagesBytes} byte libfx frame limit`);
+    }
+    encoded.push({ type: "image", data: bytesToBase64(bytes), mimeType: block.mimeType });
+  }
+  return normalizePromptInput(encoded);
+}
 function normalizeSteeringInput(input) {
   const blocks = normalizePromptInput(input);
   if (blocks.some((block) => block.type !== "text")) {
@@ -1823,6 +1904,7 @@ async function createFxAgent(options = {}) {
   let sessionId = null;
   let activeTurn = null;
   let closing = false;
+  let coreExitError = null;
   const isCurrentTurn = (turn) => turn && activeTurn === turn && !turn.cancelled && !closing;
   const emit = (type, detail = {}) => {
     try {
@@ -1962,8 +2044,12 @@ async function createFxAgent(options = {}) {
     if (closing)
       throw new Error("fx agent is closing");
     emit("acp.send", { message });
+    if (message.method === "session/prompt" && activeTurn?.cancelled)
+      throw new Error("Cancelled");
     runtime.write(`${JSON.stringify(message)}
 `);
+    if (message.method === "session/prompt")
+      activeTurn?.promptWritten();
   };
   const request = (method, params = {}) => new Promise((resolve, reject) => {
     const id = nextId++;
@@ -1976,12 +2062,14 @@ async function createFxAgent(options = {}) {
     }
   });
   runtime.exited.then((code) => {
-    emit("runtime.exit", { code });
     closing = true;
     const error = runtime.error ?? new Error(`fx-core exited with code ${code} before completing the ACP request`);
+    coreExitError = error;
+    activeTurn?.failPendingBlob(error);
     for (const waiter of pending.values())
       waiter.reject(error);
     pending.clear();
+    emit("runtime.exit", { code });
   });
   runtime.setLineHandler((message, size) => {
     emit("acp.receive", { message });
@@ -2028,7 +2116,7 @@ async function createFxAgent(options = {}) {
       return;
     pending.delete(message.id);
     if (message.error)
-      waiter.reject(new Error(message.error.message));
+      waiter.reject(agentRpcError(message.error));
     else
       waiter.resolve(message.result);
   }
@@ -2058,7 +2146,7 @@ async function createFxAgent(options = {}) {
     try {
       await runtime.exited;
     } catch {}
-    throw agentBootstrapError(error);
+    throw error;
   }
   const agent = {
     prompt(input, promptOptions = {}) {
@@ -2201,7 +2289,8 @@ async function createFxAgent(options = {}) {
   }
   function startTurn(input, promptOptions) {
     const prompt = normalizePromptInput(input);
-    if (encoder.encode(JSON.stringify({ sessionId: "", prompt })).length + promptFrameEnvelopeBytes > maxPromptFrameBytes) {
+    const hasBlobs = prompt.some((block) => block.type === "image" && typeof block.data !== "string");
+    if (promptFrameSize(prompt) > maxPromptFrameBytes) {
       throw new RangeError(`prompt exceeds the ${maxPromptFrameBytes} byte libfx frame limit`);
     }
     const signal = promptOptions.signal;
@@ -2215,6 +2304,14 @@ async function createFxAgent(options = {}) {
     let terminalError;
     let reportedPressure = false;
     let discardedBytes = 0;
+    let cancelBlobRead = null;
+    let rejectBlobRead = null;
+    let resolvePromptStart = null;
+    const promptStarted = hasBlobs ? new Promise((resolve) => {
+      resolvePromptStart = resolve;
+    }) : null;
+    let pendingSteeringCount = 0;
+    let pendingSteeringBytes = 0;
     const toolControllers = new Set;
     let finished = false;
     let cancelled = false;
@@ -2251,35 +2348,72 @@ async function createFxAgent(options = {}) {
       get cancelled() {
         return cancelled;
       },
+      failPendingBlob(error) {
+        rejectBlobRead?.(error);
+        rejectBlobRead = null;
+        cancelBlobRead = null;
+      },
+      promptWritten() {
+        resolvePromptStart?.(true);
+        resolvePromptStart = null;
+      },
       steer(text) {
         if (finished || cancelled || activeTurn !== turn) {
           return Promise.reject(new Error("no prompt is running"));
         }
-        let accepted;
-        try {
-          if (typeof runtime.steer === "function") {
-            runtime.steer(text);
-            turn.push({
-              sessionUpdate: "user_message_chunk",
-              content: { type: "text", text }
-            });
-            accepted = Promise.resolve();
-          } else {
-            accepted = request("libfx/steer", { sessionId, text });
+        if (closing)
+          return Promise.reject(coreExitError ?? new Error("fx agent is closing"));
+        const apply = () => {
+          if (finished || cancelled || activeTurn !== turn) {
+            return Promise.reject(new Error("no prompt is running"));
+          }
+          if (closing)
+            return Promise.reject(coreExitError ?? new Error("fx agent is closing"));
+          try {
+            if (typeof runtime.steer === "function") {
+              runtime.steer(text);
+              turn.push({
+                sessionUpdate: "user_message_chunk",
+                content: { type: "text", text }
+              });
+              return Promise.resolve();
+            }
+            return request("libfx/steer", { sessionId, text });
+          } catch (error) {
+            return Promise.reject(error);
           }
-        } catch (error) {
-          return Promise.reject(error);
+        };
+        if (!resolvePromptStart)
+          return apply();
+        const bytes = encoder.encode(text).length;
+        if (pendingSteeringCount >= maxSteeringMessages || bytes > maxSteeringQueueBytes - pendingSteeringBytes) {
+          return Promise.reject(new Error("steering queue is full"));
         }
-        return accepted;
+        pendingSteeringCount++;
+        pendingSteeringBytes += bytes;
+        return promptStarted.then((started) => {
+          if (coreExitError && !cancelled)
+            throw coreExitError;
+          return started === true ? apply() : Promise.reject(started instanceof Error ? started : new Error("no prompt is running"));
+        }).finally(() => {
+          pendingSteeringCount--;
+          pendingSteeringBytes -= bytes;
+        });
       },
       cancel() {
         if (finished || cancelled)
           return;
         cancelled = true;
+        cancelBlobRead?.();
+        cancelBlobRead = null;
+        rejectBlobRead = null;
+        resolvePromptStart?.(false);
+        resolvePromptStart = null;
         runtime.closeSteering?.();
         resumeOutput?.();
         resumeOutput = null;
-        send({ jsonrpc: "2.0", method: "session/cancel", params: { sessionId } });
+        if (!closing)
+          send({ jsonrpc: "2.0", method: "session/cancel", params: { sessionId } });
         for (const controller of toolControllers)
           controller.abort();
         runtime.abortHostEffects();
@@ -2311,6 +2445,8 @@ async function createFxAgent(options = {}) {
       }
     };
     if (signal?.aborted) {
+      resolvePromptStart?.(false);
+      resolvePromptStart = null;
       finished = true;
       turn.result = Promise.resolve({ stopReason: "cancelled" });
       return turn;
@@ -2319,8 +2455,34 @@ async function createFxAgent(options = {}) {
     runtime.openSteering?.();
     const abort = () => turn.cancel();
     signal?.addEventListener("abort", abort, { once: true });
-    turn.result = request("session/prompt", { sessionId, prompt }).then((response) => ({ stopReason: cancelled ? "cancelled" : response.stopReason, usage: response.usage })).catch((error) => {
-      if (error.message === "Cancelled")
+    const blobReadCancelled = hasBlobs ? new Promise((resolve, reject) => {
+      cancelBlobRead = () => resolve(null);
+      rejectBlobRead = reject;
+    }) : null;
+    const response = hasBlobs ? Promise.race([
+      Promise.resolve().then(() => materializePromptBlobs(prompt, () => cancelled || closing)),
+      blobReadCancelled
+    ]).then((encodedPrompt) => {
+      cancelBlobRead = null;
+      rejectBlobRead = null;
+      if (coreExitError && !cancelled)
+        throw coreExitError;
+      if (encodedPrompt === null || cancelled || closing) {
+        resolvePromptStart?.(false);
+        resolvePromptStart = null;
+        return { stopReason: "cancelled" };
+      }
+      if (promptFrameSize(encodedPrompt) > maxPromptFrameBytes) {
+        throw new RangeError(`prompt exceeds the ${maxPromptFrameBytes} byte libfx frame limit`);
+      }
+      return request("session/prompt", { sessionId, prompt: encodedPrompt });
+    }) : request("session/prompt", { sessionId, prompt });
+    turn.result = response.then((value) => ({ stopReason: cancelled ? "cancelled" : value.stopReason, usage: value.usage })).catch((error) => {
+      cancelBlobRead = null;
+      rejectBlobRead = null;
+      resolvePromptStart?.(error);
+      resolvePromptStart = null;
+      if (cancelled && error.message === "Cancelled")
         return { stopReason: "cancelled" };
       terminalError = error;
       throw error;
@@ -2848,10 +3010,11 @@ function createNativeAgent(addon, options) {
   });
 }
 async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWasm, options) {
-  const { nativeAddon, backend = "auto", ...runtimeOptions } = options ?? {};
+  const { nativeAddon, backend = "auto", ...unvalidatedOptions } = options ?? {};
   if (!new Set(["auto", "native", "wasm"]).has(backend)) {
     throw new TypeError('backend must be "auto", "native", or "wasm"');
   }
+  const runtimeOptions = surface === "agent" ? normalizeAgentOptions(unvalidatedOptions) : unvalidatedOptions;
   let nativeError;
   let nativeAttempted = false;
   if (backend !== "wasm") {
@@ -2865,7 +3028,7 @@ async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWas
         return await native.backend[nativeMethod](runtimeOptions);
       } catch (error) {
         nativeError = error;
-        if (backend === "native")
+        if (backend === "native" || error?.code === "LIBFX_MODEL_UNSUPPORTED_FAST" || error?.code === "LIBFX_MODEL_UNSUPPORTED_EFFORT")
           throw error;
       }
     }
@@ -2884,9 +3047,6 @@ async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWas
   return wasmFactory({ ...runtimeOptions, wasm: await wasmInput(wasmSource) });
 }
 async function createFxAgent2(options = {}) {
-  if (options != null && Object.hasOwn(Object(options), "env")) {
-    throw new TypeError("createFxAgent() does not accept env; pass apiKey and model directly");
-  }
   return createWithFallback("agent", "createCore", createFxAgent, defaultCoreWasm, options);
 }
 function createFxTerminal2(options = {}) {

package/node.js

@@ -13,6 +13,7 @@ import {
   encodeXtermKeyEvent,
   fxSdkApiVersion,
   listModels,
+  normalizeAgentOptions,
   supportsJspi,
   xtermAdapter,
 } from "./fx-sdk.js";
@@ -513,10 +514,11 @@ function createNativeAgent(addon, options) {
 }
 
 async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWasm, options) {
-  const { nativeAddon, backend = "auto", ...runtimeOptions } = options ?? {};
+  const { nativeAddon, backend = "auto", ...unvalidatedOptions } = options ?? {};
   if (!new Set(["auto", "native", "wasm"]).has(backend)) {
     throw new TypeError('backend must be "auto", "native", or "wasm"');
   }
+  const runtimeOptions = surface === "agent" ? normalizeAgentOptions(unvalidatedOptions) : unvalidatedOptions;
 
   let nativeError;
   let nativeAttempted = false;
@@ -530,7 +532,8 @@ async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWas
         return await native.backend[nativeMethod](runtimeOptions);
       } catch (error) {
         nativeError = error;
-        if (backend === "native") throw error;
+        if (backend === "native" || error?.code === "LIBFX_MODEL_UNSUPPORTED_FAST" ||
+          error?.code === "LIBFX_MODEL_UNSUPPORTED_EFFORT") throw error;
       }
     }
     if (backend === "native") {
@@ -549,9 +552,6 @@ async function createWithFallback(surface, nativeMethod, wasmFactory, defaultWas
 }
 
 export async function createFxAgent(options = {}) {
-  if (options != null && Object.hasOwn(Object(options), "env")) {
-    throw new TypeError("createFxAgent() does not accept env; pass apiKey and model directly");
-  }
   return createWithFallback(
     "agent",
     "createCore",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libfx",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "Embed fx agents and terminals in JavaScript hosts",
   "type": "module",
   "repository": {