akanjs 3.0.0-beta.2 → 3.0.0-beta.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akanjs",
-  "version": "3.0.0-beta.2",
+  "version": "3.0.0-beta.4",
   "sourceType": "module",
   "type": "module",
   "publishConfig": {

package/service/agent.service.ts

@@ -98,14 +98,18 @@ export class AgentService extends serve("agent" as const, ({ plug }) => ({
   private static isReadable(attachment: AgentWireAttachment, accepts: LlmAccepts): boolean {
     if (attachment.text) return true;
     if (!attachment.data && !attachment.url) return false;
+
+    if (typeof attachment.mimeType !== "string") return false;
     return attachment.mimeType.startsWith("image/") ? !!accepts.image : !!accepts.document;
   }
 
   private static note(attachment: AgentWireAttachment): string {
     const why =
-      attachment.data || attachment.url
-        ? "this model cannot read that type"
-        : "its content is no longer available, as a reloaded conversation keeps the name and not the bytes";
+      !attachment.data && !attachment.url
+        ? "its content is no longer available, as a reloaded conversation keeps the name and not the bytes"
+        : typeof attachment.mimeType === "string"
+          ? "this model cannot read that type"
+          : "it names no type it could be read as";
     return `[Attachment not read: ${attachment.name} (${attachment.mimeType}) — ${why}. Tell the user it was not read instead of guessing what it holds, and ask for the text if the answer needs it.]`;
   }
 }

package/service/predefinedAdaptor/anthropicLlm.ts

@@ -58,6 +58,18 @@ export class AnthropicLlm
    */
   static readonly defaultMaxTokens = 8192;
 
+  /**
+   * The four the API's image block reads. An exact set rather than an `image/*` prefix, because by the time an
+   * attachment reaches here `accepts.image` has already carried it past `AgentService.readable`: a phone's
+   * `image/heic` — the iPhone camera default, so the likeliest non-canonical image an app sees — arrives as bytes,
+   * becomes a block the API refuses, and takes the **whole turn** down on a 400 rather than going unread.
+   *
+   * The app cannot gate it either: `AttachReader` answers `null` for "not mine", which falls through to the
+   * built-in reader that base64s any `image/*`, so there is no way for a reader to refuse one. The check belongs
+   * where the block vocabulary is known, which is here.
+   */
+  static readonly imageTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+
   get #host() {
     return this.llmOption.host ?? "https://api.anthropic.com/v1";
   }
@@ -266,9 +278,13 @@ export class AnthropicLlm
         ];
       const source = AnthropicLlm.sourceOf(attachment);
       if (!source) return [];
-      if (accepts?.image && attachment.mimeType.startsWith("image/")) return [{ type: "image", source }];
+      
+      const mimeType = attachment.mimeType.split(";")[0].trim().toLowerCase();
+      if (accepts?.image && AnthropicLlm.imageTypes.has(mimeType))
+        return [{ type: "image", source: AnthropicLlm.typed(source, mimeType) }];
 
-      if (accepts?.document && attachment.mimeType === "application/pdf") return [{ type: "document", source }];
+      if (accepts?.document && mimeType === "application/pdf")
+        return [{ type: "document", source: AnthropicLlm.typed(source, mimeType) }];
       notes.push(`[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API has no block for it.]`);
       return [];
     });
@@ -276,6 +292,11 @@ export class AnthropicLlm
     return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
   }
 
+  /** The block's `media_type` is the essence, not whatever parameters the browser attached to it. */
+  static typed(source: AnthropicSource, mimeType: string): AnthropicSource {
+    return source.type === "base64" ? { ...source, media_type: mimeType } : source;
+  }
+
   static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null {
     if (attachment.url) return { type: "url", url: attachment.url };
     if (attachment.data) return { type: "base64", media_type: attachment.mimeType, data: attachment.data };

package/service/predefinedAdaptor/openaiDialect.ts

@@ -34,6 +34,14 @@ export interface OpenaiMessage {
  * read to a note in the text.
  */
 export class OpenaiDialect {
+  /**
+   * The types this dialect's image part reads. Exact rather than an `image/*` prefix and declared apart from
+   * Anthropic's identical-looking set, because the two are each a provider's own list and only happen to agree:
+   * an unsupported one passed through is a refused *request*, not an unread attachment, so the safe direction is
+   * to name what is known to work and note the rest.
+   */
+  static readonly imageTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+
   static requestBody(
     model: string,
     request: LlmTurnRequest,
@@ -118,16 +126,28 @@ export class OpenaiDialect {
    */
   static userContent(message: AgentWireMessage, accepts?: LlmAccepts): string | OpenaiContentPart[] {
     const attachments = message.attachments ?? [];
+    const notes: string[] = [];
     const blocks = attachments.flatMap((attachment) =>
       attachment.text ? [`--- attachment: ${attachment.name} (${attachment.mimeType}) ---\n${attachment.text}`] : [],
     );
-    const text = [message.text, ...blocks].filter(Boolean).join("\n\n");
-    if (!accepts?.image) return text;
-    const images = attachments.flatMap((attachment) => {
-      if (!attachment.mimeType.startsWith("image/")) return [];
-      const url = attachment.url ?? (attachment.data ? `data:${attachment.mimeType};base64,${attachment.data}` : "");
-      return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
-    });
+    const images = !accepts?.image
+      ? []
+      : attachments.flatMap((attachment) => {
+          if (attachment.text) return [];
+          
+          const mimeType = attachment.mimeType.split(";")[0].trim().toLowerCase();
+
+          if (!mimeType.startsWith("image/")) return [];
+          if (!OpenaiDialect.imageTypes.has(mimeType)) {
+            notes.push(
+              `[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API reads no image of that type.]`,
+            );
+            return [];
+          }
+          const url = attachment.url ?? (attachment.data ? `data:${mimeType};base64,${attachment.data}` : "");
+          return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
+        });
+    const text = [message.text, ...blocks, ...notes].filter(Boolean).join("\n\n");
     if (!images.length) return text;
     return [...(text ? [{ type: "text" as const, text }] : []), ...images];
   }

package/types/service/predefinedAdaptor/anthropicLlm.d.ts

@@ -64,6 +64,17 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
      * model refusing, so it is `option.setLlm({ maxTokens })` and not a constant.
      */
     static readonly defaultMaxTokens = 8192;
+    /**
+     * The four the API's image block reads. An exact set rather than an `image/*` prefix, because by the time an
+     * attachment reaches here `accepts.image` has already carried it past `AgentService.readable`: a phone's
+     * `image/heic` — the iPhone camera default, so the likeliest non-canonical image an app sees — arrives as bytes,
+     * becomes a block the API refuses, and takes the **whole turn** down on a 400 rather than going unread.
+     *
+     * The app cannot gate it either: `AttachReader` answers `null` for "not mine", which falls through to the
+     * built-in reader that base64s any `image/*`, so there is no way for a reader to refuse one. The check belongs
+     * where the block vocabulary is known, which is here.
+     */
+    static readonly imageTypes: Set<string>;
     /** What the API's blocks carry. A model of the family that reads neither takes the `accepts` override. */
     get accepts(): LlmAccepts;
     chat(request: LlmTurnRequest, onDelta?: (delta: string) => void): Promise<LlmTurnAnswer | null>;
@@ -96,6 +107,8 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
     static providerMessages(messages: AgentWireMessage[], accepts?: LlmAccepts): AnthropicMessage[];
     static providerMessage(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicMessage;
     static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[];
+    /** The block's `media_type` is the essence, not whatever parameters the browser attached to it. */
+    static typed(source: AnthropicSource, mimeType: string): AnthropicSource;
     static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null;
     static turnAnswer(answer: AnthropicAnswer): LlmTurnAnswer;
     /** The ceiling wins over the calls that did arrive — see `OpenaiDialect.stopOf` for why. */

package/types/service/predefinedAdaptor/openaiDialect.d.ts

@@ -47,6 +47,13 @@ export interface OpenaiMessage {
  * read to a note in the text.
  */
 export declare class OpenaiDialect {
+    /**
+     * The types this dialect's image part reads. Exact rather than an `image/*` prefix and declared apart from
+     * Anthropic's identical-looking set, because the two are each a provider's own list and only happen to agree:
+     * an unsupported one passed through is a refused *request*, not an unread attachment, so the safe direction is
+     * to name what is known to work and note the rest.
+     */
+    static readonly imageTypes: Set<string>;
     static requestBody(model: string, request: LlmTurnRequest, { accepts, stream }?: {
         accepts?: LlmAccepts;
         stream?: boolean;

package/types/ui/Agent/Chat.d.ts

@@ -70,6 +70,11 @@ export interface ChatProps {
      * (images as bytes, text as text). This is where an app puts what needs a parser — a PDF's text, a spreadsheet's
      * cells — since the framework carries attachments but depends on nothing that can extract one. It runs before
      * the built-in, so it can also replace how an image is prepared.
+     *
+     * **A `url` is handed to the provider as the address it will fetch**, so answer `data` whenever the provider
+     * cannot reach it. A reader that uploads is the natural place to get this wrong: the default storage backend
+     * serves a path only this app can resolve, and a model handed one answers about a picture it never saw, with
+     * nothing anywhere reporting a failure.
      */
     attach?: AttachReader;
     /**

package/types/ui/index.d.ts

@@ -1,4 +1,4 @@
-export { AgentProvider, type AgentProviderProps, type AgentRunner, AgentSession, type AgentSessionOptions, type ChatMessage, type CompactOptions, type ContextBlock, httpRunner, type PublishedTool, type RunnerEvent, type RunnerRequest, SessionContext, type SessionHistory, type SurfaceView, useAgent, } from "../vendor/use-agentic.d.ts";
+export { AgentProvider, type AgentProviderProps, type AgentRunner, AgentSession, type AgentSessionOptions, type ChatMessage, type CompactOptions, type ContextBlock, httpRunner, type MessageAttachment, type PublishedTool, type RunnerEvent, type RunnerRequest, SessionContext, type SessionHistory, type SurfaceView, useAgent, } from "../vendor/use-agentic.d.ts";
 export { Agent } from "./Agent.d.ts";
 export { type ApprovalProps, DefaultApproval } from "./Agent/Approval.d.ts";
 export { type AgentSessionSetup, agentSessionOf } from "./Agent/agentSessionOf.d.ts";

package/types/vendor/use-agentic/Transcript.d.ts

@@ -10,9 +10,10 @@ import type { ChatMessage } from "./types.d.ts";
  * transcript is assembled rather than where each hole is made.
  */
 export declare class Transcript {
-    #private;
     static readonly unanswered = "The turn was stopped before this call ran.";
     /** What one turn posts: no host-only message, no unanswered call, no result answering a call nobody sees. */
     static wire(messages: readonly ChatMessage[]): ChatMessage[];
     static sanitize(messages: readonly ChatMessage[]): ChatMessage[];
+    /** An empty assistant message is a draft a reload or an abort caught before it said anything. */
+    static carries(message: ChatMessage): boolean;
 }

package/ui/Agent/Attach.tsx

@@ -40,6 +40,7 @@ export const Attach = ({ className, label, onPick }: AttachProps) => {
 
 interface ChipsProps {
   className?: string;
+
   attachments: readonly MessageAttachment[];
   /** Omitted for a sent message: what is already on the wire cannot be taken back. */
   onRemove?: (index: number) => void;
@@ -56,7 +57,7 @@ export const Chips = ({ className, attachments, onRemove, removeLabel, pending =
         className="flex items-center gap-1 rounded-field bg-muted px-2 py-0.5 text-xs"
         key={`${attachment.name}-${idx}`}
       >
-        {attachment.data && attachment.mimeType.startsWith("image/") ? (
+        {attachment.data && attachment.mimeType?.startsWith("image/") ? (
           <img
             alt={attachment.name}
             className="size-6 rounded-field object-cover"

package/ui/Agent/Chat.tsx

@@ -109,6 +109,11 @@ export interface ChatProps {
    * (images as bytes, text as text). This is where an app puts what needs a parser — a PDF's text, a spreadsheet's
    * cells — since the framework carries attachments but depends on nothing that can extract one. It runs before
    * the built-in, so it can also replace how an image is prepared.
+   *
+   * **A `url` is handed to the provider as the address it will fetch**, so answer `data` whenever the provider
+   * cannot reach it. A reader that uploads is the natural place to get this wrong: the default storage backend
+   * serves a path only this app can resolve, and a model handed one answers about a picture it never saw, with
+   * nothing anywhere reporting a failure.
    */
   attach?: AttachReader;
   /**

package/ui/index.ts

@@ -10,6 +10,7 @@ export {
   type CompactOptions,
   type ContextBlock,
   httpRunner,
+  type MessageAttachment,
   type PublishedTool,
   type RunnerEvent,
   type RunnerRequest,

package/vendor/use-agentic/AgentSession.ts

@@ -293,6 +293,9 @@ export class AgentSession {
       this.#pending = null;
       this.#question = null;
       this.#progress = null;
+
+      const draft = this.#messages[this.#messages.length - 1];
+      if (draft?.role === "assistant" && !Transcript.carries(draft)) this.#messages = this.#messages.slice(0, -1);
       this.#notify();
     }
   }

package/vendor/use-agentic/Transcript.ts

@@ -28,7 +28,7 @@ export class Transcript {
         if (results.length) kept.push({ ...message, toolResults: results });
         continue;
       }
-      if (message.role === "assistant" && !Transcript.#carries(message)) continue;
+      if (message.role === "assistant" && !Transcript.carries(message)) continue;
       kept.push(message);
       const calls = message.toolCalls ?? [];
       for (const call of calls) called.add(call.id);
@@ -44,7 +44,7 @@ export class Transcript {
   }
 
   /** An empty assistant message is a draft a reload or an abort caught before it said anything. */
-  static #carries(message: ChatMessage) {
+  static carries(message: ChatMessage) {
     return !!message.text || !!message.error || !!message.toolCalls?.length || !!message.attachments?.length;
   }
 }