akanjs 3.0.0-beta.2 → 3.0.0-beta.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "akanjs",
-  "version": "3.0.0-beta.2",
+  "version": "3.0.0-beta.3",
   "sourceType": "module",
   "type": "module",
   "publishConfig": {

package/service/predefinedAdaptor/anthropicLlm.ts

@@ -58,6 +58,18 @@ export class AnthropicLlm
    */
   static readonly defaultMaxTokens = 8192;
 
+  /**
+   * The four the API's image block reads. An exact set rather than an `image/*` prefix, because by the time an
+   * attachment reaches here `accepts.image` has already carried it past `AgentService.readable`: a phone's
+   * `image/heic` — the iPhone camera default, so the likeliest non-canonical image an app sees — arrives as bytes,
+   * becomes a block the API refuses, and takes the **whole turn** down on a 400 rather than going unread.
+   *
+   * The app cannot gate it either: `AttachReader` answers `null` for "not mine", which falls through to the
+   * built-in reader that base64s any `image/*`, so there is no way for a reader to refuse one. The check belongs
+   * where the block vocabulary is known, which is here.
+   */
+  static readonly imageTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+
   get #host() {
     return this.llmOption.host ?? "https://api.anthropic.com/v1";
   }
@@ -266,9 +278,13 @@ export class AnthropicLlm
         ];
       const source = AnthropicLlm.sourceOf(attachment);
       if (!source) return [];
-      if (accepts?.image && attachment.mimeType.startsWith("image/")) return [{ type: "image", source }];
+      
+      const mimeType = attachment.mimeType.split(";")[0].trim().toLowerCase();
+      if (accepts?.image && AnthropicLlm.imageTypes.has(mimeType))
+        return [{ type: "image", source: AnthropicLlm.typed(source, mimeType) }];
 
-      if (accepts?.document && attachment.mimeType === "application/pdf") return [{ type: "document", source }];
+      if (accepts?.document && mimeType === "application/pdf")
+        return [{ type: "document", source: AnthropicLlm.typed(source, mimeType) }];
       notes.push(`[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API has no block for it.]`);
       return [];
     });
@@ -276,6 +292,11 @@ export class AnthropicLlm
     return [...(text ? [{ type: "text" as const, text }] : []), ...blocks];
   }
 
+  /** The block's `media_type` is the essence, not whatever parameters the browser attached to it. */
+  static typed(source: AnthropicSource, mimeType: string): AnthropicSource {
+    return source.type === "base64" ? { ...source, media_type: mimeType } : source;
+  }
+
   static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null {
     if (attachment.url) return { type: "url", url: attachment.url };
     if (attachment.data) return { type: "base64", media_type: attachment.mimeType, data: attachment.data };

package/service/predefinedAdaptor/openaiDialect.ts

@@ -34,6 +34,14 @@ export interface OpenaiMessage {
  * read to a note in the text.
  */
 export class OpenaiDialect {
+  /**
+   * The types this dialect's image part reads. Exact rather than an `image/*` prefix and declared apart from
+   * Anthropic's identical-looking set, because the two are each a provider's own list and only happen to agree:
+   * an unsupported one passed through is a refused *request*, not an unread attachment, so the safe direction is
+   * to name what is known to work and note the rest.
+   */
+  static readonly imageTypes = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"]);
+
   static requestBody(
     model: string,
     request: LlmTurnRequest,
@@ -118,16 +126,28 @@ export class OpenaiDialect {
    */
   static userContent(message: AgentWireMessage, accepts?: LlmAccepts): string | OpenaiContentPart[] {
     const attachments = message.attachments ?? [];
+    const notes: string[] = [];
     const blocks = attachments.flatMap((attachment) =>
       attachment.text ? [`--- attachment: ${attachment.name} (${attachment.mimeType}) ---\n${attachment.text}`] : [],
     );
-    const text = [message.text, ...blocks].filter(Boolean).join("\n\n");
-    if (!accepts?.image) return text;
-    const images = attachments.flatMap((attachment) => {
-      if (!attachment.mimeType.startsWith("image/")) return [];
-      const url = attachment.url ?? (attachment.data ? `data:${attachment.mimeType};base64,${attachment.data}` : "");
-      return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
-    });
+    const images = !accepts?.image
+      ? []
+      : attachments.flatMap((attachment) => {
+          if (attachment.text) return [];
+          
+          const mimeType = attachment.mimeType.split(";")[0].trim().toLowerCase();
+
+          if (!mimeType.startsWith("image/")) return [];
+          if (!OpenaiDialect.imageTypes.has(mimeType)) {
+            notes.push(
+              `[Attachment not read: ${attachment.name} (${attachment.mimeType}) — this API reads no image of that type.]`,
+            );
+            return [];
+          }
+          const url = attachment.url ?? (attachment.data ? `data:${mimeType};base64,${attachment.data}` : "");
+          return url ? [{ type: "image_url" as const, image_url: { url } }] : [];
+        });
+    const text = [message.text, ...blocks, ...notes].filter(Boolean).join("\n\n");
     if (!images.length) return text;
     return [...(text ? [{ type: "text" as const, text }] : []), ...images];
   }

package/types/service/predefinedAdaptor/anthropicLlm.d.ts

@@ -64,6 +64,17 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
      * model refusing, so it is `option.setLlm({ maxTokens })` and not a constant.
      */
     static readonly defaultMaxTokens = 8192;
+    /**
+     * The four the API's image block reads. An exact set rather than an `image/*` prefix, because by the time an
+     * attachment reaches here `accepts.image` has already carried it past `AgentService.readable`: a phone's
+     * `image/heic` — the iPhone camera default, so the likeliest non-canonical image an app sees — arrives as bytes,
+     * becomes a block the API refuses, and takes the **whole turn** down on a 400 rather than going unread.
+     *
+     * The app cannot gate it either: `AttachReader` answers `null` for "not mine", which falls through to the
+     * built-in reader that base64s any `image/*`, so there is no way for a reader to refuse one. The check belongs
+     * where the block vocabulary is known, which is here.
+     */
+    static readonly imageTypes: Set<string>;
     /** What the API's blocks carry. A model of the family that reads neither takes the `accepts` override. */
     get accepts(): LlmAccepts;
     chat(request: LlmTurnRequest, onDelta?: (delta: string) => void): Promise<LlmTurnAnswer | null>;
@@ -96,6 +107,8 @@ export declare class AnthropicLlm extends AnthropicLlm_base implements LlmAdapto
     static providerMessages(messages: AgentWireMessage[], accepts?: LlmAccepts): AnthropicMessage[];
     static providerMessage(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicMessage;
     static userContent(message: AgentWireMessage, accepts?: LlmAccepts): AnthropicBlock[];
+    /** The block's `media_type` is the essence, not whatever parameters the browser attached to it. */
+    static typed(source: AnthropicSource, mimeType: string): AnthropicSource;
     static sourceOf(attachment: AgentWireAttachment): AnthropicSource | null;
     static turnAnswer(answer: AnthropicAnswer): LlmTurnAnswer;
     /** The ceiling wins over the calls that did arrive — see `OpenaiDialect.stopOf` for why. */

package/types/service/predefinedAdaptor/openaiDialect.d.ts

@@ -47,6 +47,13 @@ export interface OpenaiMessage {
  * read to a note in the text.
  */
 export declare class OpenaiDialect {
+    /**
+     * The types this dialect's image part reads. Exact rather than an `image/*` prefix and declared apart from
+     * Anthropic's identical-looking set, because the two are each a provider's own list and only happen to agree:
+     * an unsupported one passed through is a refused *request*, not an unread attachment, so the safe direction is
+     * to name what is known to work and note the rest.
+     */
+    static readonly imageTypes: Set<string>;
     static requestBody(model: string, request: LlmTurnRequest, { accepts, stream }?: {
         accepts?: LlmAccepts;
         stream?: boolean;

package/types/ui/Agent/Chat.d.ts

@@ -70,6 +70,11 @@ export interface ChatProps {
      * (images as bytes, text as text). This is where an app puts what needs a parser — a PDF's text, a spreadsheet's
      * cells — since the framework carries attachments but depends on nothing that can extract one. It runs before
      * the built-in, so it can also replace how an image is prepared.
+     *
+     * **A `url` is handed to the provider as the address it will fetch**, so answer `data` whenever the provider
+     * cannot reach it. A reader that uploads is the natural place to get this wrong: the default storage backend
+     * serves a path only this app can resolve, and a model handed one answers about a picture it never saw, with
+     * nothing anywhere reporting a failure.
      */
     attach?: AttachReader;
     /**

package/ui/Agent/Chat.tsx

@@ -109,6 +109,11 @@ export interface ChatProps {
    * (images as bytes, text as text). This is where an app puts what needs a parser — a PDF's text, a spreadsheet's
    * cells — since the framework carries attachments but depends on nothing that can extract one. It runs before
    * the built-in, so it can also replace how an image is prepared.
+   *
+   * **A `url` is handed to the provider as the address it will fetch**, so answer `data` whenever the provider
+   * cannot reach it. A reader that uploads is the natural place to get this wrong: the default storage backend
+   * serves a path only this app can resolve, and a model handed one answers about a picture it never saw, with
+   * nothing anywhere reporting a failure.
    */
   attach?: AttachReader;
   /**