@saadnvd1/agent-os 0.3.790 → 0.3.794

package/components/Chat/composer/shrinkImage.ts

@@ -0,0 +1,95 @@
+import { needsShrink, renamed, SHRINK_STEPS, targetSize } from "./shrinkPlan";
+
+type Canvas = OffscreenCanvas | HTMLCanvasElement;
+
+function makeCanvas(width: number, height: number): Canvas {
+  if (typeof OffscreenCanvas !== "undefined")
+    return new OffscreenCanvas(width, height);
+  const c = document.createElement("canvas");
+  c.width = width;
+  c.height = height;
+  return c;
+}
+
+function encode(
+  canvas: Canvas,
+  type: string,
+  quality?: number
+): Promise<Blob | null> {
+  if ("convertToBlob" in canvas) return canvas.convertToBlob({ type, quality });
+  return new Promise((resolve) => canvas.toBlob(resolve, type, quality));
+}
+
+function hasAlpha(
+  ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+  w: number,
+  h: number
+) {
+  const data = ctx.getImageData(0, 0, w, h).data;
+  for (let i = 3; i < data.length; i += 4) if (data[i] < 255) return true;
+  return false;
+}
+
+export type ShrinkResult = { ok: true; file: File } | { ok: false };
+
+// Downscale an image that is over the byte cap or 2000px on its long edge.
+// Returns the file untouched when it is already fine, and { ok: false } when
+// it can't be decoded or still doesn't fit after every step.
+export async function shrinkImage(
+  file: File,
+  cap: number
+): Promise<ShrinkResult> {
+  let bitmap: ImageBitmap;
+  try {
+    bitmap = await createImageBitmap(file, { imageOrientation: "from-image" });
+  } catch {
+    // A file we can't decode is only a problem if it was too big anyway.
+    return file.size > cap ? { ok: false } : { ok: true, file };
+  }
+  try {
+    if (!needsShrink(bitmap.width, bitmap.height, file.size, cap))
+      return { ok: true, file };
+    // Animated GIFs would lose their frames; only touch one that can't go as is.
+    if (file.type === "image/gif" && file.size <= cap)
+      return { ok: true, file };
+
+    let keepPng: boolean | null = null;
+    for (const { maxEdge, quality } of SHRINK_STEPS) {
+      const { width, height } = targetSize(
+        bitmap.width,
+        bitmap.height,
+        maxEdge
+      );
+      const canvas = makeCanvas(width, height);
+      const ctx = canvas.getContext("2d") as CanvasRenderingContext2D | null;
+      if (!ctx) return { ok: false };
+      ctx.drawImage(bitmap, 0, 0, width, height);
+      if (keepPng === null)
+        keepPng = file.type === "image/png" && hasAlpha(ctx, width, height);
+      if (keepPng) {
+        const png = await encode(canvas, "image/png");
+        if (png && png.size <= cap)
+          return {
+            ok: true,
+            file: new File([png], file.name, { type: "image/png" }),
+          };
+        keepPng = false;
+      }
+      // JPEG has no alpha: put the picture on white first.
+      ctx.globalCompositeOperation = "destination-over";
+      ctx.fillStyle = "#fff";
+      ctx.fillRect(0, 0, width, height);
+      const jpeg = await encode(canvas, "image/jpeg", quality);
+      if (jpeg && jpeg.size <= cap)
+        return {
+          ok: true,
+          file: new File([jpeg], renamed(file.name, "jpg"), {
+            type: "image/jpeg",
+          }),
+        };
+    }
+    return { ok: false };
+  } finally {
+    bitmap.close();
+  }
+}

package/components/Chat/composer/shrinkPlan.test.ts

@@ -0,0 +1,33 @@
+import { describe, expect, it } from "vitest";
+import { needsShrink, renamed, SHRINK_STEPS, targetSize } from "./shrinkPlan";
+
+describe("shrink plan", () => {
+  it("shrinks a big photo and leaves a small one alone", () => {
+    expect(needsShrink(5712, 4284, 8_000_000, 3_750_000)).toBe(true);
+    expect(needsShrink(1200, 800, 4_000_000, 3_750_000)).toBe(true);
+    expect(needsShrink(3000, 1000, 100_000, 3_750_000)).toBe(true);
+    expect(needsShrink(2000, 1500, 500_000, 3_750_000)).toBe(false);
+  });
+
+  it("fits the long edge and keeps the aspect ratio", () => {
+    expect(targetSize(5712, 4284, 2000)).toEqual({ width: 2000, height: 1500 });
+    expect(targetSize(4284, 5712, 1600)).toEqual({ width: 1200, height: 1600 });
+    expect(targetSize(800, 600, 2000)).toEqual({ width: 800, height: 600 });
+  });
+
+  it("steps quality down before size", () => {
+    expect(SHRINK_STEPS.map((s) => `${s.maxEdge}@${s.quality}`)).toEqual([
+      "2000@0.85",
+      "2000@0.75",
+      "2000@0.65",
+      "1600@0.75",
+      "1600@0.65",
+    ]);
+  });
+
+  it("swaps the extension", () => {
+    expect(renamed("IMG_1.HEIC", "jpg")).toBe("IMG_1.jpg");
+    expect(renamed("a.b.png", "jpg")).toBe("a.b.jpg");
+    expect(renamed("noext", "jpg")).toBe("noext.jpg");
+  });
+});

package/components/Chat/composer/shrinkPlan.ts

@@ -0,0 +1,39 @@
+// The decisions behind shrinking a big photo before it is attached, kept pure
+// so they can be tested without a canvas.
+
+export const MAX_EDGE = 2000;
+const FALLBACK_EDGE = 1600;
+
+export interface ShrinkStep {
+  maxEdge: number;
+  quality: number;
+}
+
+// Tried in order until the result fits the cap.
+export const SHRINK_STEPS: ShrinkStep[] = [
+  { maxEdge: MAX_EDGE, quality: 0.85 },
+  { maxEdge: MAX_EDGE, quality: 0.75 },
+  { maxEdge: MAX_EDGE, quality: 0.65 },
+  { maxEdge: FALLBACK_EDGE, quality: 0.75 },
+  { maxEdge: FALLBACK_EDGE, quality: 0.65 },
+];
+
+export const needsShrink = (
+  width: number,
+  height: number,
+  bytes: number,
+  cap: number
+) => bytes > cap || Math.max(width, height) > MAX_EDGE;
+
+// Fit inside maxEdge on the long side; never enlarge.
+export function targetSize(width: number, height: number, maxEdge: number) {
+  const scale = Math.min(1, maxEdge / Math.max(width, height));
+  return {
+    width: Math.max(1, Math.round(width * scale)),
+    height: Math.max(1, Math.round(height * scale)),
+  };
+}
+
+// photo.HEIC -> photo.jpg; a name without an extension just gains one.
+export const renamed = (name: string, ext: "jpg" | "png") =>
+  `${name.replace(/\.[^./\\]+$/, "")}.${ext}`;

package/components/Chat/composer/useAttachFiles.ts

@@ -16,6 +16,7 @@ import {
 import type { ChatFile, ChatImage } from "@/lib/chat/events";
 import { uploadFile } from "@/lib/file-upload";
 import { isImage } from "./paste";
+import { shrinkImage } from "./shrinkImage";
 
 export interface Upload {
   key: string;
@@ -79,9 +80,9 @@ export function useAttachFiles({
       if (held.current + images.length + others.length >= limits.max)
         error = `You can attach up to ${limits.max} files to one message.`;
       else if (isImage(f)) {
-        if (f.size > limits.maxImageBytes)
-          error = tooBigMessage(f.name, f.size, limits.maxImageBytes);
-        else images.push(f);
+        const shrunk = await shrinkImage(f, limits.maxImageBytes);
+        if (shrunk.ok) images.push(shrunk.file);
+        else error = tooBigMessage(f.name, f.size, limits.maxImageBytes);
       } else if (!typeFor(f.name, limits.types))
         error = unsupportedMessage(f.name);
       else if (f.size > limits.maxBytes)

package/content/whats-new.json

@@ -1777,6 +1777,22 @@
       "area": "AgentOS",
       "text": "What's New now sits as a small icon in the sidebar footer.",
       "url": "https://github.com/saadnvd1/agent-os/pull/528"
+    },
+    {
+      "id": 529,
+      "date": "2026-10-31",
+      "kind": "fixed",
+      "area": "Chat",
+      "text": "Large photos from your phone now attach to a chat instead of being refused.",
+      "url": "https://github.com/saadnvd1/agent-os/pull/529"
+    },
+    {
+      "id": 530,
+      "date": "2026-10-31",
+      "kind": "new",
+      "area": "Chat",
+      "text": "GitHub Copilot chats can now ask you questions with clickable answer cards, like Claude chats do.",
+      "url": "https://github.com/saadnvd1/agent-os/pull/530"
     }
   ]
 }

package/lib/chat/drivers/copilot-approvals.ts

@@ -1,6 +1,7 @@
 import type { ChatAccess, DriverEvent, FileDiff } from "../events";
 import { unifiedToDiff } from "../diff";
 import { toolTitle } from "../tools";
+import { isAskUserCall } from "./copilot-ask";
 import { PendingApprovals } from "./pending-approvals";
 
 type P = Record<string, unknown>;
@@ -71,6 +72,7 @@ export class CopilotApprovals {
         : { outcome: { outcome: "cancelled" } };
 
     if (
+      isAskUserCall(call) ||
       this.access === "full" ||
       (this.access === "edits" && EDITS_KINDS.has(kind))
     )

package/lib/chat/drivers/copilot-ask.ts

@@ -0,0 +1,232 @@
+import crypto from "node:crypto";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { z } from "zod";
+import type { ChatQuestion } from "../events";
+import {
+  answersInOrder,
+  type ApprovalOutcome,
+  type PendingApprovals,
+} from "./pending-approvals";
+
+type P = Record<string, unknown>;
+const str = (v: unknown) => (typeof v === "string" ? v : "");
+
+export const ASK_SERVER = "agentos";
+export const ASK_TOOL = "ask_user";
+const MAX_BODY = 64 * 1024;
+
+// Copilot CLI doesn't offer its own ask_user over ACP, so each conversation
+// serves one: a question the agent asks becomes the chat's question card.
+export const askInput = {
+  question: z.string().min(1).describe("The question to show the user"),
+  options: z
+    .array(
+      z.object({
+        label: z.string().min(1),
+        description: z.string().optional(),
+      })
+    )
+    .optional()
+    .describe("Choices to offer; leave out for a free-text question"),
+  multiSelect: z
+    .boolean()
+    .optional()
+    .describe("Let the user pick more than one option"),
+  allowFreeText: z
+    .boolean()
+    .optional()
+    .describe("The user can always type their own answer too"),
+  header: z.string().optional().describe("A short label for the question"),
+};
+
+type AskArgs = {
+  question: string;
+  options?: { label: string; description?: string }[];
+  multiSelect?: boolean;
+  header?: string;
+};
+
+export function askUserQuestion(a: AskArgs): ChatQuestion {
+  const seen = new Set<string>();
+  const options = (a.options ?? []).flatMap((o) => {
+    const label = o.label.trim();
+    if (!label || seen.has(label)) return [];
+    seen.add(label);
+    return [{ label, description: o.description?.trim() ?? "" }];
+  });
+  return {
+    question: a.question.trim(),
+    header: a.header?.trim() ?? "",
+    multiSelect: !!a.multiSelect && options.length > 0,
+    options,
+  };
+}
+
+const text = (t: string, isError = false): CallToolResult => ({
+  content: [{ type: "text", text: t }],
+  ...(isError ? { isError } : {}),
+});
+
+export function askUserResult(
+  q: ChatQuestion,
+  o: ApprovalOutcome
+): CallToolResult {
+  if (o.decision === "answer") {
+    const [list] = answersInOrder([q], o.answers);
+    return text(
+      JSON.stringify(
+        q.multiSelect ? { answers: list } : { answer: list[0] ?? "" }
+      )
+    );
+  }
+  if (o.decision === "expired")
+    return text("The question was cancelled before the user answered.", true);
+  return text("The user skipped this question without answering.");
+}
+
+// Whether a Copilot tool call (a tool_call update or a permission request) is
+// our ask_user. Its title carries the server and tool name; the input is the
+// fallback for a title that doesn't.
+export function isAskUserCall(call: P): boolean {
+  const raw = (call.rawInput ?? {}) as P;
+  const title = str(call.title);
+  return (
+    title.includes(ASK_TOOL) ||
+    (title.includes(ASK_SERVER) && typeof raw.question === "string")
+  );
+}
+
+// One private MCP server per conversation: loopback only, behind a secret
+// path, so nothing else can reach it or answer its cards.
+export class CopilotAskServer {
+  private http?: http.Server;
+  private secret = crypto.randomBytes(32).toString("hex");
+  private next = 0;
+  private live = new Set<{ close(): Promise<void> }>();
+
+  constructor(private pending: PendingApprovals) {}
+
+  private build(): McpServer {
+    const server = new McpServer({ name: ASK_SERVER, version: "1.0.0" });
+    server.registerTool(
+      ASK_TOOL,
+      {
+        description:
+          "Ask the user a question and wait for the answer. Use it when you need a decision or missing information instead of asking in plain text. Offer options for a choice (multiSelect to allow several); with no options the user types an answer. Blocks until they answer.",
+        inputSchema: askInput,
+      },
+      async (args) => {
+        const q = askUserQuestion(args);
+        const o = await this.pending.ask({
+          id: `approval-ask-${++this.next}`,
+          toolName: "AskUserQuestion",
+          title: "Questions",
+          input: { questions: [q] },
+          questions: [q],
+          canAlways: false,
+        });
+        return askUserResult(q, o);
+      }
+    );
+    return server;
+  }
+
+  private allowed(req: http.IncomingMessage): number {
+    if (req.headers.origin) return 403;
+    const host = (req.headers.host ?? "").replace(/:\d+$/, "");
+    if (host !== "127.0.0.1" && host !== "localhost") return 403;
+    const want = Buffer.from(`/mcp/${this.secret}`);
+    const got = Buffer.from((req.url ?? "").split("?")[0]);
+    return got.length === want.length && crypto.timingSafeEqual(got, want)
+      ? 200
+      : 404;
+  }
+
+  private async handle(
+    req: http.IncomingMessage,
+    res: http.ServerResponse
+  ): Promise<void> {
+    const status = this.allowed(req);
+    if (status !== 200) {
+      res.writeHead(status).end();
+      return;
+    }
+    if (req.method !== "POST") {
+      res.writeHead(405, { allow: "POST" }).end();
+      return;
+    }
+    let size = 0;
+    const chunks: Buffer[] = [];
+    for await (const c of req) {
+      size += (c as Buffer).length;
+      if (size > MAX_BODY) {
+        res.writeHead(413).end();
+        return;
+      }
+      chunks.push(c as Buffer);
+    }
+    let body: unknown;
+    try {
+      body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+    } catch {
+      res.writeHead(400).end();
+      return;
+    }
+    const server = this.build();
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: undefined,
+      enableJsonResponse: true,
+    });
+    const handle = {
+      close: async () => {
+        await transport.close();
+        await server.close();
+      },
+    };
+    this.live.add(handle);
+    res.on("close", () => {
+      this.live.delete(handle);
+      void handle.close().catch(() => {});
+    });
+    await server.connect(transport);
+    await transport.handleRequest(req, res, body);
+  }
+
+  // The `mcpServers` entry to give ACP session/new and session/load.
+  async start(): Promise<P> {
+    const server = http.createServer((req, res) => {
+      this.handle(req, res).catch(() => {
+        if (!res.headersSent) res.writeHead(500);
+        res.end();
+      });
+    });
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(0, "127.0.0.1", resolve);
+    });
+    this.http = server;
+    const { port } = server.address() as AddressInfo;
+    return {
+      type: "http",
+      name: ASK_SERVER,
+      url: `http://127.0.0.1:${port}/mcp/${this.secret}`,
+      headers: [],
+    };
+  }
+
+  // Cancels any question still waiting (its call returns a cancellation)
+  // and stops listening.
+  close(): void {
+    this.pending.expireAll();
+    const server = this.http;
+    this.http = undefined;
+    server?.close();
+    server?.closeAllConnections();
+    for (const h of this.live) void h.close().catch(() => {});
+    this.live.clear();
+  }
+}

package/lib/chat/drivers/copilot-mapper.ts

@@ -1,5 +1,6 @@
 import type { ChatContext, DriverEvent, UsageTotals } from "../events";
 import { toolTitle } from "../tools";
+import { isAskUserCall } from "./copilot-ask";
 import { TurnItems, type ToolStart } from "./turn-items";
 
 type P = Record<string, unknown>;
@@ -135,14 +136,22 @@ export class CopilotMapper {
     this.block++;
   }
 
+  // The card is the question's only trace, so its tool call stays hidden.
+  private asks = new Set<string>();
+
   private started(update: P): void {
     this.items.startTurn();
+    if (isAskUserCall(update)) {
+      this.asks.add(str(update.toolCallId));
+      return;
+    }
     this.close();
     this.items.tool(str(update.toolCallId), toolStart(update));
   }
 
   private updated(update: P): void {
     const id = str(update.toolCallId);
+    if (this.asks.has(id)) return;
     const status = str(update.status);
     if (status !== "completed" && status !== "failed") return;
     const start = toolStart(update);

package/lib/chat/drivers/copilot.ts

@@ -6,6 +6,7 @@ import type { ChatConversation, ChatDriver, ChatStartOptions } from "../driver";
 import type { ChatImage, ChatModel, DriverEvent } from "../events";
 import { InputQueue } from "../queue";
 import { AGENT_DEFAULT_MODEL } from "../../providers/registry";
+import { ASK_SERVER, ASK_TOOL, CopilotAskServer } from "./copilot-ask";
 import { CopilotApprovals } from "./copilot-approvals";
 import { CopilotMapper } from "./copilot-mapper";
 import { CopilotRpc } from "./copilot-rpc";
@@ -33,7 +34,10 @@ export function copilotArgs(
   fallback = copilotDefaultModel({ ...process.env, ...o.env })
 ): string[] {
   const model = o.model && o.model !== AGENT_DEFAULT_MODEL ? o.model : fallback;
-  return model ? ["--model", model] : [];
+  return [
+    ...(model ? ["--model", model] : []),
+    `--allow-tool=${ASK_SERVER}(${ASK_TOOL})`,
+  ];
 }
 
 // The models a new ACP session offers (`models.availableModels`), with
@@ -117,6 +121,7 @@ export const copilotDriver: ChatDriver = {
     const emit = (e: DriverEvent) => out.push(e);
     const mapper = new CopilotMapper(emit);
     const approvals = new CopilotApprovals(emit, options.access);
+    const ask = new CopilotAskServer(approvals.pending);
     let sessionId: string | null = null;
     let busy = false;
     let intro = options.resumeId ? "" : (options.systemAppend ?? "").trim();
@@ -136,7 +141,7 @@ export const copilotDriver: ChatDriver = {
           void approvals.handle(params).then((result) => rpc.reply(id, result));
         },
         exit(error) {
-          approvals.pending.expireAll();
+          ask.close();
           mapper.endTurn({ error });
           out.end();
         },
@@ -145,7 +150,15 @@ export const copilotDriver: ChatDriver = {
     emit({ type: "usage_start", totals: { ...mapper.totals } });
 
     const ready = rpc.initialize().then(async () => {
-      const params = { cwd: options.cwd, mcpServers: [] };
+      // Without it the agent just can't ask: the chat still works.
+      const mcpServers = await ask.start().then(
+        (s) => [s],
+        (e: Error) => {
+          mapper.items.error(`Questions aren't available: ${e.message}`);
+          return [];
+        }
+      );
+      const params = { cwd: options.cwd, mcpServers };
       if (options.resumeId) {
         mapper.replaying = true;
         try {
@@ -170,6 +183,7 @@ export const copilotDriver: ChatDriver = {
       emit({ type: "resume_id", id: sessionId });
     });
     ready.catch((error: Error) => {
+      ask.close();
       mapper.items.error(`GitHub Copilot couldn't start: ${error.message}`);
       mapper.endTurn({});
       out.end();
@@ -243,7 +257,7 @@ export const copilotDriver: ChatDriver = {
         };
       },
       close() {
-        approvals.pending.expireAll();
+        ask.close();
         rpc.close();
         out.end();
       },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.790",
+  "version": "0.3.794",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"