create-interview-cockpit 0.17.1 → 0.17.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-interview-cockpit",
-  "version": "0.17.1",
+  "version": "0.17.2",
   "description": "Scaffold a personal AI-powered interview prep cockpit",
   "type": "module",
   "bin": {

package/template/client/src/api.ts

@@ -2,6 +2,7 @@ import type {
   Topic,
   Question,
   ContextFile,
+  ContextFileOrigin,
   WorkspacesRegistry,
   InfraLabWorkspace,
 } from "./types";
@@ -342,15 +343,7 @@ export async function saveCodeSnippet(
   code: string,
   language: string,
   label: string,
-  origin:
-    | "user"
-    | "ai"
-    | "sandbox"
-    | "browser-security"
-    | "infra"
-    | "react"
-    | "nextjs"
-    | "module-federation",
+  origin: Exclude<ContextFileOrigin, "upload">,
 ): Promise<ContextFile> {
   const res = await fetch(`${BASE}/questions/${questionId}/save-code-snippet`, {
     method: "POST",

package/template/client/src/components/ChatMessage.tsx

@@ -27,6 +27,7 @@ interface Props {
     newSpec: string,
   ) => void;
   onDeleteMessage?: (messageId: string) => void;
+  isStreaming?: boolean;
 }
 
 function getTextContent(message: UIMessage): string {
@@ -90,10 +91,11 @@ const ChatMessage = memo(function ChatMessage({
   preferenceSuffix,
   onSpecRefined,
   onDeleteMessage,
+  isStreaming = false,
 }: Props) {
   const isUser = message.role === "user";
   const content = getTextContent(message);
-  const reasoning = !isUser ? getReasoningContent(message) : "";
+  const reasoning = !isUser && !isStreaming ? getReasoningContent(message) : "";
   const [copied, setCopied] = useState(false);
 
   // Stable wrappers so MarkdownRenderer's `components` useMemo doesn't invalidate
@@ -196,19 +198,26 @@ const ChatMessage = memo(function ChatMessage({
           ) : (
             <>
               {reasoning && <ThinkingBlock reasoning={reasoning} />}
-              <TextAnnotator
-                content={content}
-                messageId={message.id}
-                annotations={annotations}
-                onAnnotationCreate={onAnnotationCreate ?? (() => {})}
-                onAnnotationUpdate={onAnnotationUpdate ?? (() => {})}
-                bookmarkedBlockIndex={bookmarkedBlockIndex}
-                onBookmarkBlock={
-                  onSetBookmark ? stableBookmarkBlock : undefined
-                }
-                preferenceSuffix={preferenceSuffix}
-                onSpecRefined={onSpecRefined ? stableSpecRefined : undefined}
-              />
+              {isStreaming ? (
+                <div className="whitespace-pre-wrap break-words text-slate-200">
+                  {content}
+                  <span className="ml-1 inline-block h-3 w-1 animate-pulse rounded-sm bg-cyan-400/70 align-middle" />
+                </div>
+              ) : (
+                <TextAnnotator
+                  content={content}
+                  messageId={message.id}
+                  annotations={annotations}
+                  onAnnotationCreate={onAnnotationCreate ?? (() => {})}
+                  onAnnotationUpdate={onAnnotationUpdate ?? (() => {})}
+                  bookmarkedBlockIndex={bookmarkedBlockIndex}
+                  onBookmarkBlock={
+                    onSetBookmark ? stableBookmarkBlock : undefined
+                  }
+                  preferenceSuffix={preferenceSuffix}
+                  onSpecRefined={onSpecRefined ? stableSpecRefined : undefined}
+                />
+              )}
             </>
           )}
         </div>

package/template/client/src/components/ChatView.tsx

@@ -227,6 +227,11 @@ const ChatTranscript = memo(function ChatTranscript({
               preferenceSuffix={preferenceSuffix}
               onSpecRefined={onSpecRefined}
               onDeleteMessage={!isLoading ? onDeleteMessage : undefined}
+              isStreaming={
+                status === "streaming" &&
+                message.role === "assistant" &&
+                message.id === messages[messages.length - 1]?.id
+              }
             />
           </div>
         ))}
@@ -504,6 +509,7 @@ export default function ChatView({ question }: Props) {
     id: question.id,
     transport,
     messages: initialMessages,
+    experimental_throttle: 80,
     onError: handleChatError,
     onFinish: handleChatFinish,
   });
@@ -542,6 +548,7 @@ export default function ChatView({ question }: Props) {
   // Only show the Continue button if the last assistant message looks truncated
   // (doesn't end with sentence-terminating punctuation or a closing code fence).
   const lastResponseLooksTruncated = useMemo(() => {
+    if (status !== "ready") return false;
     if (messages.length === 0) return false;
     const last = messages[messages.length - 1];
     if (last.role !== "assistant") return false;
@@ -562,7 +569,7 @@ export default function ChatView({ question }: Props) {
     const lastLine = (lines.filter(Boolean).pop() ?? "").trimEnd();
     // Complete if ends with sentence punctuation or structural close chars
     return !/[.!?`\])>]$/.test(lastLine);
-  }, [messages]);
+  }, [messages, status]);
 
   // Smooth scroll to bottom whenever a NEW message is added (length changes).
   // Using messages.length instead of the full messages array avoids firing on

package/template/client/src/components/CodeRunnerModal.tsx

@@ -1,6 +1,7 @@
 import {
   useCallback,
   useEffect,
+  useMemo,
   useRef,
   useState,
   useLayoutEffect,
@@ -36,7 +37,12 @@ import {
 import { useStore } from "../store";
 import Editor from "react-simple-code-editor";
 import MonacoEditorLib from "@monaco-editor/react";
-import type { OnMount, BeforeMount, Monaco } from "@monaco-editor/react";
+import type {
+  OnMount,
+  BeforeMount,
+  Monaco,
+  OnChange,
+} from "@monaco-editor/react";
 import { setupTypeAcquisition } from "@typescript/ata";
 import ts from "typescript";
 import Prism from "prismjs";
@@ -430,8 +436,17 @@ function MonacoEditorWrapper({
 }) {
   const fontSizePx = parseInt(fontSize) || 12;
   const monacoRef = useRef<Monaco | null>(null);
+  const onChangeRef = useRef(onChange);
   const { initATA, acquireTypes } = useATA(monacoRef);
 
+  useEffect(() => {
+    onChangeRef.current = onChange;
+  }, [onChange]);
+
+  const handleEditorChange = useCallback<OnChange>((val) => {
+    onChangeRef.current(val ?? "");
+  }, []);
+
   const handleBeforeMount: BeforeMount = (monaco) => {
     monacoRef.current = monaco;
 
@@ -659,7 +674,8 @@ declare module 'next/server' {
 
   // Re-scan for new imports whenever code changes
   useEffect(() => {
-    acquireTypes(value);
+    const id = window.setTimeout(() => acquireTypes(value), 400);
+    return () => window.clearTimeout(id);
   }, [value, acquireTypes]);
 
   const modelPath =
@@ -669,6 +685,30 @@ declare module 'next/server' {
         : `file:///${path.replace(/^\/+/, "")}`
       : undefined;
 
+  const editorOptions = useMemo(
+    () => ({
+      fontSize: fontSizePx,
+      fontFamily: EDITOR_FONT,
+      minimap: { enabled: false },
+      scrollBeyondLastLine: false,
+      lineNumbers: "on" as const,
+      glyphMargin: false,
+      folding: false,
+      lineDecorationsWidth: 4,
+      lineNumbersMinChars: 3,
+      renderLineHighlight: "line" as const,
+      overviewRulerLanes: 0,
+      hideCursorInOverviewRuler: true,
+      overviewRulerBorder: false,
+      scrollbar: { vertical: "auto" as const, horizontal: "auto" as const },
+      padding: { top: 8, bottom: 8 },
+      wordWrap: "off" as const,
+      automaticLayout: true,
+      placeholder,
+    }),
+    [fontSizePx, placeholder],
+  );
+
   return (
     <div className="absolute inset-0">
       <MonacoEditorLib
@@ -678,30 +718,11 @@ declare module 'next/server' {
         language={language.includes("typescript") ? "typescript" : "javascript"}
         path={modelPath}
         theme="vs-dark"
-        value={value}
-        onChange={(val) => onChange(val ?? "")}
+        defaultValue={value}
+        onChange={handleEditorChange}
         beforeMount={handleBeforeMount}
         onMount={handleMount}
-        options={{
-          fontSize: fontSizePx,
-          fontFamily: EDITOR_FONT,
-          minimap: { enabled: false },
-          scrollBeyondLastLine: false,
-          lineNumbers: "on",
-          glyphMargin: false,
-          folding: false,
-          lineDecorationsWidth: 4,
-          lineNumbersMinChars: 3,
-          renderLineHighlight: "line",
-          overviewRulerLanes: 0,
-          hideCursorInOverviewRuler: true,
-          overviewRulerBorder: false,
-          scrollbar: { vertical: "auto", horizontal: "auto" },
-          padding: { top: 8, bottom: 8 },
-          wordWrap: "off",
-          automaticLayout: true,
-          placeholder,
-        }}
+        options={editorOptions}
       />
     </div>
   );
@@ -2574,6 +2595,18 @@ export default function CodeRunnerModal() {
         : clientType === "module-federation"
           ? "module-federation"
           : "react";
+    let pendingDelta = "";
+    let flushTimer: number | null = null;
+    const flushPendingDelta = () => {
+      if (!pendingDelta || abort.aborted) return;
+      const chunk = pendingDelta;
+      pendingDelta = "";
+      setSbxChatMessages((prev) =>
+        prev.map((m) =>
+          m.id === aId ? { ...m, content: m.content + chunk } : m,
+        ),
+      );
+    };
     try {
       const history = [...sbxChatMessages, userMsg].map((m) => ({
         role: m.role,
@@ -2589,14 +2622,17 @@ export default function CodeRunnerModal() {
         },
         (delta) => {
           if (abort.aborted) return;
-          setSbxChatMessages((prev) =>
-            prev.map((m) =>
-              m.id === aId ? { ...m, content: m.content + delta } : m,
-            ),
-          );
+          pendingDelta += delta;
+          if (flushTimer !== null) return;
+          flushTimer = window.setTimeout(() => {
+            flushTimer = null;
+            flushPendingDelta();
+          }, 80);
         },
       );
+      flushPendingDelta();
     } catch (err: unknown) {
+      flushPendingDelta();
       if (!abort.aborted)
         setSbxChatMessages((prev) =>
           prev.map((m) =>
@@ -2606,6 +2642,7 @@ export default function CodeRunnerModal() {
           ),
         );
     } finally {
+      if (flushTimer !== null) window.clearTimeout(flushTimer);
       if (!abort.aborted) setSbxChatLoading(false);
     }
   }, [
@@ -7016,7 +7053,7 @@ export default function CodeRunnerModal() {
                       )}
                     </p>
                   )}
-                  {sbxChatMessages.map((msg) => (
+                  {sbxChatMessages.map((msg, index) => (
                     <div
                       key={msg.id}
                       className={`flex flex-col gap-0.5 ${msg.role === "user" ? "items-end" : "items-start"}`}
@@ -7031,67 +7068,76 @@ export default function CodeRunnerModal() {
                         {msg.role === "user" ? (
                           msg.content
                         ) : msg.content ? (
-                          <ReactMarkdown
-                            remarkPlugins={[remarkGfm]}
-                            components={{
-                              code({ className, children, ...props }) {
-                                const isBlock =
-                                  className?.startsWith("language-");
-                                return isBlock ? (
-                                  <pre className="bg-slate-900/80 rounded p-2 overflow-x-auto my-1">
+                          sbxChatLoading &&
+                          msg.role === "assistant" &&
+                          index === sbxChatMessages.length - 1 ? (
+                            <div className="whitespace-pre-wrap break-words">
+                              {msg.content}
+                              <span className="ml-1 inline-block h-2.5 w-1 animate-pulse rounded-sm bg-violet-400/70 align-middle" />
+                            </div>
+                          ) : (
+                            <ReactMarkdown
+                              remarkPlugins={[remarkGfm]}
+                              components={{
+                                code({ className, children, ...props }) {
+                                  const isBlock =
+                                    className?.startsWith("language-");
+                                  return isBlock ? (
+                                    <pre className="bg-slate-900/80 rounded p-2 overflow-x-auto my-1">
+                                      <code
+                                        className={`${className ?? ""} text-[11px]`}
+                                        {...props}
+                                      >
+                                        {children}
+                                      </code>
+                                    </pre>
+                                  ) : (
                                     <code
-                                      className={`${className ?? ""} text-[11px]`}
+                                      className="bg-slate-900/60 px-1 rounded text-violet-300 text-[11px]"
                                       {...props}
                                     >
                                       {children}
                                     </code>
-                                  </pre>
-                                ) : (
-                                  <code
-                                    className="bg-slate-900/60 px-1 rounded text-violet-300 text-[11px]"
-                                    {...props}
-                                  >
-                                    {children}
-                                  </code>
-                                );
-                              },
-                              p({ children }) {
-                                return (
-                                  <p className="mb-1 last:mb-0">{children}</p>
-                                );
-                              },
-                              ul({ children }) {
-                                return (
-                                  <ul className="list-disc list-inside mb-1 space-y-0.5">
-                                    {children}
-                                  </ul>
-                                );
-                              },
-                              ol({ children }) {
-                                return (
-                                  <ol className="list-decimal list-inside mb-1 space-y-0.5">
-                                    {children}
-                                  </ol>
-                                );
-                              },
-                              h2({ children }) {
-                                return (
-                                  <h2 className="text-xs font-semibold text-slate-200 mt-2 mb-0.5">
-                                    {children}
-                                  </h2>
-                                );
-                              },
-                              h3({ children }) {
-                                return (
-                                  <h3 className="text-xs font-semibold text-slate-300 mt-1.5 mb-0.5">
-                                    {children}
-                                  </h3>
-                                );
-                              },
-                            }}
-                          >
-                            {msg.content}
-                          </ReactMarkdown>
+                                  );
+                                },
+                                p({ children }) {
+                                  return (
+                                    <p className="mb-1 last:mb-0">{children}</p>
+                                  );
+                                },
+                                ul({ children }) {
+                                  return (
+                                    <ul className="list-disc list-inside mb-1 space-y-0.5">
+                                      {children}
+                                    </ul>
+                                  );
+                                },
+                                ol({ children }) {
+                                  return (
+                                    <ol className="list-decimal list-inside mb-1 space-y-0.5">
+                                      {children}
+                                    </ol>
+                                  );
+                                },
+                                h2({ children }) {
+                                  return (
+                                    <h2 className="text-xs font-semibold text-slate-200 mt-2 mb-0.5">
+                                      {children}
+                                    </h2>
+                                  );
+                                },
+                                h3({ children }) {
+                                  return (
+                                    <h3 className="text-xs font-semibold text-slate-300 mt-1.5 mb-0.5">
+                                      {children}
+                                    </h3>
+                                  );
+                                },
+                              }}
+                            >
+                              {msg.content}
+                            </ReactMarkdown>
+                          )
                         ) : (
                           <span className="flex items-center gap-1.5 text-slate-500">
                             <Loader2 className="w-3 h-3 animate-spin" />{" "}

package/template/client/src/components/InfraLabModal.tsx

@@ -365,7 +365,7 @@ export default function InfraLabModal() {
     };
   }, [activeInfraId]);
 
-  const runInfra = async (action: InfraRunAction) => {
+  const runInfra = async (action: Exclude<InfraRunAction, "command">) => {
     if (!currentQuestion) return;
 
     setRunError(null);

package/template/client/src/components/WorkspaceSwitcher.tsx

@@ -15,6 +15,7 @@ import {
   Loader2,
 } from "lucide-react";
 import { useStore } from "../store";
+import type { DriveFolder } from "../api";
 import type { WorkspaceMeta } from "../types";
 
 // ── Workspace Switcher ─────────────────────────────────────────────────────────
@@ -78,7 +79,7 @@ export default function WorkspaceSwitcher() {
   // Subfolder picker state (export destination)
   const [folderPicker, setFolderPicker] = useState<{
     ws: WorkspaceMeta;
-    folders: { id: string; name: string }[];
+    folders: DriveFolder[];
     loading: boolean;
   } | null>(null);
   const [newFolderName, setNewFolderName] = useState("");
@@ -370,7 +371,7 @@ export default function WorkspaceSwitcher() {
                         folderPicker.ws.id,
                         name,
                       );
-                      if ("needsAuth" in created && created.needsAuth) {
+                      if ("needsAuth" in created) {
                         window.location.href = created.authUrl;
                         return;
                       }

package/template/client/src/store.ts

@@ -6,6 +6,7 @@ import type {
   WorkspaceMeta,
   InfraLabWorkspace,
   FrontendLabWorkspace,
+  ContextFileOrigin,
 } from "./types";
 import type { AiSettings } from "./api";
 import * as api from "./api";
@@ -216,16 +217,7 @@ interface Store {
     code: string,
     language: string,
     label: string,
-    origin:
-      | "user"
-      | "ai"
-      | "sandbox"
-      | "browser-security"
-      | "infra"
-      | "react"
-      | "nextjs"
-      | "module-federation"
-      | "canvas",
+    origin: Exclude<ContextFileOrigin, "upload">,
   ) => Promise<import("./types").ContextFile>;
   clearMessages: (questionId: string) => Promise<void>;
 

package/template/client/tsconfig.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/app.tsx","./src/api.ts","./src/browsersecuritytemplates.ts","./src/infralab.ts","./src/main.tsx","./src/reactlab.ts","./src/store.ts","./src/types.ts","./src/vite-env.d.ts","./src/components/aisettingsmodal.tsx","./src/components/annotationdialog.tsx","./src/components/browsersecuritylabmodal.tsx","./src/components/chatmessage.tsx","./src/components/chatview.tsx","./src/components/codecontextpanel.tsx","./src/components/codelineannotationpopup.tsx","./src/components/coderunnermodal.tsx","./src/components/deploymentlabmodal.tsx","./src/components/docrefmodal.tsx","./src/components/fileattachments.tsx","./src/components/filepickermodal.tsx","./src/components/fileviewermodal.tsx","./src/components/infralabmodal.tsx","./src/components/labspanel.tsx","./src/components/linkedconvospicker.tsx","./src/components/markdownrenderer.tsx","./src/components/mermaiddiagram.tsx","./src/components/notesmodal.tsx","./src/components/plotembed.tsx","./src/components/sidebar.tsx","./src/components/textannotator.tsx","./src/components/vizcraftembed.tsx","./src/components/workspaceswitcher.tsx"],"errors":true,"version":"5.9.3"}
+{"root":["./src/app.tsx","./src/api.ts","./src/browsersecuritytemplates.ts","./src/infralab.ts","./src/main.tsx","./src/reactlab.ts","./src/store.ts","./src/types.ts","./src/vite-env.d.ts","./src/components/aisettingsmodal.tsx","./src/components/annotationdialog.tsx","./src/components/browsersecuritylabmodal.tsx","./src/components/canvaslabmodal.tsx","./src/components/chatmessage.tsx","./src/components/chatview.tsx","./src/components/codecontextpanel.tsx","./src/components/codelineannotationpopup.tsx","./src/components/coderunnermodal.tsx","./src/components/deploymentlabmodal.tsx","./src/components/docrefmodal.tsx","./src/components/fileattachments.tsx","./src/components/filepickermodal.tsx","./src/components/fileviewermodal.tsx","./src/components/infralabmodal.tsx","./src/components/labspanel.tsx","./src/components/linkedconvospicker.tsx","./src/components/markdownrenderer.tsx","./src/components/mermaiddiagram.tsx","./src/components/notesmodal.tsx","./src/components/plotembed.tsx","./src/components/sidebar.tsx","./src/components/textannotator.tsx","./src/components/vizcraftembed.tsx","./src/components/workspaceswitcher.tsx"],"version":"5.9.3"}

package/template/cockpit.json

@@ -1,3 +1,3 @@
 {
-  "version": "0.16.0"
+  "version": "0.17.1"
 }

package/template/server/src/index.ts

@@ -56,9 +56,26 @@ const upload = multer({
   },
 });
 
-// Extract text from uploaded files (docx, pdf, plain text, or images)
+const MODEL_READABLE_IMAGE_EXTS = new Set([
+  "png",
+  "jpg",
+  "jpeg",
+  "gif",
+  "webp",
+]);
+const MAX_IMAGE_BYTES_FOR_MODEL = 10 * 1024 * 1024;
+
+function extensionForFilename(filename: string): string {
+  return filename.split(".").pop()?.toLowerCase() ?? "";
+}
+
+function isModelReadableImage(filename: string): boolean {
+  return MODEL_READABLE_IMAGE_EXTS.has(extensionForFilename(filename));
+}
+
+// Extract text from uploaded files (docx, pdf, plain text; images stay visual assets)
 async function extractText(buffer: Buffer, filename: string): Promise<string> {
-  const ext = filename.split(".").pop()?.toLowerCase();
+  const ext = extensionForFilename(filename);
   if (ext === "docx") {
     try {
       const result = await mammoth.extractRawText({ buffer });
@@ -76,17 +93,14 @@ async function extractText(buffer: Buffer, filename: string): Promise<string> {
       return `[PDF extraction failed: ${e?.message ?? "unknown error"}. The original file is stored and can be downloaded.]`;
     }
   }
-  const imageExts = new Set(["png", "jpg", "jpeg", "gif", "webp", "svg"]);
-  if (ext && imageExts.has(ext)) {
-    // Images can't be read as text; store original for download.
-    // Return a descriptor so the LLM knows the file exists.
-    return `[Image file: ${filename} — the original is stored and available for download, but cannot be read as text by the AI.]`;
+  if (isModelReadableImage(filename)) {
+    return `[Image file: ${filename} — call readFile for this file id to inspect the image visually.]`;
   }
   return buffer.toString("utf-8");
 }
 
 function mimeForFilename(filename: string): string {
-  const ext = filename.split(".").pop()?.toLowerCase() ?? "";
+  const ext = extensionForFilename(filename);
   const map: Record<string, string> = {
     pdf: "application/pdf",
     docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
@@ -106,6 +120,130 @@ function mimeForFilename(filename: string): string {
   return map[ext] ?? "application/octet-stream";
 }
 
+type ReferenceFileEntry = {
+  label: string;
+  originalName: string;
+  reader: () => Promise<string>;
+  mediaType?: string;
+  imageReader?: () => Promise<Buffer | null>;
+};
+
+function makeReferenceFileEntry({
+  scope,
+  file,
+  reader,
+  blobReader,
+}: {
+  scope: "workspace" | "topic" | "question";
+  file: storage.ContextFile;
+  reader: () => Promise<string>;
+  blobReader: () => Promise<Buffer>;
+}): ReferenceFileEntry {
+  const label = `[${scope}] ${file.originalName}`;
+  return {
+    label,
+    originalName: file.originalName,
+    reader,
+    ...(isModelReadableImage(file.originalName)
+      ? {
+          mediaType: mimeForFilename(file.originalName),
+          imageReader: async () =>
+            (await storage.readOriginalBlob(file.id)) ?? (await blobReader()),
+        }
+      : {}),
+  };
+}
+
+function makeCodeReferenceFileEntry(
+  label: string,
+  reader: () => Promise<string>,
+): ReferenceFileEntry {
+  return { label, originalName: label, reader };
+}
+
+function createReadFileTool(fileRegistry: Map<string, ReferenceFileEntry>) {
+  return tool({
+    description:
+      "Read an available reference file. Text documents return text; images return visual image data to inspect.",
+    inputSchema: z.object({
+      fileId: z
+        .string()
+        .describe("The id of the file to read, from the available files list."),
+    }),
+    execute: async ({ fileId }) => {
+      const entry = fileRegistry.get(fileId);
+      if (!entry) return { type: "error", error: "File not found" };
+      try {
+        if (entry.imageReader) {
+          return {
+            type: "image",
+            fileId,
+            fileName: entry.label,
+            mediaType: entry.mediaType ?? mimeForFilename(entry.originalName),
+          };
+        }
+        const content = await entry.reader();
+        return { type: "text", fileName: entry.label, content };
+      } catch {
+        return { type: "error", error: "Could not read file" };
+      }
+    },
+    toModelOutput: async ({ output }) => {
+      if ((output as any)?.type === "image") {
+        const imageOutput = output as {
+          fileId: string;
+          fileName: string;
+          mediaType: string;
+        };
+        const entry = fileRegistry.get(imageOutput.fileId);
+        let buffer: Buffer | null | undefined;
+        try {
+          buffer = await entry?.imageReader?.();
+        } catch {
+          buffer = null;
+        }
+        if (!buffer) {
+          return {
+            type: "text",
+            value: `${imageOutput.fileName}\n\n[Image bytes are unavailable.]`,
+          };
+        }
+        if (buffer.byteLength > MAX_IMAGE_BYTES_FOR_MODEL) {
+          return {
+            type: "text",
+            value: `${imageOutput.fileName}\n\n[Image is ${(buffer.byteLength / (1024 * 1024)).toFixed(1)} MB, which is too large to send to the model. Ask the user to upload a smaller image.]`,
+          };
+        }
+        return {
+          type: "content",
+          value: [
+            {
+              type: "text",
+              text: `${imageOutput.fileName}\nInspect the attached image directly and answer using what is visible in it.`,
+            },
+            {
+              type: "image-data",
+              data: buffer.toString("base64"),
+              mediaType: imageOutput.mediaType,
+            },
+          ],
+        };
+      }
+
+      if ((output as any)?.type === "text") {
+        const textOutput = output as { fileName: string; content: string };
+        return {
+          type: "text",
+          value: `${textOutput.fileName}\n\n${textOutput.content}`,
+        };
+      }
+
+      const error = (output as any)?.error ?? "Could not read file";
+      return { type: "text", value: `[readFile error: ${error}]` };
+    },
+  });
+}
+
 const PORT = process.env.PORT || 3001;
 const CODE_CONTEXT_DIR = process.env.CODE_CONTEXT_DIR || "";
 
@@ -1374,20 +1512,22 @@ app.post("/api/chat", async (req, res) => {
     }
   }
 
-  // Build a file registry: id → { label, reader }
+  // Build a file registry: id → reference entry.
   // The model sees the list of file names and can call readFile(id) for any of them.
-  const fileRegistry = new Map<
-    string,
-    { label: string; reader: () => Promise<string> }
-  >();
+  const fileRegistry = new Map<string, ReferenceFileEntry>();
 
   // Workspace-level uploaded files (apply to all topics)
   const workspaceFiles = await storage.getWorkspaceContextFiles();
   for (const cf of workspaceFiles) {
-    fileRegistry.set(cf.id, {
-      label: `[workspace] ${cf.originalName}`,
-      reader: () => storage.readWorkspaceContextFileContent(cf.id),
-    });
+    fileRegistry.set(
+      cf.id,
+      makeReferenceFileEntry({
+        scope: "workspace",
+        file: cf,
+        reader: () => storage.readWorkspaceContextFileContent(cf.id),
+        blobReader: () => storage.readWorkspaceContextFileBlob(cf.id),
+      }),
+    );
   }
 
   // Topic-level uploaded files + topic-wide system prompt
@@ -1399,10 +1539,15 @@ app.post("/api/chat", async (req, res) => {
     }
     if (topic?.contextFiles?.length) {
       for (const cf of topic.contextFiles) {
-        fileRegistry.set(cf.id, {
-          label: `[topic] ${cf.originalName}`,
-          reader: () => storage.readContextFileContent(cf.id),
-        });
+        fileRegistry.set(
+          cf.id,
+          makeReferenceFileEntry({
+            scope: "topic",
+            file: cf,
+            reader: () => storage.readContextFileContent(cf.id),
+            blobReader: () => storage.readContextFileBlob(cf.id),
+          }),
+        );
       }
     }
   }
@@ -1414,10 +1559,15 @@ app.post("/api/chat", async (req, res) => {
       for (const cf of question.contextFiles.filter(
         (c) => c.inContext !== false,
       )) {
-        fileRegistry.set(cf.id, {
-          label: `[question] ${cf.originalName}`,
-          reader: () => storage.readContextFileContent(cf.id),
-        });
+        fileRegistry.set(
+          cf.id,
+          makeReferenceFileEntry({
+            scope: "question",
+            file: cf,
+            reader: () => storage.readContextFileContent(cf.id),
+            blobReader: () => storage.readContextFileBlob(cf.id),
+          }),
+        );
       }
     }
   }
@@ -1428,10 +1578,12 @@ app.post("/api/chat", async (req, res) => {
       const fullPath = path.join(CODE_CONTEXT_DIR, filePath);
       const resolved = path.resolve(fullPath);
       if (!resolved.startsWith(path.resolve(CODE_CONTEXT_DIR))) continue;
-      fileRegistry.set(`code:${filePath}`, {
-        label: `[code] ${filePath}`,
-        reader: () => fs.readFile(resolved, "utf-8"),
-      });
+      fileRegistry.set(
+        `code:${filePath}`,
+        makeCodeReferenceFileEntry(`[code] ${filePath}`, () =>
+          fs.readFile(resolved, "utf-8"),
+        ),
+      );
     }
   }
 
@@ -1445,6 +1597,7 @@ app.post("/api/chat", async (req, res) => {
 
     system += `\n\n--- Available Reference Files ---
 The following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant — you do not need to read them all.
+For image files, readFile returns visual image data so you can inspect what is visible rather than treating the image as text.
 
 `;
     for (const [id, { label }] of fileRegistry) {
@@ -1581,27 +1734,7 @@ Examples (illustrative only — use real ids and names from the list above):
         }),
         ...(fileRegistry.size > 0
           ? {
-              readFile: tool({
-                description:
-                  "Read the content of an available reference file. Use this to get file contents when they are relevant to the user's question.",
-                inputSchema: z.object({
-                  fileId: z
-                    .string()
-                    .describe(
-                      "The id of the file to read, from the available files list.",
-                    ),
-                }),
-                execute: async ({ fileId }) => {
-                  const entry = fileRegistry.get(fileId);
-                  if (!entry) return { error: "File not found" };
-                  try {
-                    const content = await entry.reader();
-                    return { fileName: entry.label, content };
-                  } catch {
-                    return { error: "Could not read file" };
-                  }
-                },
-              }),
+              readFile: createReadFileTool(fileRegistry),
             }
           : {}),
       },
@@ -1881,17 +2014,19 @@ app.post("/api/code-line-ask", async (req, res) => {
     const aiSettings = await storage.getAiSettings();
 
     // Build a file registry identical to /api/chat so the model has the same context
-    const fileRegistry = new Map<
-      string,
-      { label: string; reader: () => Promise<string> }
-    >();
+    const fileRegistry = new Map<string, ReferenceFileEntry>();
 
     const workspaceFiles = await storage.getWorkspaceContextFiles();
     for (const cf of workspaceFiles) {
-      fileRegistry.set(cf.id, {
-        label: `[workspace] ${cf.originalName}`,
-        reader: () => storage.readWorkspaceContextFileContent(cf.id),
-      });
+      fileRegistry.set(
+        cf.id,
+        makeReferenceFileEntry({
+          scope: "workspace",
+          file: cf,
+          reader: () => storage.readWorkspaceContextFileContent(cf.id),
+          blobReader: () => storage.readWorkspaceContextFileBlob(cf.id),
+        }),
+      );
     }
 
     if (topicId) {
@@ -1899,10 +2034,15 @@ app.post("/api/code-line-ask", async (req, res) => {
       const topic = topics.find((t: any) => t.id === topicId);
       if (topic?.contextFiles?.length) {
         for (const cf of topic.contextFiles) {
-          fileRegistry.set(cf.id, {
-            label: `[topic] ${cf.originalName}`,
-            reader: () => storage.readContextFileContent(cf.id),
-          });
+          fileRegistry.set(
+            cf.id,
+            makeReferenceFileEntry({
+              scope: "topic",
+              file: cf,
+              reader: () => storage.readContextFileContent(cf.id),
+              blobReader: () => storage.readContextFileBlob(cf.id),
+            }),
+          );
         }
       }
     }
@@ -1911,10 +2051,15 @@ app.post("/api/code-line-ask", async (req, res) => {
       const question = await storage.getQuestion(questionId);
       if (question?.contextFiles?.length) {
         for (const cf of question.contextFiles) {
-          fileRegistry.set(cf.id, {
-            label: `[question] ${cf.originalName}`,
-            reader: () => storage.readContextFileContent(cf.id),
-          });
+          fileRegistry.set(
+            cf.id,
+            makeReferenceFileEntry({
+              scope: "question",
+              file: cf,
+              reader: () => storage.readContextFileContent(cf.id),
+              blobReader: () => storage.readContextFileBlob(cf.id),
+            }),
+          );
         }
       }
     }
@@ -1924,10 +2069,12 @@ app.post("/api/code-line-ask", async (req, res) => {
         const fullPath = path.join(CODE_CONTEXT_DIR, fp);
         const resolved = path.resolve(fullPath);
         if (!resolved.startsWith(path.resolve(CODE_CONTEXT_DIR))) continue;
-        fileRegistry.set(`code:${fp}`, {
-          label: `[code] ${fp}`,
-          reader: () => fs.readFile(resolved, "utf-8"),
-        });
+        fileRegistry.set(
+          `code:${fp}`,
+          makeCodeReferenceFileEntry(`[code] ${fp}`, () =>
+            fs.readFile(resolved, "utf-8"),
+          ),
+        );
       }
     }
 
@@ -1950,7 +2097,7 @@ app.post("/api/code-line-ask", async (req, res) => {
           codeFilePaths.push(id.slice("code:".length));
       }
 
-      system += `\n\n--- Available Reference Files ---\nThe following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant.\n\n`;
+      system += `\n\n--- Available Reference Files ---\nThe following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant. For image files, readFile returns visual image data so you can inspect what is visible.\n\n`;
       for (const [id, { label }] of fileRegistry) {
         system += `• ${label}  (id: "${id}")\n`;
       }
@@ -1992,27 +2139,7 @@ app.post("/api/code-line-ask", async (req, res) => {
       tools:
         fileRegistry.size > 0
           ? {
-              readFile: tool({
-                description:
-                  "Read the content of an available reference file. Use this to get file contents when relevant to the question.",
-                inputSchema: z.object({
-                  fileId: z
-                    .string()
-                    .describe(
-                      "The id of the file to read, from the available files list.",
-                    ),
-                }),
-                execute: async ({ fileId }) => {
-                  const entry = fileRegistry.get(fileId);
-                  if (!entry) return { error: "File not found" };
-                  try {
-                    const content = await entry.reader();
-                    return { fileName: entry.label, content };
-                  } catch {
-                    return { error: "Could not read file" };
-                  }
-                },
-              }),
+              readFile: createReadFileTool(fileRegistry),
             }
           : undefined,
       stopWhen: stepCountIs(4),

package/template/server/src/storage.ts

@@ -534,6 +534,13 @@ export async function readContextFileContent(fileId: string): Promise<string> {
   return fs.readFile(path.join(contextFilesDirPath(), fileId), "utf-8");
 }
 
+export async function readContextFileBlob(
+  fileId: string,
+  workspaceId = _activeWorkspaceId,
+): Promise<Buffer> {
+  return fs.readFile(path.join(contextFilesDirPath(workspaceId), fileId));
+}
+
 /**
  * Store the raw original file bytes at {fileId}.orig so downloads can serve the
  * real file rather than the extracted-text version used by the LLM.
@@ -646,6 +653,13 @@ export async function readWorkspaceContextFileContent(
   );
 }
 
+export async function readWorkspaceContextFileBlob(
+  fileId: string,
+  workspaceId = _activeWorkspaceId,
+): Promise<Buffer> {
+  return fs.readFile(path.join(contextFilesDirPath(workspaceId), fileId));
+}
+
 export async function getQuestion(id: string): Promise<Question | null> {
   await ensureWorkspaceDirs();
   try {