create-interview-cockpit 0.17.1 → 0.17.3

package/template/server/src/index.ts

@@ -56,9 +56,26 @@ const upload = multer({
   },
 });
 
-// Extract text from uploaded files (docx, pdf, plain text, or images)
+const MODEL_READABLE_IMAGE_EXTS = new Set([
+  "png",
+  "jpg",
+  "jpeg",
+  "gif",
+  "webp",
+]);
+const MAX_IMAGE_BYTES_FOR_MODEL = 10 * 1024 * 1024;
+
+function extensionForFilename(filename: string): string {
+  return filename.split(".").pop()?.toLowerCase() ?? "";
+}
+
+function isModelReadableImage(filename: string): boolean {
+  return MODEL_READABLE_IMAGE_EXTS.has(extensionForFilename(filename));
+}
+
+// Extract text from uploaded files (docx, pdf, plain text; images stay visual assets)
 async function extractText(buffer: Buffer, filename: string): Promise<string> {
-  const ext = filename.split(".").pop()?.toLowerCase();
+  const ext = extensionForFilename(filename);
   if (ext === "docx") {
     try {
       const result = await mammoth.extractRawText({ buffer });
@@ -76,17 +93,14 @@ async function extractText(buffer: Buffer, filename: string): Promise<string> {
       return `[PDF extraction failed: ${e?.message ?? "unknown error"}. The original file is stored and can be downloaded.]`;
     }
   }
-  const imageExts = new Set(["png", "jpg", "jpeg", "gif", "webp", "svg"]);
-  if (ext && imageExts.has(ext)) {
-    // Images can't be read as text; store original for download.
-    // Return a descriptor so the LLM knows the file exists.
-    return `[Image file: ${filename} — the original is stored and available for download, but cannot be read as text by the AI.]`;
+  if (isModelReadableImage(filename)) {
+    return `[Image file: ${filename} — call readFile for this file id to inspect the image visually.]`;
   }
   return buffer.toString("utf-8");
 }
 
 function mimeForFilename(filename: string): string {
-  const ext = filename.split(".").pop()?.toLowerCase() ?? "";
+  const ext = extensionForFilename(filename);
   const map: Record<string, string> = {
     pdf: "application/pdf",
     docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
@@ -106,6 +120,130 @@ function mimeForFilename(filename: string): string {
   return map[ext] ?? "application/octet-stream";
 }
 
+type ReferenceFileEntry = {
+  label: string;
+  originalName: string;
+  reader: () => Promise<string>;
+  mediaType?: string;
+  imageReader?: () => Promise<Buffer | null>;
+};
+
+function makeReferenceFileEntry({
+  scope,
+  file,
+  reader,
+  blobReader,
+}: {
+  scope: "workspace" | "topic" | "question";
+  file: storage.ContextFile;
+  reader: () => Promise<string>;
+  blobReader: () => Promise<Buffer>;
+}): ReferenceFileEntry {
+  const label = `[${scope}] ${file.originalName}`;
+  return {
+    label,
+    originalName: file.originalName,
+    reader,
+    ...(isModelReadableImage(file.originalName)
+      ? {
+          mediaType: mimeForFilename(file.originalName),
+          imageReader: async () =>
+            (await storage.readOriginalBlob(file.id)) ?? (await blobReader()),
+        }
+      : {}),
+  };
+}
+
+function makeCodeReferenceFileEntry(
+  label: string,
+  reader: () => Promise<string>,
+): ReferenceFileEntry {
+  return { label, originalName: label, reader };
+}
+
+function createReadFileTool(fileRegistry: Map<string, ReferenceFileEntry>) {
+  return tool({
+    description:
+      "Read an available reference file. Text documents return text; images return visual image data to inspect.",
+    inputSchema: z.object({
+      fileId: z
+        .string()
+        .describe("The id of the file to read, from the available files list."),
+    }),
+    execute: async ({ fileId }) => {
+      const entry = fileRegistry.get(fileId);
+      if (!entry) return { type: "error", error: "File not found" };
+      try {
+        if (entry.imageReader) {
+          return {
+            type: "image",
+            fileId,
+            fileName: entry.label,
+            mediaType: entry.mediaType ?? mimeForFilename(entry.originalName),
+          };
+        }
+        const content = await entry.reader();
+        return { type: "text", fileName: entry.label, content };
+      } catch {
+        return { type: "error", error: "Could not read file" };
+      }
+    },
+    toModelOutput: async ({ output }) => {
+      if ((output as any)?.type === "image") {
+        const imageOutput = output as {
+          fileId: string;
+          fileName: string;
+          mediaType: string;
+        };
+        const entry = fileRegistry.get(imageOutput.fileId);
+        let buffer: Buffer | null | undefined;
+        try {
+          buffer = await entry?.imageReader?.();
+        } catch {
+          buffer = null;
+        }
+        if (!buffer) {
+          return {
+            type: "text",
+            value: `${imageOutput.fileName}\n\n[Image bytes are unavailable.]`,
+          };
+        }
+        if (buffer.byteLength > MAX_IMAGE_BYTES_FOR_MODEL) {
+          return {
+            type: "text",
+            value: `${imageOutput.fileName}\n\n[Image is ${(buffer.byteLength / (1024 * 1024)).toFixed(1)} MB, which is too large to send to the model. Ask the user to upload a smaller image.]`,
+          };
+        }
+        return {
+          type: "content",
+          value: [
+            {
+              type: "text",
+              text: `${imageOutput.fileName}\nInspect the attached image directly and answer using what is visible in it.`,
+            },
+            {
+              type: "image-data",
+              data: buffer.toString("base64"),
+              mediaType: imageOutput.mediaType,
+            },
+          ],
+        };
+      }
+
+      if ((output as any)?.type === "text") {
+        const textOutput = output as { fileName: string; content: string };
+        return {
+          type: "text",
+          value: `${textOutput.fileName}\n\n${textOutput.content}`,
+        };
+      }
+
+      const error = (output as any)?.error ?? "Could not read file";
+      return { type: "text", value: `[readFile error: ${error}]` };
+    },
+  });
+}
+
 const PORT = process.env.PORT || 3001;
 const CODE_CONTEXT_DIR = process.env.CODE_CONTEXT_DIR || "";
 
@@ -1374,20 +1512,22 @@ app.post("/api/chat", async (req, res) => {
     }
   }
 
-  // Build a file registry: id → { label, reader }
+  // Build a file registry: id → reference entry.
   // The model sees the list of file names and can call readFile(id) for any of them.
-  const fileRegistry = new Map<
-    string,
-    { label: string; reader: () => Promise<string> }
-  >();
+  const fileRegistry = new Map<string, ReferenceFileEntry>();
 
   // Workspace-level uploaded files (apply to all topics)
   const workspaceFiles = await storage.getWorkspaceContextFiles();
   for (const cf of workspaceFiles) {
-    fileRegistry.set(cf.id, {
-      label: `[workspace] ${cf.originalName}`,
-      reader: () => storage.readWorkspaceContextFileContent(cf.id),
-    });
+    fileRegistry.set(
+      cf.id,
+      makeReferenceFileEntry({
+        scope: "workspace",
+        file: cf,
+        reader: () => storage.readWorkspaceContextFileContent(cf.id),
+        blobReader: () => storage.readWorkspaceContextFileBlob(cf.id),
+      }),
+    );
   }
 
   // Topic-level uploaded files + topic-wide system prompt
@@ -1399,10 +1539,15 @@ app.post("/api/chat", async (req, res) => {
     }
     if (topic?.contextFiles?.length) {
       for (const cf of topic.contextFiles) {
-        fileRegistry.set(cf.id, {
-          label: `[topic] ${cf.originalName}`,
-          reader: () => storage.readContextFileContent(cf.id),
-        });
+        fileRegistry.set(
+          cf.id,
+          makeReferenceFileEntry({
+            scope: "topic",
+            file: cf,
+            reader: () => storage.readContextFileContent(cf.id),
+            blobReader: () => storage.readContextFileBlob(cf.id),
+          }),
+        );
       }
     }
   }
@@ -1414,10 +1559,15 @@ app.post("/api/chat", async (req, res) => {
       for (const cf of question.contextFiles.filter(
         (c) => c.inContext !== false,
       )) {
-        fileRegistry.set(cf.id, {
-          label: `[question] ${cf.originalName}`,
-          reader: () => storage.readContextFileContent(cf.id),
-        });
+        fileRegistry.set(
+          cf.id,
+          makeReferenceFileEntry({
+            scope: "question",
+            file: cf,
+            reader: () => storage.readContextFileContent(cf.id),
+            blobReader: () => storage.readContextFileBlob(cf.id),
+          }),
+        );
       }
     }
   }
@@ -1428,10 +1578,12 @@ app.post("/api/chat", async (req, res) => {
       const fullPath = path.join(CODE_CONTEXT_DIR, filePath);
       const resolved = path.resolve(fullPath);
       if (!resolved.startsWith(path.resolve(CODE_CONTEXT_DIR))) continue;
-      fileRegistry.set(`code:${filePath}`, {
-        label: `[code] ${filePath}`,
-        reader: () => fs.readFile(resolved, "utf-8"),
-      });
+      fileRegistry.set(
+        `code:${filePath}`,
+        makeCodeReferenceFileEntry(`[code] ${filePath}`, () =>
+          fs.readFile(resolved, "utf-8"),
+        ),
+      );
     }
   }
 
@@ -1445,6 +1597,7 @@ app.post("/api/chat", async (req, res) => {
 
     system += `\n\n--- Available Reference Files ---
 The following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant — you do not need to read them all.
+For image files, readFile returns visual image data so you can inspect what is visible rather than treating the image as text.
 
 `;
     for (const [id, { label }] of fileRegistry) {
@@ -1581,27 +1734,7 @@ Examples (illustrative only — use real ids and names from the list above):
         }),
         ...(fileRegistry.size > 0
           ? {
-              readFile: tool({
-                description:
-                  "Read the content of an available reference file. Use this to get file contents when they are relevant to the user's question.",
-                inputSchema: z.object({
-                  fileId: z
-                    .string()
-                    .describe(
-                      "The id of the file to read, from the available files list.",
-                    ),
-                }),
-                execute: async ({ fileId }) => {
-                  const entry = fileRegistry.get(fileId);
-                  if (!entry) return { error: "File not found" };
-                  try {
-                    const content = await entry.reader();
-                    return { fileName: entry.label, content };
-                  } catch {
-                    return { error: "Could not read file" };
-                  }
-                },
-              }),
+              readFile: createReadFileTool(fileRegistry),
             }
           : {}),
       },
@@ -1881,17 +2014,19 @@ app.post("/api/code-line-ask", async (req, res) => {
     const aiSettings = await storage.getAiSettings();
 
     // Build a file registry identical to /api/chat so the model has the same context
-    const fileRegistry = new Map<
-      string,
-      { label: string; reader: () => Promise<string> }
-    >();
+    const fileRegistry = new Map<string, ReferenceFileEntry>();
 
     const workspaceFiles = await storage.getWorkspaceContextFiles();
     for (const cf of workspaceFiles) {
-      fileRegistry.set(cf.id, {
-        label: `[workspace] ${cf.originalName}`,
-        reader: () => storage.readWorkspaceContextFileContent(cf.id),
-      });
+      fileRegistry.set(
+        cf.id,
+        makeReferenceFileEntry({
+          scope: "workspace",
+          file: cf,
+          reader: () => storage.readWorkspaceContextFileContent(cf.id),
+          blobReader: () => storage.readWorkspaceContextFileBlob(cf.id),
+        }),
+      );
     }
 
     if (topicId) {
@@ -1899,10 +2034,15 @@ app.post("/api/code-line-ask", async (req, res) => {
       const topic = topics.find((t: any) => t.id === topicId);
       if (topic?.contextFiles?.length) {
         for (const cf of topic.contextFiles) {
-          fileRegistry.set(cf.id, {
-            label: `[topic] ${cf.originalName}`,
-            reader: () => storage.readContextFileContent(cf.id),
-          });
+          fileRegistry.set(
+            cf.id,
+            makeReferenceFileEntry({
+              scope: "topic",
+              file: cf,
+              reader: () => storage.readContextFileContent(cf.id),
+              blobReader: () => storage.readContextFileBlob(cf.id),
+            }),
+          );
         }
       }
     }
@@ -1911,10 +2051,15 @@ app.post("/api/code-line-ask", async (req, res) => {
       const question = await storage.getQuestion(questionId);
       if (question?.contextFiles?.length) {
         for (const cf of question.contextFiles) {
-          fileRegistry.set(cf.id, {
-            label: `[question] ${cf.originalName}`,
-            reader: () => storage.readContextFileContent(cf.id),
-          });
+          fileRegistry.set(
+            cf.id,
+            makeReferenceFileEntry({
+              scope: "question",
+              file: cf,
+              reader: () => storage.readContextFileContent(cf.id),
+              blobReader: () => storage.readContextFileBlob(cf.id),
+            }),
+          );
         }
       }
     }
@@ -1924,10 +2069,12 @@ app.post("/api/code-line-ask", async (req, res) => {
         const fullPath = path.join(CODE_CONTEXT_DIR, fp);
         const resolved = path.resolve(fullPath);
         if (!resolved.startsWith(path.resolve(CODE_CONTEXT_DIR))) continue;
-        fileRegistry.set(`code:${fp}`, {
-          label: `[code] ${fp}`,
-          reader: () => fs.readFile(resolved, "utf-8"),
-        });
+        fileRegistry.set(
+          `code:${fp}`,
+          makeCodeReferenceFileEntry(`[code] ${fp}`, () =>
+            fs.readFile(resolved, "utf-8"),
+          ),
+        );
       }
     }
 
@@ -1950,7 +2097,7 @@ app.post("/api/code-line-ask", async (req, res) => {
           codeFilePaths.push(id.slice("code:".length));
       }
 
-      system += `\n\n--- Available Reference Files ---\nThe following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant.\n\n`;
+      system += `\n\n--- Available Reference Files ---\nThe following files are available to you. Use the readFile tool to retrieve a file's content when it would help answer the question. Only read files that are relevant. For image files, readFile returns visual image data so you can inspect what is visible.\n\n`;
       for (const [id, { label }] of fileRegistry) {
         system += `• ${label}  (id: "${id}")\n`;
       }
@@ -1992,27 +2139,7 @@ app.post("/api/code-line-ask", async (req, res) => {
       tools:
         fileRegistry.size > 0
           ? {
-              readFile: tool({
-                description:
-                  "Read the content of an available reference file. Use this to get file contents when relevant to the question.",
-                inputSchema: z.object({
-                  fileId: z
-                    .string()
-                    .describe(
-                      "The id of the file to read, from the available files list.",
-                    ),
-                }),
-                execute: async ({ fileId }) => {
-                  const entry = fileRegistry.get(fileId);
-                  if (!entry) return { error: "File not found" };
-                  try {
-                    const content = await entry.reader();
-                    return { fileName: entry.label, content };
-                  } catch {
-                    return { error: "Could not read file" };
-                  }
-                },
-              }),
+              readFile: createReadFileTool(fileRegistry),
             }
           : undefined,
       stopWhen: stepCountIs(4),

package/template/server/src/storage.ts

@@ -434,6 +434,7 @@ export async function clearWorkspaceData(workspaceId: string): Promise<void> {
   await ensureWorkspaceDirs(workspaceId);
   const wid = workspaceId;
   await fs.writeFile(topicsFilePath(wid), JSON.stringify([], null, 2));
+  await fs.writeFile(workspaceFilesFilePath(wid), JSON.stringify([], null, 2));
   await Promise.all([
     fs
       .readdir(questionsDirPath(wid))
@@ -534,6 +535,13 @@ export async function readContextFileContent(fileId: string): Promise<string> {
   return fs.readFile(path.join(contextFilesDirPath(), fileId), "utf-8");
 }
 
+export async function readContextFileBlob(
+  fileId: string,
+  workspaceId = _activeWorkspaceId,
+): Promise<Buffer> {
+  return fs.readFile(path.join(contextFilesDirPath(workspaceId), fileId));
+}
+
 /**
  * Store the raw original file bytes at {fileId}.orig so downloads can serve the
  * real file rather than the extracted-text version used by the LLM.
@@ -646,6 +654,13 @@ export async function readWorkspaceContextFileContent(
   );
 }
 
+export async function readWorkspaceContextFileBlob(
+  fileId: string,
+  workspaceId = _activeWorkspaceId,
+): Promise<Buffer> {
+  return fs.readFile(path.join(contextFilesDirPath(workspaceId), fileId));
+}
+
 export async function getQuestion(id: string): Promise<Question | null> {
   await ensureWorkspaceDirs();
   try {