create-interview-cockpit 0.17.2 → 0.18.0

package/template/server/src/google-drive.ts

@@ -137,6 +137,86 @@ async function downloadFileAuthed(
   return Buffer.from(res.data as ArrayBuffer);
 }
 
+const WORKSPACE_FILES_FOLDER_NAME = "workspace-files";
+const RESERVED_EXPORT_FOLDER_NAMES = new Set([WORKSPACE_FILES_FOLDER_NAME]);
+
+function extensionForFilename(filename: string): string {
+  return filename.split(".").pop()?.toLowerCase() ?? "";
+}
+
+function mimeForFilename(filename: string): string {
+  const ext = extensionForFilename(filename);
+  const map: Record<string, string> = {
+    pdf: "application/pdf",
+    docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    doc: "application/msword",
+    json: "application/json",
+    csv: "text/csv",
+    html: "text/html; charset=utf-8",
+    xml: "application/xml",
+    txt: "text/plain; charset=utf-8",
+    md: "text/markdown; charset=utf-8",
+    png: "image/png",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+  };
+  return map[ext] ?? "application/octet-stream";
+}
+
+type SerializedContextFile = storage.ContextFile & {
+  content?: string;
+  originalBase64?: string;
+  originalMimeType?: string;
+};
+
+async function readContextFileExportBytes(
+  cf: storage.ContextFile,
+  workspaceId: string,
+  ctxDir: string,
+): Promise<Buffer> {
+  const original = await storage.readOriginalBlob(cf.id, workspaceId);
+  return original ?? fs.readFile(path.join(ctxDir, cf.id));
+}
+
+async function serializeContextFileForExport(
+  cf: storage.ContextFile,
+  workspaceId: string,
+  ctxDir: string,
+): Promise<SerializedContextFile> {
+  const exported: SerializedContextFile = { ...cf };
+  try {
+    exported.content = await fs.readFile(path.join(ctxDir, cf.id), "utf-8");
+  } catch {
+    /* Keep metadata even if the extracted-text blob is missing. */
+  }
+  const original = await storage.readOriginalBlob(cf.id, workspaceId);
+  if (original) {
+    exported.originalBase64 = original.toString("base64");
+    exported.originalMimeType = mimeForFilename(cf.originalName);
+  }
+  return exported;
+}
+
+function contextFileMetadataForImport(
+  cf: SerializedContextFile,
+  fallbackName: string,
+): storage.ContextFile {
+  return {
+    id: cf.id,
+    name: cf.name || fallbackName,
+    originalName: cf.originalName || fallbackName,
+    ...(cf.driveFileId ? { driveFileId: cf.driveFileId } : {}),
+    createdAt: cf.createdAt || new Date().toISOString(),
+    ...(cf.origin ? { origin: cf.origin } : {}),
+    ...(cf.language ? { language: cf.language } : {}),
+    ...(cf.label ? { label: cf.label } : {}),
+    ...(typeof cf.inContext === "boolean" ? { inContext: cf.inContext } : {}),
+  };
+}
+
 export interface SyncResult {
   topicsUpserted: number;
   filesImported: number;
@@ -181,42 +261,51 @@ export async function syncWorkspace(
     errors: [],
   };
 
-  // ── Phase 0: fast wipe (parallel deletes instead of serial deleteTopic calls) ──
-  await storage.clearWorkspaceData(workspaceId);
-
   const importedQuestionKeys = new Set<string>();
 
   // ── Phase 1: fetch all folder listings in parallel ──────────────────────────
   const subfolders = await doListFolders(syncRoot);
+  const workspaceFilesFolder = subfolders.find(
+    (folder) => folder.name === WORKSPACE_FILES_FOLDER_NAME,
+  );
+  const topicFolders = subfolders.filter(
+    (folder) => !RESERVED_EXPORT_FOLDER_NAMES.has(folder.name),
+  );
 
-  const folderData = await Promise.all(
-    subfolders.map(async (folder) => {
-      const innerFolders = await doListFolders(folder.id);
-      const questionsFolder = innerFolders.find((f) => f.name === "questions");
-      const ctxFilesFolder = innerFolders.find(
-        (f) => f.name === "context-files",
-      );
+  const [folderData, workspaceFileItems] = await Promise.all([
+    Promise.all(
+      topicFolders.map(async (folder) => {
+        const innerFolders = await doListFolders(folder.id);
+        const questionsFolder = innerFolders.find(
+          (f) => f.name === "questions",
+        );
+        const ctxFilesFolder = innerFolders.find(
+          (f) => f.name === "context-files",
+        );
 
-      const [qFiles, cfFiles] = await Promise.all([
-        questionsFolder
-          ? doListFiles(questionsFolder.id)
-          : doListFiles(folder.id),
-        ctxFilesFolder ? doListFiles(ctxFilesFolder.id) : Promise.resolve([]),
-      ]);
+        const [qFiles, cfFiles] = await Promise.all([
+          questionsFolder
+            ? doListFiles(questionsFolder.id)
+            : doListFiles(folder.id),
+          ctxFilesFolder ? doListFiles(ctxFilesFolder.id) : Promise.resolve([]),
+        ]);
 
-      return { folder, qFiles, cfFiles };
-    }),
-  );
+        return { folder, qFiles, cfFiles };
+      }),
+    ),
+    workspaceFilesFolder
+      ? doListFiles(workspaceFilesFolder.id)
+      : Promise.resolve([]),
+  ]);
 
-  // ── Phase 2: write all topics in one batch (one topics.json write) ───────────
+  // ── Phase 2: build topic records in memory ─────────────────────────────────
   const topicRecords: storage.Topic[] = folderData.map(({ folder }) => ({
     id: randomUUID(),
     name: folder.name,
     contextFiles: [],
     createdAt: new Date().toISOString(),
   }));
-  await storage.replaceAllTopics(topicRecords);
-  result.topicsUpserted = topicRecords.length;
+  const extraTopicRecords: storage.Topic[] = [];
 
   const topicIdByFolderId = new Map(
     folderData.map(({ folder }, i) => [folder.id, topicRecords[i].id]),
@@ -229,8 +318,13 @@ export async function syncWorkspace(
     buffer: Buffer;
     isContextFile: boolean;
   };
+  type PendingWorkspaceFile = {
+    filename: string;
+    buffer: Buffer;
+  };
 
   const pending: Pending[] = [];
+  const pendingWorkspaceFiles: PendingWorkspaceFile[] = [];
 
   await Promise.all(
     folderData.flatMap(({ folder, qFiles, cfFiles }) => {
@@ -274,6 +368,19 @@ export async function syncWorkspace(
     }),
   );
 
+  await Promise.all(
+    workspaceFileItems.map(async (file) => {
+      try {
+        const buffer = await doDownload(file.id, file.mimeType);
+        pendingWorkspaceFiles.push({ filename: file.name, buffer });
+      } catch (err: any) {
+        result.errors.push(
+          `Workspace file "${file.name}": ${err?.message ?? "failed"}`,
+        );
+      }
+    }),
+  );
+
   // Root-level files in syncRoot → "General" topic
   const rootFiles = await doListFiles(syncRoot);
   if (rootFiles.length > 0) {
@@ -283,8 +390,7 @@ export async function syncWorkspace(
       contextFiles: [],
       createdAt: new Date().toISOString(),
     };
-    await storage.saveTopic(generalTopic);
-    result.topicsUpserted++;
+    extraTopicRecords.push(generalTopic);
 
     await Promise.all(
       rootFiles.map(async (file) => {
@@ -309,6 +415,18 @@ export async function syncWorkspace(
     );
   }
 
+  // If downloads hit private Drive files or an old token lacks enough scope,
+  // stop before wiping the local workspace. The route will prompt re-auth.
+  if (result.errors.some((error) => /\b403\b/.test(error))) {
+    return result;
+  }
+
+  // ── Phase 3b: fast wipe + write all topics in one batch ────────────────────
+  await storage.clearWorkspaceData(workspaceId);
+  const allTopicRecords = [...topicRecords, ...extraTopicRecords];
+  await storage.replaceAllTopics(allTopicRecords);
+  result.topicsUpserted = allTopicRecords.length;
+
   // ── Phase 4: save questions in parallel (each is an individual file) ─────────
   const questions = pending.filter((p) => !p.isContextFile);
   const contextFiles = pending.filter((p) => p.isContextFile);
@@ -345,42 +463,53 @@ export async function syncWorkspace(
             parentTitle = parsed.parentTitle || null;
             if (parsed.createdAt) restoredCreatedAt = parsed.createdAt;
 
-            // Restore code snippet context files (user/ai origin)
-            if (
-              Array.isArray(parsed.codeSnippets) &&
-              parsed.codeSnippets.length > 0
-            ) {
+            // Restore all question-attached context files. New exports use
+            // contextFiles; older exports used codeSnippets for lab/code blobs.
+            const parsedContextFiles: SerializedContextFile[] = Array.isArray(
+              parsed.contextFiles,
+            )
+              ? parsed.contextFiles
+              : Array.isArray(parsed.codeSnippets)
+                ? parsed.codeSnippets
+                : [];
+            if (parsedContextFiles.length > 0) {
               const ctxDir =
                 storage.getContextFilesDirForWorkspace(workspaceId);
               await Promise.all(
-                parsed.codeSnippets.map(
-                  async (cs: storage.ContextFile & { content?: string }) => {
-                    if (
-                      cs.content &&
-                      (cs.origin === "user" ||
-                        cs.origin === "ai" ||
-                        cs.origin === "sandbox" ||
-                        cs.origin === "browser-security" ||
-                        cs.origin === "react" ||
-                        cs.origin === "nextjs" ||
-                        cs.origin === "module-federation" ||
-                        cs.origin === "infra")
-                    ) {
-                      try {
-                        await fs.mkdir(ctxDir, { recursive: true });
-                        await fs.writeFile(
-                          path.join(ctxDir, cs.id),
-                          Buffer.from(cs.content, "utf-8"),
-                        );
-                        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-                        const { content: _content, ...cfMeta } = cs;
-                        restoredContextFiles.push(cfMeta);
-                      } catch {
-                        /* skip bad blobs */
-                      }
+                parsedContextFiles.map(async (cs) => {
+                  if (!cs.id) return;
+                  const originalName = cs.originalName || cs.name || cs.id;
+                  try {
+                    const originalBuffer = cs.originalBase64
+                      ? Buffer.from(cs.originalBase64, "base64")
+                      : null;
+                    const content =
+                      typeof cs.content === "string"
+                        ? cs.content
+                        : originalBuffer
+                          ? await extractText(originalBuffer, originalName)
+                          : null;
+
+                    if (content == null) return;
+                    await fs.mkdir(ctxDir, { recursive: true });
+                    await fs.writeFile(
+                      path.join(ctxDir, cs.id),
+                      Buffer.from(content, "utf-8"),
+                    );
+                    if (originalBuffer) {
+                      await storage.writeOriginalBlob(
+                        cs.id,
+                        originalBuffer,
+                        workspaceId,
+                      );
                     }
-                  },
-                ),
+                    restoredContextFiles.push(
+                      contextFileMetadataForImport(cs, originalName),
+                    );
+                  } catch {
+                    /* skip bad blobs */
+                  }
+                }),
               );
             }
           } catch {
@@ -439,13 +568,36 @@ export async function syncWorkspace(
     }
   }
 
-  // ── Phase 5: save context file blobs in parallel, then update topics.json once ─
+  // ── Phase 5: save workspace/topic context file blobs ───────────────────────
+  if (pendingWorkspaceFiles.length > 0) {
+    // saveWorkspaceContextFile updates workspace-files.json; do this
+    // sequentially so concurrent reads/writes do not overwrite metadata.
+    for (const { filename, buffer } of pendingWorkspaceFiles) {
+      const fileId = randomUUID();
+      const text = await extractText(buffer, filename);
+      await storage.writeOriginalBlob(fileId, buffer, workspaceId);
+      await storage.saveWorkspaceContextFile(
+        fileId,
+        filename,
+        Buffer.from(text, "utf-8"),
+        workspaceId,
+      );
+    }
+    result.filesImported += pendingWorkspaceFiles.length;
+  }
+
+  // Save topic context file blobs in parallel, then update topics.json once.
   if (contextFiles.length > 0) {
-    // Write binary files in parallel
     const cfRecords = await Promise.all(
       contextFiles.map(async ({ topicId, filename, buffer }) => {
         const fileId = randomUUID();
-        await storage.writeContextFileBlob(workspaceId, fileId, buffer);
+        const text = await extractText(buffer, filename);
+        await storage.writeOriginalBlob(fileId, buffer, workspaceId);
+        await storage.writeContextFileBlob(
+          workspaceId,
+          fileId,
+          Buffer.from(text, "utf-8"),
+        );
         return { topicId, fileId, filename };
       }),
     );
@@ -486,6 +638,10 @@ export async function listDriveSubfolders(
   if (!ws?.driveConfig?.folderId) {
     throw new Error("No Drive folder linked to this workspace");
   }
+  if (await isExportAuthed()) {
+    const drive = await getExportDriveClient();
+    return listFoldersAuthed(drive, ws.driveConfig.folderId);
+  }
   return listFolders(ws.driveConfig.folderId);
 }
 
@@ -518,11 +674,11 @@ const EXPORT_TOKENS_FILE = path.resolve(
 );
 
 function createExportOAuthClient() {
+  const defaultRedirectUri = `http://localhost:${process.env.GOOGLE_EXPORT_REDIRECT_PORT || "3001"}/api/drive/export-callback`;
   return new google.auth.OAuth2(
     process.env.GOOGLE_CLIENT_ID,
     process.env.GOOGLE_CLIENT_SECRET,
-    process.env.GOOGLE_EXPORT_REDIRECT_URI ||
-      "http://localhost:3001/api/drive/export-callback",
+    process.env.GOOGLE_EXPORT_REDIRECT_URI || defaultRedirectUri,
   );
 }
 
@@ -530,7 +686,7 @@ export function getExportAuthUrl(): string {
   const client = createExportOAuthClient();
   return client.generateAuthUrl({
     access_type: "offline",
-    scope: ["https://www.googleapis.com/auth/drive.file"],
+    scope: ["https://www.googleapis.com/auth/drive"],
     prompt: "consent",
   });
 }
@@ -697,6 +853,31 @@ async function uploadFileToFolder(
   const buffer = Buffer.isBuffer(content)
     ? content
     : Buffer.from(content, "utf-8");
+  const safeName = name.replace(/'/g, "\\'");
+  const existing = await drive.files.list({
+    q: `'${folderId}' in parents and name = '${safeName}' and trashed = false`,
+    fields: "files(id)",
+    pageSize: 10,
+  });
+  const [first, ...duplicates] = existing.data.files || [];
+  if (first?.id) {
+    await drive.files.update({
+      fileId: first.id,
+      requestBody: { name },
+      media: { mimeType, body: Readable.from(buffer) },
+      fields: "id",
+    });
+    await Promise.all(
+      duplicates.map((file) =>
+        file.id
+          ? drive.files.delete({ fileId: file.id }).catch(() => {})
+          : null,
+      ),
+    );
+    await ensurePublicRead(drive, first.id);
+    return;
+  }
+
   const created = await drive.files.create({
     requestBody: { name, parents: [folderId] },
     media: { mimeType, body: Readable.from(buffer) },
@@ -731,6 +912,39 @@ export async function exportWorkspace(
   const exportFolderId =
     targetFolderId ?? (await getOrCreateFolder(drive, folderId, "_export"));
   const topics = await storage.getTopicsForWorkspace(workspaceId);
+  const ctxDir = storage.getContextFilesDirForWorkspace(workspaceId);
+
+  const workspaceFiles = await storage.getWorkspaceContextFiles(workspaceId);
+  if (workspaceFiles.length > 0) {
+    const workspaceFilesFolderId = await getOrCreateFolder(
+      drive,
+      exportFolderId,
+      WORKSPACE_FILES_FOLDER_NAME,
+    );
+    await Promise.all(
+      workspaceFiles.map(async (cf) => {
+        try {
+          const buffer = await readContextFileExportBytes(
+            cf,
+            workspaceId,
+            ctxDir,
+          );
+          await uploadFileToFolder(
+            drive,
+            workspaceFilesFolderId,
+            cf.originalName,
+            buffer,
+            mimeForFilename(cf.originalName),
+          );
+          result.filesExported++;
+        } catch (err: any) {
+          result.errors.push(
+            `Workspace file "${cf.originalName}": ${err?.message || "upload failed"}`,
+          );
+        }
+      }),
+    );
+  }
 
   for (const topic of topics) {
     try {
@@ -761,34 +975,16 @@ export async function exportWorkspace(
                 ? questions.find((p) => p.id === q.parentQuestionId)?.title
                 : undefined;
 
-              // Read code snippet blobs so they survive drive sync
-              const ctxDir =
-                storage.getContextFilesDirForWorkspace(workspaceId);
-              const contextFilesWithContent: Array<
-                storage.ContextFile & { content?: string }
-              > = [];
-              for (const cf of q.contextFiles || []) {
-                if (
-                  cf.origin === "user" ||
-                  cf.origin === "ai" ||
-                  cf.origin === "sandbox" ||
-                  cf.origin === "browser-security" ||
-                  cf.origin === "react" ||
-                  cf.origin === "nextjs" ||
-                  cf.origin === "module-federation" ||
-                  cf.origin === "infra"
-                ) {
-                  try {
-                    const content = await fs.readFile(
-                      path.join(ctxDir, cf.id),
-                      "utf-8",
-                    );
-                    contextFilesWithContent.push({ ...cf, content });
-                  } catch {
-                    contextFilesWithContent.push(cf);
-                  }
-                }
-              }
+              // Read every question-attached context file so uploaded docs,
+              // images, and generated code snippets survive Drive sync.
+              const contextFilesWithContent = await Promise.all(
+                (q.contextFiles || []).map((cf) =>
+                  serializeContextFileForExport(cf, workspaceId, ctxDir),
+                ),
+              );
+              const legacyCodeSnippets = contextFilesWithContent.filter(
+                (cf) => cf.origin && cf.origin !== "upload",
+              );
 
               // Export full question as JSON so nothing is lost on sync-back
               const payload = JSON.stringify(
@@ -799,7 +995,8 @@ export async function exportWorkspace(
                   messages: q.messages,
                   codeContextFiles: q.codeContextFiles,
                   codeAnnotations: q.codeAnnotations ?? {},
-                  codeSnippets: contextFilesWithContent,
+                  contextFiles: contextFilesWithContent,
+                  codeSnippets: legacyCodeSnippets,
                   createdAt: q.createdAt,
                 },
                 null,
@@ -829,17 +1026,20 @@ export async function exportWorkspace(
           topicFolderId,
           "context-files",
         );
-        const ctxDir = storage.getContextFilesDirForWorkspace(workspaceId);
         await Promise.all(
           topic.contextFiles.map(async (cf) => {
             try {
-              const buffer = await fs.readFile(path.join(ctxDir, cf.id));
+              const buffer = await readContextFileExportBytes(
+                cf,
+                workspaceId,
+                ctxDir,
+              );
               await uploadFileToFolder(
                 drive,
                 ctxFolderId,
                 cf.originalName,
                 buffer,
-                "application/octet-stream",
+                mimeForFilename(cf.originalName),
               );
               result.filesExported++;
             } catch (err: any) {