create-interview-cockpit 0.1.0 → 0.2.0

package/template/server/src/google-drive.ts

@@ -0,0 +1,714 @@
+import { randomUUID } from "crypto";
+import { google } from "googleapis";
+import type { drive_v3 } from "googleapis";
+import { Readable } from "stream";
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
+import * as storage from "./storage.js";
+
+const DRIVE_API = "https://www.googleapis.com/drive/v3";
+
+function getApiKey(): string {
+  const key = process.env.GOOGLE_DRIVE_API_KEY || process.env.GOOGLE_API_KEY;
+  if (!key) throw new Error("No Google API key set (GOOGLE_DRIVE_API_KEY)");
+  return key;
+}
+
+/** Extract a folder ID from any Google Drive share URL format. */
+export function extractFolderIdFromUrl(url: string): string | null {
+  // e.g. https://drive.google.com/drive/folders/FOLDER_ID?usp=sharing
+  const m = url.match(/\/folders\/([a-zA-Z0-9_-]+)/);
+  return m ? m[1] : null;
+}
+
+/** Validate that the folder is publicly accessible and return its metadata. */
+export async function getFolderMeta(
+  folderId: string,
+): Promise<{ id: string; name: string }> {
+  const key = getApiKey();
+  const res = await fetch(
+    `${DRIVE_API}/files/${folderId}?fields=id,name&key=${encodeURIComponent(key)}`,
+  );
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(
+      `Cannot access folder (HTTP ${res.status}). Make sure the folder is shared as "Anyone with the link". Details: ${body}`,
+    );
+  }
+  return res.json() as Promise<{ id: string; name: string }>;
+}
+
+export interface DriveItem {
+  id: string;
+  name: string;
+  mimeType: string;
+}
+
+async function listFolders(parentId: string): Promise<DriveItem[]> {
+  const key = getApiKey();
+  const q = encodeURIComponent(
+    `'${parentId}' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false`,
+  );
+  const res = await fetch(
+    `${DRIVE_API}/files?q=${q}&fields=files(id,name,mimeType)&orderBy=name&pageSize=100&key=${encodeURIComponent(key)}`,
+  );
+  if (!res.ok) throw new Error(`Drive listFolders failed: ${res.status}`);
+  const data = (await res.json()) as { files?: DriveItem[] };
+  return data.files || [];
+}
+
+async function listFiles(folderId: string): Promise<DriveItem[]> {
+  const key = getApiKey();
+  const q = encodeURIComponent(
+    `'${folderId}' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed = false`,
+  );
+  const res = await fetch(
+    `${DRIVE_API}/files?q=${q}&fields=files(id,name,mimeType)&orderBy=name&pageSize=200&key=${encodeURIComponent(key)}`,
+  );
+  if (!res.ok) throw new Error(`Drive listFiles failed: ${res.status}`);
+  const data = (await res.json()) as { files?: DriveItem[] };
+  return data.files || [];
+}
+
+async function downloadFile(fileId: string, mimeType: string): Promise<Buffer> {
+  const key = getApiKey();
+
+  if (mimeType === "application/vnd.google-apps.document") {
+    // Export Google Doc as plain text
+    const res = await fetch(
+      `${DRIVE_API}/files/${fileId}/export?mimeType=text%2Fplain&key=${encodeURIComponent(key)}`,
+    );
+    if (!res.ok) throw new Error(`Drive export failed: ${res.status}`);
+    return Buffer.from(await res.arrayBuffer());
+  }
+
+  const res = await fetch(
+    `${DRIVE_API}/files/${fileId}?alt=media&key=${encodeURIComponent(key)}`,
+  );
+  if (!res.ok) throw new Error(`Drive download failed: ${res.status}`);
+  return Buffer.from(await res.arrayBuffer());
+}
+
+/** OAuth-authenticated versions — used during sync when export tokens are available.
+ *  These work regardless of public sharing settings. */
+async function listFoldersAuthed(
+  drive: drive_v3.Drive,
+  parentId: string,
+): Promise<DriveItem[]> {
+  const res = await drive.files.list({
+    q: `'${parentId}' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false`,
+    fields: "files(id,name,mimeType)",
+    orderBy: "name",
+    pageSize: 200,
+  });
+  return (res.data.files || []) as DriveItem[];
+}
+
+async function listFilesAuthed(
+  drive: drive_v3.Drive,
+  folderId: string,
+): Promise<DriveItem[]> {
+  const res = await drive.files.list({
+    q: `'${folderId}' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed = false`,
+    fields: "files(id,name,mimeType)",
+    orderBy: "name",
+    pageSize: 200,
+  });
+  return (res.data.files || []) as DriveItem[];
+}
+
+async function downloadFileAuthed(
+  drive: drive_v3.Drive,
+  fileId: string,
+  mimeType: string,
+): Promise<Buffer> {
+  if (mimeType === "application/vnd.google-apps.document") {
+    const res = await drive.files.export(
+      { fileId, mimeType: "text/plain" },
+      { responseType: "arraybuffer" },
+    );
+    return Buffer.from(res.data as ArrayBuffer);
+  }
+  const res = await drive.files.get(
+    { fileId, alt: "media" },
+    { responseType: "arraybuffer" },
+  );
+  return Buffer.from(res.data as ArrayBuffer);
+}
+
+export interface SyncResult {
+  topicsUpserted: number;
+  filesImported: number;
+  filesSkipped: number;
+  errors: string[];
+}
+
+export async function syncWorkspace(
+  workspaceId: string,
+  extractText: (buffer: Buffer, filename: string) => Promise<string>,
+): Promise<SyncResult> {
+  const registry = await storage.getWorkspaces();
+  const ws = registry.workspaces.find((w) => w.id === workspaceId);
+  if (!ws?.driveConfig?.folderId) {
+    throw new Error("No Drive folder linked to this workspace");
+  }
+
+  const { folderId, subFolderId } = ws.driveConfig;
+  const syncRoot = subFolderId || folderId;
+
+  storage.setActiveWorkspaceId(workspaceId);
+
+  // Use OAuth client when available — it bypasses public sharing restrictions.
+  // Fall back to API-key functions for read-only public workspaces.
+  const authed = await isExportAuthed();
+  const drive = authed ? await getExportDriveClient() : null;
+
+  const doListFolders = drive
+    ? (id: string) => listFoldersAuthed(drive, id)
+    : listFolders;
+  const doListFiles = drive
+    ? (id: string) => listFilesAuthed(drive, id)
+    : listFiles;
+  const doDownload = drive
+    ? (id: string, mime: string) => downloadFileAuthed(drive, id, mime)
+    : downloadFile;
+
+  const result: SyncResult = {
+    topicsUpserted: 0,
+    filesImported: 0,
+    filesSkipped: 0,
+    errors: [],
+  };
+
+  // ── Phase 0: fast wipe (parallel deletes instead of serial deleteTopic calls) ──
+  await storage.clearWorkspaceData(workspaceId);
+
+  const importedQuestionKeys = new Set<string>();
+
+  // ── Phase 1: fetch all folder listings in parallel ──────────────────────────
+  const subfolders = await doListFolders(syncRoot);
+
+  const folderData = await Promise.all(
+    subfolders.map(async (folder) => {
+      const innerFolders = await doListFolders(folder.id);
+      const questionsFolder = innerFolders.find((f) => f.name === "questions");
+      const ctxFilesFolder = innerFolders.find(
+        (f) => f.name === "context-files",
+      );
+
+      const [qFiles, cfFiles] = await Promise.all([
+        questionsFolder
+          ? doListFiles(questionsFolder.id)
+          : doListFiles(folder.id),
+        ctxFilesFolder ? doListFiles(ctxFilesFolder.id) : Promise.resolve([]),
+      ]);
+
+      return { folder, qFiles, cfFiles };
+    }),
+  );
+
+  // ── Phase 2: write all topics in one batch (one topics.json write) ───────────
+  const topicRecords: storage.Topic[] = folderData.map(({ folder }) => ({
+    id: randomUUID(),
+    name: folder.name,
+    contextFiles: [],
+    createdAt: new Date().toISOString(),
+  }));
+  await storage.replaceAllTopics(topicRecords);
+  result.topicsUpserted = topicRecords.length;
+
+  const topicIdByFolderId = new Map(
+    folderData.map(({ folder }, i) => [folder.id, topicRecords[i].id]),
+  );
+
+  // ── Phase 3: download all files in parallel ──────────────────────────────────
+  type Pending = {
+    topicId: string;
+    filename: string;
+    buffer: Buffer;
+    isContextFile: boolean;
+  };
+
+  const pending: Pending[] = [];
+
+  await Promise.all(
+    folderData.flatMap(({ folder, qFiles, cfFiles }) => {
+      const topicId = topicIdByFolderId.get(folder.id)!;
+      return [
+        ...qFiles.map(async (file) => {
+          const title = file.name.replace(/\.[^/.]+$/, "") || file.name;
+          if (importedQuestionKeys.has(`${topicId}:${title}`)) {
+            result.filesSkipped++;
+            return;
+          }
+          try {
+            const buffer = await doDownload(file.id, file.mimeType);
+            pending.push({
+              topicId,
+              filename: file.name,
+              buffer,
+              isContextFile: false,
+            });
+            importedQuestionKeys.add(`${topicId}:${title}`);
+          } catch (err: any) {
+            result.errors.push(`${file.name}: ${err?.message ?? "failed"}`);
+          }
+        }),
+        ...cfFiles.map(async (cf) => {
+          try {
+            const buffer = await doDownload(cf.id, cf.mimeType);
+            pending.push({
+              topicId,
+              filename: cf.name,
+              buffer,
+              isContextFile: true,
+            });
+          } catch (err: any) {
+            result.errors.push(
+              `Context file "${cf.name}": ${err?.message ?? "failed"}`,
+            );
+          }
+        }),
+      ];
+    }),
+  );
+
+  // Root-level files in syncRoot → "General" topic
+  const rootFiles = await doListFiles(syncRoot);
+  if (rootFiles.length > 0) {
+    const generalTopic: storage.Topic = {
+      id: randomUUID(),
+      name: "General",
+      contextFiles: [],
+      createdAt: new Date().toISOString(),
+    };
+    await storage.saveTopic(generalTopic);
+    result.topicsUpserted++;
+
+    await Promise.all(
+      rootFiles.map(async (file) => {
+        const title = file.name.replace(/\.[^/.]+$/, "") || file.name;
+        if (importedQuestionKeys.has(`${generalTopic.id}:${title}`)) {
+          result.filesSkipped++;
+          return;
+        }
+        try {
+          const buffer = await doDownload(file.id, file.mimeType);
+          pending.push({
+            topicId: generalTopic.id,
+            filename: file.name,
+            buffer,
+            isContextFile: false,
+          });
+          importedQuestionKeys.add(`${generalTopic.id}:${title}`);
+        } catch (err: any) {
+          result.errors.push(`${file.name}: ${err?.message ?? "failed"}`);
+        }
+      }),
+    );
+  }
+
+  // ── Phase 4: save questions in parallel (each is an individual file) ─────────
+  const questions = pending.filter((p) => !p.isContextFile);
+  const contextFiles = pending.filter((p) => p.isContextFile);
+
+  await Promise.all(
+    questions.map(async ({ topicId, filename, buffer }) => {
+      try {
+        let title = filename.replace(/\.[^/.]+$/, "") || filename;
+        let systemContext = "";
+        let messages: storage.Question["messages"] = [];
+        let codeContextFiles: string[] = [];
+
+        if (filename.endsWith(".json")) {
+          // Our own exported JSON — restore fields directly, no text extraction
+          try {
+            const parsed = JSON.parse(buffer.toString("utf-8"));
+            title = parsed.title || title;
+            systemContext = parsed.systemContext || "";
+            messages = parsed.messages || [];
+            codeContextFiles = parsed.codeContextFiles || [];
+          } catch {
+            // Malformed JSON — fall through to raw text
+            systemContext = buffer.toString("utf-8");
+          }
+        } else {
+          // Non-JSON file (legacy .txt, .pdf, etc.) — extract text as before
+          systemContext = await extractText(buffer, filename);
+        }
+
+        const q: storage.Question = {
+          id: randomUUID(),
+          topicId,
+          title,
+          systemContext,
+          codeContextFiles,
+          contextFiles: [],
+          messages,
+          createdAt: new Date().toISOString(),
+        };
+        await storage.saveQuestion(q);
+        result.filesImported++;
+      } catch (err: any) {
+        result.errors.push(`${filename}: ${err?.message ?? "failed"}`);
+      }
+    }),
+  );
+
+  // ── Phase 5: save context file blobs in parallel, then update topics.json once ─
+  if (contextFiles.length > 0) {
+    // Write binary files in parallel
+    const cfRecords = await Promise.all(
+      contextFiles.map(async ({ topicId, filename, buffer }) => {
+        const fileId = randomUUID();
+        await storage.writeContextFileBlob(workspaceId, fileId, buffer);
+        return { topicId, fileId, filename };
+      }),
+    );
+    // Update topics.json once with all context file metadata
+    const allTopics = await storage.getTopics();
+    for (const { topicId, fileId, filename } of cfRecords) {
+      const topic = allTopics.find((t) => t.id === topicId);
+      if (topic) {
+        topic.contextFiles.push({
+          id: fileId,
+          name: filename,
+          originalName: filename,
+          createdAt: new Date().toISOString(),
+        });
+      }
+    }
+    await storage.replaceAllTopics(allTopics);
+    result.filesImported += cfRecords.length;
+  }
+
+  // Update lastSyncedAt
+  await storage.updateWorkspace(workspaceId, {
+    driveConfig: {
+      ...ws.driveConfig,
+      lastSyncedAt: new Date().toISOString(),
+    },
+  });
+
+  return result;
+}
+
+/** List subfolders directly inside the workspace's linked Drive folder (public API key read). */
+export async function listDriveSubfolders(
+  workspaceId: string,
+): Promise<DriveItem[]> {
+  const registry = await storage.getWorkspaces();
+  const ws = registry.workspaces.find((w) => w.id === workspaceId);
+  if (!ws?.driveConfig?.folderId) {
+    throw new Error("No Drive folder linked to this workspace");
+  }
+  return listFolders(ws.driveConfig.folderId);
+}
+
+/** Create a new subfolder inside the workspace's linked Drive folder using OAuth. */
+export async function createDriveSubfolder(
+  workspaceId: string,
+  name: string,
+): Promise<DriveItem> {
+  const registry = await storage.getWorkspaces();
+  const ws = registry.workspaces.find((w) => w.id === workspaceId);
+  if (!ws?.driveConfig?.folderId) {
+    throw new Error("No Drive folder linked to this workspace");
+  }
+  const drive = await getExportDriveClient();
+  const created = await drive.files.create({
+    requestBody: {
+      name,
+      mimeType: "application/vnd.google-apps.folder",
+      parents: [ws.driveConfig.folderId],
+    },
+    fields: "id,name,mimeType",
+  });
+  return created.data as DriveItem;
+}
+
+const __filedir = path.dirname(fileURLToPath(import.meta.url));
+const EXPORT_TOKENS_FILE = path.resolve(
+  __filedir,
+  "../../data/export-tokens.json",
+);
+
+function createExportOAuthClient() {
+  return new google.auth.OAuth2(
+    process.env.GOOGLE_CLIENT_ID,
+    process.env.GOOGLE_CLIENT_SECRET,
+    process.env.GOOGLE_EXPORT_REDIRECT_URI ||
+      "http://localhost:3001/api/drive/export-callback",
+  );
+}
+
+export function getExportAuthUrl(): string {
+  const client = createExportOAuthClient();
+  return client.generateAuthUrl({
+    access_type: "offline",
+    scope: ["https://www.googleapis.com/auth/drive.file"],
+    prompt: "consent",
+  });
+}
+
+export async function handleExportCallback(code: string): Promise<void> {
+  const client = createExportOAuthClient();
+  const { tokens } = await client.getToken(code);
+  await fs.mkdir(path.dirname(EXPORT_TOKENS_FILE), { recursive: true });
+  await fs.writeFile(EXPORT_TOKENS_FILE, JSON.stringify(tokens, null, 2));
+}
+
+export async function isExportAuthed(): Promise<boolean> {
+  try {
+    await fs.access(EXPORT_TOKENS_FILE);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function getExportDriveClient(): Promise<drive_v3.Drive> {
+  const client = createExportOAuthClient();
+  const data = await fs.readFile(EXPORT_TOKENS_FILE, "utf-8");
+  const tokens = JSON.parse(data);
+  client.setCredentials(tokens);
+  client.on("tokens", async (newTokens) => {
+    try {
+      const existing = JSON.parse(
+        await fs.readFile(EXPORT_TOKENS_FILE, "utf-8"),
+      );
+      await fs.writeFile(
+        EXPORT_TOKENS_FILE,
+        JSON.stringify({ ...existing, ...newTokens }, null, 2),
+      );
+    } catch {
+      /* ok */
+    }
+  });
+  return google.drive({ version: "v3", auth: client });
+}
+
+export interface ExportResult {
+  topicsExported: number;
+  questionsExported: number;
+  filesExported: number;
+  errors: string[];
+}
+
+/**
+ * Walk every file and folder inside a Drive folder tree and grant
+ * "anyone reader" on anything that doesn't already have it.
+ * Used to repair exports that were created before the permission fix.
+ */
+export async function repairDriveFolderPermissions(
+  folderId: string,
+): Promise<number> {
+  const drive = await getExportDriveClient();
+  let fixed = 0;
+
+  async function walk(id: string): Promise<void> {
+    await ensurePublicRead(drive, id);
+    fixed++;
+
+    // List all children (files + folders)
+    const res = await drive.files.list({
+      q: `'${id}' in parents and trashed = false`,
+      fields: "files(id,mimeType)",
+      pageSize: 200,
+    });
+    const children = res.data.files || [];
+    await Promise.all(
+      children.map(async (child) => {
+        if (child.mimeType === "application/vnd.google-apps.folder") {
+          await walk(child.id!);
+        } else {
+          await ensurePublicRead(drive, child.id!);
+          fixed++;
+        }
+      }),
+    );
+  }
+
+  await walk(folderId);
+  return fixed;
+}
+
+async function ensurePublicRead(
+  drive: drive_v3.Drive,
+  fileId: string,
+): Promise<void> {
+  try {
+    await drive.permissions.create({
+      fileId,
+      requestBody: { type: "anyone", role: "reader" },
+    });
+  } catch {
+    // Already has the permission or not applicable — safe to ignore
+  }
+}
+
+async function getOrCreateFolder(
+  drive: drive_v3.Drive,
+  parentId: string,
+  name: string,
+): Promise<string> {
+  const safeName = name.replace(/'/g, "\\'");
+  const res = await drive.files.list({
+    q: `'${parentId}' in parents and name = '${safeName}' and mimeType = 'application/vnd.google-apps.folder' and trashed = false`,
+    fields: "files(id)",
+    pageSize: 1,
+  });
+  const folderId = res.data.files?.length
+    ? res.data.files[0].id!
+    : (
+        await drive.files.create({
+          requestBody: {
+            name,
+            mimeType: "application/vnd.google-apps.folder",
+            parents: [parentId],
+          },
+          fields: "id",
+        })
+      ).data.id!;
+  // Always ensure public read — covers both newly created and pre-existing folders
+  await ensurePublicRead(drive, folderId);
+  return folderId;
+}
+
+async function uploadFileToFolder(
+  drive: drive_v3.Drive,
+  folderId: string,
+  name: string,
+  content: Buffer | string,
+  mimeType: string,
+): Promise<void> {
+  const buffer = Buffer.isBuffer(content)
+    ? content
+    : Buffer.from(content, "utf-8");
+  const created = await drive.files.create({
+    requestBody: { name, parents: [folderId] },
+    media: { mimeType, body: Readable.from(buffer) },
+    fields: "id",
+  });
+  // Always ensure public read — covers re-uploaded and brand-new files
+  if (created.data.id) {
+    await ensurePublicRead(drive, created.data.id);
+  }
+}
+
+export async function exportWorkspace(
+  workspaceId: string,
+  targetFolderId?: string,
+): Promise<ExportResult> {
+  const registry = await storage.getWorkspaces();
+  const ws = registry.workspaces.find((w) => w.id === workspaceId);
+  if (!ws?.driveConfig?.folderId) {
+    throw new Error("No Drive folder linked to this workspace");
+  }
+
+  const drive = await getExportDriveClient();
+  const { folderId } = ws.driveConfig;
+  const result: ExportResult = {
+    topicsExported: 0,
+    questionsExported: 0,
+    filesExported: 0,
+    errors: [],
+  };
+
+  // Use the chosen subfolder directly, or fall back to an "_export" subfolder
+  const exportFolderId =
+    targetFolderId ?? (await getOrCreateFolder(drive, folderId, "_export"));
+  const topics = await storage.getTopicsForWorkspace(workspaceId);
+
+  for (const topic of topics) {
+    try {
+      const topicFolderId = await getOrCreateFolder(
+        drive,
+        exportFolderId,
+        topic.name,
+      );
+      const questions = await storage.getQuestionsByTopicForWorkspace(
+        workspaceId,
+        topic.id,
+      );
+
+      // Upload questions into a questions/ subfolder (parallel uploads)
+      if (questions.length > 0) {
+        const questionsFolderId = await getOrCreateFolder(
+          drive,
+          topicFolderId,
+          "questions",
+        );
+        await Promise.all(
+          questions.map(async (q) => {
+            try {
+              const safeName =
+                q.title.replace(/[/\\:*?"<>|]/g, "-").trim() || q.id;
+              // Export full question as JSON so nothing is lost on sync-back
+              const payload = JSON.stringify(
+                {
+                  title: q.title,
+                  systemContext: q.systemContext || "",
+                  messages: q.messages,
+                  codeContextFiles: q.codeContextFiles,
+                },
+                null,
+                2,
+              );
+              await uploadFileToFolder(
+                drive,
+                questionsFolderId,
+                `${safeName}.json`,
+                payload,
+                "application/json",
+              );
+              result.questionsExported++;
+            } catch (err: any) {
+              result.errors.push(
+                `Question "${q.title}": ${err?.message || "upload failed"}`,
+              );
+            }
+          }),
+        );
+      }
+
+      // Upload context files into a context-files/ subfolder (parallel uploads)
+      if (topic.contextFiles.length > 0) {
+        const ctxFolderId = await getOrCreateFolder(
+          drive,
+          topicFolderId,
+          "context-files",
+        );
+        const ctxDir = storage.getContextFilesDirForWorkspace(workspaceId);
+        await Promise.all(
+          topic.contextFiles.map(async (cf) => {
+            try {
+              const buffer = await fs.readFile(path.join(ctxDir, cf.id));
+              await uploadFileToFolder(
+                drive,
+                ctxFolderId,
+                cf.originalName,
+                buffer,
+                "application/octet-stream",
+              );
+              result.filesExported++;
+            } catch (err: any) {
+              result.errors.push(
+                `File "${cf.originalName}": ${err?.message || "upload failed"}`,
+              );
+            }
+          }),
+        );
+      }
+
+      result.topicsExported++;
+    } catch (err: any) {
+      result.errors.push(
+        `Topic "${topic.name}": ${err?.message || "export failed"}`,
+      );
+    }
+  }
+
+  return result;
+}