@vellumai/assistant 0.6.0 → 0.6.2

package/src/runtime/routes/log-export-routes.ts

@@ -2,14 +2,12 @@
  * HTTP route handler for exporting audit data and daemon log files.
  *
  * A single POST /v1/export endpoint allows clients (e.g. macOS Export Logs)
- * to retrieve audit database records, daemon log files, workspace contents,
- * and a sanitized config snapshot as a tar.gz archive.
+ * to retrieve audit database records, daemon log files, and a sanitized
+ * config snapshot as a tar.gz archive.
  */
 
-import { spawnSync } from "node:child_process";
 import {
   existsSync,
-  lstatSync,
   mkdirSync,
   mkdtempSync,
   readdirSync,
@@ -19,7 +17,7 @@ import {
   writeFileSync,
 } from "node:fs";
 import { tmpdir } from "node:os";
-import { join, relative } from "node:path";
+import { join } from "node:path";
 
 import { and, desc, eq, gte, lte } from "drizzle-orm";
 import { z } from "zod";
@@ -31,25 +29,23 @@ import {
   messages,
   toolInvocations,
 } from "../../memory/schema.js";
-import { getLogger } from "../../util/logger.js";
+import { getLogger, LOG_FILE_PATTERN } from "../../util/logger.js";
 import {
   getDaemonStderrLogPath,
   getDataDir,
   getWorkspaceConfigPath,
-  getWorkspaceDir,
 } from "../../util/platform.js";
 import { APP_VERSION, COMMIT_SHA } from "../../version.js";
 import { httpError } from "../http-errors.js";
 import type { RouteDefinition } from "../http-router.js";
+import { createTarGz } from "./archive-utils.js";
+import { collectWorkspaceData } from "./log-export/workspace-allowlist.js";
 
 const log = getLogger("log-export-routes");
 
 /** Maximum total payload size for log file contents (10 MB). */
 const MAX_LOG_PAYLOAD_BYTES = 10 * 1024 * 1024;
 
-/** Maximum compressed archive size before pruning workspace directories (50 MB). */
-const MAX_ARCHIVE_BYTES = 50 * 1024 * 1024;
-
 interface ExportRequestBody {
   auditLimit?: number;
   conversationId?: string; // scope to a single conversation
@@ -58,14 +54,14 @@ interface ExportRequestBody {
 }
 
 /**
- * Collect audit data, daemon log files, workspace contents, and a sanitized
- * config snapshot, then package everything into a tar.gz archive.
+ * Collect audit data, daemon log files, and a sanitized config snapshot,
+ * then package everything into a tar.gz archive.
  *
  * Archive layout:
- *   audit-data.json          — tool invocation records
- *   config-snapshot.json     — sanitized workspace config
- *   daemon-logs/<name>       — daemon log files
- *   workspace/<relpath>      — workspace file tree
+ *   audit-data.json                 — tool invocation records
+ *   config-snapshot.json            — sanitized workspace config
+ *   daemon-logs/<name>              — daemon log files
+ *   workspace/conversations/<dir>/  — allowlisted workspace data (see ./log-export/AGENTS.md)
  */
 async function handleExport(body: ExportRequestBody): Promise<Response> {
   const staging = mkdtempSync(join(tmpdir(), "vellum-export-"));
@@ -167,9 +163,19 @@ async function handleExport(body: ExportRequestBody): Promise<Response> {
 
     const logsDir = join(getDataDir(), "logs");
     const collectedLogFiles: string[] = [];
+    const startDate = startTime ? new Date(startTime) : undefined;
+    const endDate = endTime ? new Date(endTime) : undefined;
     if (existsSync(logsDir)) {
       const entries = readdirSync(logsDir);
       for (const entry of entries) {
+        // Filter dated log files by time range
+        const dateMatch = entry.match(LOG_FILE_PATTERN);
+        if (dateMatch && (startDate || endDate)) {
+          const fileDate = new Date(dateMatch[1] + "T23:59:59.999Z"); // end of day
+          const fileDateStart = new Date(dateMatch[1] + "T00:00:00.000Z");
+          if (startDate && fileDate < startDate) continue; // entire day is before range
+          if (endDate && fileDateStart > endDate) continue; // entire day is after range
+        }
         const filePath = join(logsDir, entry);
         try {
           const stat = statSync(filePath);
@@ -237,6 +243,17 @@ async function handleExport(body: ExportRequestBody): Promise<Response> {
       }
     }
 
+    // --- Workspace allowlist ---
+    // Includes specific subpaths from <workspace>/ governed by the rules in
+    // ./log-export/AGENTS.md. Honors the same time + conversation filters as
+    // the rest of the export.
+    const workspaceResult = collectWorkspaceData({
+      staging,
+      conversationId: conversationId || undefined,
+      startTime: startTime || undefined,
+      endTime: endTime || undefined,
+    });
+
     // --- Sanitized config snapshot ---
     const configSnapshot = readSanitizedConfig();
     if (configSnapshot) {
@@ -247,20 +264,6 @@ async function handleExport(body: ExportRequestBody): Promise<Response> {
       );
     }
 
-    // --- Workspace files (skip for conversation-scoped exports) ---
-    let workspaceFileCount = 0;
-    if (!conversationId) {
-      const workspaceFiles = collectWorkspaceFiles();
-      const workspaceDir = join(staging, "workspace");
-      mkdirSync(workspaceDir, { recursive: true });
-      for (const [relPath, content] of Object.entries(workspaceFiles)) {
-        const dest = join(workspaceDir, relPath);
-        mkdirSync(join(dest, ".."), { recursive: true });
-        writeFileSync(dest, content, "utf-8");
-      }
-      workspaceFileCount = Object.keys(workspaceFiles).length;
-    }
-
     // --- Export manifest ---
     const manifest = conversationId
       ? {
@@ -290,64 +293,17 @@ async function handleExport(body: ExportRequestBody): Promise<Response> {
         logFileCount,
         totalBytes,
         hasConfig: configSnapshot !== undefined,
-        workspaceFileCount,
         conversationId: conversationId ?? null,
+        workspaceEntries: workspaceResult.entries.length,
+        workspaceBytes: workspaceResult.totalBytes,
       },
       "Export collected, creating tar.gz archive",
     );
 
-    // --- Create tar.gz archive, pruning workspace dirs if too large ---
-    const excludedDirs: string[] = [];
-    let archiveBytes = createTarGz(staging);
-
-    while (!archiveBytes) {
-      // Conversation-scoped exports have no workspace directory to prune —
-      // if the archive still exceeds the size limit, report a clear error.
-      if (conversationId) {
-        log.error(
-          "Conversation-scoped export exceeds archive size limit with no workspace dirs to prune",
-        );
-        return httpError(
-          "INTERNAL_ERROR",
-          "Conversation export exceeds the maximum archive size",
-          500,
-        );
-      }
-
-      // Find the largest top-level directory under workspace/ and remove it
-      const wsDir = join(staging, "workspace");
-      const largest = findLargestSubdirectory(wsDir);
-      if (!largest) {
-        log.error("tar command failed and no workspace dirs to prune");
-        return httpError("INTERNAL_ERROR", "Failed to create archive", 500);
-      }
-
-      log.warn(
-        { dir: largest.name, bytes: largest.bytes },
-        "Archive exceeds size limit, removing largest workspace directory",
-      );
-      excludedDirs.push(
-        `workspace/${largest.name} (${formatBytes(largest.bytes)})`,
-      );
-      rmSync(join(wsDir, largest.name), { recursive: true, force: true });
-      archiveBytes = createTarGz(staging);
-    }
-
-    if (excludedDirs.length > 0) {
-      const errorLines = [
-        "The following workspace directories were excluded because the archive exceeded the size limit:",
-        "",
-        ...excludedDirs.map((d) => `  - ${d}`),
-        "",
-        "Use the streaming export endpoint for full workspace exports.",
-      ];
-      writeFileSync(join(staging, "error.log"), errorLines.join("\n"), "utf-8");
-
-      // Re-create the archive now that error.log is included
-      const withErrorLog = createTarGz(staging);
-      if (withErrorLog) {
-        archiveBytes = withErrorLog;
-      }
+    // --- Create tar.gz archive ---
+    const archiveBytes = createTarGz(staging);
+    if (!archiveBytes) {
+      return httpError("INTERNAL_ERROR", "Failed to create archive", 500);
     }
 
     return new Response(archiveBytes, {
@@ -371,199 +327,6 @@ async function handleExport(body: ExportRequestBody): Promise<Response> {
   }
 }
 
-/**
- * Attempts to create a tar.gz archive of `staging` into a Buffer.
- * Returns the Buffer on success, or `undefined` if the archive exceeds
- * the size limit or tar otherwise fails.
- */
-function createTarGz(staging: string): ArrayBuffer | undefined {
-  const proc = spawnSync("tar", ["czf", "-", "-C", staging, "."], {
-    maxBuffer: MAX_ARCHIVE_BYTES,
-    timeout: 30_000,
-  });
-  if (proc.status !== 0) return undefined;
-  const buf = Buffer.isBuffer(proc.stdout)
-    ? proc.stdout
-    : Buffer.from(proc.stdout);
-  return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
-}
-
-/**
- * Returns the name and total byte size of the largest top-level subdirectory
- * inside `dir`, or `undefined` if `dir` has no subdirectories.
- */
-function findLargestSubdirectory(
-  dir: string,
-): { name: string; bytes: number } | undefined {
-  if (!existsSync(dir)) return undefined;
-
-  let largest: { name: string; bytes: number } | undefined;
-
-  for (const entry of readdirSync(dir)) {
-    const fullPath = join(dir, entry);
-    try {
-      if (!statSync(fullPath).isDirectory()) continue;
-    } catch {
-      continue;
-    }
-    const bytes = directorySize(fullPath);
-    if (!largest || bytes > largest.bytes) {
-      largest = { name: entry, bytes };
-    }
-  }
-
-  return largest;
-}
-
-/** Recursively sums the byte size of all files in `dir`. */
-function directorySize(dir: string): number {
-  let total = 0;
-  try {
-    for (const entry of readdirSync(dir)) {
-      const fullPath = join(dir, entry);
-      try {
-        const stat = statSync(fullPath);
-        if (stat.isDirectory()) {
-          total += directorySize(fullPath);
-        } else if (stat.isFile()) {
-          total += stat.size;
-        }
-      } catch {
-        // skip
-      }
-    }
-  } catch {
-    // skip
-  }
-  return total;
-}
-
-/** Formats a byte count as a human-readable string (e.g. "12.3 MB"). */
-function formatBytes(bytes: number): string {
-  if (bytes < 1024) return `${bytes} B`;
-  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
-  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
-}
-
-/** Directory prefixes to skip when collecting workspace files. */
-const WORKSPACE_SKIP_DIRS = new Set([
-  "embedding-models",
-  "data/qdrant",
-  "data/attachments",
-  "data/sounds",
-  "conversations",
-  "signals",
-  "deprecated",
-]);
-
-/** Files at the workspace root to skip (already covered by sanitized fields). */
-const WORKSPACE_SKIP_ROOT_FILES = new Set(["config.json"]);
-
-/** Maximum cumulative size for workspace file contents (10 MB). */
-const MAX_WORKSPACE_PAYLOAD_BYTES = 10 * 1024 * 1024;
-
-/**
- * Recursively collects files from the workspace directory into a
- * `Record<string, string>` map of relative path to content.
- *
- * - Skips `config.json` at the workspace root (already exported as a
- *   sanitized `configSnapshot`; the raw file contains secrets).
- * - Skips symlinks to prevent reading files outside the workspace.
- * - Skips directories in `WORKSPACE_SKIP_DIRS`.
- * - For `.db` files, shells out to `sqlite3 <path> .dump` and stores the
- *   SQL text output with a `.sql` suffix appended to the key.
- * - Skips binary files (detected via null-byte heuristic).
- * - Stops collecting once `MAX_WORKSPACE_PAYLOAD_BYTES` is reached.
- */
-function collectWorkspaceFiles(): Record<string, string> {
-  const wsDir = getWorkspaceDir();
-  if (!existsSync(wsDir)) return {};
-
-  const result: Record<string, string> = {};
-  let totalBytes = 0;
-
-  function walk(dir: string): void {
-    let entries: string[];
-    try {
-      entries = readdirSync(dir);
-    } catch {
-      return;
-    }
-
-    for (const entry of entries) {
-      const fullPath = join(dir, entry);
-      const relPath = relative(wsDir, fullPath);
-
-      // Check if this path falls under a skipped directory prefix
-      if (
-        [...WORKSPACE_SKIP_DIRS].some(
-          (prefix) => relPath === prefix || relPath.startsWith(prefix + "/"),
-        )
-      ) {
-        continue;
-      }
-
-      // Skip root-level files that are already exported separately
-      if (dir === wsDir && WORKSPACE_SKIP_ROOT_FILES.has(entry)) {
-        continue;
-      }
-
-      try {
-        // Use lstatSync to avoid following symlinks
-        const stat = lstatSync(fullPath);
-
-        // Skip symlinks — they could point outside the workspace
-        if (stat.isSymbolicLink()) continue;
-
-        if (stat.isDirectory()) {
-          walk(fullPath);
-          continue;
-        }
-        if (!stat.isFile()) continue;
-
-        // SQLite DB handling: dump as SQL text, then enforce size cap
-        if (entry.endsWith(".db")) {
-          // Skip the dump entirely if the budget is already exhausted
-          if (totalBytes >= MAX_WORKSPACE_PAYLOAD_BYTES) continue;
-          try {
-            const proc = spawnSync("sqlite3", [fullPath, ".dump"], {
-              timeout: 10_000,
-            });
-            if (proc.status === 0 && proc.stdout) {
-              const output =
-                proc.stdout instanceof Buffer
-                  ? proc.stdout.toString("utf-8")
-                  : String(proc.stdout);
-              const outputBytes = Buffer.byteLength(output, "utf-8");
-              if (totalBytes + outputBytes > MAX_WORKSPACE_PAYLOAD_BYTES)
-                continue;
-              result[relPath + ".sql"] = output;
-              totalBytes += outputBytes;
-            }
-          } catch {
-            // Skip if dump fails
-          }
-          continue;
-        }
-
-        // Enforce cumulative size cap for non-DB files
-        if (totalBytes + stat.size > MAX_WORKSPACE_PAYLOAD_BYTES) continue;
-
-        // Read as UTF-8 and skip binary files (null-byte heuristic)
-        const content = readFileSync(fullPath, "utf-8");
-        if (content.includes("\0")) continue;
-        result[relPath] = content;
-        totalBytes += stat.size;
-      } catch {
-        // Skip unreadable files
-      }
-    }
-  }
-
-  walk(wsDir);
-  return result;
-}
-
 /**
  * Replaces a string value with a presence flag: "(set)" if truthy, "(empty)" otherwise.
  */
@@ -689,7 +452,7 @@ export function logExportRouteDefinitions(): RouteDefinition[] {
       policyKey: "export",
       summary: "Export logs and audit data",
       description:
-        "Export audit records, assistant logs, workspace contents, and config as a tar.gz archive.",
+        "Export audit records, assistant logs, and config as a tar.gz archive.",
       tags: ["export"],
       requestBody: exportRequestBody,
       handler: async ({ req }) => {
@@ -703,7 +466,7 @@ export function logExportRouteDefinitions(): RouteDefinition[] {
       policyKey: "export",
       summary: "Export logs and audit data (alias)",
       description:
-        "Alias for /v1/export. Export audit records, assistant logs, workspace contents, and config as a tar.gz archive.",
+        "Alias for /v1/export. Export audit records, assistant logs, and config as a tar.gz archive.",
       tags: ["export"],
       requestBody: exportRequestBody,
       handler: async ({ req }) => {