@vellumai/assistant 0.5.5 → 0.5.7

package/src/runtime/routes/diagnostics-routes.ts

@@ -1,25 +1,7 @@
 /**
- * HTTP route handlers for diagnostics export and dictation processing.
- *
- * Handles diagnostics export and dictation processing requests.
+ * HTTP route handlers for dictation processing.
  */
 
-import { randomBytes } from "node:crypto";
-import {
-  createWriteStream,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  rmSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { basename, join } from "node:path";
-
-import archiver from "archiver";
-import { and, asc, desc, eq, gte, lte } from "drizzle-orm";
-
 import {
   type ProfileResolution,
   resolveProfile,
@@ -31,14 +13,6 @@ import {
 import { detectDictationModeHeuristic } from "../../daemon/handlers/dictation.js";
 import type { DictationRequest } from "../../daemon/message-types/diagnostics.js";
 import type { DictationContext } from "../../daemon/message-types/shared.js";
-import { resolveConversationId } from "../../memory/conversation-key-store.js";
-import { getDb } from "../../memory/db.js";
-import {
-  llmRequestLogs,
-  llmUsageEvents,
-  messages,
-  toolInvocations,
-} from "../../memory/schema.js";
 import {
   createTimeout,
   extractToolUse,
@@ -51,444 +25,6 @@ import type { RouteDefinition } from "../http-router.js";
 
 const log = getLogger("diagnostics-routes");
 
-// ---------------------------------------------------------------------------
-// Diagnostics export — redaction helpers
-// ---------------------------------------------------------------------------
-
-const MAX_CONTENT_LENGTH = 500;
-
-const REDACT_PATTERNS = [
-  /\b(sk|key|api[_-]?key|token|secret|password|passwd|credential)[_\-]?[a-zA-Z0-9]{16,}\b/gi,
-  /Bearer\s+[A-Za-z0-9\-._~+\/]+=*/gi,
-  /[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g,
-  /\b(AKIA|ASIA)[A-Z0-9]{16}\b/g,
-  /\b[A-Fa-f0-9]{32,}\b/g,
-];
-
-function redact(text: string): string {
-  let result = text;
-  for (const pattern of REDACT_PATTERNS) {
-    result = result.replace(pattern, "[REDACTED]");
-  }
-  return result;
-}
-
-function truncateAndRedact(text: string): string {
-  const truncated =
-    text.length > MAX_CONTENT_LENGTH
-      ? text.slice(0, MAX_CONTENT_LENGTH) + "...[truncated]"
-      : text;
-  return redact(truncated);
-}
-
-const SENSITIVE_KEYS = new Set([
-  "api_key",
-  "apikey",
-  "api-key",
-  "authorization",
-  "x-api-key",
-  "secret",
-  "password",
-  "token",
-  "credential",
-  "credentials",
-]);
-
-function redactDeep(value: unknown): unknown {
-  if (typeof value === "string") return redact(value);
-  if (Array.isArray(value)) return value.map(redactDeep);
-  if (value != null && typeof value === "object") {
-    const out: Record<string, unknown> = {};
-    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
-      if (SENSITIVE_KEYS.has(k.toLowerCase())) {
-        out[k] = "[REDACTED]";
-      } else {
-        out[k] = redactDeep(v);
-      }
-    }
-    return out;
-  }
-  return value;
-}
-
-// ---------------------------------------------------------------------------
-// Crash report discovery
-// ---------------------------------------------------------------------------
-
-const CRASH_REPORT_EXTENSIONS = new Set([".crash", ".ips", ".diag"]);
-const CRASH_REPORT_TAR_GZ = ".tar.gz";
-const CRASH_REPORT_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
-
-function findRecentCrashReports(): string[] {
-  const diagnosticReportsDir = join(
-    homedir(),
-    "Library",
-    "Logs",
-    "DiagnosticReports",
-  );
-
-  try {
-    const entries = readdirSync(diagnosticReportsDir);
-    const now = Date.now();
-    const results: string[] = [];
-
-    for (const entry of entries) {
-      // Case-insensitive prefix match for "vellum-assistant"
-      if (!entry.toLowerCase().startsWith("vellum-assistant")) continue;
-
-      // Check extension
-      const lowerEntry = entry.toLowerCase();
-      const hasValidExt =
-        CRASH_REPORT_EXTENSIONS.has(
-          lowerEntry.slice(lowerEntry.lastIndexOf(".")),
-        ) || lowerEntry.endsWith(CRASH_REPORT_TAR_GZ);
-
-      if (!hasValidExt) continue;
-
-      const filePath = join(diagnosticReportsDir, entry);
-      try {
-        const stat = statSync(filePath);
-        if (!stat.isFile()) continue;
-        if (now - stat.mtimeMs > CRASH_REPORT_MAX_AGE_MS) continue;
-        results.push(filePath);
-      } catch {
-        // Skip files we can't stat
-      }
-    }
-
-    return results;
-  } catch {
-    // Directory doesn't exist or can't be read — not an error
-    return [];
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Diagnostics export handler
-// ---------------------------------------------------------------------------
-
-async function handleDiagnosticsExport(body: {
-  conversationId?: string;
-  anchorMessageId?: string;
-}): Promise<Response> {
-  if (!body.conversationId) {
-    return httpError("BAD_REQUEST", "conversationId is required", 400);
-  }
-
-  // The client may send a conversation key (client-side UUID) rather than
-  // the daemon's internal conversation ID. Resolve to the canonical ID.
-  const conversationId =
-    resolveConversationId(body.conversationId) ?? body.conversationId;
-  const { anchorMessageId } = body;
-
-  try {
-    const db = getDb();
-
-    // 1. Find the anchor message.
-    // Try in order: specific ID → most recent assistant message → any message.
-    // The final fallback handles the race condition where the user clicks
-    // "export" before message_complete fires and the assistant message has
-    // been persisted — the user message and in-flight tool/usage data are
-    // still captured.
-    let anchorMessage;
-    let anchorIsFallback = false;
-    if (anchorMessageId) {
-      anchorMessage = db
-        .select()
-        .from(messages)
-        .where(
-          and(
-            eq(messages.id, anchorMessageId),
-            eq(messages.conversationId, conversationId),
-          ),
-        )
-        .get();
-    }
-    if (!anchorMessage) {
-      anchorMessage = db
-        .select()
-        .from(messages)
-        .where(
-          and(
-            eq(messages.conversationId, conversationId),
-            eq(messages.role, "assistant"),
-          ),
-        )
-        .orderBy(desc(messages.createdAt))
-        .limit(1)
-        .get();
-    }
-    if (!anchorMessage) {
-      anchorMessage = db
-        .select()
-        .from(messages)
-        .where(eq(messages.conversationId, conversationId))
-        .orderBy(desc(messages.createdAt))
-        .limit(1)
-        .get();
-      anchorIsFallback = true;
-    }
-
-    // 2. Compute the export time range.
-    // When an anchor message exists, scope from the earliest message in the
-    // conversation through the anchor so the full conversation context is
-    // captured. When no messages exist at all (empty conversation or race
-    // condition), use the current timestamp so the export still captures any
-    // in-flight usage/tool data.
-    const now = Date.now();
-    let rangeEnd: number;
-    let rangeStart: number;
-    let usageRangeEnd: number;
-
-    if (anchorMessage) {
-      const earliestMessage = db
-        .select()
-        .from(messages)
-        .where(eq(messages.conversationId, conversationId))
-        .orderBy(asc(messages.createdAt))
-        .limit(1)
-        .get();
-
-      rangeStart = earliestMessage?.createdAt ?? anchorMessage.createdAt - 2000;
-
-      // When the anchor was selected via the fallback "any message" path
-      // (because the assistant reply hasn't been persisted yet), extend the
-      // range to the current time so in-flight tool invocations and usage
-      // recorded after the user message are captured. An explicit anchor to a
-      // non-assistant message uses the message's own timestamp.
-      rangeEnd = anchorIsFallback ? now : anchorMessage.createdAt;
-      usageRangeEnd = anchorIsFallback
-        ? now + 5000
-        : anchorMessage.createdAt + 5000;
-    } else {
-      // No messages at all — use the current time so we capture any
-      // in-flight LLM usage or tool invocations.
-      rangeStart = now - 60_000;
-      rangeEnd = now;
-      usageRangeEnd = now + 5000;
-    }
-
-    // 3. Query all messages in the range
-    const rangeMessages = db
-      .select()
-      .from(messages)
-      .where(
-        and(
-          eq(messages.conversationId, conversationId),
-          gte(messages.createdAt, rangeStart),
-          lte(messages.createdAt, rangeEnd),
-        ),
-      )
-      .orderBy(messages.createdAt)
-      .all();
-
-    // 4. Query tool invocations in the range
-    const rangeToolInvocations = db
-      .select()
-      .from(toolInvocations)
-      .where(
-        and(
-          eq(toolInvocations.conversationId, conversationId),
-          gte(toolInvocations.createdAt, rangeStart),
-          lte(toolInvocations.createdAt, rangeEnd),
-        ),
-      )
-      .orderBy(toolInvocations.createdAt)
-      .all();
-
-    // 5. Query LLM usage events
-    const rangeUsageEvents = db
-      .select()
-      .from(llmUsageEvents)
-      .where(
-        and(
-          eq(llmUsageEvents.conversationId, conversationId),
-          gte(llmUsageEvents.createdAt, rangeStart),
-          lte(llmUsageEvents.createdAt, usageRangeEnd),
-        ),
-      )
-      .orderBy(llmUsageEvents.createdAt)
-      .all();
-
-    // 5b. Query raw LLM request/response logs
-    const rangeRequestLogs = db
-      .select()
-      .from(llmRequestLogs)
-      .where(
-        and(
-          eq(llmRequestLogs.conversationId, conversationId),
-          gte(llmRequestLogs.createdAt, rangeStart),
-          lte(llmRequestLogs.createdAt, usageRangeEnd),
-        ),
-      )
-      .orderBy(llmRequestLogs.createdAt)
-      .all();
-
-    // 6. Write export files to a temp directory
-    const exportId = `diagnostics-${new Date().toISOString().replace(/[:.]/g, "-")}-${randomBytes(4).toString("hex")}`;
-    const tempDir = join(tmpdir(), exportId);
-    mkdirSync(tempDir, { recursive: true });
-
-    try {
-      const manifest = {
-        version: "1.1",
-        exportedAt: new Date().toISOString(),
-        conversationId,
-        messageId: anchorMessage?.id ?? null,
-      };
-      writeFileSync(
-        join(tempDir, "manifest.json"),
-        JSON.stringify(manifest, null, 2),
-      );
-
-      const messagesLines = rangeMessages.map((m) =>
-        JSON.stringify({
-          id: m.id,
-          conversationId: m.conversationId,
-          role: m.role,
-          content: truncateAndRedact(m.content),
-          createdAt: m.createdAt,
-        }),
-      );
-      writeFileSync(
-        join(tempDir, "messages.jsonl"),
-        messagesLines.join("\n") + (messagesLines.length > 0 ? "\n" : ""),
-      );
-
-      const toolLines = rangeToolInvocations.map((t) =>
-        JSON.stringify({
-          id: t.id,
-          conversationId: t.conversationId,
-          toolName: t.toolName,
-          input: truncateAndRedact(t.input),
-          result: truncateAndRedact(t.result),
-          decision: t.decision,
-          riskLevel: t.riskLevel,
-          durationMs: t.durationMs,
-          createdAt: t.createdAt,
-        }),
-      );
-      writeFileSync(
-        join(tempDir, "tool_invocations.jsonl"),
-        toolLines.join("\n") + (toolLines.length > 0 ? "\n" : ""),
-      );
-
-      const usageLines = rangeUsageEvents.map((u) =>
-        JSON.stringify({
-          id: u.id,
-          conversationId: u.conversationId,
-          actor: u.actor,
-          provider: u.provider,
-          model: u.model,
-          inputTokens: u.inputTokens,
-          outputTokens: u.outputTokens,
-          cacheCreationInputTokens: u.cacheCreationInputTokens,
-          cacheReadInputTokens: u.cacheReadInputTokens,
-          estimatedCostUsd: u.estimatedCostUsd,
-          pricingStatus: u.pricingStatus,
-          createdAt: u.createdAt,
-        }),
-      );
-      writeFileSync(
-        join(tempDir, "usage.jsonl"),
-        usageLines.join("\n") + (usageLines.length > 0 ? "\n" : ""),
-      );
-
-      const requestLogLines = rangeRequestLogs.map((r) => {
-        let request: unknown;
-        let response: unknown;
-        try {
-          request = JSON.parse(r.requestPayload);
-        } catch {
-          request = r.requestPayload;
-        }
-        try {
-          response = JSON.parse(r.responsePayload);
-        } catch {
-          response = r.responsePayload;
-        }
-        return JSON.stringify({
-          id: r.id,
-          conversationId: r.conversationId,
-          provider: r.provider,
-          request: redactDeep(request),
-          response: redactDeep(response),
-          createdAt: r.createdAt,
-        });
-      });
-      writeFileSync(
-        join(tempDir, "llm_requests.jsonl"),
-        requestLogLines.join("\n") + (requestLogLines.length > 0 ? "\n" : ""),
-      );
-
-      // 7. Zip the temp directory
-      const downloadsDir = join(homedir(), "Downloads");
-      mkdirSync(downloadsDir, { recursive: true });
-      const zipFilename = `${exportId}.zip`;
-      const zipPath = join(downloadsDir, zipFilename);
-
-      await new Promise<void>((resolve, reject) => {
-        const output = createWriteStream(zipPath);
-        const archive = archiver("zip", { zlib: { level: 9 } });
-
-        output.on("close", () => resolve());
-        output.on("error", (err: Error) => reject(err));
-        archive.on("error", (err: Error) => reject(err));
-        archive.on("warning", (err: Error) => {
-          log.warn({ err }, "Archiver warning during diagnostics export");
-        });
-
-        archive.pipe(output);
-        archive.directory(tempDir, false);
-
-        // Add recent crash report files under crash-reports/.
-        // Text-based crash files (.crash, .ips, .diag) are redacted using the
-        // same patterns as conversation data. Binary archives (.tar.gz) are
-        // added as-is since they can't be meaningfully text-redacted.
-        const crashReportFiles = findRecentCrashReports();
-        for (const filePath of crashReportFiles) {
-          try {
-            const fileName = basename(filePath);
-            if (fileName.toLowerCase().endsWith(CRASH_REPORT_TAR_GZ)) {
-              archive.file(filePath, { name: "crash-reports/" + fileName });
-            } else {
-              const content = readFileSync(filePath, "utf-8");
-              archive.append(redact(content), {
-                name: "crash-reports/" + fileName,
-              });
-            }
-          } catch {
-            // Skip files that can't be read
-          }
-        }
-
-        archive.finalize();
-      });
-
-      log.info(
-        { conversationId, zipPath, messageCount: rangeMessages.length },
-        "Diagnostics export completed via HTTP",
-      );
-
-      return Response.json({ success: true, filePath: zipPath });
-    } finally {
-      try {
-        rmSync(tempDir, { recursive: true, force: true });
-      } catch {
-        // Best-effort cleanup
-      }
-    }
-  } catch (err) {
-    const errorMessage = err instanceof Error ? err.message : String(err);
-    log.error({ err, conversationId }, "Failed to export diagnostics");
-    return httpError(
-      "INTERNAL_ERROR",
-      `Failed to export diagnostics: ${errorMessage}`,
-      500,
-    );
-  }
-}
-
 // ---------------------------------------------------------------------------
 // Dictation
 // ---------------------------------------------------------------------------
@@ -895,18 +431,6 @@ async function handleCommandMode(
 
 export function diagnosticsRouteDefinitions(): RouteDefinition[] {
   return [
-    {
-      endpoint: "diagnostics/export",
-      method: "POST",
-      policyKey: "diagnostics/export",
-      handler: async ({ req }) => {
-        const body = (await req.json()) as {
-          conversationId?: string;
-          anchorMessageId?: string;
-        };
-        return handleDiagnosticsExport(body);
-      },
-    },
     {
       endpoint: "dictation",
       method: "POST",

package/src/runtime/routes/identity-routes.ts

@@ -9,7 +9,10 @@ import { fileURLToPath } from "node:url";
 
 import { getBaseDataDir } from "../../config/env-registry.js";
 import { parseIdentityFields } from "../../daemon/handlers/identity.js";
-import { getWorkspacePromptPath, readLockfile } from "../../util/platform.js";
+import { getMaxMigrationVersion } from "../../memory/migrations/registry.js";
+import { getWorkspacePromptPath } from "../../util/platform.js";
+import { WORKSPACE_MIGRATIONS } from "../../workspace/migrations/registry.js";
+import { getLastWorkspaceMigrationId } from "../../workspace/migrations/runner.js";
 import { httpError } from "../http-errors.js";
 import type { RouteDefinition } from "../http-router.js";
 import { getCachedIntro } from "./identity-intro-cache.js";
@@ -133,6 +136,10 @@ function getPackageVersion(): string | undefined {
 }
 
 export function handleHealth(): Response {
+  return Response.json({ status: "ok" });
+}
+
+export function handleDetailedHealth(): Response {
   return Response.json({
     status: "healthy",
     timestamp: new Date().toISOString(),
@@ -140,9 +147,18 @@ export function handleHealth(): Response {
     disk: getDiskSpaceInfo(),
     memory: getMemoryInfo(),
     cpu: getCpuInfo(),
+    migrations: {
+      dbVersion: getMaxMigrationVersion(),
+      lastWorkspaceMigrationId:
+        getLastWorkspaceMigrationId(WORKSPACE_MIGRATIONS),
+    },
   });
 }
 
+export function handleReadyz(): Response {
+  return Response.json({ status: "ok" });
+}
+
 export function handleGetIdentity(): Response {
   const identityPath = getWorkspacePromptPath("IDENTITY.md");
   if (!existsSync(identityPath)) {
@@ -163,31 +179,6 @@ export function handleGetIdentity(): Response {
     // ignore
   }
 
-  // Read lockfile for assistantId, cloud, and originSystem
-  let assistantId: string | undefined;
-  let cloud: string | undefined;
-  let originSystem: string | undefined;
-  try {
-    const lockData = readLockfile();
-    const assistants = lockData?.assistants as
-      | Array<Record<string, unknown>>
-      | undefined;
-    if (assistants && assistants.length > 0) {
-      // Use the most recently hatched assistant
-      const sorted = [...assistants].sort((a, b) => {
-        const dateA = new Date((a.hatchedAt as string) || 0).getTime();
-        const dateB = new Date((b.hatchedAt as string) || 0).getTime();
-        return dateB - dateA;
-      });
-      const latest = sorted[0];
-      assistantId = latest.assistantId as string | undefined;
-      cloud = latest.cloud as string | undefined;
-      originSystem = cloud === "local" ? "local" : cloud;
-    }
-  } catch {
-    // ignore -- lockfile may not exist
-  }
-
   return Response.json({
     name: fields.name ?? "",
     role: fields.role ?? "",
@@ -195,9 +186,7 @@ export function handleGetIdentity(): Response {
     emoji: fields.emoji ?? "",
     home: fields.home ?? "",
     version,
-    assistantId,
     createdAt,
-    originSystem,
   });
 }
 
@@ -255,7 +244,7 @@ export function identityRouteDefinitions(): RouteDefinition[] {
     {
       endpoint: "health",
       method: "GET",
-      handler: () => handleHealth(),
+      handler: () => handleDetailedHealth(),
     },
     {
       endpoint: "identity",

package/src/runtime/routes/inbound-stages/secret-ingress-check.ts

@@ -1,12 +1,6 @@
 /**
- * Secret ingress check stage: persists the raw inbound payload, runs the
- * secret detection scan, and records a conversation-seen signal for
- * Telegram messages.
- *
- * The payload is stored before the scan so dead-lettered events can be
- * replayed. If the scan detects embedded secrets the stored payload is
- * cleared before the IngressBlockedError propagates, ensuring
- * secret-bearing content is never left on disk.
+ * Inbound payload persistence stage: persists the raw inbound payload and
+ * records a conversation-seen signal for Telegram messages.
  *
  * Extracted from inbound-message-handler.ts to keep the top-level handler
  * focused on orchestration.
@@ -15,8 +9,6 @@ import type { ChannelId } from "../../../channels/types.js";
 import type { TrustContext } from "../../../daemon/conversation-runtime-assembly.js";
 import { recordConversationSeenSignal } from "../../../memory/conversation-attention-store.js";
 import * as deliveryCrud from "../../../memory/delivery-crud.js";
-import { checkIngressForSecrets } from "../../../security/secret-ingress.js";
-import { IngressBlockedError } from "../../../util/errors.js";
 import { getLogger } from "../../../util/logger.js";
 
 const log = getLogger("runtime-http");
@@ -44,10 +36,7 @@ export interface SecretIngressCheckParams {
 }
 
 /**
- * Persist the raw payload, run the secret ingress scan, and record a
- * Telegram seen signal.
- *
- * Throws IngressBlockedError if the content contains secrets.
+ * Persist the raw payload and record a Telegram seen signal.
  */
 export function runSecretIngressCheck(params: SecretIngressCheckParams): void {
   const {
@@ -68,9 +57,7 @@ export function runSecretIngressCheck(params: SecretIngressCheckParams): void {
     canonicalAssistantId,
   } = params;
 
-  // Persist the raw payload first so dead-lettered events can always be
-  // replayed. If the ingress check later detects secrets we clear it
-  // before throwing, so secret-bearing content is never left on disk.
+  // Persist the raw payload so dead-lettered events can always be replayed.
   deliveryCrud.storePayload(eventId, {
     sourceChannel,
     externalChatId: conversationExternalId,
@@ -86,22 +73,6 @@ export function runSecretIngressCheck(params: SecretIngressCheckParams): void {
     assistantId: canonicalAssistantId,
   });
 
-  const contentToCheck = content ?? "";
-  let ingressCheck: ReturnType<typeof checkIngressForSecrets>;
-  try {
-    ingressCheck = checkIngressForSecrets(contentToCheck);
-  } catch (checkErr) {
-    deliveryCrud.clearPayload(eventId);
-    throw checkErr;
-  }
-  if (ingressCheck.blocked) {
-    deliveryCrud.clearPayload(eventId);
-    throw new IngressBlockedError(
-      ingressCheck.userNotice!,
-      ingressCheck.detectedTypes,
-    );
-  }
-
   // Record inferred seen signal for non-duplicate Telegram inbound messages
   if (sourceChannel === "telegram") {
     try {

package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts

@@ -24,7 +24,7 @@ mock.module("../../../config/assistant-feature-flags.js", () => ({
 }));
 
 mock.module("../../../config/loader.js", () => ({
-  getConfig: () => ({ assistantFeatureFlagValues: {} }),
+  getConfig: () => ({}),
 }));
 
 mock.module("../../../memory/attachments-store.js", () => ({