@vellumai/assistant 0.5.16 → 0.6.1

package/src/workspace/migrations/026-backfill-install-meta.ts

@@ -0,0 +1,325 @@
+/**
+ * Workspace migration 026: Backfill install-meta.json for existing skills
+ *
+ * Scans ~/.vellum/workspace/skills/ for installed skill directories and writes
+ * an install-meta.json for each skill that lacks one, inferring the origin
+ * from legacy version.json and .integrity.json files.
+ *
+ * Idempotent: safe to re-run after interruption at any point.
+ */
+
+import { randomUUID } from "node:crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+import type { WorkspaceMigration } from "./types.js";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const INSTALL_META_FILENAME = "install-meta.json";
+const VERSION_JSON_FILENAME = "version.json";
+const INTEGRITY_JSON_FILENAME = ".integrity.json";
+const SKILL_MD_FILENAME = "SKILL.md";
+
+// ---------------------------------------------------------------------------
+// Inlined helpers (self-contained per migrations/AGENTS.md)
+// ---------------------------------------------------------------------------
+
+interface SkillInstallMeta {
+  origin: "vellum" | "clawhub" | "skillssh" | "custom";
+  installedAt: string;
+  installedBy?: string;
+  backfilledBy?: string;
+  version?: string;
+  slug?: string;
+  sourceRepo?: string;
+  contentHash?: string;
+}
+
+/**
+ * Atomically write a file: write to a temp file, then rename.
+ */
+function atomicWriteFile(filePath: string, content: string): void {
+  const dir = dirname(filePath);
+  mkdirSync(dir, { recursive: true });
+  const tmpPath = join(dir, `.tmp-${randomUUID()}`);
+  writeFileSync(tmpPath, content, "utf-8");
+  renameSync(tmpPath, filePath);
+}
+
+/**
+ * Write install-meta.json into a skill directory.
+ */
+function writeInstallMeta(skillDir: string, meta: SkillInstallMeta): void {
+  const filePath = join(skillDir, INSTALL_META_FILENAME);
+  atomicWriteFile(filePath, JSON.stringify(meta, null, 2) + "\n");
+}
+
+/**
+ * Metadata files excluded from content hashing.
+ */
+const METADATA_FILENAMES = new Set([
+  INSTALL_META_FILENAME,
+  VERSION_JSON_FILENAME,
+]);
+
+/**
+ * Collect all file contents in a directory tree, sorted by relative path.
+ * Metadata files at root level are excluded.
+ */
+function collectFileContents(
+  dir: string,
+  prefix = "",
+): Array<{ relPath: string; content: Buffer }> {
+  const results: Array<{ relPath: string; content: Buffer }> = [];
+  if (!existsSync(dir)) return results;
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    if (!prefix && METADATA_FILENAMES.has(entry.name)) continue;
+
+    const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      results.push(...collectFileContents(fullPath, relPath));
+    } else if (entry.isFile()) {
+      results.push({ relPath, content: readFileSync(fullPath) });
+    }
+  }
+  return results.sort((a, b) => a.relPath.localeCompare(b.relPath));
+}
+
+/**
+ * Compute SHA-256 content hash over all non-metadata files in a skill dir.
+ * Returns format: "v2:sha256hex".
+ */
+function computeSkillHash(skillDir: string): string | null {
+  if (!existsSync(skillDir) || !statSync(skillDir).isDirectory()) return null;
+
+  const files = collectFileContents(skillDir);
+  if (files.length === 0) return null;
+
+  const hasher = new Bun.CryptoHasher("sha256");
+  for (const file of files) {
+    const pathBuf = Buffer.from(file.relPath, "utf-8");
+    hasher.update(`${pathBuf.length}:`);
+    hasher.update(pathBuf);
+    hasher.update(`${file.content.length}:`);
+    hasher.update(file.content);
+  }
+  return `v2:${hasher.digest("hex")}`;
+}
+
+// ---------------------------------------------------------------------------
+// Integrity manifest helpers
+// ---------------------------------------------------------------------------
+
+interface IntegrityRecord {
+  sha256: string;
+  installedAt: string;
+}
+
+type IntegrityManifest = Record<string, IntegrityRecord>;
+
+function loadIntegrityManifest(skillsDir: string): IntegrityManifest {
+  const path = join(skillsDir, INTEGRITY_JSON_FILENAME);
+  if (!existsSync(path)) return {};
+  try {
+    return JSON.parse(readFileSync(path, "utf-8")) as IntegrityManifest;
+  } catch {
+    return {};
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Origin inference
+// ---------------------------------------------------------------------------
+
+/**
+ * Infer SkillInstallMeta for a skill directory that has no install-meta.json.
+ *
+ * Decision tree:
+ * 1. version.json with `origin: "skills.sh"` -> skillssh
+ * 2. version.json with `version` but no `origin` field:
+ *    - Has entry in .integrity.json -> clawhub
+ *    - Otherwise -> vellum
+ * 3. version.json exists but doesn't match above -> custom
+ * 4. No version.json:
+ *    - Has entry in .integrity.json -> clawhub
+ *    - Otherwise -> custom
+ */
+function inferInstallMeta(
+  skillDir: string,
+  skillId: string,
+  integrityManifest: IntegrityManifest,
+): SkillInstallMeta {
+  const versionJsonPath = join(skillDir, VERSION_JSON_FILENAME);
+  const hasIntegrityEntry = skillId in integrityManifest;
+
+  if (existsSync(versionJsonPath)) {
+    let raw: Record<string, unknown>;
+    try {
+      raw = JSON.parse(readFileSync(versionJsonPath, "utf-8")) as Record<
+        string,
+        unknown
+      >;
+    } catch {
+      // Malformed version.json — treat as if it doesn't exist
+      return buildFallbackMeta(skillDir, skillId, hasIntegrityEntry);
+    }
+
+    // Case 1: skills.sh origin
+    if (raw.origin === "skills.sh") {
+      return {
+        origin: "skillssh",
+        installedAt:
+          typeof raw.installedAt === "string"
+            ? raw.installedAt
+            : getDirectoryMtime(skillDir),
+        sourceRepo: typeof raw.source === "string" ? raw.source : undefined,
+        slug: typeof raw.skillSlug === "string" ? raw.skillSlug : undefined,
+        contentHash: computeSkillHash(skillDir) ?? undefined,
+      };
+    }
+
+    // Case 2: has version but no origin field
+    if (typeof raw.version === "string" && !("origin" in raw)) {
+      return {
+        origin: hasIntegrityEntry ? "clawhub" : "vellum",
+        installedAt:
+          typeof raw.installedAt === "string"
+            ? raw.installedAt
+            : getDirectoryMtime(skillDir),
+        version: raw.version,
+        contentHash: computeSkillHash(skillDir) ?? undefined,
+      };
+    }
+
+    // Case 3: version.json exists but doesn't match known patterns
+    return {
+      origin: "custom",
+      installedAt:
+        typeof raw.installedAt === "string"
+          ? raw.installedAt
+          : getDirectoryMtime(skillDir),
+      contentHash: computeSkillHash(skillDir) ?? undefined,
+    };
+  }
+
+  // Case 4: no version.json
+  return buildFallbackMeta(skillDir, skillId, hasIntegrityEntry);
+}
+
+function buildFallbackMeta(
+  skillDir: string,
+  _skillId: string,
+  hasIntegrityEntry: boolean,
+): SkillInstallMeta {
+  return {
+    origin: hasIntegrityEntry ? "clawhub" : "custom",
+    installedAt: getDirectoryMtime(skillDir),
+    contentHash: computeSkillHash(skillDir) ?? undefined,
+  };
+}
+
+/**
+ * Get directory mtime as ISO 8601 string. Falls back to current time.
+ */
+function getDirectoryMtime(dir: string): string {
+  try {
+    return statSync(dir).mtime.toISOString();
+  } catch {
+    return new Date().toISOString();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Migration
+// ---------------------------------------------------------------------------
+
+export const backfillInstallMetaMigration: WorkspaceMigration = {
+  id: "026-backfill-install-meta",
+  description:
+    "Backfill install-meta.json with origin field for existing skill directories",
+
+  run(workspaceDir: string): void {
+    const skillsDir = join(workspaceDir, "skills");
+    if (!existsSync(skillsDir)) return;
+
+    // Load the integrity manifest once — shared across all skills
+    const integrityManifest = loadIntegrityManifest(skillsDir);
+
+    // Enumerate skill directories (each subdirectory containing a SKILL.md)
+    let dirNames: string[];
+    try {
+      dirNames = readdirSync(skillsDir, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name);
+    } catch {
+      return;
+    }
+
+    for (const name of dirNames) {
+      const skillDir = join(skillsDir, name);
+      const skillMdPath = join(skillDir, SKILL_MD_FILENAME);
+      const installMetaPath = join(skillDir, INSTALL_META_FILENAME);
+
+      // Only process directories that contain SKILL.md
+      if (!existsSync(skillMdPath)) continue;
+
+      // Skip if install-meta.json already exists (idempotency)
+      if (existsSync(installMetaPath)) continue;
+
+      const meta = inferInstallMeta(skillDir, name, integrityManifest);
+
+      // Mark as backfilled so down() can safely identify and remove only
+      // migration-created files without touching CLI-installed ones.
+      writeInstallMeta(skillDir, { ...meta, backfilledBy: "migration-026" });
+    }
+  },
+
+  down(workspaceDir: string): void {
+    const skillsDir = join(workspaceDir, "skills");
+    if (!existsSync(skillsDir)) return;
+
+    let dirNames: string[];
+    try {
+      dirNames = readdirSync(skillsDir, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => entry.name);
+    } catch {
+      return;
+    }
+
+    for (const name of dirNames) {
+      const installMetaPath = join(skillsDir, name, INSTALL_META_FILENAME);
+      if (existsSync(installMetaPath)) {
+        try {
+          const meta = JSON.parse(
+            readFileSync(installMetaPath, "utf-8"),
+          ) as SkillInstallMeta;
+
+          // Only remove install-meta.json that were backfilled by this migration.
+          // Files written by the normal install flow (CLI, daemon, etc.) won't
+          // have backfilledBy set and are safely preserved on rollback.
+          if (meta.backfilledBy === "migration-026") {
+            unlinkSync(installMetaPath);
+          }
+        } catch {
+          // Malformed file — skip
+        }
+      }
+    }
+  },
+};

package/src/workspace/migrations/027-remove-orphaned-optimized-images-cache.ts

@@ -0,0 +1,42 @@
+/**
+ * Workspace migration 027: Remove orphaned optimized-images cache directory.
+ *
+ * The optimized image cache was moved from `workspace/cache/optimized-images/`
+ * to `os.tmpdir()/vellum-optimized-images/`. This migration cleans up the old
+ * directory and removes the parent `cache/` directory if it is now empty.
+ *
+ * Idempotent: safe to re-run after interruption at any point.
+ */
+
+import { existsSync, readdirSync, rmdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+
+import type { WorkspaceMigration } from "./types.js";
+
+export const removeOrphanedOptimizedImagesCacheMigration: WorkspaceMigration = {
+  id: "027-remove-orphaned-optimized-images-cache",
+  description:
+    "Remove orphaned cache/optimized-images/ directory after cache moved to tmpdir",
+
+  run(workspaceDir: string): void {
+    const cacheDir = join(workspaceDir, "cache");
+    const optimizedImagesDir = join(cacheDir, "optimized-images");
+
+    if (existsSync(optimizedImagesDir)) {
+      rmSync(optimizedImagesDir, { recursive: true, force: true });
+    }
+
+    // Remove the parent cache/ directory if it is now empty
+    if (existsSync(cacheDir)) {
+      const remaining = readdirSync(cacheDir);
+      if (remaining.length === 0) {
+        rmdirSync(cacheDir);
+      }
+    }
+  },
+
+  down(_workspaceDir: string): void {
+    // The old cache directory contained ephemeral cached data that does not
+    // need to be restored. The cache will be rebuilt in tmpdir on next use.
+  },
+};

package/src/workspace/migrations/028-recover-conversations-from-disk-view.ts

@@ -0,0 +1,270 @@
+/**
+ * Workspace migration 028: Recover conversations from disk-view directories.
+ *
+ * If the SQLite database was recreated empty but the disk-view directories
+ * under `workspace/conversations/` still exist, this migration reads each
+ * conversation's `meta.json` and `messages.jsonl` and re-inserts the rows
+ * into the database.
+ *
+ * Idempotent: conversations already present in the DB are skipped.
+ * Malformed files are skipped with warnings — they do not crash the migration.
+ */
+
+import { randomUUID } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+import { eq } from "drizzle-orm";
+
+import { getDb } from "../../memory/db.js";
+import { conversations, messages } from "../../memory/schema/conversations.js";
+import { getLogger } from "../../util/logger.js";
+import type { WorkspaceMigration } from "./types.js";
+
+const log = getLogger("workspace-migrations");
+
+interface DiskMeta {
+  id: string;
+  title?: string;
+  type?: string;
+  channel?: string;
+  createdAt?: string;
+  updatedAt?: string;
+}
+
+interface DiskToolCall {
+  name?: string;
+  input?: unknown;
+}
+
+interface DiskToolResult {
+  content?: unknown;
+}
+
+interface DiskMessageRecord {
+  role: string;
+  ts?: string;
+  content?: string;
+  toolCalls?: DiskToolCall[];
+  toolResults?: DiskToolResult[];
+  attachments?: unknown[];
+}
+
+function parseEpochMs(isoString: string | undefined): number | null {
+  if (!isoString) return null;
+  const ms = new Date(isoString).getTime();
+  return Number.isNaN(ms) ? null : ms;
+}
+
+function buildContentBlocks(record: DiskMessageRecord): unknown[] {
+  const blocks: unknown[] = [];
+
+  if (record.content) {
+    blocks.push({ type: "text", text: record.content });
+  }
+
+  if (Array.isArray(record.toolCalls)) {
+    for (const tc of record.toolCalls) {
+      blocks.push({
+        type: "tool_use",
+        id: randomUUID(),
+        name: tc.name ?? "unknown",
+        input: tc.input ?? {},
+      });
+    }
+  }
+
+  if (Array.isArray(record.toolResults)) {
+    for (const tr of record.toolResults) {
+      blocks.push({
+        type: "tool_result",
+        tool_use_id: "",
+        content:
+          typeof tr.content === "string"
+            ? tr.content
+            : JSON.stringify(tr.content),
+      });
+    }
+  }
+
+  // content column is NOT NULL — ensure at least one block
+  if (blocks.length === 0) {
+    blocks.push({ type: "text", text: "" });
+  }
+
+  return blocks;
+}
+
+export const recoverConversationsFromDiskViewMigration: WorkspaceMigration = {
+  id: "028-recover-conversations-from-disk-view",
+  description:
+    "Recover conversations from disk-view directories into the database",
+
+  run(workspaceDir: string): void {
+    const conversationsDir = join(workspaceDir, "conversations");
+    if (!existsSync(conversationsDir)) return;
+
+    const db = getDb();
+
+    let entries: string[];
+    try {
+      entries = readdirSync(conversationsDir);
+    } catch (err) {
+      log.warn(`Failed to read conversations directory: ${err}`);
+      return;
+    }
+
+    let recovered = 0;
+    let skipped = 0;
+    let errors = 0;
+
+    for (const entry of entries) {
+      const dirPath = join(conversationsDir, entry);
+
+      // Skip non-directories
+      try {
+        if (!statSync(dirPath).isDirectory()) {
+          continue;
+        }
+      } catch {
+        continue;
+      }
+
+      // Read and parse meta.json
+      const metaPath = join(dirPath, "meta.json");
+      if (!existsSync(metaPath)) {
+        log.warn(
+          `Skipping ${entry}: missing meta.json`,
+        );
+        skipped++;
+        continue;
+      }
+
+      let meta: DiskMeta;
+      try {
+        meta = JSON.parse(readFileSync(metaPath, "utf-8")) as DiskMeta;
+      } catch (err) {
+        log.warn(
+          `Skipping ${entry}: malformed meta.json: ${err}`,
+        );
+        skipped++;
+        continue;
+      }
+
+      if (!meta.id) {
+        log.warn(
+          `Skipping ${entry}: meta.json missing id`,
+        );
+        skipped++;
+        continue;
+      }
+
+      // Check if conversation already exists in DB (idempotency)
+      const existing = db
+        .select()
+        .from(conversations)
+        .where(eq(conversations.id, meta.id))
+        .get();
+
+      if (existing) {
+        skipped++;
+        continue;
+      }
+
+      // Parse messages.jsonl
+      const messagesPath = join(dirPath, "messages.jsonl");
+      const messageRecords: DiskMessageRecord[] = [];
+
+      if (existsSync(messagesPath)) {
+        try {
+          const raw = readFileSync(messagesPath, "utf-8");
+          for (const line of raw.split("\n")) {
+            const trimmed = line.trim();
+            if (!trimmed) continue;
+            try {
+              messageRecords.push(
+                JSON.parse(trimmed) as DiskMessageRecord,
+              );
+            } catch {
+              log.warn(
+                `Skipping malformed JSONL line in ${entry}/messages.jsonl`,
+              );
+            }
+          }
+        } catch (err) {
+          log.warn(
+            `Failed to read messages.jsonl for ${entry}: ${err}`,
+          );
+        }
+      }
+
+      // Compute timestamps
+      const createdAt = parseEpochMs(meta.createdAt) ?? Date.now();
+      const updatedAt = parseEpochMs(meta.updatedAt) ?? createdAt;
+
+      // Insert conversation + messages in a transaction
+      try {
+        db.transaction((tx) => {
+          tx.insert(conversations)
+            .values({
+              id: meta.id,
+              title: meta.title ?? null,
+              createdAt,
+              updatedAt,
+              conversationType: meta.type ?? "standard",
+              originChannel: meta.channel ?? null,
+              source: "user",
+              memoryScopeId: "default",
+              isAutoTitle: 1,
+              totalInputTokens: 0,
+              totalOutputTokens: 0,
+              totalEstimatedCost: 0,
+              contextSummary: null,
+              contextCompactedMessageCount: 0,
+              contextCompactedAt: null,
+              originInterface: null,
+              forkParentConversationId: null,
+              forkParentMessageId: null,
+              scheduleJobId: null,
+            })
+            .run();
+
+          for (const record of messageRecords) {
+            const contentBlocks = buildContentBlocks(record);
+            const msgCreatedAt =
+              parseEpochMs(record.ts) ?? createdAt;
+
+            tx.insert(messages)
+              .values({
+                id: randomUUID(),
+                conversationId: meta.id,
+                role: record.role,
+                content: JSON.stringify(contentBlocks),
+                createdAt: msgCreatedAt,
+                metadata: null,
+              })
+              .run();
+          }
+        });
+
+        recovered++;
+      } catch (err) {
+        log.warn(
+          `Failed to insert conversation ${meta.id} (${entry}): ${err}`,
+        );
+        errors++;
+      }
+    }
+
+    if (recovered > 0 || errors > 0) {
+      log.info(
+        `Recover conversations from disk-view: recovered=${recovered}, skipped=${skipped}, errors=${errors}`,
+      );
+    }
+  },
+
+  // No-op: deleting recovered conversation data from the database would cause
+  // data loss — the disk-view files are the only remaining copy after the
+  // original DB was lost.
+  down(_workspaceDir: string): void {},
+};