@vellumai/assistant 0.4.46 → 0.4.49

package/src/security/token-manager.ts

@@ -1,15 +1,18 @@
 /**
- * Metadata-driven token manager for OAuth2 credentials.
+ * Token manager for OAuth2 credentials.
  *
- * Reads refresh configuration (tokenUrl, clientId) from credential metadata
- * rather than requiring an IntegrationDefinition, enabling autonomous token
- * refresh for any OAuth2 service that stores its config in metadata.
+ * Reads refresh configuration (tokenUrl, clientId, authMethod) exclusively
+ * from the SQLite oauth-store (provider + app + connection rows). After a
+ * successful refresh, writes tokens to new-format secure key paths and
+ * updates the oauth_connection row.
  */
 
 import {
-  getCredentialMetadata,
-  upsertCredentialMetadata,
-} from "../tools/credentials/metadata-store.js";
+  getApp,
+  getConnectionByProvider,
+  getProvider,
+  updateConnection,
+} from "../oauth/oauth-store.js";
 import { getLogger } from "../util/logger.js";
 import { refreshOAuth2Token, type TokenEndpointAuthMethod } from "./oauth2.js";
 import { getSecureKey, setSecureKeyAsync } from "./secure-keys.js";
@@ -106,11 +109,34 @@ function recordRefreshFailure(service: string): void {
   }
 }
 
+// ── Per-service refresh deduplication ─────────────────────────────────
+// When multiple concurrent `withValidToken` calls detect an expired or
+// 401-rejected token for the same service, only one actual refresh
+// attempt is made. Other callers join the in-flight promise.
+
+const inflightRefreshes = new Map<string, Promise<string>>();
+
+function deduplicatedRefresh(service: string): Promise<string> {
+  const existing = inflightRefreshes.get(service);
+  if (existing) return existing;
+
+  const promise = doRefresh(service).finally(() => {
+    inflightRefreshes.delete(service);
+  });
+  inflightRefreshes.set(service, promise);
+  return promise;
+}
+
 /** @internal Test-only: reset all circuit breaker state */
 export function _resetRefreshBreakers(): void {
   refreshBreakers.clear();
 }
 
+/** @internal Test-only: reset in-flight refresh deduplication state */
+export function _resetInflightRefreshes(): void {
+  inflightRefreshes.clear();
+}
+
 /** @internal Test-only: get breaker state for a service */
 export function _getRefreshBreakerState(
   service: string,
@@ -132,54 +158,113 @@ export class TokenExpiredError extends Error {
 
 /**
  * Check whether the access token for a service is expired or will expire
- * within the buffer window, based on the `expiresAt` field in credential metadata.
+ * within the buffer window, based on the `expiresAt` field in the
+ * oauth_connection row.
  */
 function isTokenExpired(service: string): boolean {
-  const meta = getCredentialMetadata(service, "access_token");
-  if (!meta?.expiresAt) return false;
-  return Date.now() >= meta.expiresAt - EXPIRY_BUFFER_MS;
+  try {
+    const conn = getConnectionByProvider(service);
+    if (!conn?.expiresAt) return false;
+    return Date.now() >= conn.expiresAt - EXPIRY_BUFFER_MS;
+  } catch {
+    return false;
+  }
+}
+
+// ── Refresh config resolution ─────────────────────────────────────────
+
+/** Shared shape for resolved refresh configuration. */
+interface RefreshConfig {
+  tokenUrl: string;
+  clientId: string;
+  /** OAuth client secret (optional — PKCE flows may omit it). */
+  secret?: string;
+  refreshToken?: string;
+  authMethod?: TokenEndpointAuthMethod;
+  connId: string;
 }
 
 /**
- * Attempt to refresh the OAuth2 access token for a service using the
- * refresh token and OAuth2 config stored in credential metadata.
+ * Resolve refresh configuration from the SQLite oauth-store.
  *
- * Returns the new access token on success.
- * Throws `TokenExpiredError` if refresh is not possible.
+ * Looks up connection -> app -> provider to read tokenUrl, clientId, and
+ * authMethod. Throws `TokenExpiredError` if the connection is not found
+ * or incomplete.
  */
-async function doRefresh(service: string): Promise<string> {
-  const refreshToken = getSecureKey(`credential:${service}:refresh_token`);
-  if (!refreshToken) {
+function resolveRefreshConfig(service: string): RefreshConfig {
+  const conn = getConnectionByProvider(service);
+  if (!conn) {
     throw new TokenExpiredError(
       service,
-      `No refresh token available for "${service}". Re-authorization required.${recoveryHint(service)}`,
+      `No OAuth connection found for "${service}". Re-authorization required.${recoveryHint(service)}`,
     );
   }
 
-  const meta = getCredentialMetadata(service, "access_token");
-  const tokenUrl = meta?.oauth2TokenUrl;
-  const clientId = meta?.oauth2ClientId;
+  const app = getApp(conn.oauthAppId);
+  if (!app) {
+    throw new TokenExpiredError(
+      service,
+      `No OAuth app found for "${service}". Re-authorization required.${recoveryHint(service)}`,
+    );
+  }
+
+  const provider = getProvider(conn.providerKey);
+  if (!provider) {
+    throw new TokenExpiredError(
+      service,
+      `No OAuth provider found for "${service}". Re-authorization required.${recoveryHint(service)}`,
+    );
+  }
 
+  const tokenUrl = provider.tokenUrl;
+  const clientId = app.clientId;
   if (!tokenUrl || !clientId) {
-    // Legacy credentials created by the old integration flow don't store
-    // oauth2TokenUrl/oauth2ClientId in metadata. The client ID is user-specific
-    // (from their Google Cloud Console) and cannot be recovered, so the only
-    // path forward is re-authorization via the new oauth2_connect flow.
-    const isLegacy = service === "integration:gmail" && !tokenUrl && !clientId;
-    const hint = isLegacy
-      ? ` This is a one-time migration: your old Gmail connection needs to be re-authorized. Ask me to "reconnect Gmail" to set it up again.`
-      : "";
     throw new TokenExpiredError(
       service,
-      `Missing OAuth2 refresh config for "${service}".${hint}${recoveryHint(service)}`,
+      `Missing OAuth2 refresh config for "${service}".${recoveryHint(service)}`,
     );
   }
 
-  const clientSecret = meta?.oauth2ClientSecret as string | undefined;
-  const authMethod = meta?.oauth2TokenEndpointAuthMethod as
+  const secret = getSecureKey(`oauth_app/${app.id}/client_secret`);
+
+  const refreshToken = getSecureKey(
+    `oauth_connection/${conn.id}/refresh_token`,
+  );
+
+  const authMethod = provider.tokenEndpointAuthMethod as
     | TokenEndpointAuthMethod
     | undefined;
-  const resolvedTokenUrl = tokenUrl;
+
+  return {
+    connId: conn.id,
+    tokenUrl,
+    clientId,
+    secret,
+    refreshToken,
+    authMethod,
+  };
+}
+
+/**
+ * Attempt to refresh the OAuth2 access token for a service.
+ *
+ * Reads refresh config exclusively from the SQLite oauth-store (provider,
+ * app, connection).
+ *
+ * Returns the new access token on success.
+ * Throws `TokenExpiredError` if refresh is not possible.
+ */
+async function doRefresh(service: string): Promise<string> {
+  const refreshConfig = resolveRefreshConfig(service);
+  const { tokenUrl, clientId, secret, authMethod, connId, refreshToken } =
+    refreshConfig;
+
+  if (!refreshToken) {
+    throw new TokenExpiredError(
+      service,
+      `No refresh token available for "${service}". Re-authorization required.${recoveryHint(service)}`,
+    );
+  }
 
   if (isRefreshBreakerOpen(service)) {
     const state = refreshBreakers.get(service)!;
@@ -196,10 +281,10 @@ async function doRefresh(service: string): Promise<string> {
   let result;
   try {
     result = await refreshOAuth2Token(
-      resolvedTokenUrl,
+      tokenUrl,
       clientId,
       refreshToken,
-      clientSecret,
+      secret,
       authMethod,
     );
   } catch (err) {
@@ -217,9 +302,10 @@ async function doRefresh(service: string): Promise<string> {
     throw err;
   }
 
+  // ----- Store refreshed access_token -----
   if (
     !(await setSecureKeyAsync(
-      `credential:${service}:access_token`,
+      `oauth_connection/${connId}/access_token`,
       result.accessToken,
     ))
   ) {
@@ -232,7 +318,7 @@ async function doRefresh(service: string): Promise<string> {
   if (result.refreshToken) {
     if (
       !(await setSecureKeyAsync(
-        `credential:${service}:refresh_token`,
+        `oauth_connection/${connId}/refresh_token`,
         result.refreshToken,
       ))
     ) {
@@ -243,7 +329,7 @@ async function doRefresh(service: string): Promise<string> {
     }
   }
 
-  // Update metadata with new expiry.
+  // Update oauth_connection row with new expiry.
   // Use null to explicitly clear a stale expiresAt when the provider omits
   // expires_in (or returns 0), so isTokenExpired won't keep forcing refreshes.
   const expiresAt =
@@ -251,7 +337,17 @@ async function doRefresh(service: string): Promise<string> {
       ? Date.now() + result.expiresIn * 1000
       : null;
 
-  upsertCredentialMetadata(service, "access_token", { expiresAt });
+  try {
+    updateConnection(connId, {
+      expiresAt,
+      hasRefreshToken: !!result.refreshToken,
+    });
+  } catch (err) {
+    log.warn(
+      { err, service },
+      "Failed to update oauth_connection after refresh",
+    );
+  }
 
   recordRefreshSuccess(service);
   log.info({ service }, "OAuth2 access token refreshed successfully");
@@ -265,12 +361,18 @@ async function doRefresh(service: string): Promise<string> {
  * 1. Retrieves the stored access token (throws if none exists).
  * 2. If the token is expired or near-expiry, refreshes it before calling the callback.
  * 3. If the callback throws with a 401 status, attempts one refresh-and-retry cycle.
+ *
+ * Retained only for BYO connection internals — prefer
+ * `resolveOAuthConnection(service).request()` for new code.
  */
 export async function withValidToken<T>(
   service: string,
   callback: (token: string) => Promise<T>,
 ): Promise<T> {
-  let token = getSecureKey(`credential:${service}:access_token`);
+  const conn = getConnectionByProvider(service);
+  let token = conn
+    ? getSecureKey(`oauth_connection/${conn.id}/access_token`)
+    : undefined;
   if (!token) {
     throw new TokenExpiredError(
       service,
@@ -280,14 +382,14 @@ export async function withValidToken<T>(
 
   // Proactively refresh if expired or about to expire.
   if (isTokenExpired(service)) {
-    token = await doRefresh(service);
+    token = await deduplicatedRefresh(service);
   }
 
   try {
     return await callback(token);
   } catch (err: unknown) {
     if (is401Error(err)) {
-      token = await doRefresh(service);
+      token = await deduplicatedRefresh(service);
       return callback(token);
     }
     throw err;

package/src/skills/catalog-install.ts

@@ -0,0 +1,358 @@
+import { execSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+import { gunzipSync } from "node:zlib";
+
+import { getLogger } from "../util/logger.js";
+import {
+  getWorkspaceConfigPath,
+  getWorkspaceSkillsDir,
+  readPlatformToken,
+} from "../util/platform.js";
+
+const log = getLogger("catalog-install");
+
+// ─── Types ───────────────────────────────────────────────────────────────────
+
+export interface CatalogSkill {
+  id: string;
+  name: string;
+  description: string;
+  emoji?: string;
+  includes?: string[];
+  version?: string;
+}
+
+export interface CatalogManifest {
+  version: number;
+  skills: CatalogSkill[];
+}
+
+// ─── Path helpers ────────────────────────────────────────────────────────────
+
+export function getSkillsIndexPath(): string {
+  return join(getWorkspaceSkillsDir(), "SKILLS.md");
+}
+
+/**
+ * Resolve the repo-level skills/ directory when running in dev mode.
+ * Returns the path if VELLUM_DEV is set and the directory exists, or undefined.
+ */
+export function getRepoSkillsDir(): string | undefined {
+  if (!process.env.VELLUM_DEV) return undefined;
+
+  // assistant/src/skills/catalog-install.ts -> ../../../skills/
+  const candidate = join(import.meta.dir, "..", "..", "..", "skills");
+  if (existsSync(join(candidate, "catalog.json"))) {
+    return candidate;
+  }
+  return undefined;
+}
+
+// ─── Platform API ────────────────────────────────────────────────────────────
+
+function getConfigPlatformUrl(): string | undefined {
+  try {
+    const configPath = getWorkspaceConfigPath();
+    if (!existsSync(configPath)) return undefined;
+    const raw = JSON.parse(readFileSync(configPath, "utf-8")) as Record<
+      string,
+      unknown
+    >;
+    const platform = raw.platform as Record<string, unknown> | undefined;
+    const baseUrl = platform?.baseUrl;
+    if (typeof baseUrl === "string" && baseUrl.trim()) return baseUrl.trim();
+  } catch {
+    // ignore
+  }
+  return undefined;
+}
+
+function getPlatformUrl(): string {
+  return (
+    process.env.VELLUM_PLATFORM_URL ??
+    getConfigPlatformUrl() ??
+    "https://platform.vellum.ai"
+  );
+}
+
+function buildHeaders(): Record<string, string> {
+  const headers: Record<string, string> = {};
+  const token = readPlatformToken();
+  if (token) {
+    headers["X-Session-Token"] = token;
+  }
+  return headers;
+}
+
+// ─── Catalog operations ──────────────────────────────────────────────────────
+
+export async function fetchCatalog(): Promise<CatalogSkill[]> {
+  const url = `${getPlatformUrl()}/v1/skills/`;
+  const response = await fetch(url, {
+    headers: buildHeaders(),
+    signal: AbortSignal.timeout(10000),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Platform API error ${response.status}: ${response.statusText}`,
+    );
+  }
+
+  const manifest = (await response.json()) as CatalogManifest;
+  if (!Array.isArray(manifest.skills)) {
+    throw new Error("Platform catalog has invalid skills array");
+  }
+  return manifest.skills;
+}
+
+export function readLocalCatalog(repoSkillsDir: string): CatalogSkill[] {
+  try {
+    const raw = readFileSync(join(repoSkillsDir, "catalog.json"), "utf-8");
+    const manifest = JSON.parse(raw) as CatalogManifest;
+    if (!Array.isArray(manifest.skills)) return [];
+    return manifest.skills;
+  } catch {
+    return [];
+  }
+}
+
+// ─── Tar extraction ──────────────────────────────────────────────────────────
+
+/**
+ * Extract all files from a tar archive (uncompressed) into a directory.
+ * Returns true if a SKILL.md was found in the archive.
+ */
+export function extractTarToDir(tarBuffer: Buffer, destDir: string): boolean {
+  let foundSkillMd = false;
+  let offset = 0;
+  while (offset + 512 <= tarBuffer.length) {
+    const header = tarBuffer.subarray(offset, offset + 512);
+
+    // End-of-archive (two consecutive zero blocks)
+    if (header.every((b) => b === 0)) break;
+
+    // Filename (bytes 0-99, null-terminated)
+    const nameEnd = header.indexOf(0, 0);
+    const name = header
+      .subarray(0, Math.min(nameEnd >= 0 ? nameEnd : 100, 100))
+      .toString("utf-8");
+
+    // File type (byte 156): '5' = directory, '0' or '\0' = regular file
+    const typeFlag = header[156];
+
+    // File size (bytes 124-135, octal)
+    const sizeStr = header.subarray(124, 136).toString("utf-8").trim();
+    const size = parseInt(sizeStr, 8) || 0;
+
+    offset += 512; // past header
+
+    // Skip directories and empty names
+    if (name && typeFlag !== 53 /* '5' */) {
+      // Prevent path traversal
+      const normalizedName = name.replace(/^\.\//, "");
+      if (!normalizedName.startsWith("..") && !normalizedName.includes("/..")) {
+        const destPath = join(destDir, normalizedName);
+        mkdirSync(dirname(destPath), { recursive: true });
+        writeFileSync(destPath, tarBuffer.subarray(offset, offset + size));
+
+        if (
+          normalizedName === "SKILL.md" ||
+          normalizedName.endsWith("/SKILL.md")
+        ) {
+          foundSkillMd = true;
+        }
+      }
+    }
+
+    // Skip to next header (data padded to 512 bytes)
+    offset += Math.ceil(size / 512) * 512;
+  }
+  return foundSkillMd;
+}
+
+export async function fetchAndExtractSkill(
+  skillId: string,
+  destDir: string,
+): Promise<void> {
+  const url = `${getPlatformUrl()}/v1/skills/${encodeURIComponent(skillId)}/`;
+  const response = await fetch(url, {
+    headers: buildHeaders(),
+    signal: AbortSignal.timeout(15000),
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch skill "${skillId}": HTTP ${response.status}`,
+    );
+  }
+
+  const gzipBuffer = Buffer.from(await response.arrayBuffer());
+  const tarBuffer = gunzipSync(gzipBuffer);
+  const foundSkillMd = extractTarToDir(tarBuffer, destDir);
+
+  if (!foundSkillMd) {
+    throw new Error(`SKILL.md not found in archive for "${skillId}"`);
+  }
+}
+
+// ─── SKILLS.md index management ──────────────────────────────────────────────
+
+function atomicWriteFile(filePath: string, content: string): void {
+  const dir = dirname(filePath);
+  mkdirSync(dir, { recursive: true });
+  const tmpPath = join(dir, `.tmp-${randomUUID()}`);
+  writeFileSync(tmpPath, content, "utf-8");
+  renameSync(tmpPath, filePath);
+}
+
+export function upsertSkillsIndex(id: string): void {
+  const indexPath = getSkillsIndexPath();
+  let lines: string[] = [];
+  if (existsSync(indexPath)) {
+    lines = readFileSync(indexPath, "utf-8").split("\n");
+  }
+
+  const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const pattern = new RegExp(`^[-*]\\s+(?:\`)?${escaped}(?:\`)?\\s*$`);
+  if (lines.some((line) => pattern.test(line))) return;
+
+  const nonEmpty = lines.filter((l) => l.trim());
+  nonEmpty.push(`- ${id}`);
+  const content = nonEmpty.join("\n");
+  atomicWriteFile(indexPath, content.endsWith("\n") ? content : content + "\n");
+}
+
+export function removeSkillsIndexEntry(id: string): void {
+  const indexPath = getSkillsIndexPath();
+  if (!existsSync(indexPath)) return;
+
+  const lines = readFileSync(indexPath, "utf-8").split("\n");
+  const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const pattern = new RegExp(`^[-*]\\s+(?:\`)?${escaped}(?:\`)?\\s*$`);
+  const filtered = lines.filter((line) => !pattern.test(line));
+
+  // If nothing changed, skip the write
+  if (filtered.length === lines.length) return;
+
+  const content = filtered.join("\n");
+  atomicWriteFile(indexPath, content.endsWith("\n") ? content : content + "\n");
+}
+
+// ─── Install / uninstall ─────────────────────────────────────────────────────
+
+export function uninstallSkillLocally(skillId: string): void {
+  const skillDir = join(getWorkspaceSkillsDir(), skillId);
+
+  if (!existsSync(skillDir)) {
+    throw new Error(`Skill "${skillId}" is not installed.`);
+  }
+
+  rmSync(skillDir, { recursive: true, force: true });
+  removeSkillsIndexEntry(skillId);
+}
+
+export async function installSkillLocally(
+  skillId: string,
+  catalogEntry: CatalogSkill,
+  overwrite: boolean,
+): Promise<void> {
+  const skillDir = join(getWorkspaceSkillsDir(), skillId);
+  const skillFilePath = join(skillDir, "SKILL.md");
+
+  if (existsSync(skillFilePath) && !overwrite) {
+    throw new Error(
+      `Skill "${skillId}" is already installed. Use --overwrite to replace it.`,
+    );
+  }
+
+  mkdirSync(skillDir, { recursive: true });
+
+  // In dev mode, install from the local repo skills directory if available
+  const repoSkillsDir = getRepoSkillsDir();
+  const repoSkillSource = repoSkillsDir
+    ? join(repoSkillsDir, skillId)
+    : undefined;
+
+  if (repoSkillSource && existsSync(join(repoSkillSource, "SKILL.md"))) {
+    cpSync(repoSkillSource, skillDir, { recursive: true });
+  } else {
+    await fetchAndExtractSkill(skillId, skillDir);
+  }
+
+  // Write version metadata
+  if (catalogEntry.version) {
+    const meta = {
+      version: catalogEntry.version,
+      installedAt: new Date().toISOString(),
+    };
+    atomicWriteFile(
+      join(skillDir, "version.json"),
+      JSON.stringify(meta, null, 2) + "\n",
+    );
+  }
+
+  // Install npm dependencies if the skill has a package.json
+  if (existsSync(join(skillDir, "package.json"))) {
+    const bunPath = `${homedir()}/.bun/bin`;
+    execSync("bun install", {
+      cwd: skillDir,
+      stdio: "inherit",
+      env: { ...process.env, PATH: `${bunPath}:${process.env.PATH}` },
+    });
+  }
+
+  // Register in SKILLS.md only after all steps succeed
+  upsertSkillsIndex(skillId);
+}
+
+// ─── Auto-install (for skill_load) ──────────────────────────────────────────
+
+/**
+ * Attempt to find and install a skill from the first-party catalog.
+ * Returns true if the skill was installed, false if not found in catalog.
+ * Throws on install failures (network, filesystem, etc).
+ */
+export async function autoInstallFromCatalog(
+  skillId: string,
+): Promise<boolean> {
+  // Check local catalog first (dev mode), then remote
+  const repoSkillsDir = getRepoSkillsDir();
+  let entry: CatalogSkill | undefined;
+
+  if (repoSkillsDir) {
+    const localCatalog = readLocalCatalog(repoSkillsDir);
+    entry = localCatalog.find((s) => s.id === skillId);
+  }
+
+  if (!entry) {
+    try {
+      const remoteCatalog = await fetchCatalog();
+      entry = remoteCatalog.find((s) => s.id === skillId);
+    } catch (err) {
+      log.warn(
+        { err, skillId },
+        "Failed to fetch remote catalog for auto-install",
+      );
+      return false;
+    }
+  }
+
+  if (!entry) {
+    return false;
+  }
+
+  await installSkillLocally(skillId, entry, false);
+  return true;
+}

package/src/skills/include-graph.ts

@@ -151,3 +151,35 @@ export function traverseIncludes(
   dfs(rootId);
   return { visited };
 }
+
+/**
+ * Collect all missing skill IDs reachable from the root's include graph.
+ * DFS traversal that tracks visited nodes to prevent infinite loops on cycles.
+ * The root itself is never reported as missing (it's already loaded by the caller).
+ */
+export function collectAllMissing(
+  rootId: string,
+  catalogIndex: Map<string, SkillSummary>,
+): Set<string> {
+  const missing = new Set<string>();
+  const visited = new Set<string>();
+
+  function dfs(id: string): void {
+    if (visited.has(id)) return;
+    visited.add(id);
+
+    const skill = catalogIndex.get(id);
+    if (!skill?.includes) return;
+
+    for (const childId of skill.includes) {
+      if (!catalogIndex.has(childId)) {
+        missing.add(childId);
+      } else if (!visited.has(childId)) {
+        dfs(childId);
+      }
+    }
+  }
+
+  dfs(rootId);
+  return missing;
+}