@vellumai/assistant 0.8.8 → 0.8.9-staging.2

package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Tests for {@link getPluginCatalog}.
+ *
+ * The underlying `loadPluginCatalog` is mocked so we exercise the cache's
+ * TTL window, refresh-on-expiry, and stale-on-error fallback in isolation —
+ * no network and no dependence on the real GitHub-backed loader.
+ */
+
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  mock,
+  setSystemTime,
+  test,
+} from "bun:test";
+
+import type { PluginCatalog, SearchPluginsDeps } from "../search-plugins.js";
+import { PluginCatalogUnavailableError } from "../search-plugins.js";
+
+// Mock the loader the cache wraps. `loadSpy` records every call and returns
+// (or throws) whatever the test configures. The real error class is passed
+// through so propagation preserves its identity.
+const loadSpy = mock(
+  async (
+    _opts: { ref: string },
+    _deps: SearchPluginsDeps,
+  ): Promise<PluginCatalog> => {
+    throw new Error("loadSpy default impl not configured");
+  },
+);
+
+mock.module("../search-plugins.js", () => ({
+  PluginCatalogUnavailableError,
+  loadPluginCatalog: loadSpy,
+}));
+
+const {
+  PLUGIN_CATALOG_CACHE_TTL_MS,
+  getPluginCatalog,
+  invalidatePluginCatalogCache,
+} = await import("../plugin-catalog-cache.js");
+
+const deps: SearchPluginsDeps = {
+  fetch: (async () => new Response("", { status: 200 })) as never,
+};
+
+// A non-zero base time. Bun treats `setSystemTime(new Date(0))` as "reset to
+// the real clock", so the fake clock must start from a positive epoch.
+const BASE_TIME_MS = 1_700_000_000_000;
+
+function catalog(ref: string, names: string[]): PluginCatalog {
+  return {
+    ref,
+    matches: names.map((name) => ({
+      name,
+      path: `experimental/plugins/${name}`,
+      source: { kind: "first-party" as const },
+    })),
+  };
+}
+
+describe("getPluginCatalog", () => {
+  beforeEach(() => {
+    loadSpy.mockClear();
+    invalidatePluginCatalogCache();
+  });
+
+  afterEach(() => {
+    // Restore real time in case a test installed a fake clock.
+    setSystemTime();
+  });
+
+  test("loads once and serves the cached catalog within the TTL window", async () => {
+    // GIVEN a loader that returns a catalog
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["a"]));
+
+    // WHEN we request the same ref twice in quick succession
+    const first = await getPluginCatalog("main", deps);
+    const second = await getPluginCatalog("main", deps);
+
+    // THEN the loader is consulted exactly once — the second call is cached
+    expect(loadSpy).toHaveBeenCalledTimes(1);
+    expect(first).toBe(second);
+    expect(first.matches.map((m) => m.name)).toEqual(["a"]);
+  });
+
+  test("caches per ref independently", async () => {
+    // GIVEN a loader that echoes the requested ref
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, [ref]));
+
+    // WHEN we request two different refs
+    const main = await getPluginCatalog("main", deps);
+    const branch = await getPluginCatalog("feature", deps);
+
+    // THEN each ref triggers its own load and is keyed separately
+    expect(loadSpy).toHaveBeenCalledTimes(2);
+    expect(main.ref).toBe("main");
+    expect(branch.ref).toBe("feature");
+  });
+
+  test("refreshes after the TTL elapses", async () => {
+    // GIVEN a fresh load at t0
+    setSystemTime(new Date(BASE_TIME_MS));
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["v1"]));
+    await getPluginCatalog("main", deps);
+
+    // AND the loader now returns updated data
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["v2"]));
+
+    // WHEN the TTL has elapsed and we request again
+    setSystemTime(new Date(BASE_TIME_MS + PLUGIN_CATALOG_CACHE_TTL_MS + 1));
+    const refreshed = await getPluginCatalog("main", deps);
+
+    // THEN the loader is consulted again and the new catalog is returned
+    expect(loadSpy).toHaveBeenCalledTimes(2);
+    expect(refreshed.matches.map((m) => m.name)).toEqual(["v2"]);
+  });
+
+  test("serves the stale catalog when a refresh fails", async () => {
+    // GIVEN a good catalog cached at t0
+    setSystemTime(new Date(BASE_TIME_MS));
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["good"]));
+    const fresh = await getPluginCatalog("main", deps);
+
+    // AND the upstream later fails (rate-limited)
+    loadSpy.mockImplementation(async () => {
+      throw new PluginCatalogUnavailableError("rate limited", 403);
+    });
+
+    // WHEN the TTL has elapsed and the refresh fails
+    setSystemTime(new Date(BASE_TIME_MS + PLUGIN_CATALOG_CACHE_TTL_MS + 1));
+    const stale = await getPluginCatalog("main", deps);
+
+    // THEN the last good catalog is served instead of throwing
+    expect(stale).toBe(fresh);
+    expect(stale.matches.map((m) => m.name)).toEqual(["good"]);
+  });
+
+  test("resets the TTL window after serving stale so it doesn't re-fetch every call", async () => {
+    // GIVEN a cached catalog and a now-failing upstream
+    setSystemTime(new Date(BASE_TIME_MS));
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["good"]));
+    await getPluginCatalog("main", deps);
+    loadSpy.mockImplementation(async () => {
+      throw new PluginCatalogUnavailableError("rate limited", 403);
+    });
+
+    // WHEN the TTL elapses (one failed refresh) and we immediately ask again
+    setSystemTime(new Date(BASE_TIME_MS + PLUGIN_CATALOG_CACHE_TTL_MS + 1));
+    await getPluginCatalog("main", deps);
+    const callsAfterFirstStale = loadSpy.mock.calls.length;
+    await getPluginCatalog("main", deps);
+
+    // THEN the second request is served from the reset window without
+    // re-entering the failing loader
+    expect(loadSpy.mock.calls.length).toBe(callsAfterFirstStale);
+  });
+
+  test("propagates a hard error even with a cache, and does not serve stale", async () => {
+    // GIVEN a good catalog cached at t0
+    setSystemTime(new Date(BASE_TIME_MS));
+    loadSpy.mockImplementation(async ({ ref }) => catalog(ref, ["good"]));
+    await getPluginCatalog("main", deps);
+
+    // AND the upstream later fails with a HARD error (e.g. the prefix is
+    // gone), NOT a transient PluginCatalogUnavailableError
+    loadSpy.mockImplementation(async () => {
+      throw new Error("GitHub contents listing failed: HTTP 404");
+    });
+
+    // WHEN the TTL has elapsed and the refresh hard-fails
+    setSystemTime(new Date(BASE_TIME_MS + PLUGIN_CATALOG_CACHE_TTL_MS + 1));
+    const err = await getPluginCatalog("main", deps).catch((e: unknown) => e);
+
+    // THEN the hard error propagates instead of masking the misconfiguration
+    // with stale data
+    expect(err).toBeInstanceOf(Error);
+    expect(err).not.toBeInstanceOf(PluginCatalogUnavailableError);
+    expect((err as Error).message).toMatch(/HTTP 404/);
+  });
+
+  test("propagates the error when there is no cache to fall back on", async () => {
+    // GIVEN no prior successful load and a failing upstream
+    loadSpy.mockImplementation(async () => {
+      throw new PluginCatalogUnavailableError("rate limited", 403);
+    });
+
+    // WHEN we request a cold catalog
+    const err = await getPluginCatalog("main", deps).catch((e: unknown) => e);
+
+    // THEN the failure propagates so the caller can map it to 503
+    expect(err).toBeInstanceOf(PluginCatalogUnavailableError);
+  });
+});

package/src/cli/lib/__tests__/plugin-details.test.ts

@@ -0,0 +1,372 @@
+/**
+ * Tests for {@link getPluginDetails}.
+ *
+ * Network is replaced with an in-memory GitHub Contents API fixture passed via
+ * the `fetch` dependency, and the installed-copy path is exercised against a
+ * real temp directory passed via `workspacePluginsDir` — no globals are
+ * monkey-patched. The fixture answers three URL shapes:
+ *   - the marketplace manifest file (raw JSON body),
+ *   - directory listings (`/contents/<path>` → entry array or 404),
+ *   - raw file downloads (a listing entry's `download_url`).
+ */
+
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import type { FetchLike } from "../install-from-github.js";
+import {
+  getPluginDetails,
+  PluginDetailsNotFoundError,
+} from "../plugin-details.js";
+
+interface ContentEntry {
+  name: string;
+  path: string;
+  type: "file" | "dir" | "symlink" | "submodule";
+  download_url: string | null;
+}
+
+function fileEntry(name: string, downloadUrl: string): ContentEntry {
+  return { name, path: name, type: "file", download_url: downloadUrl };
+}
+
+interface FixtureConfig {
+  /** Manifest object served at `experimental/plugins/marketplace.json`. Omit for 404. */
+  marketplace?: unknown;
+  /** Directory listings keyed by `<owner>/<repo>[/<path>]`. Missing key → 404. */
+  listings?: Record<string, ContentEntry[]>;
+  /** Raw file bodies keyed by `download_url`. Missing key → 404. */
+  raw?: Record<string, string>;
+  /** URL substrings that should reject with a network error. */
+  failOn?: string[];
+}
+
+/**
+ * Build a `fetch` that routes GitHub Contents API requests against in-memory
+ * fixtures. Anything unrecognised returns 500 so test bugs surface loudly.
+ */
+function makeFetch(config: FixtureConfig): FetchLike {
+  return (async (input: RequestInfo | URL) => {
+    const url = typeof input === "string" ? input : input.toString();
+
+    for (const needle of config.failOn ?? []) {
+      if (url.includes(needle)) throw new Error(`network down: ${needle}`);
+    }
+
+    if (url.includes("marketplace.json")) {
+      if (config.marketplace === undefined) {
+        return new Response("not found", { status: 404 });
+      }
+      return new Response(JSON.stringify(config.marketplace), { status: 200 });
+    }
+
+    if (config.raw && url in config.raw) {
+      return new Response(config.raw[url], { status: 200 });
+    }
+
+    if (url.includes("/contents")) {
+      const key = listingKey(url);
+      const entries = config.listings?.[key];
+      if (!entries) return new Response("not found", { status: 404 });
+      return new Response(JSON.stringify(entries), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }
+
+    return new Response("unexpected url: " + url, { status: 500 });
+  }) as FetchLike;
+}
+
+/** Derive the `<owner>/<repo>[/<path>]` listing key from a contents URL. */
+function listingKey(url: string): string {
+  const afterRepos = url.split("/repos/")[1] ?? "";
+  const [ownerRepo, rest = ""] = splitOnce(afterRepos, "/contents");
+  const pathPart = rest.split("?")[0] ?? ""; // leading "/" or ""
+  return `${ownerRepo}${pathPart}`;
+}
+
+function splitOnce(s: string, sep: string): [string, string] {
+  const i = s.indexOf(sep);
+  if (i === -1) return [s, ""];
+  return [s.slice(0, i), s.slice(i + sep.length)];
+}
+
+let workspace: string;
+
+beforeEach(() => {
+  workspace = mkdtempSync(join(tmpdir(), "plugin-details-"));
+});
+
+afterEach(() => {
+  rmSync(workspace, { recursive: true, force: true });
+});
+
+describe("getPluginDetails", () => {
+  test("resolves an external plugin: manifest metadata + repo README/package.json", async () => {
+    // GIVEN a marketplace entry for an external, not-installed plugin
+    const fetch = makeFetch({
+      marketplace: {
+        name: "vellum",
+        plugins: [
+          {
+            name: "caveman",
+            source: {
+              source: "github",
+              repo: "example-org/caveman",
+              ref: "v1",
+            },
+            description: "manifest description",
+            homepage: "https://example.com/caveman",
+            license: "MIT",
+          },
+        ],
+      },
+      // AND no first-party directory for the name (404), but the external repo
+      // root lists a README and package.json
+      listings: {
+        "example-org/caveman": [
+          fileEntry("README.md", "raw://caveman/readme"),
+          fileEntry("package.json", "raw://caveman/pkg"),
+        ],
+      },
+      raw: {
+        "raw://caveman/readme": "# Caveman\n\nGrug brain plugin.",
+        "raw://caveman/pkg": JSON.stringify({
+          version: "1.8.2",
+          description: "package description",
+          homepage: "https://pkg.example.com",
+        }),
+      },
+    });
+
+    // WHEN we resolve the detail view at a ref
+    const details = await getPluginDetails(
+      { name: "caveman", ref: "v1" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    // THEN the source is the external repo and the README comes from it
+    expect(details.source).toEqual({
+      kind: "github",
+      repo: "example-org/caveman",
+      ref: "v1",
+    });
+    expect(details.installed).toBe(false);
+    expect(details.readme).toContain("Grug brain plugin");
+    // AND manifest fields win over the repo package.json for description/homepage
+    expect(details.description).toBe("manifest description");
+    expect(details.homepage).toBe("https://example.com/caveman");
+    expect(details.license).toBe("MIT");
+    // AND version falls back to the repo package.json (manifest has none)
+    expect(details.version).toBe("1.8.2");
+    expect(details.ref).toBe("v1");
+  });
+
+  test("reads an external plugin at its pinned source ref, not the catalog ref", async () => {
+    // GIVEN a marketplace entry pinned to a ref that differs from the catalog
+    // ref the detail view is resolved at
+    const marketplace = {
+      name: "vellum",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "example-org/caveman",
+            ref: "v1.8.2",
+          },
+        },
+      ],
+    };
+    // AND a fetch that records the ref query param of every contents request
+    const contentsRefs = new Map<string, string>();
+    const fetch = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("marketplace.json")) {
+        return new Response(JSON.stringify(marketplace), { status: 200 });
+      }
+      if (url.includes("/contents")) {
+        const ref = new URL(url).searchParams.get("ref") ?? "";
+        const key = url.includes("example-org/caveman")
+          ? "external"
+          : "first-party";
+        contentsRefs.set(key, ref);
+        if (key === "external") {
+          return new Response(
+            JSON.stringify([fileEntry("README.md", "raw://caveman/readme")]),
+            { status: 200, headers: { "content-type": "application/json" } },
+          );
+        }
+        return new Response("not found", { status: 404 });
+      }
+      if (url === "raw://caveman/readme") {
+        return new Response("# Caveman", { status: 200 });
+      }
+      return new Response("unexpected url: " + url, { status: 500 });
+    }) as FetchLike;
+
+    // WHEN we resolve the detail view at the catalog ref `main`
+    const details = await getPluginDetails(
+      { name: "caveman", ref: "main" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    // THEN the external repo is read at the plugin's pinned ref, not `main`
+    expect(contentsRefs.get("external")).toBe("v1.8.2");
+    // AND the first-party collision probe still uses the catalog ref
+    expect(contentsRefs.get("first-party")).toBe("main");
+    // AND the README from the pinned ref is surfaced
+    expect(details.readme).toContain("Caveman");
+  });
+
+  test("first-party directory wins a name also claimed by the marketplace", async () => {
+    // GIVEN both a marketplace entry AND a first-party in-repo directory for
+    // the same name
+    const fetch = makeFetch({
+      marketplace: {
+        name: "vellum",
+        plugins: [
+          {
+            name: "simple-memory",
+            source: {
+              source: "github",
+              repo: "example-org/simple-memory",
+              ref: "v9",
+            },
+          },
+        ],
+      },
+      listings: {
+        "vellum-ai/vellum-assistant/experimental/plugins/simple-memory": [
+          fileEntry("README.md", "raw://fp/readme"),
+        ],
+      },
+      raw: { "raw://fp/readme": "# Simple Memory (first-party)" },
+    });
+
+    // WHEN we resolve the detail view
+    const details = await getPluginDetails(
+      { name: "simple-memory" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    // THEN the first-party source is used, not the external one
+    expect(details.source).toEqual({ kind: "first-party" });
+    expect(details.readme).toContain("first-party");
+  });
+
+  test("prefers an installed copy's README and package.json over the repo", async () => {
+    // GIVEN an installed copy on disk with its own README + package.json
+    const target = join(workspace, "caveman");
+    mkdirSync(target, { recursive: true });
+    writeFileSync(join(target, "README.md"), "# Installed Caveman");
+    writeFileSync(
+      join(target, "package.json"),
+      JSON.stringify({ version: "2.0.0", license: "Apache-2.0" }),
+    );
+
+    // AND a marketplace entry + external repo that would otherwise be used
+    const fetch = makeFetch({
+      marketplace: {
+        name: "vellum",
+        plugins: [
+          {
+            name: "caveman",
+            source: {
+              source: "github",
+              repo: "example-org/caveman",
+              ref: "v1",
+            },
+            description: "manifest description",
+          },
+        ],
+      },
+      listings: {
+        "example-org/caveman": [fileEntry("README.md", "raw://caveman/readme")],
+      },
+      raw: { "raw://caveman/readme": "# Remote Caveman" },
+    });
+
+    // WHEN we resolve the detail view
+    const details = await getPluginDetails(
+      { name: "caveman" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    // THEN the installed README + version/license win; manifest fills the gap
+    expect(details.installed).toBe(true);
+    expect(details.readme).toBe("# Installed Caveman");
+    expect(details.version).toBe("2.0.0");
+    expect(details.license).toBe("Apache-2.0");
+    expect(details.description).toBe("manifest description");
+  });
+
+  test("throws PluginDetailsNotFoundError when nothing claims the name", async () => {
+    // GIVEN no install, no first-party directory, and an empty marketplace
+    const fetch = makeFetch({
+      marketplace: { name: "vellum", plugins: [] },
+    });
+
+    // WHEN / THEN resolving an unknown name rejects with the not-found error
+    await expect(
+      getPluginDetails(
+        { name: "ghost" },
+        { fetch, workspacePluginsDir: workspace },
+      ),
+    ).rejects.toBeInstanceOf(PluginDetailsNotFoundError);
+  });
+
+  test("degrades to manifest metadata when the repo listing fails", async () => {
+    // GIVEN a marketplace entry whose external repo listing errors out
+    const fetch = makeFetch({
+      marketplace: {
+        name: "vellum",
+        plugins: [
+          {
+            name: "caveman",
+            source: {
+              source: "github",
+              repo: "example-org/caveman",
+              ref: "v1",
+            },
+            description: "manifest description",
+            license: "MIT",
+          },
+        ],
+      },
+      failOn: ["example-org/caveman"],
+    });
+
+    // WHEN we resolve the detail view
+    const details = await getPluginDetails(
+      { name: "caveman", ref: "v1" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    // THEN it still renders manifest metadata with a null README rather than throwing
+    expect(details.readme).toBeNull();
+    expect(details.description).toBe("manifest description");
+    expect(details.license).toBe("MIT");
+    expect(details.source).toEqual({
+      kind: "github",
+      repo: "example-org/caveman",
+      ref: "v1",
+    });
+  });
+
+  test("rejects an invalid (path-traversal) plugin name", async () => {
+    // GIVEN a name that fails the install-name sanitizer
+    const fetch = makeFetch({ marketplace: { name: "vellum", plugins: [] } });
+
+    // WHEN / THEN resolution throws before any lookup
+    await expect(
+      getPluginDetails(
+        { name: "../escape" },
+        { fetch, workspacePluginsDir: workspace },
+      ),
+    ).rejects.toThrow();
+  });
+});