@vellumai/assistant 0.8.8 → 0.8.9-dev.202606091926.ebb2d62

package/src/cli/lib/__tests__/plugin-marketplace.test.ts

@@ -0,0 +1,270 @@
+/**
+ * Tests for {@link fetchMarketplaceEntries} and {@link resolveMarketplaceSource}.
+ *
+ * Network is replaced with an in-memory fixture passed via the `fetch`
+ * dependency — no globals are monkey-patched.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import type { FetchLike } from "../install-from-github.js";
+import {
+  fetchMarketplaceEntries,
+  type MarketplaceEntry,
+  MarketplaceFetchError,
+  resolveMarketplaceSource,
+} from "../plugin-marketplace.js";
+
+const MANIFEST_URL_PREFIX =
+  "https://api.github.com/repos/vellum-ai/vellum-assistant/contents/experimental/plugins/marketplace.json";
+
+// External marketplace refs must be full commit SHAs (immutable). Fixtures use
+// realistic 40-char hex object names rather than tags/branches.
+const CAVEMAN_SHA = "63a91ecadbf4c4719a4602a5abb00883f9966034";
+const NESTED_SHA = "0123456789abcdef0123456789abcdef01234567";
+
+/** Serve `body` (any value) as the raw manifest file at the manifest URL. */
+function manifestFetch(body: unknown, status = 200, raw?: string): FetchLike {
+  return (async (input: RequestInfo | URL) => {
+    const url = typeof input === "string" ? input : input.toString();
+    if (!url.startsWith(MANIFEST_URL_PREFIX)) {
+      return new Response("unexpected url: " + url, { status: 500 });
+    }
+    if (status !== 200) {
+      return new Response("error", { status });
+    }
+    return new Response(raw ?? JSON.stringify(body), { status: 200 });
+  }) as FetchLike;
+}
+
+const VALID_MANIFEST = {
+  name: "vellum-assistant",
+  owner: { name: "Vellum" },
+  plugins: [
+    {
+      name: "caveman",
+      source: {
+        source: "github",
+        repo: "JuliusBrussee/caveman",
+        ref: CAVEMAN_SHA,
+      },
+      description: "Ultra-compressed communication mode.",
+      category: "productivity",
+    },
+    {
+      name: "nested",
+      source: {
+        source: "github",
+        repo: "acme/monorepo",
+        path: "packages/nested",
+        ref: NESTED_SHA,
+      },
+    },
+  ],
+};
+
+describe("fetchMarketplaceEntries", () => {
+  test("parses a valid manifest and returns its plugin entries", async () => {
+    // GIVEN a canonical repo serving a valid marketplace manifest
+    const fetch = manifestFetch(VALID_MANIFEST);
+
+    // WHEN we fetch the marketplace entries at a ref
+    const entries = await fetchMarketplaceEntries({ fetch }, { ref: "main" });
+
+    // THEN both whitelisted plugins are returned with their pinned sources
+    expect(entries.map((e) => e.name)).toEqual(["caveman", "nested"]);
+    expect(entries[0]!.source).toEqual({
+      source: "github",
+      repo: "JuliusBrussee/caveman",
+      ref: CAVEMAN_SHA,
+    });
+    expect(entries[1]!.source.path).toBe("packages/nested");
+  });
+
+  test("forwards the ref to the GitHub Contents API", async () => {
+    // GIVEN a fixture that records the ref query parameter
+    let seenRef: string | undefined;
+    const fetch: FetchLike = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      seenRef = /[?&]ref=([^&]+)/.exec(url)?.[1];
+      return new Response(JSON.stringify(VALID_MANIFEST), { status: 200 });
+    }) as FetchLike;
+
+    // WHEN we fetch at an explicit ref
+    await fetchMarketplaceEntries({ fetch }, { ref: "feat-branch" });
+
+    // THEN the ref reaches GitHub
+    expect(seenRef).toBe("feat-branch");
+  });
+
+  test("treats a missing manifest (404) as an empty whitelist", async () => {
+    // GIVEN no manifest exists at this ref
+    const fetch = manifestFetch(null, 404);
+
+    // WHEN we fetch the entries
+    const entries = await fetchMarketplaceEntries({ fetch }, { ref: "main" });
+
+    // THEN the absence is a normal empty result, not an error
+    expect(entries).toEqual([]);
+  });
+
+  test("throws MarketplaceFetchError on non-404 HTTP failure", async () => {
+    // GIVEN GitHub returns a server error
+    const fetch = manifestFetch(null, 503);
+
+    // WHEN / THEN the failure surfaces as a typed error
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+
+  test("throws MarketplaceFetchError on non-JSON body", async () => {
+    // GIVEN the manifest body is not valid JSON
+    const fetch = manifestFetch(undefined, 200, "{ not json");
+
+    // WHEN / THEN parsing fails loudly
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+
+  test("rejects a mutable tag or branch ref (must be a full commit SHA)", async () => {
+    // GIVEN an entry whose source pins a mutable tag rather than a commit SHA
+    const fetch = manifestFetch({
+      name: "x",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "JuliusBrussee/caveman",
+            ref: "v1.8.2",
+          },
+        },
+      ],
+    });
+
+    // WHEN / THEN the mutable ref is rejected: a retag could repoint it at
+    // attacker code the daemon later dynamically imports.
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+
+  test("rejects an abbreviated commit SHA", async () => {
+    // GIVEN an entry whose source uses a short (ambiguous, non-pinning) SHA
+    const fetch = manifestFetch({
+      name: "x",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "JuliusBrussee/caveman",
+            ref: "63a91ec",
+          },
+        },
+      ],
+    });
+
+    // WHEN / THEN only a full object name pins the install immutably
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+
+  test("rejects a manifest entry missing a pinned ref", async () => {
+    // GIVEN an entry whose source omits the required ref
+    const fetch = manifestFetch({
+      name: "x",
+      plugins: [
+        { name: "floating", source: { source: "github", repo: "a/b" } },
+      ],
+    });
+
+    // WHEN / THEN schema validation rejects the unpinned source
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+
+  test("rejects a path that escapes the repo root", async () => {
+    // GIVEN an entry whose path contains a parent-segment escape
+    const fetch = manifestFetch({
+      name: "x",
+      plugins: [
+        {
+          name: "escape",
+          source: {
+            source: "github",
+            repo: "a/b",
+            path: "../../etc",
+            ref: "main",
+          },
+        },
+      ],
+    });
+
+    // WHEN / THEN the unsafe path is rejected
+    await expect(
+      fetchMarketplaceEntries({ fetch }, { ref: "main" }),
+    ).rejects.toBeInstanceOf(MarketplaceFetchError);
+  });
+});
+
+describe("resolveMarketplaceSource", () => {
+  const entries: readonly MarketplaceEntry[] = [
+    {
+      name: "caveman",
+      source: {
+        source: "github",
+        repo: "JuliusBrussee/caveman",
+        ref: CAVEMAN_SHA,
+      },
+    },
+    {
+      name: "nested",
+      source: {
+        source: "github",
+        repo: "acme/monorepo",
+        path: "packages/nested",
+        ref: NESTED_SHA,
+      },
+    },
+  ];
+
+  test("resolves a repo-root plugin to owner/repo with empty path", () => {
+    // GIVEN whitelisted entries
+    // WHEN resolving a name whose source has no sub-path
+    const resolved = resolveMarketplaceSource("caveman", entries);
+
+    // THEN the owner/repo split out and path defaults to repo root
+    expect(resolved).toEqual({
+      owner: "JuliusBrussee",
+      repo: "caveman",
+      path: "",
+      ref: CAVEMAN_SHA,
+    });
+  });
+
+  test("resolves a sub-path plugin to its directory", () => {
+    // GIVEN whitelisted entries
+    // WHEN resolving a name whose source pins a sub-path
+    const resolved = resolveMarketplaceSource("nested", entries);
+
+    // THEN the sub-path is preserved
+    expect(resolved).toEqual({
+      owner: "acme",
+      repo: "monorepo",
+      path: "packages/nested",
+      ref: NESTED_SHA,
+    });
+  });
+
+  test("returns null for a name not in the whitelist", () => {
+    // GIVEN whitelisted entries
+    // WHEN resolving an unknown name
+    // THEN no source is returned (caller falls back to first-party)
+    expect(resolveMarketplaceSource("unknown", entries)).toBeNull();
+  });
+});

package/src/cli/lib/__tests__/search-plugins.test.ts

@@ -11,6 +11,7 @@ import { describe, expect, test } from "bun:test";
 import {
   type FetchLike,
   InvalidSearchPatternError,
+  PluginCatalogUnavailableError,
   searchPlugins,
 } from "../search-plugins.js";
 
@@ -180,44 +181,73 @@ describe("searchPlugins", () => {
     expect(result.matches.map((m) => m.name)).toEqual(["simple-memory"]);
   });
 
-  test("HTTP 5xx from GitHub propagates with the status code", async () => {
-    await expect(
-      searchPlugins(
-        { query: "memory" },
-        {
-          fetch: (async () =>
-            new Response("upstream broken", { status: 503 })) as FetchLike,
-        },
-      ),
-    ).rejects.toThrow(/HTTP 503/);
+  test("HTTP 5xx is a transient PluginCatalogUnavailableError", async () => {
+    // GIVEN GitHub returns a 5xx (upstream outage)
+    // WHEN we search
+    // THEN the failure is classified transient so the caller can serve a
+    // stale cache and the route can map it to 503 (not a misleading 500).
+    const err = await searchPlugins(
+      { query: "memory" },
+      {
+        fetch: (async () =>
+          new Response("upstream broken", { status: 503 })) as FetchLike,
+      },
+    ).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(PluginCatalogUnavailableError);
+    expect((err as PluginCatalogUnavailableError).status).toBe(503);
   });
 
-  test("HTTP 403 (rate-limited / forbidden) surfaces as an error", async () => {
-    await expect(
-      searchPlugins(
-        { query: "memory" },
-        {
-          fetch: (async () =>
-            new Response("rate limit exceeded", { status: 403 })) as FetchLike,
-        },
-      ),
-    ).rejects.toThrow(/HTTP 403/);
+  test("rate-limited 403 (x-ratelimit-remaining: 0) is transient", async () => {
+    // GIVEN GitHub returns 403 with the rate-limit budget exhausted
+    // WHEN we search
+    // THEN it's transient — this is exactly the unauthenticated 60 req/hr
+    // exhaustion that empties the catalog, and it must surface as 503.
+    const err = await searchPlugins(
+      { query: "memory" },
+      {
+        fetch: (async () =>
+          new Response("rate limit exceeded", {
+            status: 403,
+            headers: { "x-ratelimit-remaining": "0" },
+          })) as FetchLike,
+      },
+    ).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(PluginCatalogUnavailableError);
+    expect((err as PluginCatalogUnavailableError).status).toBe(403);
   });
 
-  test("404 on the plugins prefix surfaces as an error (not silently empty)", async () => {
+  test("a 403 without the rate-limit signal is a hard error", async () => {
+    // GIVEN a bare 403 (genuine permissions problem, not rate limiting)
+    // WHEN we search
+    // THEN it is NOT transient — serving a stale catalog would mask a real
+    // misconfiguration, so it propagates as a plain error.
+    const err = await searchPlugins(
+      { query: "memory" },
+      {
+        fetch: (async () =>
+          new Response("forbidden", { status: 403 })) as FetchLike,
+      },
+    ).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(Error);
+    expect(err).not.toBeInstanceOf(PluginCatalogUnavailableError);
+    expect((err as Error).message).toMatch(/HTTP 403/);
+  });
+
+  test("404 on the plugins prefix is a hard error (not silently empty)", async () => {
     // Distinct from `installPlugin`, where 404 on a specific plugin name is
     // normal "not found". For the search, 404 on the prefix means the
-    // canonical source path itself is gone — that's an upstream problem
-    // worth surfacing, not a clean empty result.
-    await expect(
-      searchPlugins(
-        { query: "memory" },
-        {
-          fetch: (async () =>
-            new Response("not found", { status: 404 })) as FetchLike,
-        },
-      ),
-    ).rejects.toThrow(/HTTP 404/);
+    // canonical source path itself is gone — that's a real misconfiguration
+    // worth surfacing, not a clean empty result and not a transient outage.
+    const err = await searchPlugins(
+      { query: "memory" },
+      {
+        fetch: (async () =>
+          new Response("not found", { status: 404 })) as FetchLike,
+      },
+    ).catch((e: unknown) => e);
+    expect(err).toBeInstanceOf(Error);
+    expect(err).not.toBeInstanceOf(PluginCatalogUnavailableError);
+    expect((err as Error).message).toMatch(/HTTP 404/);
   });
 
   test("returns matches sorted by name", async () => {
@@ -238,6 +268,177 @@ describe("searchPlugins", () => {
     ]);
   });
 
+  test("merges whitelisted marketplace entries with first-party dirs", async () => {
+    // GIVEN the canonical repo serves both a first-party plugin listing and a
+    // marketplace manifest whitelisting an external plugin
+    const manifest = {
+      name: "vellum-assistant",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "JuliusBrussee/caveman",
+            ref: "63a91ecadbf4c4719a4602a5abb00883f9966034",
+          },
+          description: "Ultra-compressed communication mode.",
+        },
+      ],
+    };
+    const fetch: FetchLike = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("marketplace.json")) {
+        return new Response(JSON.stringify(manifest), { status: 200 });
+      }
+      return new Response(
+        JSON.stringify([
+          {
+            name: "simple-memory",
+            path: "experimental/plugins/simple-memory",
+            type: "dir",
+            size: 0,
+            download_url: null,
+          },
+        ]),
+        { status: 200, headers: { "content-type": "application/json" } },
+      );
+    }) as FetchLike;
+
+    // WHEN we search the catalog with a match-all query
+    const result = await searchPlugins({ query: "" }, { fetch });
+
+    // THEN both sources appear, sorted by name, each tagged with its origin
+    expect(result.matches).toEqual([
+      {
+        name: "caveman",
+        path: "github:JuliusBrussee/caveman@63a91ecadbf4c4719a4602a5abb00883f9966034",
+        description: "Ultra-compressed communication mode.",
+        source: {
+          kind: "github",
+          repo: "JuliusBrussee/caveman",
+          ref: "63a91ecadbf4c4719a4602a5abb00883f9966034",
+        },
+      },
+      {
+        name: "simple-memory",
+        path: "experimental/plugins/simple-memory",
+        source: { kind: "first-party" },
+      },
+    ]);
+  });
+
+  test("filters marketplace entries by the query too", async () => {
+    // GIVEN a marketplace whitelisting an external plugin and no first-party dirs
+    const manifest = {
+      name: "vellum-assistant",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "JuliusBrussee/caveman",
+            ref: "63a91ecadbf4c4719a4602a5abb00883f9966034",
+          },
+        },
+      ],
+    };
+    const fetch: FetchLike = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("marketplace.json")) {
+        return new Response(JSON.stringify(manifest), { status: 200 });
+      }
+      return new Response(JSON.stringify([]), { status: 200 });
+    }) as FetchLike;
+
+    // WHEN the query matches no plugin name
+    const result = await searchPlugins({ query: "memory" }, { fetch });
+
+    // THEN the non-matching marketplace entry is excluded
+    expect(result.matches).toEqual([]);
+  });
+
+  test("a marketplace entry owns a name shared by an in-repo stub dir", async () => {
+    // GIVEN a marketplace entry named "caveman"
+    const manifest = {
+      name: "vellum-assistant",
+      plugins: [
+        {
+          name: "caveman",
+          source: {
+            source: "github",
+            repo: "JuliusBrussee/caveman",
+            ref: "63a91ecadbf4c4719a4602a5abb00883f9966034",
+          },
+        },
+      ],
+    };
+    // AND a same-named in-repo directory, which is caveman's adapter stub
+    // rather than a standalone first-party plugin
+    const fetch: FetchLike = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("marketplace.json")) {
+        return new Response(JSON.stringify(manifest), { status: 200 });
+      }
+      return new Response(
+        JSON.stringify([
+          {
+            name: "caveman",
+            path: "experimental/plugins/caveman",
+            type: "dir",
+            size: 0,
+            download_url: null,
+          },
+        ]),
+        { status: 200 },
+      );
+    }) as FetchLike;
+
+    // WHEN we search
+    const result = await searchPlugins({ query: "caveman" }, { fetch });
+
+    // THEN the name surfaces once, as the external marketplace entry — the
+    // stub dir is its adapter overlay, not a competing first-party listing
+    expect(result.matches).toEqual([
+      {
+        name: "caveman",
+        path: "github:JuliusBrussee/caveman@63a91ecadbf4c4719a4602a5abb00883f9966034",
+        source: {
+          kind: "github",
+          repo: "JuliusBrussee/caveman",
+          ref: "63a91ecadbf4c4719a4602a5abb00883f9966034",
+        },
+      },
+    ]);
+  });
+
+  test("a broken marketplace manifest degrades to the first-party listing", async () => {
+    // GIVEN the manifest is malformed but the first-party listing is healthy
+    const fetch: FetchLike = (async (input: RequestInfo | URL) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("marketplace.json")) {
+        return new Response("{ not json", { status: 200 });
+      }
+      return new Response(
+        JSON.stringify([
+          {
+            name: "simple-memory",
+            path: "experimental/plugins/simple-memory",
+            type: "dir",
+            size: 0,
+            download_url: null,
+          },
+        ]),
+        { status: 200 },
+      );
+    }) as FetchLike;
+
+    // WHEN we search
+    const result = await searchPlugins({ query: "" }, { fetch });
+
+    // THEN the core catalog is unaffected by the broken whitelist
+    expect(result.matches.map((m) => m.name)).toEqual(["simple-memory"]);
+  });
+
   test("sends no Authorization header (canonical source is a public repo)", async () => {
     let seenAuth: string | undefined;
     let seenUserAgent: string | undefined;