@stigmer/cli 3.16.0 → 3.16.1-dev.20260917224426

package/src/marketplace/github.test.ts

@@ -0,0 +1,318 @@
+// Pins the GitHub source over zipballs built in-test: a good tree lands with
+// its top directory stripped; every cap refuses naming itself, from the
+// central directory before anything inflates; an escaping path, a second
+// top-level directory and a lying declared size refuse; 404 and a network
+// failure carry their own exit codes.
+
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { zipSync } from "fflate";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { ExitCode } from "../errors/index.js";
+import {
+  GITHUB_ZIPBALL_LIMITS,
+  fetchGitHubTree,
+  zipballUrl,
+} from "./github.js";
+
+let dest: string;
+
+beforeEach(() => {
+  dest = mkdtempSync(join(tmpdir(), "stigmer-github-tree-"));
+});
+
+afterEach(() => {
+  rmSync(dest, { recursive: true, force: true });
+});
+
+const encoder = new TextEncoder();
+
+function zipball(files: Record<string, string | Uint8Array>): Uint8Array {
+  const tree: Record<string, Uint8Array> = {};
+  for (const [path, content] of Object.entries(files)) {
+    tree[path] =
+      typeof content === "string" ? encoder.encode(content) : content;
+  }
+  return zipSync(tree, { level: 6 });
+}
+
+function serving(
+  bytes: Uint8Array,
+  init: { status?: number; contentLength?: number } = {},
+): typeof globalThis.fetch {
+  return async () => {
+    const headers = new Headers();
+    if (init.contentLength !== undefined)
+      headers.set("content-length", String(init.contentLength));
+    const body = bytes.buffer.slice(
+      bytes.byteOffset,
+      bytes.byteOffset + bytes.byteLength,
+    ) as ArrayBuffer;
+    return new Response(
+      init.status === undefined || init.status === 200 ? body : "nope",
+      {
+        status: init.status ?? 200,
+        headers,
+      },
+    );
+  };
+}
+
+/**
+ * The same archive with one entry's central-directory record rewritten: a
+ * declaration that lies (`uncompressedSize`) or a checksum that does not
+ * match (`crc32`). The payload is untouched, so only a reader that budgets
+ * from the central directory and verifies the CRC can refuse it.
+ */
+function declaring(
+  bytes: Uint8Array,
+  entryName: string,
+  patch: { uncompressedSize?: number; crc32?: number },
+): Uint8Array {
+  const patched = new Uint8Array(bytes);
+  const view = new DataView(
+    patched.buffer,
+    patched.byteOffset,
+    patched.byteLength,
+  );
+  const name = encoder.encode(entryName);
+  for (let offset = 0; offset + 46 <= patched.length; offset += 1) {
+    if (view.getUint32(offset, true) !== 0x02014b50) continue;
+    const nameLength = view.getUint16(offset + 28, true);
+    const recorded = patched.subarray(offset + 46, offset + 46 + nameLength);
+    if (
+      recorded.length === name.length &&
+      recorded.every((byte, i) => byte === name[i])
+    ) {
+      if (patch.crc32 !== undefined)
+        view.setUint32(offset + 16, patch.crc32, true);
+      if (patch.uncompressedSize !== undefined)
+        view.setUint32(offset + 24, patch.uncompressedSize, true);
+      return patched;
+    }
+  }
+  throw new Error(`no central directory record for ${entryName}`);
+}
+
+async function refusal(
+  promise: Promise<unknown>,
+): Promise<{ message: string; exitCode: number }> {
+  try {
+    await promise;
+  } catch (error) {
+    const e = error as { message: string; exitCode: number };
+    return { message: e.message, exitCode: e.exitCode };
+  }
+  throw new Error("expected a refusal");
+}
+
+describe("zipballUrl", () => {
+  it("names codeload's HEAD by default and the ref when given", () => {
+    expect(zipballUrl({ repo: "cursor/plugins" })).toBe(
+      "https://codeload.github.com/cursor/plugins/zip/HEAD",
+    );
+    expect(zipballUrl({ repo: "cursor/plugins", ref: "v1" })).toBe(
+      "https://codeload.github.com/cursor/plugins/zip/v1",
+    );
+  });
+});
+
+describe("fetchGitHubTree", () => {
+  it("extracts the tree with the zipball's top directory stripped", async () => {
+    const bytes = zipball({
+      "plugins-HEAD/.cursor-plugin/marketplace.json": JSON.stringify({
+        name: "cursor-plugins",
+        plugins: [],
+      }),
+      "plugins-HEAD/thermos/.cursor-plugin/plugin.json": JSON.stringify({
+        name: "thermos",
+      }),
+      "plugins-HEAD/thermos/skills/x/SKILL.md":
+        "---\nname: x\ndescription: d\n---\nbody\n",
+    });
+    await fetchGitHubTree(
+      { repo: "cursor/plugins" },
+      { destDir: dest, fetchImpl: serving(bytes) },
+    );
+    expect(existsSync(join(dest, "plugins-HEAD"))).toBe(false);
+    expect(
+      JSON.parse(
+        readFileSync(join(dest, ".cursor-plugin", "marketplace.json"), "utf8"),
+      ).name,
+    ).toBe("cursor-plugins");
+    expect(
+      readFileSync(join(dest, "thermos", "skills", "x", "SKILL.md"), "utf8"),
+    ).toContain("body");
+  });
+
+  it("refuses an archive over the download cap from content-length, before reading the body", async () => {
+    const bytes = zipball({ "r-HEAD/marketplace.json": "{}" });
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        {
+          destDir: dest,
+          fetchImpl: serving(bytes, {
+            contentLength: GITHUB_ZIPBALL_LIMITS.downloadBytes + 1,
+          }),
+        },
+      ),
+    );
+    expect(outcome.message).toMatch(/over the 64\.0 MiB download cap/);
+    expect(outcome.exitCode).toBe(ExitCode.Usage);
+  });
+
+  it("refuses more entries than the cap from the central directory", async () => {
+    const files: Record<string, string> = {};
+    for (let i = 0; i <= GITHUB_ZIPBALL_LIMITS.entries; i += 1)
+      files[`r-HEAD/f${i}`] = "";
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        { destDir: dest, fetchImpl: serving(zipball(files)) },
+      ),
+    );
+    expect(outcome.message).toMatch(/entries, over the 20000 cap/);
+    expect(existsSync(join(dest, "f0"))).toBe(false);
+  });
+
+  it("refuses a declared inflated size over the cap from the central directory, before inflating", async () => {
+    const bytes = declaring(
+      zipball({ "r-HEAD/marketplace.json": "{}", "r-HEAD/big.bin": "tiny" }),
+      "r-HEAD/big.bin",
+      {
+        uncompressedSize: GITHUB_ZIPBALL_LIMITS.inflatedBytes + 1,
+      },
+    );
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        { destDir: dest, fetchImpl: serving(bytes) },
+      ),
+    );
+    expect(outcome.message).toMatch(
+      /inflates to 256\.0 MiB, over the 256\.0 MiB cap/,
+    );
+    expect(existsSync(join(dest, "marketplace.json"))).toBe(false);
+  });
+
+  it("refuses an entry whose declared size undercounts what it inflates to", async () => {
+    const bytes = declaring(
+      zipball({
+        "r-HEAD/marketplace.json": "{}",
+        "r-HEAD/liar.txt": "twelve bytes, and then some more text",
+      }),
+      "r-HEAD/liar.txt",
+      { uncompressedSize: 3 },
+    );
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        { destDir: dest, fetchImpl: serving(bytes) },
+      ),
+    );
+    expect(outcome.message).toMatch(
+      /entry 'liar\.txt' inflated past its declared size/,
+    );
+  });
+
+  it("refuses an entry whose bytes do not match its checksum", async () => {
+    const bytes = declaring(
+      zipball({
+        "r-HEAD/marketplace.json": "{}",
+        "r-HEAD/tampered.txt": "some content here",
+      }),
+      "r-HEAD/tampered.txt",
+      { crc32: 0xdeadbeef },
+    );
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        { destDir: dest, fetchImpl: serving(bytes) },
+      ),
+    );
+    expect(outcome.message).toMatch(
+      /entry 'tampered\.txt' failed its checksum/,
+    );
+  });
+
+  it("refuses an entry that would escape the destination", async () => {
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        {
+          destDir: dest,
+          fetchImpl: serving(
+            zipball({ "r-HEAD/marketplace.json": "{}", "r-HEAD/../evil": "x" }),
+          ),
+        },
+      ),
+    );
+    expect(outcome.message).toMatch(/would escape the destination directory/);
+    expect(existsSync(join(dest, "..", "evil"))).toBe(false);
+  });
+
+  it("refuses an archive with more than one top-level directory", async () => {
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        {
+          destDir: dest,
+          fetchImpl: serving(
+            zipball({
+              "one/marketplace.json": "{}",
+              "two/marketplace.json": "{}",
+            }),
+          ),
+        },
+      ),
+    );
+    expect(outcome.message).toMatch(
+      /more than one top-level directory \('one' and 'two'\)/,
+    );
+  });
+
+  it("refuses bytes that are not a zip", async () => {
+    const outcome = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        {
+          destDir: dest,
+          fetchImpl: serving(encoder.encode("<html>not a zip</html>")),
+        },
+      ),
+    );
+    expect(outcome.message).toMatch(/not a readable zip/);
+  });
+
+  it("maps 404 to NotFound and a network failure to Connection, each with its own sentence", async () => {
+    const missing = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b", ref: "v9" },
+        {
+          destDir: dest,
+          fetchImpl: serving(new Uint8Array(), { status: 404 }),
+        },
+      ),
+    );
+    expect(missing.message).toBe(
+      "GitHub has no public repository 'a/b' at ref 'v9'",
+    );
+    expect(missing.exitCode).toBe(ExitCode.NotFound);
+
+    const offline = await refusal(
+      fetchGitHubTree(
+        { repo: "a/b" },
+        {
+          destDir: dest,
+          fetchImpl: async () => {
+            throw new Error("ENOTFOUND");
+          },
+        },
+      ),
+    );
+    expect(offline.message).toBe("could not reach GitHub for a/b");
+    expect(offline.exitCode).toBe(ExitCode.Connection);
+  });
+});

package/src/marketplace/github.ts

@@ -0,0 +1,296 @@
+// A public GitHub repository as a marketplace tree: the zipball of one ref,
+// downloaded from codeload and extracted into a directory the one plugin
+// walker then reads like any other.
+//
+// Why a zipball and not `git`: codeload needs no API call, no token and no
+// `git` binary, and the tree convention every vendor publishes (Cursor,
+// Claude Code, Codex) is "the repository at a ref", which is exactly what a
+// zipball is. Why to disk and not in memory: `install` and `push plugin`
+// must yield one digest for one tree, and the digest is a function of the
+// CLI's gitignore-aware, symlink-skipping walk over a directory. A second,
+// in-memory walk with its own rules would be a second identity.
+//
+// The archive is untrusted input, so it is read the way the server's push
+// gate reads one: the shape from the central directory (the shared
+// `@stigmer/zip-structure` parser) BEFORE a byte is inflated, entry count and
+// total declared size against the caps, every path contained; then each
+// entry inflated with node:zlib under its declared size as a hard output cap
+// and its CRC verified, because a declaration is a claim and the bytes are
+// the fact. That per-entry read mirrors `archive/inflate.ts` in the server,
+// its second consumer; when a third appears it moves to `@stigmer/zip-
+// structure` beside `crc32`. The caps are set against the three vendors'
+// real trees at HEAD (cursor/plugins 4.5 MiB down and 5.5 MiB inflated over
+// 1,169 entries; anthropics/claude-code 11.7 / 13.6 / 1,595; openai/codex
+// 18.6 / 76.8 / 9,188, its marketplace file at the repository root so the
+// whole repository is the tree), with room above the largest.
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { inflateRawSync } from "node:zlib";
+import { isContainedPath } from "@stigmer/plugin-package";
+import {
+  type ZipStructuralEntry,
+  crc32,
+  parseZipStructure,
+} from "@stigmer/zip-structure";
+import { CliExitError } from "../errors/cli-exit-error.js";
+import { ExitCode } from "../errors/exit-codes.js";
+import { log } from "../logger.js";
+
+/** ZIP compression methods this reader opens, the two the platform's gates register. */
+const METHOD_STORED = 0;
+const METHOD_DEFLATED = 8;
+
+/** The caps a repository zipball must fit under; each refusal names the one it hit. */
+export const GITHUB_ZIPBALL_LIMITS = {
+  /** The compressed archive as downloaded. */
+  downloadBytes: 64 * 1024 * 1024,
+  /** The sum of every entry's declared uncompressed size. */
+  inflatedBytes: 256 * 1024 * 1024,
+  /** Files and directories the central directory lists. */
+  entries: 20_000,
+} as const;
+
+/** The ref codeload serves when none is named: the repository's default branch. */
+const DEFAULT_REF = "HEAD";
+
+export interface GitHubTreeSource {
+  /** `owner/repo`. */
+  readonly repo: string;
+  /** Branch, tag or commit; the default branch when absent. */
+  readonly ref?: string;
+}
+
+export interface FetchGitHubTreeOptions {
+  /** Where the tree is written; created if missing. The caller owns its lifetime. */
+  readonly destDir: string;
+  /** The HTTP client (injectable for tests). */
+  readonly fetchImpl?: typeof globalThis.fetch;
+}
+
+/** The URL codeload answers with the zipball of `ref` (no API, no token). */
+export function zipballUrl(source: GitHubTreeSource): string {
+  return `https://codeload.github.com/${source.repo}/zip/${source.ref ?? DEFAULT_REF}`;
+}
+
+/**
+ * Download the zipball and extract it into `destDir` with the zipball's
+ * single top-level directory (`<repo>-<ref>/`) stripped, so `destDir` IS the
+ * repository root. Throws a `CliExitError` naming what refused: the
+ * repository, a cap, or an entry that would escape the directory.
+ */
+export async function fetchGitHubTree(
+  source: GitHubTreeSource,
+  options: FetchGitHubTreeOptions,
+): Promise<void> {
+  const bytes = await download(source, options.fetchImpl ?? globalThis.fetch);
+  const entries = inspect(bytes, source);
+
+  mkdirSync(options.destDir, { recursive: true });
+  for (const entry of entries) {
+    const content = inflate(entry, source);
+    const target = join(options.destDir, ...entry.relative.split("/"));
+    mkdirSync(dirname(target), { recursive: true });
+    writeFileSync(target, content);
+  }
+  log.debug("github tree extracted", {
+    repo: source.repo,
+    ref: source.ref ?? DEFAULT_REF,
+    files: entries.length,
+  });
+}
+
+/**
+ * One entry's bytes, inflated under its declared size as the output cap
+ * and checked against its CRC. An entry that inflates past what it declared
+ * lied about its budget; one whose bytes do not match its CRC is corrupt or
+ * tampered; a method neither the server nor Go registers is not opened.
+ */
+function inflate(entry: PlannedEntry, source: GitHubTreeSource): Uint8Array {
+  let content: Uint8Array;
+  switch (entry.zip.compressionMethod) {
+    case METHOD_STORED:
+      content = entry.zip.compressedData;
+      break;
+    case METHOD_DEFLATED:
+      try {
+        content = new Uint8Array(
+          inflateRawSync(entry.zip.compressedData, {
+            maxOutputLength: entry.declaredSize + 1,
+          }),
+        );
+      } catch (error) {
+        if (isOutputLengthError(error)) {
+          throw refusal(
+            source,
+            `entry '${entry.relative}' inflated past its declared size`,
+          );
+        }
+        throw refusal(
+          source,
+          `entry '${entry.relative}' could not be inflated (${error instanceof Error ? error.message : String(error)})`,
+        );
+      }
+      break;
+    default:
+      throw refusal(
+        source,
+        `entry '${entry.relative}' uses compression method ${entry.zip.compressionMethod}, which is not supported`,
+      );
+  }
+  if (content.length > entry.declaredSize) {
+    throw refusal(
+      source,
+      `entry '${entry.relative}' inflated past its declared size`,
+    );
+  }
+  if (crc32(content) !== entry.zip.crc32) {
+    throw refusal(source, `entry '${entry.relative}' failed its checksum`);
+  }
+  return content;
+}
+
+async function download(
+  source: GitHubTreeSource,
+  fetchImpl: typeof globalThis.fetch,
+): Promise<Uint8Array> {
+  const url = zipballUrl(source);
+  let response: Response;
+  try {
+    response = await fetchImpl(url, { redirect: "follow" });
+  } catch (error) {
+    throw new CliExitError(
+      `could not reach GitHub for ${source.repo}`,
+      ExitCode.Connection,
+      [
+        `URL: ${url}`,
+        "Check the network connection and retry.",
+        error instanceof Error ? error.message : String(error),
+      ],
+    );
+  }
+  if (response.status === 404) {
+    throw new CliExitError(
+      `GitHub has no public repository '${source.repo}'${source.ref === undefined ? "" : ` at ref '${source.ref}'`}`,
+      ExitCode.NotFound,
+      [
+        "Check the owner, the repository name and the ref.",
+        "Private repositories are not supported; a marketplace is public content.",
+      ],
+    );
+  }
+  if (!response.ok) {
+    throw new CliExitError(
+      `GitHub answered ${response.status} for ${source.repo}`,
+      ExitCode.General,
+      [`URL: ${url}`],
+    );
+  }
+  const declaredLength = Number(response.headers.get("content-length") ?? "0");
+  if (declaredLength > GITHUB_ZIPBALL_LIMITS.downloadBytes) {
+    throw refusal(
+      source,
+      `the archive is ${mib(declaredLength)}, over the ${mib(GITHUB_ZIPBALL_LIMITS.downloadBytes)} download cap`,
+    );
+  }
+  const bytes = new Uint8Array(await response.arrayBuffer());
+  if (bytes.length > GITHUB_ZIPBALL_LIMITS.downloadBytes) {
+    throw refusal(
+      source,
+      `the archive is ${mib(bytes.length)}, over the ${mib(GITHUB_ZIPBALL_LIMITS.downloadBytes)} download cap`,
+    );
+  }
+  return bytes;
+}
+
+// node:zlib's maxOutputLength violation: the output-cap arm.
+function isOutputLengthError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    (("code" in error &&
+      (error as { code?: string }).code === "ERR_BUFFER_TOO_LARGE") ||
+      error.message.includes("maxOutputLength"))
+  );
+}
+
+/** One file the archive will yield: where it lands, how large it said it is, and its record. */
+interface PlannedEntry {
+  /** Repository-relative path, the zipball's top directory stripped. */
+  readonly relative: string;
+  /** The central directory's uncompressed size, the inflate budget. */
+  readonly declaredSize: number;
+  readonly zip: ZipStructuralEntry;
+}
+
+/**
+ * The files to extract, in archive order, decided from the central directory
+ * alone. Refuses the caps, an entry outside the one top directory, and a
+ * path that would escape the destination.
+ */
+function inspect(bytes: Uint8Array, source: GitHubTreeSource): PlannedEntry[] {
+  let structure;
+  try {
+    structure = parseZipStructure(bytes);
+  } catch (error) {
+    throw refusal(
+      source,
+      `the archive is not a readable zip (${error instanceof Error ? error.message : String(error)})`,
+    );
+  }
+  if (structure.length > GITHUB_ZIPBALL_LIMITS.entries) {
+    throw refusal(
+      source,
+      `the archive lists ${structure.length} entries, over the ${GITHUB_ZIPBALL_LIMITS.entries} cap`,
+    );
+  }
+  let inflated = 0;
+  for (const entry of structure) inflated += entry.uncompressedSize;
+  if (inflated > GITHUB_ZIPBALL_LIMITS.inflatedBytes) {
+    throw refusal(
+      source,
+      `the archive inflates to ${mib(inflated)}, over the ${mib(GITHUB_ZIPBALL_LIMITS.inflatedBytes)} cap`,
+    );
+  }
+
+  const entries: PlannedEntry[] = [];
+  let top: string | undefined;
+  for (const entry of structure) {
+    const slash = entry.name.indexOf("/");
+    const head = slash === -1 ? entry.name : entry.name.slice(0, slash);
+    if (top === undefined) top = head;
+    if (head !== top) {
+      throw refusal(
+        source,
+        `the archive holds more than one top-level directory ('${top}' and '${head}')`,
+      );
+    }
+    const relative =
+      slash === -1 ? "" : entry.name.slice(slash + 1).replace(/\/+$/, "");
+    if (relative === "") continue; // the top directory itself
+    if (!isContainedPath(relative)) {
+      throw refusal(
+        source,
+        `entry '${entry.name}' would escape the destination directory`,
+      );
+    }
+    if (entry.isDirectory) continue;
+    entries.push({
+      relative,
+      declaredSize: entry.uncompressedSize,
+      zip: entry,
+    });
+  }
+  return entries;
+}
+
+function refusal(source: GitHubTreeSource, reason: string): CliExitError {
+  return new CliExitError(
+    `cannot read ${source.repo} as a marketplace: ${reason}`,
+    ExitCode.Usage,
+    ["A marketplace is a repository tree with a marketplace file at its root."],
+  );
+}
+
+function mib(bytes: number): string {
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
+}

package/src/marketplace/index.ts

@@ -0,0 +1,54 @@
+// Public surface of the marketplace module: the sources a CLI knows, the two
+// grammars a user types, one way to get a tree on disk, one way to read it,
+// one way to turn an entry into a push, and the two renderers. Commands
+// lazy-import this barrel inside their actions so `--help` stays fast.
+
+export {
+  OFFICIAL_MARKETPLACE,
+  OFFICIAL_MARKETPLACE_NAME,
+  type KnownMarketplace,
+  type MarketplaceListing,
+  type MarketplaceSource,
+  type UnreadableMarketplace,
+  addMarketplace,
+  describeSource,
+  findMarketplace,
+  listMarketplaces,
+  removeMarketplace,
+} from "./config.js";
+export {
+  type LocatedEntry,
+  assertVersion,
+  locateEntry,
+  prepareEntry,
+} from "./install.js";
+export {
+  type OpenMarketplace,
+  type OpenMarketplaceOptions,
+  openMarketplace,
+  withMarketplace,
+} from "./materialize.js";
+export {
+  type OfficialMarketplaceDir,
+  type ResolveOfficialOptions,
+  resolveOfficialMarketplace,
+} from "./official.js";
+export {
+  type ReadMarketplaceTree,
+  entryDirectory,
+  findEntry,
+  marketplaceRefusal,
+  readMarketplaceTree,
+} from "./read.js";
+export {
+  type InstallRef,
+  formatInstallRef,
+  parseAddSource,
+  parseInstallRef,
+} from "./ref.js";
+export {
+  type ShownEntry,
+  inspectEntries,
+  renderMarketplaceList,
+  renderMarketplaceShow,
+} from "./render.js";