@shortlink-org/portolan 0.1.0 → 0.2.0

package/scripts/host-plugins/fetch-csr.mjs

@@ -0,0 +1,386 @@
+// fetch-csr, run by the host: a Confluent Schema Registry on one side, schema
+// files committed to this repository on the other.
+//
+// Shaped like fetch-bsr on purpose and living by the same rules: a pinned
+// version is immutable, so a fetch is byte-reproducible; the committed copy
+// replays offline and in CI; a failed fetch falls back to the tree and, with
+// nothing to fall back to, is an error rather than a short file list. A
+// subject a fetched schema references is fetched too, pinned by the reference
+// itself, so following one adds no lottery.
+//
+// It runs in the host (portolan.0008) because it needs a socket and a
+// credential. The credential comes from the environment and is never in the
+// manifest; the registry URL is a fact about the estate and is.
+
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { join, posix } from "node:path";
+
+import optionsSchema from "./fetch-csr.options.json" with { type: "json" };
+
+export const LOCK_NAME = "csr.lock.json";
+export const OFFLINE_ENV = "PORTOLAN_OFFLINE";
+export const KEY_ENV = "CONFLUENT_SCHEMA_REGISTRY_API_KEY";
+export const SECRET_ENV = "CONFLUENT_SCHEMA_REGISTRY_API_SECRET";
+export const SHORT_KEY_ENV = "CSR_API_KEY";
+export const SHORT_SECRET_ENV = "CSR_API_SECRET";
+export const TOKEN_ENV = "CSR_TOKEN";
+
+// The registry's own media type first and plain JSON second, so the same
+// client works against Confluent's registry and the API-compatible ones.
+const ACCEPT = "application/vnd.schemaregistry.v1+json, application/json";
+
+export function describe() {
+  return {
+    name: "fetch-csr",
+    summary: "Fetches pinned subject versions from a Confluent Schema Registry into the tree, with a lock beside each, so a schema published to a registry can be read without talking to it.",
+    phases: ["extract"],
+    options: optionsSchema,
+  };
+}
+
+/**
+ * @param {{options?: object}} request
+ * @param {{env?: NodeJS.ProcessEnv, fetch?: typeof fetch}} [io]
+ */
+export async function run(request, { env = process.env, fetch: fetchFn = globalThis.fetch } = {}) {
+  const options = request.options ?? {};
+  const registry = String(options.registry ?? "").trim();
+  if (!registry) throw new Error("no registry: name the schema registry's base URL in the manifest");
+  const subjects = Array.isArray(options.subjects) ? options.subjects : [];
+  if (subjects.length === 0) throw new Error("no subjects to fetch: name at least one in the manifest");
+  if (!options.cache) {
+    throw new Error("no cache directory: set `cache` to the same path as the step's `out`, so an offline run can replay what the last online one wrote");
+  }
+
+  const out = new Builder();
+  const queue = [...subjects].map((want) => ({ subject: String(want.subject ?? ""), version: Number(want.version ?? 0) || 0 }))
+    .sort((a, b) => (a.subject < b.subject ? -1 : a.subject > b.subject ? 1 : 0));
+  const client = new Client(registry, authorization(env), fetchFn);
+  const skip = offline(env);
+
+  // Two subjects that slug to the same directory would overwrite each other,
+  // and one subject wanted at two versions cannot be held in a tree that
+  // keeps one version per subject. Both are refused by name.
+  const claimed = new Map();
+  const done = new Set();
+
+  while (queue.length) {
+    const want = queue.shift();
+    if (!want.subject) throw new Error("a subject entry names no subject");
+    if (claimed.has(want.subject)) {
+      const held = claimed.get(want.subject);
+      if (held.version !== want.version) {
+        throw new Error(`${want.subject} is wanted at version ${pinOf(held.version)} and at version ${pinOf(want.version)}; the tree keeps one version per subject, so pin one`);
+      }
+      continue;
+    }
+    const dir = slugOf(want.subject);
+    if (done.has(dir)) throw new Error(`${want.subject} and another subject both vendor into ${dir}; the tree cannot hold both`);
+    claimed.set(want.subject, want);
+    done.add(dir);
+    const at = join(options.cache, dir);
+
+    let references;
+    if (skip) {
+      references = emitCached(out, registry, dir, at, want, "offline");
+    } else {
+      try {
+        references = await live(client, out, registry, dir, want);
+      } catch (cause) {
+        try {
+          references = emitCached(out, registry, dir, at, want, cause.message);
+        } catch (cacheCause) {
+          throw new Error(`${want.subject} could not be fetched (${cause.message}) and there is no usable copy in the tree (${cacheCause.message})`);
+        }
+      }
+    }
+
+    // A referenced subject is pinned by the schema that references it, so
+    // following one needs no permission from the manifest.
+    const following = references
+      .filter((ref) => ref.subject && !claimed.has(ref.subject))
+      .map((ref) => ({ subject: ref.subject, version: Number(ref.version) || 0 }))
+      .sort((a, b) => (a.subject < b.subject ? -1 : a.subject > b.subject ? 1 : 0));
+    queue.push(...following);
+  }
+
+  return out.response();
+}
+
+export function offline(env = process.env) {
+  if (String(env[OFFLINE_ENV] ?? "").trim()) return true;
+  return !["", "0", "false"].includes(String(env.CI ?? "").trim().toLowerCase());
+}
+
+/** The Authorization value, or "" for an anonymous registry. */
+export function authorization(env = process.env) {
+  const read = (name) => String(env[name] ?? "").trim();
+  const raw = read(TOKEN_ENV);
+  if (raw) {
+    // Passed through when it already names its scheme, so the same variable
+    // serves "Bearer ..." and an opaque token a proxy expects bare.
+    return raw.includes(" ") ? raw : `Bearer ${raw}`;
+  }
+  const key = read(KEY_ENV) || read(SHORT_KEY_ENV);
+  const secret = read(SECRET_ENV) || read(SHORT_SECRET_ENV);
+  if (!key || !secret) return "";
+  return `Basic ${Buffer.from(`${key}:${secret}`).toString("base64")}`;
+}
+
+/**
+ * The directory a subject vendors into. A subject name is whatever the
+ * producer registered; everything outside a safe set becomes an underscore,
+ * and the true subject is written in the lock beside the file.
+ */
+export function slugOf(subject) {
+  const slug = [...String(subject)].map((char) => (/[A-Za-z0-9._-]/.test(char) ? char : "_")).join("").replace(/^\.+|\.+$/g, "");
+  return slug || "_";
+}
+
+/** The empty answer is AVRO by the registry's own rule. */
+export function schemaType(declared) {
+  const clean = String(declared ?? "").trim();
+  return clean ? clean.toUpperCase() : "AVRO";
+}
+
+export function extensionFor(kind) {
+  switch (schemaType(kind)) {
+    case "AVRO": return ".avsc";
+    case "PROTOBUF": return ".proto";
+    case "JSON": return ".json";
+    default: throw new Error(`unknown schema type "${kind}"`);
+  }
+}
+
+/**
+ * The schema as it should be written. AVRO and JSON arrive minified onto one
+ * line; they are indented without reordering or rewriting a single token, so
+ * the file still says what the registry said. PROTOBUF is left as it is.
+ */
+export function body(kind, schema) {
+  if (schemaType(kind) === "PROTOBUF") return withNewline(schema);
+  try {
+    JSON.parse(schema);
+  } catch {
+    // A registry that answered with something that is not JSON has told us
+    // something, and the honest thing is to keep it so the reader can see.
+    return withNewline(schema);
+  }
+  return withNewline(indentJson(schema));
+}
+
+/** json.Indent: re-spaced, never re-parsed - every token stays as it came. */
+export function indentJson(text) {
+  let out = "";
+  let depth = 0;
+  let index = 0;
+  const skipSpace = () => { while (index < text.length && /\s/.test(text[index])) index += 1; };
+  while (index < text.length) {
+    const char = text[index];
+    if (/\s/.test(char)) { index += 1; continue; }
+    if (char === '"') {
+      let end = index + 1;
+      while (end < text.length && text[end] !== '"') end += text[end] === "\\" ? 2 : 1;
+      out += text.slice(index, end + 1);
+      index = end + 1;
+      continue;
+    }
+    if (char === "{" || char === "[") {
+      const closer = char === "{" ? "}" : "]";
+      index += 1;
+      skipSpace();
+      if (text[index] === closer) { out += char + closer; index += 1; continue; }
+      depth += 1;
+      out += `${char}\n${"  ".repeat(depth)}`;
+      continue;
+    }
+    if (char === "}" || char === "]") {
+      depth -= 1;
+      out += `\n${"  ".repeat(depth)}${char}`;
+      index += 1;
+      continue;
+    }
+    if (char === ",") { out += `,\n${"  ".repeat(depth)}`; index += 1; continue; }
+    if (char === ":") { out += ": "; index += 1; continue; }
+    let end = index;
+    while (end < text.length && !/[\s,\]}:]/.test(text[end])) end += 1;
+    out += text.slice(index, end);
+    index = end;
+  }
+  return out;
+}
+
+function withNewline(text) {
+  return text.endsWith("\n") ? text : `${text}\n`;
+}
+
+/** The lock, one subject per directory, references sorted by name. */
+export function encodeLock(registry, entry) {
+  const record = { subject: entry.subject, version: entry.version, id: entry.id };
+  if (entry.guid) record.guid = entry.guid;
+  record.schemaType = entry.schemaType;
+  if (entry.references?.length) {
+    record.references = [...entry.references]
+      .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))
+      .map((ref) => ({ name: ref.name, subject: ref.subject, version: ref.version }));
+  }
+  record.files = [...entry.files]
+    .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
+    .map((file) => ({ path: file.path, sha256: file.sha256, size: file.size }));
+  return `${JSON.stringify({ registry, subjects: [record] }, null, 2)}\n`;
+}
+
+function digestOf(contents) {
+  return createHash("sha256").update(contents).digest("hex");
+}
+
+function pinOf(version) {
+  return version === 0 ? "latest" : String(version);
+}
+
+async function live(client, out, registry, dir, want) {
+  const got = await client.version(want.subject, want.version);
+  if (want.version === 0) {
+    out.warn(want.subject, `is not pinned; "latest" resolved to version ${got.version}. Pin it in portolan.json or every run is a lottery.`);
+  }
+  let extension;
+  try {
+    extension = extensionFor(got.schemaType);
+  } catch (cause) {
+    throw new Error(`${want.subject} at version ${got.version}: ${cause.message}`);
+  }
+  const name = `v${got.version}${extension}`;
+  const contents = body(got.schemaType, got.schema);
+  const entry = {
+    subject: want.subject,
+    version: got.version,
+    id: got.id ?? 0,
+    guid: got.guid ?? "",
+    schemaType: schemaType(got.schemaType),
+    references: got.references ?? [],
+    files: [{ path: name, sha256: digestOf(contents), size: Buffer.byteLength(contents) }],
+  };
+  out.file(posix.join(dir, name), contents);
+  out.file(posix.join(dir, LOCK_NAME), encodeLock(registry, entry));
+  return entry.references;
+}
+
+function emitCached(out, registry, dir, at, want, why) {
+  if (want.version === 0) throw new Error(`${want.subject} is not pinned to a version, so there is nothing to replay`);
+  const held = replay(at);
+  if (held.lock.version !== want.version) throw new Error(`${want.subject} holds version ${held.lock.version} but the manifest pins ${want.version}; fetch it`);
+  if (held.lock.subject !== want.subject) throw new Error(`${want.subject} vendors into ${at.split("\\").join("/")}, which holds ${held.lock.subject}`);
+  if (held.registry && held.registry !== registry) throw new Error(`${want.subject} was fetched from ${held.registry} but the manifest names ${registry}; fetch it again`);
+  for (const file of held.lock.files) out.file(posix.join(dir, file.path), held.files.get(file.path).toString("utf8"));
+  out.file(posix.join(dir, LOCK_NAME), encodeLock(registry, held.lock));
+  out.warn(want.subject, `not fetched (${why}); the copy committed in this repository is used unchanged`);
+  return held.lock.references ?? [];
+}
+
+function replay(dir) {
+  const lockPath = join(dir, LOCK_NAME);
+  let raw;
+  try {
+    raw = readFileSync(lockPath, "utf8");
+  } catch (cause) {
+    if (cause.code === "ENOENT") throw new Error(`no ${LOCK_NAME} in ${dir.split("\\").join("/")}`);
+    throw cause;
+  }
+  let lock;
+  try {
+    lock = JSON.parse(raw);
+  } catch (cause) {
+    throw new Error(`${lockPath}: ${cause.message}`);
+  }
+  const subjects = Array.isArray(lock.subjects) ? lock.subjects : [];
+  if (subjects.length !== 1) throw new Error(`${lockPath} names ${subjects.length} subjects; expected exactly one`);
+  const entry = subjects[0];
+  const files = new Map();
+  for (const want of entry.files ?? []) {
+    let contents;
+    try {
+      contents = readFileSync(join(dir, ...String(want.path).split("/")));
+    } catch (cause) {
+      if (cause.code === "ENOENT") throw new Error(`${want.path} is in the lock but not on disk`);
+      throw cause;
+    }
+    if (digestOf(contents) !== want.sha256) throw new Error(`${want.path} does not match its digest; the vendored copy was edited by hand`);
+    files.set(want.path, contents);
+  }
+  return {
+    registry: String(lock.registry ?? ""),
+    lock: {
+      subject: entry.subject, version: Number(entry.version) || 0, id: entry.id ?? 0, guid: entry.guid ?? "",
+      schemaType: entry.schemaType ?? "", references: entry.references ?? [], files: entry.files ?? [],
+    },
+    files,
+  };
+}
+
+/** The registry's REST API, and only the one call this plugin makes. */
+class Client {
+  constructor(base, auth, fetchFn) {
+    this.base = base.replace(/\/+$/, "");
+    this.auth = auth;
+    this.fetch = fetchFn;
+  }
+
+  /** One registration. A zero version asks for `latest`. */
+  async version(subject, version) {
+    const which = version > 0 ? String(version) : "latest";
+    const at = `${this.base}/subjects/${encodeURIComponent(subject)}/versions/${which}`;
+    let got;
+    try {
+      got = await this.get(at);
+    } catch (cause) {
+      throw new Error(`${subject} at version ${which}: ${cause.message}`);
+    }
+    if (!String(got.schema ?? "").trim()) throw new Error(`${subject} at version ${which}: the registry answered with no schema`);
+    if (!(Number(got.version) > 0)) throw new Error(`${subject} at version ${which}: the registry did not say which version it answered with`);
+    return { ...got, version: Number(got.version) };
+  }
+
+  async get(at) {
+    const headers = { Accept: ACCEPT };
+    if (this.auth) headers.Authorization = this.auth;
+    let response;
+    try {
+      response = await this.fetch(at, { headers, signal: AbortSignal.timeout(30_000) });
+    } catch (cause) {
+      throw new Error(cause.cause?.message ?? cause.message);
+    }
+    const raw = await response.text();
+    if (!response.ok) {
+      try {
+        const fault = JSON.parse(raw);
+        if (fault?.message) throw new Error(`the registry answered ${response.status} ${response.statusText}: ${fault.message} (error_code ${fault.error_code ?? 0})`);
+      } catch (cause) {
+        if (cause.message.startsWith("the registry answered")) throw cause;
+      }
+      throw new Error(`the registry answered ${response.status} ${response.statusText}`);
+    }
+    try {
+      return JSON.parse(raw);
+    } catch (cause) {
+      throw new Error(`the registry's answer is not the shape this plugin reads: ${cause.message}`);
+    }
+  }
+}
+
+class Builder {
+  files = [];
+  warnings = [];
+
+  file(name, contents) {
+    this.files.push({ name, contents });
+  }
+
+  warn(ref, message) {
+    this.warnings.push({ severity: "warning", message, ref });
+  }
+
+  response() {
+    return { files: this.files, warnings: this.warnings };
+  }
+}

package/scripts/host-plugins/fetch-csr.test.mjs

@@ -0,0 +1,178 @@
+// The fetcher, against a recorded registry. NEVER against a live one.
+import { createHash } from "node:crypto";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+
+import { runPlugin } from "../plugin-host.mjs";
+import { KEY_ENV, OFFLINE_ENV, SECRET_ENV, TOKEN_ENV, authorization, body, indentJson, run, slugOf } from "./fetch-csr.mjs";
+
+const VALUE = "shop.oms.order-value";
+const PARTY = "shop.oms.Party";
+// The value schema references the party one, which is how a registry says a
+// record is shared between subjects.
+const ORDER = '{"type":"record","namespace":"shop.oms","name":"OrderPlaced","fields":[{"name":"order_id","type":"string"},{"name":"buyer","type":"shop.oms.Party"}]}';
+const PARTY_SCHEMA = '{"type":"record","namespace":"shop.oms","name":"Party","fields":[{"name":"id","type":"string"}]}';
+
+const cleanups = [];
+afterEach(async () => {
+  for (const cleanup of cleanups.splice(0)) await cleanup();
+});
+
+async function registry(answers = recorded()) {
+  const seen = [];
+  const server = createServer((request, response) => {
+    seen.push(request.headers.authorization ?? "");
+    response.setHeader("Content-Type", "application/json");
+    const answer = answers[decodeURIComponent(request.url)];
+    if (!answer) {
+      response.statusCode = 404;
+      response.end(JSON.stringify({ error_code: 40401, message: `Subject '${request.url}' not found.` }));
+      return;
+    }
+    if (typeof answer === "function") return answer(response);
+    response.end(JSON.stringify(answer));
+  });
+  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+  cleanups.push(() => new Promise((resolve) => server.close(resolve)));
+  return { url: `http://127.0.0.1:${server.address().port}`, seen };
+}
+
+function recorded() {
+  const order = { subject: VALUE, version: 3, id: 100021, guid: "8f0d", schemaType: "AVRO", schema: ORDER, references: [{ name: "shop.oms.Party", subject: PARTY, version: 1 }] };
+  return {
+    "/subjects/shop.oms.order-value/versions/3": order,
+    "/subjects/shop.oms.order-value/versions/latest": order,
+    "/subjects/shop.oms.Party/versions/1": { subject: PARTY, version: 1, id: 100014, schemaType: "AVRO", schema: PARTY_SCHEMA },
+  };
+}
+
+function cache() {
+  const dir = mkdtempSync(join(tmpdir(), "portolan-fetch-csr-"));
+  cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
+  return dir;
+}
+
+const isolated = { [OFFLINE_ENV]: "", CI: "", [TOKEN_ENV]: "", [KEY_ENV]: "", [SECRET_ENV]: "", CSR_API_KEY: "", CSR_API_SECRET: "" };
+const options = (base, cacheDir, ...subjects) => ({ registry: base, cache: cacheDir, subjects });
+const fetch = (opts, env = {}) => run({ options: opts }, { env: { ...isolated, ...env } });
+const names = (response) => response.files.map((file) => file.name).sort();
+const contentsOf = (response, name) => response.files.find((file) => file.name === name)?.contents ?? "";
+const write = (dir, response) => {
+  for (const file of response.files) {
+    mkdirSync(dirname(join(dir, file.name)), { recursive: true });
+    writeFileSync(join(dir, file.name), file.contents);
+  }
+};
+
+describe("fetch-csr", () => {
+  it("writes the schema, its lock, and follows a reference the schema pinned", async () => {
+    const server = await registry();
+    const response = await fetch(options(server.url, cache(), { subject: VALUE, version: 3 }));
+    expect(names(response)).toEqual(["shop.oms.Party/csr.lock.json", "shop.oms.Party/v1.avsc", "shop.oms.order-value/csr.lock.json", "shop.oms.order-value/v3.avsc"]);
+    expect(contentsOf(response, "shop.oms.Party/v1.avsc")).toContain('"name": "Party"');
+
+    // Indented rather than the one line the registry answers with, without
+    // reordering a key.
+    const schema = contentsOf(response, "shop.oms.order-value/v3.avsc");
+    expect(schema).toContain('\n  "type": "record"');
+    expect(schema.endsWith("\n")).toBe(true);
+    expect(schema.indexOf('"namespace"')).toBeLessThan(schema.indexOf('"name"'));
+    expect(JSON.parse(schema)).toEqual(JSON.parse(ORDER));
+
+    const lock = JSON.parse(contentsOf(response, "shop.oms.order-value/csr.lock.json"));
+    expect(lock).toEqual({
+      registry: server.url,
+      subjects: [{
+        subject: VALUE, version: 3, id: 100021, guid: "8f0d", schemaType: "AVRO",
+        references: [{ name: "shop.oms.Party", subject: PARTY, version: 1 }],
+        files: [{ path: "v3.avsc", sha256: createHash("sha256").update(schema).digest("hex"), size: schema.length }],
+      }],
+    });
+    expect(JSON.parse(contentsOf(response, "shop.oms.Party/csr.lock.json")).subjects[0]).not.toHaveProperty("guid");
+    expect(response.warnings).toEqual([]);
+  });
+
+  it("reads an absent schema type as AVRO and resolves latest with a warning", async () => {
+    const answers = recorded();
+    delete answers["/subjects/shop.oms.Party/versions/1"].schemaType;
+    const server = await registry(answers);
+    const response = await fetch(options(server.url, cache(), { subject: VALUE }));
+    expect(JSON.parse(contentsOf(response, "shop.oms.Party/csr.lock.json")).subjects[0].schemaType).toBe("AVRO");
+    expect(response.warnings).toHaveLength(1);
+    expect(response.warnings[0].message).toContain('"latest" resolved to version 3');
+
+    await expect(fetch(options("http://127.0.0.1:1", cache(), { subject: VALUE }), { [OFFLINE_ENV]: "1" })).rejects.toThrow(/not pinned/);
+  });
+
+  it("replays the committed copies offline, references included, byte for byte", async () => {
+    const server = await registry();
+    const cacheDir = cache();
+    const online = await fetch(options(server.url, cacheDir, { subject: VALUE, version: 3 }));
+    write(cacheDir, online);
+    const replayed = await fetch(options(server.url, cacheDir, { subject: VALUE, version: 3 }), { [OFFLINE_ENV]: "1" });
+    expect(names(replayed)).toEqual(names(online));
+    for (const file of online.files) expect(contentsOf(replayed, file.name)).toBe(file.contents);
+    expect(replayed.warnings.map((warning) => warning.ref)).toEqual([VALUE, PARTY]);
+  });
+
+  it("falls back to the tree when the registry fails, and is an error without one", async () => {
+    const server = await registry();
+    const cacheDir = cache();
+    write(cacheDir, await fetch(options(server.url, cacheDir, { subject: VALUE, version: 3 })));
+    const broken = await registry({ "/subjects/shop.oms.order-value/versions/3": (response) => { response.statusCode = 502; response.end(); } });
+    // The lock names the registry it came from; a copy from another one is
+    // not this one's to replay.
+    await expect(fetch(options(broken.url, cacheDir, { subject: VALUE, version: 3 }))).rejects.toThrow(/was fetched from .* but the manifest names/);
+
+    await expect(fetch(options("http://127.0.0.1:1", cache(), { subject: VALUE, version: 3 }))).rejects.toThrow(/no usable copy/);
+    const missing = await registry({});
+    await expect(fetch(options(missing.url, cache(), { subject: VALUE, version: 3 }))).rejects.toThrow(/Subject .* not found\. \(error_code 40401\)/);
+  });
+
+  it("reports an edited copy, a moved pin, and one subject at two versions", async () => {
+    const server = await registry();
+    const cacheDir = cache();
+    write(cacheDir, await fetch(options(server.url, cacheDir, { subject: VALUE, version: 3 })));
+    await expect(fetch(options(server.url, cacheDir, { subject: VALUE, version: 2 }), { [OFFLINE_ENV]: "1" })).rejects.toThrow(/holds version 3 but the manifest pins 2/);
+    writeFileSync(join(cacheDir, "shop.oms.order-value/v3.avsc"), "{}\n");
+    await expect(fetch(options(server.url, cacheDir, { subject: VALUE, version: 3 }), { [OFFLINE_ENV]: "1" })).rejects.toThrow(/v3\.avsc does not match its digest/);
+    await expect(fetch(options(server.url, cache(), { subject: VALUE, version: 3 }, { subject: VALUE, version: 2 }))).rejects.toThrow(/one version per subject/);
+  });
+
+  it("sends the credential from the environment and gives a bare token a scheme", async () => {
+    const server = await registry();
+    await fetch(options(server.url, cache(), { subject: PARTY, version: 1 }));
+    await fetch(options(server.url, cache(), { subject: PARTY, version: 1 }), { [KEY_ENV]: "key", [SECRET_ENV]: "secret" });
+    expect(server.seen).toEqual(["", `Basic ${Buffer.from("key:secret").toString("base64")}`]);
+    expect(authorization({ [TOKEN_ENV]: "abc" })).toBe("Bearer abc");
+    expect(authorization({ [TOKEN_ENV]: "Token abc" })).toBe("Token abc");
+    expect(authorization({ CSR_API_KEY: "k", CSR_API_SECRET: "s" })).toBe(`Basic ${Buffer.from("k:s").toString("base64")}`);
+    expect(authorization({ CSR_API_KEY: "k" })).toBe("");
+  });
+
+  it("runs through the host like any plugin", async () => {
+    const server = await registry();
+    const result = await runPlugin({ name: "csr", host: "fetch-csr" }, { portolanVersion: "0.1.0", options: options(server.url, cache(), { subject: PARTY, version: 1 }) });
+    expect(result.files.map((file) => file.name)).toEqual(["shop.oms.Party/v1.avsc", "shop.oms.Party/csr.lock.json"]);
+    const described = await runPlugin({ name: "csr", host: "fetch-csr" }, { portolanVersion: "0.1.0", kind: "describe" });
+    expect(described.describe).toMatchObject({ name: "fetch-csr", phases: ["extract"] });
+  });
+});
+
+describe("shapes", () => {
+  it("keeps a subject readable as a directory", () => {
+    expect(slugOf("shop.oms.order-value")).toBe("shop.oms.order-value");
+    expect(slugOf("shop.oms.OrderPlaced")).toBe("shop.oms.OrderPlaced");
+    expect(slugOf("a/b c")).toBe("a_b_c");
+    expect(slugOf("...")).toBe("_");
+  });
+
+  it("indents JSON without touching a token, and leaves protobuf alone", () => {
+    expect(indentJson('{"a":[1,2.50,"x\\"y",{"b":null}],"c":{},"d":[],"e":true}')).toBe('{\n  "a": [\n    1,\n    2.50,\n    "x\\"y",\n    {\n      "b": null\n    }\n  ],\n  "c": {},\n  "d": [],\n  "e": true\n}');
+    expect(body("PROTOBUF", 'syntax = "proto3";')).toBe('syntax = "proto3";\n');
+    expect(body("JSON", "not json")).toBe("not json\n");
+  });
+});