@canra/cli 0.2.0 → 0.3.0

package/README.md

@@ -23,14 +23,20 @@ canra search json
 |---|---|
 | `canra search <query>` | Search the registry |
 | `canra info <owner>/<name>` | Show a spec's metadata |
-| `canra fetch <owner>/<name>[@version]` | Save a spec's metadata to `.canra/specs/<owner>/<name>/` |
-| `canra outdated` | Check fetched specs for newer versions |
+| `canra fetch <owner>/<name>[@version]` | Download a spec (latest or a pinned version), verify its SHA-256, and save it to `.canra/specs/<owner>/<name>/` |
+| `canra outdated [--json]` | Check fetched specs for newer versions; exits 1 if any is outdated |
+| `canra update [<owner>/<name>]` | Re-fetch the latest version of every outdated spec (or the named one) |
+| `canra remove <owner>/<name>` | Delete a fetched spec from `.canra/specs/` |
 | `canra login` | Save a registry URL and API token |
 | `canra whoami` | Show which account your saved token belongs to |
 | `canra publish <file.spec.md>` | Publish a spec (needs a token) |
 | `canra logout` | Remove saved credentials |
 | `canra version` / `canra help` | Version and usage |
 
+## Vendoring specs
+
+Commit `.canra/` to source control. The saved specs are the record of which spec versions your project implemented (a version is exact and immutable). After `canra update`, `git diff .canra/` shows what changed in each spec.
+
 ## Publishing
 
 Create an API token at **canra.dev → Account → API tokens**, then:

package/canra.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 
+import { createHash } from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
@@ -148,43 +149,128 @@ function createPrompter() {
 
 // --- Commands ---
 
-async function cmdFetch(specArg) {
-  if (!specArg) {
-    console.error("Usage: canra fetch <owner>/<name>[@version]");
-    process.exit(1);
+function specFrontmatter(text) {
+  const match = /^\+\+\+\r?\n([\s\S]*?)\r?\n\+\+\+/.exec(text);
+  return match ? match[1] : "";
+}
+
+// Registry strings are untrusted and end up in a terminal: control and bidi
+// characters would let them inject escape sequences or reorder the output.
+function printable(value) {
+  return [...String(value ?? "")]
+    .filter((c) => {
+      const code = c.codePointAt(0);
+      return (
+        code > 31 &&
+        code !== 127 &&
+        !(code >= 0x80 && code <= 0x9f) &&
+        !(code >= 0x202a && code <= 0x202e) &&
+        !(code >= 0x2066 && code <= 0x2069)
+      );
+    })
+    .join("");
+}
+
+// Reads only the top-level keys (before the first [table]), so a `version` inside
+// a table cannot win. A bare integer is accepted because toml.js accepts `version = 1`.
+function frontmatterVersion(text) {
+  const topLevel = specFrontmatter(text).split(/^\[/m)[0];
+  const match = /^[ \t]*version[ \t]*=[ \t]*(?:"([^"\n]*)"|(\d+))[ \t\r]*(?:#.*)?$/m.exec(topLevel);
+  const version = match ? (match[1] ?? match[2]) : null;
+  return version !== null && printable(version) === version ? version : null;
+}
+
+// `.canra/` is committed and git stores symlinks, so a cloned repo can point
+// these paths anywhere; writing or deleting through one would escape `.canra/`.
+function refuseSymlinks(paths) {
+  for (const p of paths) {
+    let stat;
+    try {
+      stat = fs.lstatSync(p);
+    } catch {
+      continue;
+    }
+    if (stat.isSymbolicLink()) throw new Error(`${printable(p)} is a symbolic link; refusing to follow it`);
   }
+}
 
-  const ref = parseSpecRef(specArg, "canra fetch <owner>/<name>[@version]");
+function localSpecPaths(owner, name) {
+  const specsDir = path.join(LOCAL_DIR, "specs");
+  const ownerDir = path.join(specsDir, owner);
+  return [LOCAL_DIR, specsDir, ownerDir, path.join(ownerDir, name)];
+}
 
-  const registry = getRegistry();
-  const url = specApiUrl(registry, ref);
+function frontmatterFunctionCount(text) {
+  const api = /^\[api\][^\n]*\n([\s\S]*?)(?=^\[|$(?![\s\S]))/m.exec(specFrontmatter(text));
+  const list = api ? /^functions\s*=\s*\[([\s\S]*?)\]/m.exec(api[1]) : null;
+  return list ? (list[1].match(/"(?:[^"\\]|\\.)*"/g) || []).length : 0;
+}
 
-  const res = await fetchRegistry(url);
+function localSpecDir(owner, name) {
+  return path.join(LOCAL_DIR, "specs", owner, name);
+}
+
+async function downloadSpec(registry, { owner, name, version }) {
+  const base = `${registry}/specs/${encodeURIComponent(owner)}/${encodeURIComponent(name)}`;
+  const res = await fetchRegistry(version ? `${base}/${encodeURIComponent(version)}.md` : `${base}.md`);
   if (!res.ok) {
-    const body = await res.text();
-    console.error(`Error: ${res.status} - ${body}`);
-    process.exit(1);
+    const body = printable((await res.text()).trim());
+    throw new Error(`${res.status} - ${body}`);
+  }
+
+  // The hash covers the raw response bytes and those same bytes are written, so
+  // no decode/encode round trip can change what was verified. It detects
+  // corruption in transit, in a CDN or in the blob store; a malicious registry
+  // supplies a matching hash too.
+  const bytes = Buffer.from(await res.arrayBuffer());
+  const expected = res.headers.get("x-content-sha256");
+  if (!expected) {
+    throw new Error(`the registry sent no content hash for ${owner}/${name}; refusing to save unverified content`);
+  }
+  const actual = createHash("sha256").update(bytes).digest("hex");
+  if (actual !== expected.trim().toLowerCase()) {
+    throw new Error(
+      `content hash mismatch for ${owner}/${name} (expected ${printable(expected)}, got ${actual}); nothing was saved`,
+    );
   }
 
-  const spec = await res.json();
-
-  // The registry serves only the latest version; never save a different one under
-  // a pinned request.
-  if (ref.version && ref.version !== spec.version) {
-    console.error(
-      `Error: ${ref.owner}/${ref.name}@${ref.version} is not available (the registry serves the latest, ${spec.version})`,
+  const text = bytes.toString("utf8");
+  const fetchedVersion = frontmatterVersion(text);
+  if (!fetchedVersion) throw new Error(`${owner}/${name} has no version in its frontmatter`);
+  if (version && version !== fetchedVersion) {
+    throw new Error(
+      `${owner}/${name}@${version} was requested but the registry returned version ${printable(fetchedVersion)}`,
     );
-    process.exit(1);
   }
 
-  // Save metadata locally
-  const specDir = path.join(LOCAL_DIR, "specs", ref.owner, ref.name);
+  const specDir = localSpecDir(owner, name);
+  refuseSymlinks([
+    ...localSpecPaths(owner, name),
+    path.join(specDir, `${name}.spec.md`),
+    path.join(specDir, "metadata.json"),
+  ]);
   fs.mkdirSync(specDir, { recursive: true });
-  fs.writeFileSync(path.join(specDir, "metadata.json"), `${JSON.stringify(spec, null, 2)}\n`);
+  fs.writeFileSync(path.join(specDir, `${name}.spec.md`), bytes);
+  fs.writeFileSync(
+    path.join(specDir, "metadata.json"),
+    `${JSON.stringify({ owner, name, version: fetchedVersion, content_hash: actual }, null, 2)}\n`,
+  );
+  return { owner, name, version: fetchedVersion, specDir, functions: frontmatterFunctionCount(text) };
+}
+
+async function cmdFetch(specArg) {
+  if (!specArg) {
+    console.error("Usage: canra fetch <owner>/<name>[@version]");
+    process.exit(1);
+  }
+
+  const ref = parseSpecRef(specArg, "canra fetch <owner>/<name>[@version]");
+  const saved = await downloadSpec(getRegistry(), ref);
 
-  const fnCount = (spec.functions || []).length;
-  console.log(`Fetched ${ref.owner}/${ref.name}@${spec.version} (${fnCount} function${fnCount !== 1 ? "s" : ""})`);
-  console.log(`  Saved to ${specDir}/metadata.json`);
+  console.log(
+    `Fetched ${saved.owner}/${saved.name}@${printable(saved.version)} (${saved.functions} function${saved.functions !== 1 ? "s" : ""})`,
+  );
+  console.log(`  Saved to ${saved.specDir}/${saved.name}.spec.md`);
 }
 
 async function cmdSearch(query) {
@@ -372,70 +458,235 @@ async function cmdInfo(specArg) {
   if (spec.updated_at) console.log(`  Updated:     ${spec.updated_at}`);
 }
 
-async function cmdOutdated() {
-  const specsDir = path.join(LOCAL_DIR, "specs");
-  if (!fs.existsSync(specsDir)) {
-    console.log("No local specs found. Run `canra fetch <owner>/<name>` first.");
-    return;
-  }
+// The server rejects more than MAX_REFS (website/outdated.js) entries per request.
+const OUTDATED_BATCH = 50;
 
-  const registry = getRegistry();
-  // Fetched specs live at .canra/specs/<owner>/<name>/metadata.json.
-  const refs = [];
+function localVersion(owner, name) {
+  const dir = localSpecDir(owner, name);
+  let version = null;
   try {
-    for (const owner of fs.readdirSync(specsDir)) {
-      const ownerDir = path.join(specsDir, owner);
-      if (!fs.statSync(ownerDir).isDirectory()) continue;
-      for (const name of fs.readdirSync(ownerDir)) {
-        if (fs.existsSync(path.join(ownerDir, name, "metadata.json"))) refs.push({ owner, name });
-      }
-    }
+    version = JSON.parse(fs.readFileSync(path.join(dir, "metadata.json"), "utf8")).version;
   } catch {
-    console.log("No local specs found.");
-    return;
+    version = null;
   }
-
-  if (refs.length === 0) {
-    console.log("No local specs found.");
-    return;
+  if (!usableVersion(version)) {
+    try {
+      version = frontmatterVersion(fs.readFileSync(path.join(dir, `${name}.spec.md`), "utf8"));
+    } catch {
+      version = null;
+    }
   }
+  return usableVersion(version) ? version : null;
+}
 
-  const nameWidth = 20;
-  const versionWidth = 12;
-  console.log(`${"SPEC".padEnd(nameWidth)} ${"LOCAL".padEnd(versionWidth)} LATEST`);
-  console.log(`${"-".repeat(nameWidth)} ${"-".repeat(versionWidth)} ${"-".repeat(versionWidth)}`);
-
-  let outdatedCount = 0;
-  for (const ref of refs) {
-    const label = `${ref.owner}/${ref.name}`;
-    const metaPath = path.join(specsDir, ref.owner, ref.name, "metadata.json");
-    let local;
+// The server rejects the whole batch (400) if any entry is empty or over its
+// MAX_FIELD_LENGTH (website/outdated.js), so one bad local version is skipped here.
+function usableVersion(version) {
+  return typeof version === "string" && version.length > 0 && version.length <= 200 && printable(version) === version;
+}
+
+function scanLocalSpecs() {
+  const specsDir = path.join(LOCAL_DIR, "specs");
+  const refs = [];
+  let owners;
+  try {
+    owners = fs.readdirSync(specsDir);
+  } catch {
+    return refs;
+  }
+  for (const owner of owners) {
+    if (!OWNER_RE.test(owner)) continue;
     try {
-      local = JSON.parse(fs.readFileSync(metaPath, "utf8"));
+      refuseSymlinks([LOCAL_DIR, specsDir, path.join(specsDir, owner)]);
     } catch {
       continue;
     }
-
-    let res;
+    let names;
     try {
-      res = await fetch(specApiUrl(registry, ref));
+      names = fs.readdirSync(path.join(specsDir, owner));
     } catch {
       continue;
     }
-    if (!res.ok) continue;
+    for (const name of names) {
+      if (!NAME_RE.test(name)) continue;
+      try {
+        refuseSymlinks([path.join(specsDir, owner, name)]);
+      } catch {
+        continue;
+      }
+      const version = localVersion(owner, name);
+      if (version) refs.push({ owner, name, version });
+    }
+  }
+  return refs;
+}
 
-    const remote = await res.json();
-    if (local.version !== remote.version) {
-      console.log(`${label.padEnd(nameWidth)} ${(local.version || "?").padEnd(versionWidth)} ${remote.version}`);
-      outdatedCount++;
+async function checkRemote(registry, refs) {
+  const results = [];
+  for (let i = 0; i < refs.length; i += OUTDATED_BATCH) {
+    const batch = refs.slice(i, i + OUTDATED_BATCH);
+    const res = await fetchRegistry(`${registry}/api/outdated`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ specs: batch }),
+    });
+    if (!res.ok) throw new Error(`${res.status} - ${printable((await res.text()).trim())}`);
+    const rows = (await res.json()).results;
+    if (!Array.isArray(rows) || rows.length !== batch.length) {
+      throw new Error("the registry returned an unexpected number of results");
     }
+    // Rows are zipped with the submitted refs, and the local owner/name (never the
+    // registry's) are what every later path is built from.
+    rows.forEach((row, j) => {
+      if (row?.owner !== batch[j].owner || row?.name !== batch[j].name) {
+        throw new Error("the registry returned an unexpected result that does not match the request");
+      }
+      results.push({ ...row, owner: batch[j].owner, name: batch[j].name, current: batch[j].version });
+    });
   }
+  return results;
+}
+
+function printOutdatedTable(results) {
+  const labels = results.map((r) => `${r.owner}/${r.name}`);
+  const currents = results.map((r) => printable(r.current));
+  const latests = results.map((r) => printable(r.latest ?? "-"));
+  const specWidth = Math.max(4, ...labels.map((l) => l.length)) + 2;
+  const currentWidth = Math.max(7, ...currents.map((c) => c.length)) + 2;
+  const latestWidth = Math.max(6, ...latests.map((l) => l.length)) + 2;
+  console.log(`${"SPEC".padEnd(specWidth)}${"CURRENT".padEnd(currentWidth)}${"LATEST".padEnd(latestWidth)}BEHIND`);
+  results.forEach((r, i) => {
+    let behind = "-";
+    if (r.status === "outdated") behind = String((r.behind || []).length);
+    else if (r.status !== "current") behind = printable(r.status);
+    const moved = r.moved_to ? `  moved → ${printable(r.moved_to)}` : "";
+    console.log(
+      `${labels[i].padEnd(specWidth)}${currents[i].padEnd(currentWidth)}${latests[i].padEnd(latestWidth)}${behind}${moved}`,
+    );
+  });
+}
 
-  if (outdatedCount === 0) {
-    console.log("\nAll specs are up to date.");
+function needsUpdate(row) {
+  return row.status === "outdated" || (row.status === "current" && Boolean(row.moved_to));
+}
+
+async function cmdOutdated(args) {
+  const asJson = args.includes("--json");
+  const refs = scanLocalSpecs();
+  if (refs.length === 0) {
+    console.log(asJson ? "[]" : "No local specs found. Run `canra fetch <owner>/<name>` first.");
+    return;
+  }
+
+  const results = await checkRemote(getRegistry(), refs);
+  const outdated = results.filter(needsUpdate).length;
+
+  if (asJson) {
+    console.log(JSON.stringify(results, null, 2));
   } else {
-    console.log(`\n${outdatedCount} outdated spec${outdatedCount !== 1 ? "s" : ""}.`);
+    printOutdatedTable(results);
+    if (results.some(needsUpdate)) {
+      console.log(`\n${outdated} outdated spec${outdated !== 1 ? "s" : ""}.`);
+      console.log("Run canra update, then git diff .canra/ to review the changes.");
+    } else {
+      console.log("\nAll specs are up to date.");
+    }
+  }
+  if (outdated > 0) process.exitCode = 1;
+}
+
+function deleteLocalSpec(owner, name) {
+  const specsRoot = path.resolve(LOCAL_DIR, "specs");
+  const dir = path.resolve(localSpecDir(owner, name));
+  if (!dir.startsWith(specsRoot + path.sep)) throw new Error(`unexpected local path for ${owner}/${name}`);
+  refuseSymlinks(localSpecPaths(owner, name));
+  fs.rmSync(dir, { recursive: true, force: true });
+  const ownerDir = path.join(LOCAL_DIR, "specs", owner);
+  if (fs.readdirSync(ownerDir).length === 0) fs.rmdirSync(ownerDir);
+}
+
+function cmdRemove(specArg) {
+  const usage = "canra remove <owner>/<name>";
+  if (!specArg) {
+    console.error("Error: remove needs a spec to delete");
+    console.error(`Usage: ${usage}`);
+    process.exit(1);
+  }
+  const ref = parseSpecRef(specArg, usage);
+  if (ref.version) {
+    console.error("Error: remove deletes the whole spec; omit @version");
+    console.error(`Usage: ${usage}`);
+    process.exit(1);
   }
+  const label = `${ref.owner}/${ref.name}`;
+  if (!fs.existsSync(localSpecDir(ref.owner, ref.name))) {
+    console.error(`Error: ${label} is not fetched locally. Nothing to remove.`);
+    process.exit(1);
+  }
+  deleteLocalSpec(ref.owner, ref.name);
+  console.log(`Removed ${label}`);
+}
+
+async function cmdUpdate(specArg) {
+  const only = specArg ? parseSpecRef(specArg, "canra update [<owner>/<name>]") : null;
+  if (only?.version) {
+    console.error("Error: update always moves to the latest version; omit @version");
+    console.error("Usage: canra update [<owner>/<name>]");
+    process.exit(1);
+  }
+
+  let refs = scanLocalSpecs();
+  if (only) {
+    refs = refs.filter((r) => r.owner === only.owner && r.name === only.name);
+    if (refs.length === 0) {
+      console.error(`Error: ${only.owner}/${only.name} is not fetched locally. Run \`canra fetch\` first.`);
+      process.exit(1);
+    }
+  }
+  if (refs.length === 0) {
+    console.log("No local specs found. Run `canra fetch <owner>/<name>` first.");
+    return;
+  }
+
+  const registry = getRegistry();
+  const targets = (await checkRemote(registry, refs)).filter(needsUpdate);
+  if (targets.length === 0) {
+    console.log("All specs are up to date.");
+    return;
+  }
+
+  let failed = 0;
+  let updated = 0;
+  for (const row of targets) {
+    const label = `${row.owner}/${row.name}`;
+    try {
+      // moved_to comes from the registry and decides where files are written, so it
+      // must be exactly <owner>/<same name>.
+      const target = typeof row.moved_to === "string" ? /^([^/]+)\/([^/]+)$/.exec(row.moved_to) : null;
+      if (row.moved_to && !target) throw new Error(`unexpected move target '${printable(row.moved_to)}'`);
+      const toOwner = target ? target[1] : row.owner;
+      if (!OWNER_RE.test(toOwner) || (target && target[2] !== row.name)) {
+        throw new Error(`unexpected move target '${printable(row.moved_to)}'`);
+      }
+      const moving = toOwner !== row.owner;
+      if (moving && fs.existsSync(localSpecDir(toOwner, row.name))) {
+        throw new Error(`${toOwner}/${row.name} is already fetched locally; remove one of the two copies`);
+      }
+      const saved = await downloadSpec(registry, { owner: toOwner, name: row.name, version: null });
+      updated++;
+      if (!moving) {
+        console.log(`Updated ${label} ${row.current} → ${printable(saved.version)}`);
+        continue;
+      }
+      deleteLocalSpec(row.owner, row.name);
+      console.log(`Updated ${label} ${row.current} → ${toOwner}/${row.name}@${printable(saved.version)}`);
+    } catch (err) {
+      failed++;
+      console.error(`Error: could not update ${label}: ${printable(err.message)}`);
+    }
+  }
+  if (updated > 0) console.log("\nRun git diff .canra/ to review the changes.");
+  if (failed > 0) process.exitCode = 1;
 }
 
 function cmdHelp() {
@@ -445,7 +696,8 @@ Usage: canra <command> [args]
 
 Commands:
   fetch <owner>/<name>[@version]
-                           Fetch spec from registry, save to .canra/specs/
+                           Download a spec (latest, or a pinned version) and
+                           save it, hash-verified, to .canra/specs/
   search <query>           Search registry, print formatted results
   publish <file> [--owner <org>]
                            Publish a spec file (requires auth token); --owner
@@ -454,7 +706,11 @@ Commands:
   logout                   Remove saved credentials
   whoami                   Print the user your saved token authenticates as
   info <owner>/<name>      Print spec metadata
-  outdated                 Check local specs for newer versions
+  outdated [--json]        Check .canra/specs/ for newer versions; exits 1 if
+                           any spec is outdated; --json prints the results
+  update [<owner>/<name>]  Re-fetch the latest version of every outdated spec
+                           (or just the named one); review with git diff .canra/
+  remove <owner>/<name>    Delete a fetched spec from .canra/specs/
   help                     Print this help message
   version                  Print CLI version
 
@@ -495,7 +751,13 @@ async function main() {
       await cmdInfo(args[1]);
       break;
     case "outdated":
-      await cmdOutdated();
+      await cmdOutdated(args.slice(1));
+      break;
+    case "update":
+      await cmdUpdate(args[1]);
+      break;
+    case "remove":
+      cmdRemove(args[1]);
       break;
     case "version":
     case "--version":
@@ -516,7 +778,7 @@ async function main() {
 }
 
 main().catch((err) => {
-  console.error(`Error: ${err.message}`);
+  console.error(`Error: ${printable(err.message)}`);
   process.exit(1);
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@canra/cli",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "CLI for Canra, the registry of machine-readable specs: fetch, search, and publish specs.",
   "keywords": [
     "canra",