@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/scripts/generateCliSkillReference.ts

@@ -0,0 +1,103 @@
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+import * as commands from "../commands";
+import type { CommandDef, InputDef } from "../core/types";
+
+/**
+ * renders the CLI command reference of the catladder-cli agent skill
+ * from the command definitions, so the skill always documents the
+ * commands that actually exist. Runs as part of the cli build
+ * (`build:skill-cli-reference`) and writes into the repository's
+ * `skills/` directory — commit the result.
+ */
+
+const kebab = (name: string) => name.replace(/([A-Z])/g, "-$1").toLowerCase();
+
+const cleanMessage = (message: string) =>
+  message.replace(/\s*🤔\s*$/, "").trim();
+
+const renderInput = ([name, def]: [string, InputDef]): string => {
+  const description = cleanMessage(def.message);
+  const details = [
+    def.required ? "required" : undefined,
+    "default" in def && def.default !== undefined
+      ? `default: ${JSON.stringify(def.default)}`
+      : undefined,
+  ]
+    .filter(Boolean)
+    .join(", ");
+  const usage = def.positional
+    ? `\`${name}\` (positional)`
+    : def.type === "boolean"
+      ? `\`--${kebab(name)}\``
+      : `\`--${kebab(name)} <value>\``;
+  return `- ${usage}: ${description}${details ? ` (${details})` : ""}`;
+};
+
+const renderCommand = (def: CommandDef): string => {
+  const positionals = Object.entries(def.inputs)
+    .filter(([, input]) => input.positional)
+    .map(([name]) => ` [${name}]`)
+    .join("");
+  const inputs = Object.entries(def.inputs);
+  return [
+    `## \`catladder ${def.name}${positionals}\``,
+    "",
+    def.description,
+    ...(inputs.length > 0 ? ["", ...inputs.map(renderInput)] : []),
+  ].join("\n");
+};
+
+const isCommandDef = (value: unknown): value is CommandDef =>
+  typeof value === "object" &&
+  value !== null &&
+  "name" in value &&
+  "execute" in value;
+
+export const renderCliReference = (): string => {
+  const defs = Object.values(commands)
+    .filter(isCommandDef)
+    .sort((a, b) => a.name.localeCompare(b.name));
+  return [
+    "# catladder CLI command reference",
+    "",
+    "<!-- rendered from the command definitions by",
+    "     apps/cli/src/scripts/generateCliSkillReference.ts — do not edit -->",
+    "",
+    "All commands run non-interactively as `yarn catladder <command> ...`",
+    "(`pnpm catladder ...` in pnpm projects); see SKILL.md for how to",
+    "pass inputs.",
+    "",
+    defs.map(renderCommand).join("\n\n"),
+    "",
+  ].join("\n");
+};
+
+// executed by the cli build via tsx (node itself cannot require the
+// ESM-only deps of the command modules on older versions)
+const main = () => {
+  const repoRootCandidates = [
+    join(__dirname, "../../../.."), // source: apps/cli/src/scripts
+    join(__dirname, "../../../../../../.."), // compiled: apps/cli/dist/apps/cli/src/scripts
+  ];
+  const repoRoot = repoRootCandidates.find((candidate) =>
+    existsSync(join(candidate, "skills")),
+  );
+  if (!repoRoot) {
+    throw new Error("skills directory not found next to the cli workspace");
+  }
+  const target = join(
+    repoRoot,
+    "skills",
+    "catladder-cli",
+    "references",
+    "commands.md",
+  );
+  mkdirSync(join(target, ".."), { recursive: true });
+  writeFileSync(target, renderCliReference(), "utf-8");
+  console.log(`wrote ${target}`);
+};
+
+if (require.main === module) {
+  main();
+}

package/src/security/topics.json

@@ -17,67 +17,67 @@
   {
     "description": "No sensitive information (passwords, keys, user data, ...) is logged or traced",
     "responsibles": 1,
-    "more": "[Logging guide](https://www.notion.so/panterch/Long-story-logging-022722bb878f4724ae5b49e17667b630?pvs=4#9e5a36b7158a4953b73ec6a345bd8989), [Tracing guide](https://www.notion.so/panterch/Long-story-tracing-d8a9ec1ac2ff4fa78cefa8991233224e?pvs=4#535121b5bf9741fbaf8654b4b64d879d)"
+    "more": ""
   },
   {
     "description": "Passwords are stored hashed with salt and salt is not stored in the repository",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/hash.md)"
+    "more": ""
   },
   {
     "description": "Input that ends up in DOM is properly sanitized",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/xss.md)"
+    "more": ""
   },
   {
     "description": "All user inputs have reasonable validations",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/validation.md)"
+    "more": ""
   },
   {
     "description": "The app is not using cookies",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cookies.md)"
+    "more": ""
   },
   {
     "description": "The app is using cookies and cookies are properly configured",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cookies.md)"
+    "more": ""
   },
   {
     "description": "The app uses JWT with a secret and the secret is not stored in the repository",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cookies.md)"
+    "more": ""
   },
   {
     "description": "Authorization and user roles (RBAC) were reviewed thoroughly",
     "responsibles": 2,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/authorization.md)"
+    "more": ""
   },
   {
     "description": "CORS headers do not use `*`",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/cors.md)"
+    "more": ""
   },
   {
     "description": "CSP headers are properly configured (no `unsafe-inline` or `unsafe-eval`)",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/csp.md)"
+    "more": ""
   },
   {
     "description": "DoS defense mechanism is implemented",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/dos.md)"
+    "more": ""
   },
   {
     "description": "YAML/XML parsing is not used or used YAML/XML parsers have disabled DTD",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/dos.md)"
+    "more": ""
   },
   {
     "description": "The app implements CSRF prevention",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/csrf.md)"
+    "more": ""
   },
   {
     "description": "The app has a rate limitter",
@@ -87,12 +87,12 @@
   {
     "description": "The app has disabled GraphQL introspection and schema registry",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/graphql.md)"
+    "more": ""
   },
   {
     "description": "The app has set GraphQL complexity query limits",
     "responsibles": 1,
-    "more": "[guide](https://git.panter.ch/panter/security-guide/-/blob/main/docs/audit/graphql.md)"
+    "more": ""
   },
   {
     "description": "`sitemap.xml` does not leak any routes with sensitive data",

package/src/store/updateCatladderStore.ts

@@ -0,0 +1,33 @@
+import { mkdir, writeFile } from "fs/promises";
+import { dirname } from "path";
+
+import type { CatladderStore } from "@catladder/pipeline";
+import {
+  catladderStoreSchema,
+  getCatladderStorePath,
+  readCatladderStore,
+} from "@catladder/pipeline";
+import { stringify } from "yaml";
+
+import { getGitRoot } from "../utils/projects";
+
+const HEADER = `# managed by catladder — written by \`catladder project setup\`.
+# Non-secret, machine-fetched values needed to generate the pipeline.
+# Check this file in. If it is missing, rerun \`catladder project setup\`.
+`;
+
+/**
+ * applies an update to `.catladder-store/store.yml` at the git root,
+ * creating the folder on first use
+ */
+export const updateCatladderStore = async (
+  update: (store: CatladderStore) => CatladderStore,
+) => {
+  const root = (await getGitRoot()) ?? process.cwd();
+  const store = readCatladderStore(root);
+  const updated = catladderStoreSchema.parse(update(store));
+  const path = getCatladderStorePath(root);
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, HEADER + stringify(updated));
+  return updated;
+};

package/src/utils/concurrency.ts

@@ -0,0 +1,25 @@
+/**
+ * maps over items with at most `limit` calls in flight, preserving the
+ * input order in the result (unlike reporting-as-you-go, this lets
+ * callers run slow per-item lookups in parallel and still log the
+ * outcomes in a stable order)
+ */
+export const mapWithConcurrency = async <T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> => {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const workers = Array.from(
+    { length: Math.max(1, Math.min(limit, items.length)) },
+    async () => {
+      while (next < items.length) {
+        const index = next++;
+        results[index] = await fn(items[index], index);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+};

package/src/utils/github.ts

@@ -0,0 +1,199 @@
+import { execFile as execFileCb, spawn } from "child_process";
+import { promisify } from "util";
+
+const execFile = promisify(execFileCb);
+
+/**
+ * whether the github cli (gh) is installed and authenticated
+ */
+export const isGhAuthenticated = async (): Promise<boolean> => {
+  try {
+    await execFile("gh", ["auth", "status"]);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * the github repository (owner/name) a git remote points to.
+ * Which remote belongs to github is configured per pipeline type
+ * (`pipelines.github.gitRemote`, defaults to origin).
+ */
+export const getGithubRepoFromRemote = async (
+  remoteName: string,
+): Promise<string | undefined> => {
+  try {
+    const { stdout } = await execFile("git", ["remote", "get-url", remoteName]);
+    const match = stdout.match(
+      /github\.com[:/]([\w.-]+\/[\w.-]+?)(\.git)?\s*$/,
+    );
+    return match?.[1];
+  } catch {
+    return undefined;
+  }
+};
+
+/**
+ * runs a paginated `gh api` query and returns the jq-extracted lines
+ * (one value per line, e.g. names)
+ */
+const ghApiLines = async (path: string, jq: string): Promise<string[]> => {
+  const { stdout } = await execFile(
+    "gh",
+    ["api", "--paginate", path, "--jq", jq],
+    { maxBuffer: 10 * 1024 * 1024 },
+  );
+  return stdout
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+};
+
+/** names of all deployment environments of the repository */
+export const listGithubEnvironments = (repo: string): Promise<string[]> =>
+  ghApiLines(`repos/${repo}/environments?per_page=100`, ".environments[].name");
+
+/**
+ * names of the actions secrets — of an environment, or of the repo
+ * level when no environment is given. Github secrets are write-only:
+ * names are all that can ever be inspected.
+ */
+export const listGithubSecretNames = (
+  repo: string,
+  environment?: string,
+): Promise<string[]> =>
+  ghApiLines(
+    environment
+      ? `repos/${repo}/environments/${encodeURIComponent(environment)}/secrets?per_page=100`
+      : `repos/${repo}/actions/secrets?per_page=100`,
+    ".secrets[].name",
+  );
+
+/** names of the actions variables (environment or repo level) */
+export const listGithubVariableNames = (
+  repo: string,
+  environment?: string,
+): Promise<string[]> =>
+  ghApiLines(
+    environment
+      ? `repos/${repo}/environments/${encodeURIComponent(environment)}/variables?per_page=100`
+      : `repos/${repo}/actions/variables?per_page=100`,
+    ".variables[].name",
+  );
+
+const ghSetViaStdin = (
+  kind: "secret" | "variable",
+  repo: string,
+  name: string,
+  value: string,
+  environment?: string,
+): Promise<void> =>
+  new Promise<void>((resolve, reject) => {
+    const child = spawn(
+      "gh",
+      [
+        kind,
+        "set",
+        name,
+        "--repo",
+        repo,
+        ...(environment ? ["--env", environment] : []),
+      ],
+      { stdio: ["pipe", "ignore", "pipe"] },
+    );
+    let stderr = "";
+    child.stderr?.on("data", (data) => (stderr += data));
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0
+        ? resolve()
+        : reject(new Error(`gh ${kind} set ${name} failed: ${stderr.trim()}`)),
+    );
+    child.stdin?.end(value);
+  });
+
+/**
+ * sets a github actions repository secret via the gh cli.
+ * The value is passed via stdin so it never shows up in a process list.
+ */
+export const setGithubSecret = async (
+  repo: string,
+  name: string,
+  value: string,
+  environment?: string,
+): Promise<void> => ghSetViaStdin("secret", repo, name, value, environment);
+
+/**
+ * sets a github actions repository variable (plain, unmasked) via the
+ * gh cli — for vault values classified as not sensitive
+ */
+export const setGithubVariable = async (
+  repo: string,
+  name: string,
+  value: string,
+  environment?: string,
+): Promise<void> => ghSetViaStdin("variable", repo, name, value, environment);
+
+/**
+ * creates the github environment when it does not exist (idempotent —
+ * PUT with an empty body leaves existing settings untouched)
+ */
+export const ensureGithubEnvironment = async (
+  repo: string,
+  environment: string,
+): Promise<void> =>
+  new Promise<void>((resolve, reject) => {
+    const child = spawn(
+      "gh",
+      [
+        "api",
+        "-X",
+        "PUT",
+        `repos/${repo}/environments/${encodeURIComponent(environment)}`,
+      ],
+      { stdio: ["ignore", "ignore", "pipe"] },
+    );
+    let stderr = "";
+    child.stderr?.on("data", (data) => (stderr += data));
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0
+        ? resolve()
+        : reject(
+            new Error(
+              `creating github environment ${environment} failed: ${stderr.trim()}`,
+            ),
+          ),
+    );
+  });
+
+/** runs `gh api` with an optional JSON body and returns the parsed response */
+export const ghApiJson = async <T = any>(
+  method: "GET" | "PATCH" | "PUT",
+  path: string,
+  body?: unknown,
+): Promise<T> => {
+  const args = ["api", "-X", method, path];
+  const child = spawn("gh", [...args, ...(body ? ["--input", "-"] : [])], {
+    stdio: [body ? "pipe" : "ignore", "pipe", "pipe"],
+  });
+  let stdout = "";
+  let stderr = "";
+  child.stdout?.on("data", (data) => (stdout += data));
+  child.stderr?.on("data", (data) => (stderr += data));
+  if (body) {
+    child.stdin?.write(JSON.stringify(body));
+    child.stdin?.end();
+  }
+  return new Promise<T>((resolve, reject) => {
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0
+        ? resolve(stdout ? JSON.parse(stdout) : (undefined as T))
+        : reject(
+            new Error(`gh api ${method} ${path} failed: ${stderr.trim()}`),
+          ),
+    );
+  });
+};

package/src/utils/gitlab.ts

@@ -45,6 +45,13 @@ export const getGitlabToken = async (io: IO | null) => {
       );
       process.exit(1);
     }
+    if (!io.interactive) {
+      // don't start the setup wizard in a context that cannot prompt
+      // (direnv, turbo, CI) — fail with the remedy instead
+      throw new Error(
+        "gitlab token missing — run `catenv` (or any `catladder` command) once in a terminal to set it up",
+      );
+    }
     await setupGitlabToken(io);
   }
   return getPreference(TOKEN_KEY);
@@ -90,6 +97,30 @@ export const doGitlabRequest = async <T = any>(
   );
 };
 
+/**
+ * GET a list endpoint across ALL pages (gitlab returns 20 items per
+ * page by default — long-lived projects easily exceed that, e.g.
+ * rotated access tokens accumulate one entry per rotation)
+ */
+export const doGitlabRequestAllPages = async <T = any>(
+  io: IO | null,
+  path: string,
+): Promise<T[]> => {
+  const PER_PAGE = 100;
+  const separator = path.includes("?") ? "&" : "?";
+  const all: T[] = [];
+  for (let page = 1; ; page++) {
+    const items = await doGitlabRequest<T[]>(
+      io,
+      `${path}${separator}per_page=${PER_PAGE}&page=${page}`,
+    );
+    if (!items?.length) break;
+    all.push(...items);
+    if (items.length < PER_PAGE) break;
+  }
+  return all;
+};
+
 export const getProjectInfo = async (
   io: IO | null,
 ): Promise<{ id: string; web_url: string }> => {

package/src/utils/gitlabHost.ts

@@ -0,0 +1,23 @@
+import { getGitRemoteHostAndPath } from "../git/gitProjectInformation";
+
+/**
+ * the base url of the gitlab instance this project lives on.
+ *
+ * Never hardcode an instance: catladder is used outside panter, and the
+ * host was baked in as https://git.panter.ch in several places, which
+ * silently broke every consumer on another gitlab.
+ *
+ * - in a gitlab CI job, `CI_SERVER_URL` is the authoritative value
+ *   (predefined, already includes the scheme)
+ * - locally, derive it from the git remote
+ *
+ * @param remoteName the remote to derive from outside CI (default origin)
+ */
+export const getGitlabHostUrl = async (remoteName = "origin") => {
+  const fromCi = process.env.CI_SERVER_URL;
+  if (fromCi) {
+    return fromCi.replace(/\/+$/, "");
+  }
+  const { gitRemoteHost } = await getGitRemoteHostAndPath(remoteName);
+  return `https://${gitRemoteHost}`;
+};

package/src/utils/readStdin.ts

@@ -0,0 +1,8 @@
+/** reads the whole stdin stream (for piped input in non-interactive use) */
+export const readStdin = async (): Promise<string> => {
+  const chunks: Buffer[] = [];
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk as Buffer);
+  }
+  return Buffer.concat(chunks).toString("utf-8");
+};