@spunto/build 0.5.0 → 0.6.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@spunto/build",
-  "version": "0.5.0",
-  "description": "Spunto's shared Build engine \u2014 the devcontainer image protocol and VS Code extension registry clients, with no database, no HTTP framework and no UI.",
+  "version": "0.6.0",
+  "description": "Spunto's shared Build engine — the devcontainer image protocol and VS Code extension registry clients, with no database, no HTTP framework and no UI.",
   "type": "module",
   "license": "MIT",
   "homepage": "https://spunto.net",

package/src/script/setup-script.ts

@@ -35,6 +35,37 @@ function shQuote(v: string): string {
   return `'${v.replace(/'/g, `'\\''`)}'`
 }
 
+/**
+ * Runs a command as the workspace user instead of root.
+ *
+ * The setup script itself is the container's CMD, so it runs as root — it has to, to chown
+ * /workspace and write under /etc. Everything it *clones*, though, belongs to the user, and
+ * cloning as root has cost us twice: once as `npm install` failing with EACCES on a root-owned
+ * node_modules (hence the re-chown after the repo clones), and once as a private dotfiles repo
+ * that authenticated with no key at all, because root has no `~/.ssh` and the identity was only
+ * named on the repo clones.
+ *
+ * `su` gives the command `HOME=/home/vscode`, which is what makes ssh pick up the `~/.ssh/config`
+ * written in the credentials step — so a clone over SSH needs nothing else to find the user's key.
+ */
+function asUser(username: string, cmd: string): string {
+  return `su ${username} -c ${shQuote(cmd)}`
+}
+
+/**
+ * A dotfiles setting, as a clonable URL.
+ *
+ * Anything already naming a transport is left alone; only a bare `owner/repo` shorthand is
+ * expanded against GitHub. The previous test — `startsWith("http") || startsWith("git@")` — turned
+ * `ssh://git@host/me/dotfiles.git` into `https://github.com/ssh://git@host/me/dotfiles.git`.
+ */
+function normalizeDotfilesUrl(raw: string): string {
+  const v = raw.trim()
+  return /^(https?:\/\/|ssh:\/\/|git:\/\/|file:\/\/|[^/\s]+@[^/\s]+:)/.test(v)
+    ? v
+    : `https://github.com/${v.replace(/^\/+|\/+$/g, "")}`
+}
+
 /**
  * `export EXTENSIONS_GALLERY=…`, or nothing when the org is on the default (Open VSX) registry.
  *
@@ -105,11 +136,15 @@ function cloneRepoBlock(params: {
   index: number
   total: number
   homeDir: string
+  /** Who the checkout belongs to — the clone runs as them. */
+  username: string
   branch?: string
   githubInstallationTokens?: Record<string, string>
   userSshPrivateKey?: string
+  /** Whether that key is registered on the member's provider account — see `originSetUrl`. */
+  userSshKeyRegistered?: boolean
 }): { header: string; lines: string[] } {
-  const { repo: r, index, total, homeDir, branch, githubInstallationTokens, userSshPrivateKey } = params
+  const { repo: r, index, total, homeDir, username, branch, githubInstallationTokens, userSshPrivateKey, userSshKeyRegistered } = params
   // Pick the installation token for this repo's owner. A GitHub App installation only grants
   // access to repos owned by its account, so the owner login (== installation accountLogin)
   // uniquely selects the right token among the org's connected installations. Covers legacy
@@ -117,13 +152,22 @@ function cloneRepoBlock(params: {
   const owner = (r.project.split("/")[0] ?? "").toLowerCase()
   const repoToken = githubInstallationTokens?.[owner]
   const b = branch ? ` --branch ${shQuote(branch)}` : ""
-  const cloneCmd = r.provider === "git" && r.cloneUrl
+  const rawCloneCmd = r.provider === "git" && r.cloneUrl
     ? `GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${homeDir}/.ssh/mp_deploy_key" git clone${b} ${shQuote(r.cloneUrl)} /workspace/${r.workspacePath}`
     : r.provider === "github" && repoToken
-    ? `git clone${b} https://x-access-token:${repoToken}@github.com/${r.project}.git /workspace/${r.workspacePath} && git -C /workspace/${r.workspacePath} remote set-url origin git@github.com:${r.project}.git`
+    ? `git clone${b} https://x-access-token:${repoToken}@github.com/${r.project}.git /workspace/${r.workspacePath} && ${originSetUrl(r, `/workspace/${r.workspacePath}`, userSshKeyRegistered)}`
     : r.provider === "github" && userSshPrivateKey
     ? `GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${homeDir}/.ssh/mp_user_key" git clone${b} git@github.com:${r.project}.git /workspace/${r.workspacePath}`
-    : `git clone${b} https://github.com/${r.project} /workspace/${r.workspacePath}`
+    // Last resort: a public clone over https. Only valid for GitHub — for any other hosting
+    // provider this would quietly clone the wrong host, so say what happened instead. A provider
+    // reaches this line only when its credential could not be minted, which is worth reading in
+    // the setup log rather than debugging from a confusing 404.
+    : r.provider === "github"
+    ? `git clone${b} https://github.com/${r.project} /workspace/${r.workspacePath}`
+    : `echo "No clone credential for ${r.provider} repository ${r.project} — check the organization's integrations." >&2 && false`
+  // Cloned as the user, not as root: /workspace is already theirs (the ownership step runs
+  // first), so the checkout lands with the right owner and needs no chown afterwards.
+  const cloneCmd = asUser(username, rawCloneCmd)
 
   // Single-quoted only when a (user-provided) branch is interpolated; the plain form is kept
   // verbatim for the no-branch case so the generated script is unchanged there.
@@ -173,6 +217,74 @@ function cloneRepoBlock(params: {
  * both, and also makes the credentials usable by everything that runs as the user *during* setup
  * (postCreateCommand, dotfiles install script).
  */
+/**
+ * The credential helper's own source, kept whole rather than assembled line by line: it is a
+ * shell script inside a TypeScript string inside a heredoc, and every layer of escaping added
+ * to that is a layer someone has to debug later.
+ */
+/**
+ * Where `origin` points after an installation-token clone.
+ *
+ * The clone itself is the organization's; `origin` decides who everything *after* it acts as.
+ * Three cases, in this order:
+ *
+ *  1. **The worker has an identity** (`SPUNTO_WORKER_TOKEN`) → HTTPS, and the credential helper
+ *     answers as the member who owns it (RFC 0024). This is the normal case now: every spawn mints
+ *     one. Decided in shell rather than here so it stays locked to the same condition the helper's
+ *     own install is guarded on — two places, one signal.
+ *  2. **No identity, but the member's key is registered on their account** → SSH, the old path.
+ *  3. **Neither** → HTTPS with nothing behind it. Public repos work; a private one fails *at the
+ *     push*, which is the honest outcome, rather than an SSH refusal that says nothing.
+ *
+ * Note what case 2 keys on: the key being **registered**, not merely existing. A key is generated
+ * for every account at signup (`auth.router.ts`) and only ever registered on the provider when
+ * someone connects it — so "has a private key" is true for everyone and means nothing here.
+ */
+function originSetUrl(
+  r: { project: string },
+  path: string,
+  userSshKeyRegistered?: boolean,
+): string {
+  const https = `git -C ${path} remote set-url origin https://github.com/${r.project}.git`
+  const ssh = `git -C ${path} remote set-url origin git@github.com:${r.project}.git`
+
+  // **Undefined is a caller that predates this parameter**, and it gets exactly what it always
+  // got: the SSH rewrite. This package is consumed by published version, so a host can adopt a new
+  // release before it knows to pass the new field — and for that host there is no worker identity
+  // either, so answering HTTPS would leave it with an origin nothing can authenticate. Silence on
+  // a push, for every member with a registered key.
+  if (userSshKeyRegistered === undefined) return ssh
+
+  // Told, and the key is not registered: SSH would refuse, so stay on HTTPS and let the credential
+  // helper answer as the member.
+  if (!userSshKeyRegistered) return https
+
+  // Told, and registered: the helper still comes first where the worker has an identity.
+  return `if [ -n "\${SPUNTO_WORKER_TOKEN:-}" ]; then ${https}; else ${ssh}; fi`
+}
+
+const GIT_CREDENTIAL_HELPER = `#!/bin/sh
+# Asks Spunto for a credential belonging to this worker's owner, at the moment git needs it.
+# Only \`get\` is answered: there is nothing to store (we hold no cache) and nothing to erase.
+[ "$1" = "get" ] || exit 0
+while IFS= read -r line; do
+  case "$line" in
+    path=*) REPO="\${line#path=}" ;;
+    "") break ;;
+  esac
+done
+REPO="\${REPO%.git}"
+RESPONSE=$(curl -fsS -m 20 -X POST "$SPUNTO_API_URL/api/workers/git-credential" -H "Authorization: Bearer $SPUNTO_WORKER_TOKEN" -H "Content-Type: application/json" -d "{\\"repo\\":\\"$REPO\\"}" 2>/dev/null)
+PASSWORD=$(printf "%s" "$RESPONSE" | sed -n 's/.*"password":"\\([^"]*\\)".*/\\1/p')
+if [ -z "$PASSWORD" ]; then
+  echo "spunto: no git credential for $REPO - authorize the provider in Integrations." >&2
+  exit 0
+fi
+USERNAME=$(printf "%s" "$RESPONSE" | sed -n 's/.*"username":"\\([^"]*\\)".*/\\1/p')
+echo "username=$USERNAME"
+echo "password=$PASSWORD"
+`
+
 function credentialsBlock(params: {
   homeDir: string
   username: string
@@ -200,6 +312,30 @@ function credentialsBlock(params: {
     )
   }
 
+  // Git credential helper (RFC 0024). Git runs this at the moment of a `fetch`/`push` and reads
+  // `username=` / `password=` from its stdout; anything else it prints is ignored, so a failure
+  // message on stderr reaches the user's terminal while git falls through to asking as before.
+  //
+  // Nothing durable is stored: the credential is fetched per operation and lives only in the
+  // process that asked for it. `credential.useHttpPath` is on because the platform resolves *which*
+  // provider hosts a repository from its `owner/name`, which git only sends when asked to.
+  //
+  // Guarded on the worker token being present: a worker spawned before RFC 0024, or one whose
+  // token could not be minted, simply keeps the previous behaviour instead of failing to start.
+  lines.push(
+    'if [ -n "${SPUNTO_WORKER_TOKEN:-}" ] && [ -n "${SPUNTO_API_URL:-}" ]; then',
+    '  echo "Configuring git credential helper..."',
+    `  mkdir -p ${homeDir}/.spunto`,
+    `  cat > ${homeDir}/.spunto/git-credential-spunto <<'SPUNTO_HELPER'`,
+    GIT_CREDENTIAL_HELPER,
+    "SPUNTO_HELPER",
+    `  chmod +x ${homeDir}/.spunto/git-credential-spunto`,
+    `  git config --global credential.helper ${homeDir}/.spunto/git-credential-spunto`,
+    "  git config --global credential.useHttpPath true",
+    '  echo "Git credential helper configured"',
+    "fi",
+  )
+
   // Per-project deploy key (RFC 0013) — used only for generic "git" repos, referenced per-clone
   // via GIT_SSH_COMMAND (not added to ~/.ssh/config, so it never shadows the user's own key).
   if (projectDeployKey) {
@@ -714,6 +850,13 @@ export type SetupScriptParams = {
   workerId: string
   userInfo?: { name: string; email?: string | null }
   userSshPrivateKey?: string
+  /**
+   * Whether that key is **registered on the member's provider account**. Distinct from having one:
+   * every account is given a keypair at signup, and it is only ever registered when someone
+   * connects their provider — so `userSshPrivateKey` is set for everyone and says nothing about
+   * whether SSH would authenticate. See `originSetUrl`.
+   */
+  userSshKeyRegistered?: boolean
   sshGatewayPublicKey?: string
   skipFeatures?: boolean
   dotfilesRepo?: string | null
@@ -732,7 +875,7 @@ export type SetupScriptParams = {
 }
 
 export function buildSetupScript(params: SetupScriptParams): { script: string } {
-  const { project, workerId, userInfo, userSshPrivateKey, dotfilesRepo, userEnvSecrets, githubInstallationTokens, projectDeployKey, branch } = params
+  const { project, workerId, userInfo, userSshPrivateKey, userSshKeyRegistered, dotfilesRepo, userEnvSecrets, githubInstallationTokens, projectDeployKey, branch } = params
   const homeDir = "/home/vscode"
   const username = "vscode"
 
@@ -851,23 +994,23 @@ export function buildSetupScript(params: SetupScriptParams): { script: string }
 
   // ── 4. Dotfiles ───────────────────────────────────────────────────────────
   if (dotfilesRepo) {
-    const dotfilesUrl = dotfilesRepo.startsWith("http") || dotfilesRepo.startsWith("git@")
-      ? dotfilesRepo
-      : `https://github.com/${dotfilesRepo}`
+    const dotfilesUrl = normalizeDotfilesUrl(dotfilesRepo)
     push(...banner("SETUP: DOTFILES"))
     mpAt(mkStatus("dotfiles", allReposPending, pc0, null), "dotfiles")
     push(
       `echo "Cloning dotfiles from ${dotfilesUrl}..."`,
       `set +e`,
-      `git clone ${JSON.stringify(dotfilesUrl)} ${homeDir}/dotfiles 2>&1`,
+      // As the user, like every other clone: root has no `~/.ssh`, so a private dotfiles repo
+      // over SSH would authenticate with no key at all.
+      `${asUser(username, `git clone ${shQuote(dotfilesUrl)} ${homeDir}/dotfiles`)} 2>&1`,
       `_DOTS_EXIT=$?`,
       `set -e`,
       `if [ $_DOTS_EXIT -ne 0 ]; then`,
       `  echo "Dotfiles clone failed (exit $_DOTS_EXIT) — continuing without dotfiles"`,
       `else`,
       `  echo "Dotfiles cloned"`,
-      // Cloned as root, but the install script below runs as the user (and so does whoever edits
-      // these files later) — hand the clone over before touching it.
+      // The clone above already lands as the user. Kept for a worker whose ~/dotfiles was left
+      // root-owned by an earlier release, where the install script below would fail on it.
       `  chown -R ${username}:${username} ${homeDir}/dotfiles`,
       `  _INSTALL_SCRIPT=""`,
       `  for _candidate in install.sh bootstrap.sh setup.sh script/setup; do`,
@@ -911,9 +1054,11 @@ export function buildSetupScript(params: SetupScriptParams): { script: string }
       index: i,
       total: project.repositories.length,
       homeDir,
+      username,
       branch: resolveRepoBranch(r, branch),
       githubInstallationTokens,
       userSshPrivateKey,
+      userSshKeyRegistered,
     })
     push("")
     push(header)
@@ -924,11 +1069,10 @@ export function buildSetupScript(params: SetupScriptParams): { script: string }
   })
 
   // ── 5b. Re-own /workspace after cloning ───────────────────────────────────
-  // Repos are cloned as root (the setup script runs as root), so the cloned
-  // directories end up root-owned. The initial chown (step 1) ran *before* the
-  // clone, so it didn't cover them. postCreateCommand runs as vscode, so without
-  // this the user can't write into the repo (e.g. `npm install` → EACCES on
-  // node_modules). Must run before postCreate, not just in the final step.
+  // The clones above run as the user, so their checkouts already belong to them. This stays as a
+  // net: a repo left root-owned by an earlier release (when cloning *was* done as root) would
+  // otherwise keep failing postCreate with EACCES — `npm install` on a root-owned node_modules —
+  // and the final chown in step 8 comes too late for that.
   if (project.repositories.length > 0) {
     push("", `chown -R ${username}:${username} /workspace`)
   }
@@ -1659,7 +1803,7 @@ export function buildSetupPlan(params: SetupScriptParams): {
   initialStatus: SetupStatus
   readyStatus: SetupStatus
 } {
-  const { project, userInfo, userSshPrivateKey, dotfilesRepo, userEnvSecrets, githubInstallationTokens, projectDeployKey, branch } = params
+  const { project, userInfo, userSshPrivateKey, userSshKeyRegistered, dotfilesRepo, userEnvSecrets, githubInstallationTokens, projectDeployKey, branch } = params
   const homeDir = "/home/vscode"
   const username = "vscode"
 
@@ -1774,21 +1918,21 @@ export function buildSetupPlan(params: SetupScriptParams): {
 
   // ── dotfiles ──────────────────────────────────────────────────────────────
   if (dotfilesRepo) {
-    const dotfilesUrl = dotfilesRepo.startsWith("http") || dotfilesRepo.startsWith("git@")
-      ? dotfilesRepo
-      : `https://github.com/${dotfilesRepo}`
+    const dotfilesUrl = normalizeDotfilesUrl(dotfilesRepo)
     const l: string[] = ["set -e", ...banner("SETUP: DOTFILES"),
       `echo "Cloning dotfiles from ${dotfilesUrl}..."`,
       `set +e`,
-      `git clone ${JSON.stringify(dotfilesUrl)} ${homeDir}/dotfiles 2>&1`,
+      // As the user, like every other clone: root has no `~/.ssh`, so a private dotfiles repo
+      // over SSH would authenticate with no key at all.
+      `${asUser(username, `git clone ${shQuote(dotfilesUrl)} ${homeDir}/dotfiles`)} 2>&1`,
       `_DOTS_EXIT=$?`,
       `set -e`,
       `if [ $_DOTS_EXIT -ne 0 ]; then`,
       `  echo "Dotfiles clone failed (exit $_DOTS_EXIT) — continuing without dotfiles"`,
       `else`,
       `  echo "Dotfiles cloned"`,
-      // Cloned as root, but the install script below runs as the user (and so does whoever edits
-      // these files later) — hand the clone over before touching it.
+      // The clone above already lands as the user. Kept for a worker whose ~/dotfiles was left
+      // root-owned by an earlier release, where the install script below would fail on it.
       `  chown -R ${username}:${username} ${homeDir}/dotfiles`,
       `  _INSTALL_SCRIPT=""`,
       `  for _candidate in install.sh bootstrap.sh setup.sh script/setup; do`,
@@ -1833,9 +1977,11 @@ export function buildSetupPlan(params: SetupScriptParams): {
         index: i,
         total: project.repositories.length,
         homeDir,
+        username,
         branch: resolveRepoBranch(r, branch),
         githubInstallationTokens,
         userSshPrivateKey,
+        userSshKeyRegistered,
       })
       l.push("", header, ...cloneLines)
     })