@crombolo/cli 0.14.1 → 0.14.4

package/bin/crombolo.ts

@@ -32,6 +32,10 @@
  *   crombolo domains set            write domains.toml and apply it
  *
  *   crombolo import <file>          replace this site with a whole-site file; says what first
+ *   crombolo remote add|list|remove the remotes this site pulls from and pushes to (remotes.toml)
+ *   crombolo login|logout|whoami    sign in to a remote with a code you approve in its back office
+ *   crombolo pull [<remote>]        a copy of a remote environment: started here, or replacing this site
+ *   crombolo push [<remote>]        replace a remote environment with this site
  *   crombolo import umbraco report  read-only: what importing an Umbraco backup would and would not bring
  *   crombolo import umbraco apply   write a crombolo site from an Umbraco backup
  *
@@ -45,6 +49,7 @@
  */
 import { existsSync } from 'node:fs'
 import { mkdir, writeFile } from 'node:fs/promises'
+import { hostname } from 'node:os'
 import { basename, join, relative, resolve } from 'node:path'
 import { AuthStore, resetAdminPassword } from '@crombolo/auth'
 import { ObjectTypes } from '@crombolo/core'
@@ -108,15 +113,20 @@ import {
   describeDatabase,
   dictionaryToUdt,
   domainsFilePath,
+  forgetSignIn,
   formTypesFor,
   importDictionaryUdt,
   isSafeAlias,
   listAssets,
   listComponents,
+  listeningBodyLimit,
   loadConfig,
   mediaStoreFor,
+  osKeychain,
   placeBlobs,
   readDomainsFile,
+  readRemoteMedia,
+  readSignIn,
   resolveDomainNode,
   resolvePlaceholders,
   type ServerHandle,
@@ -129,6 +139,7 @@ import {
   writeDictionaryUdt,
   writeDomains,
   writeDomainsFile,
+  writeRemoteMedia,
   writeUdtAll,
 } from '@crombolo/server'
 import {
@@ -151,6 +162,23 @@ import {
   writeResolutions,
 } from '@crombolo/transfer'
 import * as ops from '../src/operations/commands.ts'
+import {
+  discover,
+  type Job,
+  problemOf,
+  RemoteClient,
+  RemoteError,
+  signIn,
+  TimedOut,
+} from '../src/remote/client.ts'
+import { addRemote, pickRemote, readRemotes, removeRemote } from '../src/remote/remotes-file.ts'
+import {
+  fetchReferencedMedia,
+  pullFile,
+  pushSite,
+  RefusedBeforeSending,
+  referencedBlobs,
+} from '../src/remote/transfer.ts'
 import { findTemplate, listTemplates, scaffoldFiles } from '../src/templates.ts'
 
 const cwd = process.cwd()
@@ -222,7 +250,7 @@ function stopOnSignal(
  * imported as a replace so nothing of the scaffold's own starting point stays.
  * The new administrator's password is printed here, once.
  */
-async function startFromFile(from: string): Promise<void> {
+async function startFromFile(from: string, known?: string): Promise<void> {
   const file = resolve(cwd, from)
   if (!existsSync(file)) {
     console.error(`${from} does not exist.`)
@@ -240,12 +268,16 @@ async function startFromFile(from: string): Promise<void> {
     console.error(`${from} is not a whole-site file: write one with \`crombolo export --site\`.`)
     process.exit(1)
   }
-  const passphrase = manifest.people
-    ? (process.env[PASSPHRASE_ENV] ??
-      (process.stdin.isTTY ? await readSecret('  Passphrase for its people: ') : undefined))
-    : undefined
-  if (manifest.people && !passphrase) {
-    console.error(`${from} carries people, sealed: set ${PASSPHRASE_ENV} to its passphrase.`)
+  // A pull already holds the passphrase it sealed the file with.
+  const sealed = manifest.people !== undefined || manifest.content === 'sealed'
+  const passphrase = known
+    ? known
+    : sealed
+      ? (process.env[PASSPHRASE_ENV] ??
+        (process.stdin.isTTY ? await readSecret('  Passphrase for its people: ') : undefined))
+      : undefined
+  if (sealed && !passphrase) {
+    console.error(`${from} is sealed: set ${PASSPHRASE_ENV} to its passphrase.`)
     process.exit(1)
   }
 
@@ -404,7 +436,7 @@ async function bootWithRetries(config: CromboloConfig) {
   }
   const listening = Bun.serve({
     port: config.port,
-    maxRequestBodySize: config.maxRequestBodyBytes,
+    maxRequestBodySize: listeningBodyLimit(config),
     fetch: holding,
     websocket: { message() {} },
   })
@@ -2606,6 +2638,27 @@ function help(): void {
   domains clear <path|uuid|/> unbind that page, in the file and in the database
   domains apply               apply domains.toml here, as a boot does
   domains undo                put back the file the last write replaced, and apply it
+  remote add <name> <url> [--env <id>]
+                              a remote this site pulls from and pushes to, kept in
+                              remotes.toml (URLs only; sign-ins are in your keychain)
+  remote list | remove <name>
+  login [<remote>]            sign in with a code you approve in the remote's back office
+  logout [<remote>] | whoami [<remote>]
+  pull [<remote>] [--env <id>] [--full] [--yes]
+                              a copy of the environment, sealed and anonymised unless
+                              --full: started here in an empty folder, or replacing this
+                              site (what it loses is recycled; --yes skips the question)
+                              [--no-media] [--media-info-only] fetches media files one at a
+                              time, only what changed since the last pull; [--lazy-media]
+                              fetches each the first time it is asked for
+  push [<remote>] [--env <id>] [--force] [--auto-approve] [--media-info-only]
+                              replace the environment with this site; shows what it would
+                              replace first. A protected one needs --force and its name
+                              typed back (--auto-approve skips that); the remote may also
+                              hold it for someone else's approval
+                              exit 0 done, 1 refused or failed, 2 refused before sending,
+                              3 gave up waiting
+  media fetch                 a lazily pulled site's media files, all of them now
   import <file> [--replace | --replace-all]
                               replace this site with a whole-site file. Says first what
                               would be replaced, and changes nothing without a mode:
@@ -2705,6 +2758,331 @@ function commandAudit(): ops.CommandAudit | undefined {
   }
 }
 
+// ---------------------------------------------------------------- remotes (story 108)
+
+const keychain = () => osKeychain()
+
+/** Exit codes a pull or push states: 1 the remote refused or failed, 2 refused before sending, 3 gave up waiting. */
+function remoteExit(error: unknown): never {
+  if (error instanceof RefusedBeforeSending) {
+    console.error(`  ${error.message}`)
+    process.exit(2)
+  }
+  if (error instanceof TimedOut) {
+    console.error(`  ${error.message} The job carries on at the remote.`)
+    process.exit(3)
+  }
+  if (error instanceof RemoteError) {
+    console.error(`  ${error.message}`)
+    for (const item of error.problem.items ?? []) console.error(`    ${item}`)
+    process.exit(error.problem.code === 'sign-in-required' ? 2 : 1)
+  }
+  console.error(`  ${error instanceof Error ? error.message : String(error)}`)
+  process.exit(1)
+}
+
+function openInBrowser(url: string): void {
+  if (flags.has('--no-browser')) return
+  const opener =
+    process.platform === 'darwin'
+      ? ['open']
+      : process.platform === 'win32'
+        ? ['cmd', '/c', 'start', '']
+        : ['xdg-open']
+  try {
+    Bun.spawn([...opener, url], { stdout: 'ignore', stderr: 'ignore' })
+  } catch {
+    // Printed above; opening it is a convenience.
+  }
+}
+
+/** The remote a command names, signed in or not, with the environment it goes to. */
+async function connectRemote(name: string | undefined) {
+  const entry = pickRemote(cwd, name)
+  if (!entry) {
+    const known = readRemotes(cwd)
+    console.error(
+      known.length === 0
+        ? '  No remotes yet: `crombolo remote add <name> <url>`.'
+        : `  Which remote? ${known.map((r) => r.name).join(', ')}`,
+    )
+    process.exit(2)
+  }
+  const discovery = await discover(entry.url).catch(remoteExit)
+  const client = new RemoteClient(discovery, keychain())
+  const environment =
+    flagValue('--env') ??
+    entry.environment ??
+    (discovery.environments.length === 1 ? discovery.environments[0]?.id : undefined)
+  return { entry, discovery, client, environment }
+}
+
+async function remoteCommand(): Promise<void> {
+  const [sub, name, url] = positional
+  if (sub === 'add') {
+    if (!name || !url) {
+      console.error("remote add needs a name and the site's URL.")
+      process.exit(2)
+    }
+    const discovery = await discover(url).catch(remoteExit)
+    addRemote(cwd, { name, url: new URL(url).origin, environment: flagValue('--env') })
+    console.log(`  added    ${name}  ${discovery.name} (${new URL(url).origin})`)
+    for (const env of discovery.environments)
+      console.log(`  env      ${env.id}${env.protected ? '  protected' : ''}`)
+    console.log(`\n\`crombolo login ${name}\` signs in.`)
+    return
+  }
+  if (sub === 'remove') {
+    if (!name || !removeRemote(cwd, name)) {
+      console.error(`  There is no remote "${name ?? ''}".`)
+      process.exit(2)
+    }
+    console.log(`  removed  ${name}`)
+    return
+  }
+  const remotes = readRemotes(cwd)
+  if (remotes.length === 0) console.log('  No remotes yet: `crombolo remote add <name> <url>`.')
+  for (const remote of remotes) {
+    const signed = (await readSignIn(keychain(), remote.url).catch(() => undefined)) !== undefined
+    console.log(
+      `  ${remote.name.padEnd(12)} ${remote.url}${remote.environment ? `  env ${remote.environment}` : ''}${signed ? '' : '  (not signed in)'}`,
+    )
+  }
+}
+
+async function loginCommand(): Promise<void> {
+  const { entry, discovery } = await connectRemote(positional[0])
+  await signIn(discovery, keychain(), {
+    clientName: `crombolo on ${hostname()}`,
+    onCode: (code, url) => {
+      console.log(`  Approve this sign-in at ${url}`)
+      console.log(`  and check it shows   ${code}`)
+      openInBrowser(url)
+    },
+  }).catch(remoteExit)
+  console.log(`  signed in to ${entry.name}`)
+}
+
+async function logoutCommand(): Promise<void> {
+  const { entry, discovery } = await connectRemote(positional[0])
+  const signedIn = await readSignIn(keychain(), discovery.api)
+  // Ended at the remote too, best effort: the device leaves "Signed-in devices".
+  if (signedIn)
+    await fetch(signedIn.tokenUrl.replace(/\/token$/, '/revoke'), {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: new URLSearchParams({ token: signedIn.accessToken }).toString(),
+    }).catch(() => undefined)
+  await forgetSignIn(keychain(), discovery.api)
+  console.log(`  signed out of ${entry.name}`)
+}
+
+async function whoamiCommand(): Promise<void> {
+  const { entry, client, discovery } = await connectRemote(positional[0])
+  const response = await client
+    .call(`${new URL(discovery.api).origin}/umbraco/management/api/v1/user/current`)
+    .catch(remoteExit)
+  if (!response.ok) remoteExit(await problemOf(response))
+  const me = (await response.json()) as { name: string; email: string }
+  console.log(`  ${me.name} <${me.email}> on ${entry.name}`)
+  for (const env of await client.environments().catch(remoteExit)) {
+    const may = [env.may.pull && 'pull', env.may.pullFull && 'full pull', env.may.push && 'push']
+      .filter(Boolean)
+      .join(', ')
+    console.log(`  ${env.id.padEnd(12)} ${may || 'nothing'}${env.protected ? '  (protected)' : ''}`)
+  }
+}
+
+const describeJob = (job: Job) =>
+  `  ${job.kind.padEnd(5)} ${job.state}${job.stage ? `, ${job.stage}` : ''}${job.progress != null ? ` (${job.progress}%)` : ''}`
+
+async function pullCommand(): Promise<void> {
+  const { client, environment, discovery } = await connectRemote(positional[0])
+  if (!environment) {
+    console.error(
+      `  Which environment? ${discovery.environments.map((e) => e.id).join(', ')} (--env)`,
+    )
+    process.exit(2)
+  }
+  const media = flags.has('--no-media')
+    ? ('omitted' as const)
+    : flags.has('--media-info-only') || flags.has('--lazy-media')
+      ? ('referenced' as const)
+      : ('included' as const)
+  const incoming = join(cwd, '.crombolo', 'transfer', 'incoming')
+  await mkdir(incoming, { recursive: true })
+  const out = join(incoming, `pull-${environment}-${Date.now().toString(36)}.tgz`)
+  const pulled = await pullFile(client, {
+    environment,
+    out,
+    full: flags.has('--full'),
+    media,
+    onProgress: (job) => console.log(describeJob(job)),
+  }).catch(remoteExit)
+  console.log(
+    `  pulled   ${describeBytes(pulled.size)}${flags.has('--full') ? '' : ', anonymised'}`,
+  )
+  if (media !== 'included') flags.add('--allow-missing-media')
+
+  const existing = existsSync(join(cwd, 'crombolo.config.ts'))
+  if (!existing) {
+    await startFromFile(out, pulled.passphrase)
+  } else {
+    const config = await siteConfig()
+    const { summary } = await ops.siteReplaceCheck(config, {
+      bundle: out,
+      passphrase: pulled.passphrase,
+      replace: 'replace',
+      allowMissingBlobs: media !== 'included',
+    })
+    const leaving = Object.entries(summary.leaving)
+    if (leaving.length > 0)
+      console.log(
+        `  recycled ${leaving.map(([kind, n]) => `${n} ${kind}`).join(', ')} here, not in ${environment}`,
+      )
+    if (summary.filesRemoved.length > 0)
+      console.log(`  removes  ${summary.filesRemoved.length} file(s) here, kept for a revert`)
+    if (!flags.has('--yes')) {
+      if (!process.stdin.isTTY) {
+        console.error(`\nThis replaces this site with ${environment}: pass --yes to go on.`)
+        process.exit(2)
+      }
+      const answer = prompt(`  Replace this site with ${environment}? [y/N]`)
+      if (!/^y(es)?$/i.test(answer?.trim() ?? '')) {
+        console.error('  Nothing was changed.')
+        process.exit(2)
+      }
+    }
+    const result = await ops.contentImport(config, {
+      bundle: out,
+      passphrase: pulled.passphrase,
+      replace: 'replace',
+      allowMissingBlobs: media !== 'included',
+      label: `pull from ${environment}`,
+      backupTaken: flags.has('--backup-taken'),
+    })
+    for (const problem of result.problems) console.error(`  ${problem}`)
+    if (!result.runId) {
+      printTransferFindings(result.check)
+      console.error('\npull refused. Nothing was changed.')
+      process.exit(1)
+    }
+    console.log(
+      `  replaced, run ${result.runId}: \`crombolo content revert ${result.runId}\` puts it back`,
+    )
+  }
+
+  if (media === 'referenced') {
+    const config = await siteConfig()
+    const blobs = await referencedBlobs(out)
+    if (flags.has('--lazy-media')) {
+      writeRemoteMedia(cwd, { api: discovery.api, environment })
+      await writeFile(
+        join(cwd, '.crombolo', 'remote-media-blobs.json'),
+        `${JSON.stringify(
+          blobs.map((b) => ({ key: b.key, etag: b.etag })),
+          null,
+          2,
+        )}\n`,
+      )
+      console.log(
+        `  media    ${blobs.length} file(s) fetched from ${environment} as they are first asked for`,
+      )
+    } else {
+      const fetched = await fetchReferencedMedia(client, {
+        environment,
+        blobs,
+        store: await mediaStoreFor(config),
+        siteDir: cwd,
+      }).catch(remoteExit)
+      console.log(
+        `  media    ${fetched.fetched} fetched (${describeBytes(fetched.bytes)}), ${fetched.skipped} already here`,
+      )
+      for (const key of fetched.failed)
+        console.error(`  missing  ${key}: run the pull again to fetch it`)
+    }
+  }
+  if (!existing) await start()
+}
+
+async function pushCommand(): Promise<void> {
+  const { client, environment, discovery } = await connectRemote(positional[0])
+  if (!environment) {
+    console.error(
+      `  Which environment? ${discovery.environments.map((e) => e.id).join(', ')} (--env)`,
+    )
+    process.exit(2)
+  }
+  const config = await siteConfig()
+  const result = await pushSite(config, client, {
+    environment,
+    force: flags.has('--force'),
+    autoApprove: flags.has('--auto-approve'),
+    mediaInfoOnly: flags.has('--media-info-only'),
+    onCheck: (check) => {
+      const counts = Object.entries(check.sending.counts)
+      console.log(
+        `  sending  ${counts.map(([kind, n]) => `${n} ${kind}`).join(', ') || 'no content'}, ${check.sending.files} file(s)`,
+      )
+      if (check.fromRemote) {
+        const recycles = Object.entries(check.recycles)
+        if (recycles.length > 0)
+          console.log(
+            `  recycles ${recycles.map(([kind, n]) => `${n} ${kind}`).join(', ')} on ${environment}`,
+          )
+        if (check.removesFiles.length > 0)
+          console.log(`  removes  ${check.removesFiles.join(', ')} on ${environment}`)
+      }
+    },
+    confirm: async () => {
+      if (!process.stdin.isTTY) return false
+      const answer = prompt(`  This replaces ${environment}. Type its name to go on:`)
+      return answer?.trim() === environment
+    },
+    onProgress: (job) => console.log(describeJob(job)),
+    onApproval: (job) => {
+      console.log(`  waiting  for someone else to approve it: ${job.approvalUrl}`)
+      if (job.approvalUrl) openInBrowser(job.approvalUrl)
+    },
+  }).catch(remoteExit)
+  const { job } = result
+  if (job.state === 'done') {
+    console.log(`\n  pushed   ${environment} replaced, run ${job.runId ?? '—'}`)
+    return
+  }
+  console.error(`\n  ${job.problem?.title ?? `the push ended ${job.state}`}`)
+  for (const item of job.problem?.items ?? []) console.error(`    ${item}`)
+  process.exit(1)
+}
+
+async function mediaCommand(): Promise<void> {
+  if (positional[0] !== 'fetch') return help()
+  const source = readRemoteMedia(cwd)
+  const listed = join(cwd, '.crombolo', 'remote-media-blobs.json')
+  if (!source || !existsSync(listed)) {
+    console.error('  This site was not pulled with --lazy-media: there is nothing to fetch.')
+    process.exit(2)
+  }
+  const discovery = await discover(source.api).catch(remoteExit)
+  const client = new RemoteClient(discovery, keychain())
+  const config = await siteConfig()
+  const blobs = JSON.parse(await Bun.file(listed).text()) as { key: string; etag: string | null }[]
+  const fetched = await fetchReferencedMedia(client, {
+    environment: source.environment,
+    blobs,
+    store: await mediaStoreFor(config),
+    siteDir: cwd,
+    onProgress: (done, total) => {
+      if (done % 50 === 0 || done === total) console.log(`  ${done}/${total}`)
+    },
+  }).catch(remoteExit)
+  console.log(
+    `  media    ${fetched.fetched} fetched (${describeBytes(fetched.bytes)}), ${fetched.skipped} already here`,
+  )
+  for (const key of fetched.failed) console.error(`  missing  ${key}`)
+  if (fetched.failed.length > 0) process.exit(1)
+}
+
 async function dispatch(): Promise<void> {
   switch (command) {
     case 'start':
@@ -2757,6 +3135,27 @@ async function dispatch(): Promise<void> {
     case 'export':
       await exportCommand()
       break
+    case 'remote':
+      await remoteCommand()
+      break
+    case 'login':
+      await loginCommand()
+      break
+    case 'logout':
+      await logoutCommand()
+      break
+    case 'whoami':
+      await whoamiCommand()
+      break
+    case 'pull':
+      await pullCommand()
+      break
+    case 'push':
+      await pushCommand()
+      break
+    case 'media':
+      await mediaCommand()
+      break
     case 'import':
       await importSite()
       break

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/cli",
-  "version": "0.14.1",
+  "version": "0.14.4",
   "description": "The crombolo command line: scaffolding a site, schema, views, domains, content transfer and upgrades.",
   "license": "MIT",
   "author": "Crombolo",
@@ -34,11 +34,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/auth": "0.14.1",
-    "@crombolo/core": "0.14.1",
-    "@crombolo/data": "0.14.1",
-    "@crombolo/schema": "0.14.1",
-    "@crombolo/server": "0.14.1",
-    "@crombolo/transfer": "0.14.1"
+    "@crombolo/auth": "0.14.4",
+    "@crombolo/core": "0.14.4",
+    "@crombolo/data": "0.14.4",
+    "@crombolo/schema": "0.14.4",
+    "@crombolo/server": "0.14.4",
+    "@crombolo/transfer": "0.14.4"
   }
 }

package/src/remote/client.ts

@@ -0,0 +1,288 @@
+/**
+ * The CLI's side of the remote protocol (story 108, `docs/remote-protocol.md`):
+ * discovery, signing in by the device grant, and the API — jobs, downloads,
+ * uploads and media — under a sign-in kept in the keychain and refreshed as it
+ * runs out.
+ */
+import { createHash } from 'node:crypto'
+import { createWriteStream } from 'node:fs'
+import { freshAccessToken, type Keychain, readSignIn, saveSignIn } from '@crombolo/server'
+
+export const PROTOCOL_VERSION = '1'
+
+export interface Discovery {
+  protocol: string
+  versions: string[]
+  name: string
+  api: string
+  auth: { deviceAuthorization: string; token: string; clientId: string; grantType: string }
+  capabilities: string[]
+  environments: Array<{ id: string; name: string; protected: boolean }>
+}
+
+export interface Problem {
+  status: number
+  code: string
+  title: string
+  detail?: string
+  items?: string[]
+}
+
+export class RemoteError extends Error {
+  constructor(readonly problem: Problem) {
+    super(problem.detail ? `${problem.title}: ${problem.detail}` : problem.title)
+  }
+}
+
+export interface Job {
+  id: string
+  kind: 'pull' | 'push'
+  environment: string
+  state: 'queued' | 'running' | 'approval-required' | 'done' | 'failed' | 'refused'
+  stage?: string | null
+  progress?: number | null
+  pollAfter?: number
+  approvalUrl?: string
+  download?: { url: string; expiresAt: string; size: number | null; sha256: string | null }
+  problem?: Problem
+  runId?: string
+}
+
+/** Reads a remote's discovery document; refuses a site that is not a remote, or one speaking only other versions. */
+export async function discover(url: string, http: typeof fetch = fetch): Promise<Discovery> {
+  const origin = new URL(url).origin
+  let response: Response
+  try {
+    response = await http(`${origin}/.well-known/crombolo-remote`)
+  } catch (error) {
+    throw new Error(
+      `${origin} could not be reached: ${error instanceof Error ? error.message : error}`,
+    )
+  }
+  if (response.status === 404) throw new Error(`${origin} is not a crombolo remote.`)
+  if (!response.ok)
+    throw new Error(`${origin} answered ${response.status} for its discovery document.`)
+  const body = (await response.json()) as Discovery
+  if (body.protocol !== 'crombolo-remote') throw new Error(`${origin} is not a crombolo remote.`)
+  if (!body.versions?.includes(PROTOCOL_VERSION))
+    throw new Error(
+      `${origin} speaks protocol ${body.versions?.join(', ') || 'none'}; this crombolo speaks ${PROTOCOL_VERSION}.`,
+    )
+  return body
+}
+
+/**
+ * Signs in by the device grant (RFC 8628): `onCode` shows the person the code
+ * and the page to approve it on, then this polls until they decide. The tokens
+ * go into the keychain.
+ */
+export async function signIn(
+  discovery: Discovery,
+  keychain: Keychain,
+  options: {
+    clientName: string
+    onCode(code: string, url: string): void | Promise<void>
+    http?: typeof fetch
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<void> {
+  const http = options.http ?? fetch
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms))
+  const form = (url: string, fields: Record<string, string>) =>
+    http(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: new URLSearchParams(fields).toString(),
+    })
+  const started = await form(discovery.auth.deviceAuthorization, {
+    client_id: discovery.auth.clientId,
+    client_name: options.clientName,
+  })
+  if (!started.ok) throw new Error(`The remote would not start a sign-in (${started.status}).`)
+  const codes = (await started.json()) as {
+    device_code: string
+    user_code: string
+    verification_uri: string
+    verification_uri_complete?: string
+    expires_in: number
+    interval: number
+  }
+  await options.onCode(codes.user_code, codes.verification_uri_complete ?? codes.verification_uri)
+  let interval = codes.interval * 1000
+  const deadline = Date.now() + codes.expires_in * 1000
+  for (;;) {
+    await sleep(interval)
+    const polled = await form(discovery.auth.token, {
+      grant_type: discovery.auth.grantType,
+      device_code: codes.device_code,
+      client_id: discovery.auth.clientId,
+    })
+    const body = (await polled.json().catch(() => ({}))) as {
+      error?: string
+      access_token?: string
+      refresh_token?: string
+      expires_in?: number
+    }
+    if (polled.ok && body.access_token) {
+      await saveSignIn(keychain, discovery.api, {
+        accessToken: body.access_token,
+        refreshToken: body.refresh_token ?? '',
+        expiresAt: Date.now() + (body.expires_in ?? 0) * 1000,
+        tokenUrl: discovery.auth.token,
+        clientId: discovery.auth.clientId,
+      })
+      return
+    }
+    if (body.error === 'slow_down') interval += 5000
+    else if (body.error === 'access_denied') throw new Error('The sign-in was declined.')
+    else if (body.error === 'expired_token' || Date.now() > deadline)
+      throw new Error('The code expired before it was approved. Sign in again.')
+    else if (body.error !== 'authorization_pending')
+      throw new Error(`The sign-in failed: ${body.error ?? polled.status}`)
+  }
+}
+
+/** The protocol's API for one remote, as the signed-in person. */
+export class RemoteClient {
+  readonly discovery: Discovery
+  #keychain: Keychain
+  #http: typeof fetch
+
+  constructor(discovery: Discovery, keychain: Keychain, http: typeof fetch = fetch) {
+    this.discovery = discovery
+    this.#keychain = keychain
+    this.#http = http
+  }
+
+  async signedIn(): Promise<boolean> {
+    return (await readSignIn(this.#keychain, this.discovery.api)) !== undefined
+  }
+
+  /** One authorised request; a 401 refreshes the sign-in once and tries again. */
+  async call(url: string, init: RequestInit = {}): Promise<Response> {
+    const attempt = async (force: boolean) => {
+      const token = await freshAccessToken(this.#keychain, this.discovery.api, this.#http, force)
+      if (!token)
+        throw new RemoteError({
+          status: 401,
+          code: 'sign-in-required',
+          title: `Not signed in to ${new URL(this.discovery.api).origin}`,
+          detail: 'Run `crombolo login` first.',
+        })
+      const headers = new Headers(init.headers)
+      headers.set('authorization', `Bearer ${token}`)
+      return this.#http(url, { ...init, headers })
+    }
+    const first = await attempt(false)
+    if (first.status !== 401) return first
+    return attempt(true)
+  }
+
+  async api(path: string, init: RequestInit = {}): Promise<Response> {
+    return this.call(`${this.discovery.api}/${path}`, init)
+  }
+
+  async json<T>(path: string, init: RequestInit = {}): Promise<T> {
+    const response = await this.api(path, init)
+    if (!response.ok) throw await problemOf(response)
+    return (await response.json()) as T
+  }
+
+  post<T>(path: string, body: unknown): Promise<T> {
+    return this.json<T>(path, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+    })
+  }
+
+  environments() {
+    return this.json<
+      Array<{
+        id: string
+        name: string
+        protected: boolean
+        may: { pull: boolean; pullFull: boolean; push: boolean }
+      }>
+    >('environments')
+  }
+
+  /** Waits for a job to stop moving, reporting each change; `onApproval` hears it once. */
+  async settle(
+    id: string,
+    options: {
+      onProgress?(job: Job): void
+      onApproval?(job: Job): void | Promise<void>
+      timeoutMs?: number
+      sleep?: (ms: number) => Promise<void>
+    } = {},
+  ): Promise<Job> {
+    const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms))
+    const deadline = Date.now() + (options.timeoutMs ?? 30 * 60_000)
+    let announced = false
+    let last = ''
+    for (;;) {
+      const job = await this.json<Job>(`jobs/${id}`)
+      const seen = `${job.state}|${job.stage ?? ''}|${job.progress ?? ''}`
+      if (seen !== last) options.onProgress?.(job)
+      last = seen
+      if (job.state === 'approval-required' && !announced) {
+        announced = true
+        await options.onApproval?.(job)
+      }
+      if (job.state !== 'queued' && job.state !== 'running' && job.state !== 'approval-required')
+        return job
+      if (Date.now() > deadline) throw new TimedOut(job)
+      await sleep((job.pollAfter ?? 2) * 1000)
+    }
+  }
+
+  /** Streams a finished pull to `path`, checking it against the hash the job stated. */
+  async download(job: Job, path: string): Promise<{ size: number }> {
+    if (!job.download) throw new Error('The job has nothing to download.')
+    const response = await this.call(job.download.url)
+    if (!response.ok || !response.body) throw await problemOf(response)
+    const hasher = createHash('sha256')
+    const sink = createWriteStream(path)
+    let size = 0
+    const reader = response.body.getReader()
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      hasher.update(value)
+      size += value.length
+      if (!sink.write(value)) await new Promise((r) => sink.once('drain', r))
+    }
+    await new Promise<void>((resolve, reject) =>
+      sink.end((error?: Error | null) => (error ? reject(error) : resolve())),
+    )
+    if (job.download.sha256 && hasher.digest('hex') !== job.download.sha256)
+      throw new Error('The download does not match the hash the remote stated; try again.')
+    return { size }
+  }
+}
+
+/** A job still moving when the client stopped waiting. */
+export class TimedOut extends Error {
+  constructor(readonly job: Job) {
+    super(`Still ${job.state} when the wait ran out.`)
+  }
+}
+
+export async function problemOf(response: Response): Promise<RemoteError> {
+  const body = (await response.json().catch(() => ({}))) as Partial<Problem>
+  return new RemoteError({
+    status: response.status,
+    code: body.code ?? `http-${response.status}`,
+    title: body.title ?? `The remote answered ${response.status}`,
+    ...(body.detail ? { detail: body.detail } : {}),
+    ...(body.items ? { items: body.items } : {}),
+  })
+}
+
+/** SHA-256 of a file, streamed. */
+export async function sha256File(path: string): Promise<string> {
+  const hasher = createHash('sha256')
+  for await (const chunk of Bun.file(path).stream()) hasher.update(chunk)
+  return hasher.digest('hex')
+}

package/src/remote/remotes-file.ts

@@ -0,0 +1,80 @@
+/**
+ * `remotes.toml` at a site's root (story 108): the remotes a team pulls from
+ * and pushes to, by name — URLs only, never credentials, so it is committed.
+ *
+ *   [remotes.production]
+ *   url = "https://example.com"
+ *   environment = "production"
+ */
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export const REMOTES_FILE = 'remotes.toml'
+
+export interface RemoteEntry {
+  name: string
+  url: string
+  /** The environment a pull or push goes to when `--env` is not given. */
+  environment?: string
+}
+
+const NAME = /^[a-z][a-z0-9-]{0,39}$/
+
+export function readRemotes(siteDir: string): RemoteEntry[] {
+  const file = join(siteDir, REMOTES_FILE)
+  if (!existsSync(file)) return []
+  const parsed = Bun.TOML.parse(readFileSync(file, 'utf8')) as {
+    remotes?: Record<string, { url?: unknown; environment?: unknown }>
+  }
+  return Object.entries(parsed.remotes ?? {}).flatMap(([name, entry]) =>
+    typeof entry?.url === 'string'
+      ? [
+          {
+            name,
+            url: entry.url,
+            ...(typeof entry.environment === 'string' ? { environment: entry.environment } : {}),
+          },
+        ]
+      : [],
+  )
+}
+
+const quote = (value: string) => JSON.stringify(value)
+
+export function writeRemotes(siteDir: string, remotes: readonly RemoteEntry[]): void {
+  const lines = [
+    '# The remotes `crombolo pull` and `push` reach, by name. URLs only: sign-ins',
+    '# are kept in your keychain, never here.',
+  ]
+  for (const remote of remotes) {
+    lines.push('', `[remotes.${remote.name}]`, `url = ${quote(remote.url)}`)
+    if (remote.environment) lines.push(`environment = ${quote(remote.environment)}`)
+  }
+  writeFileSync(join(siteDir, REMOTES_FILE), `${lines.join('\n')}\n`)
+}
+
+/** Adds or replaces one, refusing a name that would not survive as a TOML key. */
+export function addRemote(siteDir: string, remote: RemoteEntry): void {
+  if (!NAME.test(remote.name))
+    throw new Error(`"${remote.name}" is not a remote name: lower case letters, digits and dashes.`)
+  const others = readRemotes(siteDir).filter((r) => r.name !== remote.name)
+  writeRemotes(
+    siteDir,
+    [...others, remote].sort((a, b) => a.name.localeCompare(b.name)),
+  )
+}
+
+export function removeRemote(siteDir: string, name: string): boolean {
+  const remotes = readRemotes(siteDir)
+  const kept = remotes.filter((r) => r.name !== name)
+  if (kept.length === remotes.length) return false
+  writeRemotes(siteDir, kept)
+  return true
+}
+
+/** The remote a command names, or the only one there is. */
+export function pickRemote(siteDir: string, name: string | undefined): RemoteEntry | undefined {
+  const remotes = readRemotes(siteDir)
+  if (name) return remotes.find((r) => r.name === name)
+  return remotes.length === 1 ? remotes[0] : undefined
+}

package/src/remote/transfer.ts

@@ -0,0 +1,337 @@
+/**
+ * `crombolo pull` and `crombolo push` as functions (story 108): they take a
+ * signed-in client and options, and return what happened; the binary prints.
+ *
+ * A pull is a job on the remote ending in a sealed file, opened here with a
+ * passphrase made for that one pull and kept in memory. A push exports this
+ * site sealed the same way, checks with the remote what it would replace, and
+ * uploads it as a job — which the remote may hold for approval.
+ */
+import { randomBytes } from 'node:crypto'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+  bootstrapDatabase,
+  type CromboloConfig,
+  exportSite,
+  extractTarball,
+  type MediaStore,
+  mediaStoreFor,
+} from '@crombolo/server'
+import { type BundleBlob, loadBundle } from '@crombolo/transfer'
+import { type Job, problemOf, type RemoteClient, RemoteError, sha256File } from './client.ts'
+
+export type MediaMode = 'included' | 'omitted' | 'referenced'
+
+/** A passphrase for one transfer: never shown, never stored. */
+export const transferPassphrase = () => randomBytes(24).toString('base64url')
+
+/** Refused before anything was sent: the CLI's own guard, exit code 2. */
+export class RefusedBeforeSending extends Error {}
+
+export interface PullResult {
+  job: Job
+  file: string
+  passphrase: string
+  size: number
+}
+
+/** Asks the remote for its site and downloads the file to `out`. */
+export async function pullFile(
+  client: RemoteClient,
+  options: {
+    environment: string
+    out: string
+    full?: boolean
+    media?: MediaMode
+    onProgress?(job: Job): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PullResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} has no environment "${options.environment}": it has ${environments.map((e) => e.id).join(', ')}.`,
+    )
+  if (!env.may.pull) throw new RefusedBeforeSending(`You may not pull from ${env.name}.`)
+  if (options.full && !env.may.pullFull)
+    throw new RefusedBeforeSending(
+      `You may not pull a full copy of ${env.name}; pull without --full for an anonymised one.`,
+    )
+  if (options.media === 'referenced' && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} does not serve media by reference; pull without --media-info-only.`,
+    )
+  const passphrase = transferPassphrase()
+  const started = await client.post<Job>(`environments/${env.id}/pulls`, {
+    passphrase,
+    full: options.full === true,
+    media: options.media ?? 'included',
+  })
+  const job = await client.settle(started.id, {
+    onProgress: options.onProgress,
+    timeoutMs: options.timeoutMs,
+    sleep: options.sleep,
+  })
+  if (job.state !== 'done') throw new RemoteError(job.problem ?? jobProblem(job))
+  const { size } = await client.download(job, options.out)
+  return { job, file: options.out, passphrase, size }
+}
+
+const jobProblem = (job: Job) => ({
+  status: 0,
+  code: job.state,
+  title: `The ${job.kind} ended ${job.state}`,
+})
+
+/** What the local site has already fetched of a remote's media, by key: the remote's etag. */
+const FETCHED_FILE = join('.crombolo', 'remote-media-fetched.json')
+
+function readFetched(siteDir: string): Record<string, string> {
+  const file = join(siteDir, FETCHED_FILE)
+  if (!existsSync(file)) return {}
+  try {
+    return JSON.parse(readFileSync(file, 'utf8')) as Record<string, string>
+  } catch {
+    return {}
+  }
+}
+
+export interface MediaFetchResult {
+  fetched: number
+  skipped: number
+  bytes: number
+  failed: string[]
+}
+
+/**
+ * Fetches the media files a file names by reference into the local store, a
+ * batch of links at a time and a few files at once. A file already fetched at
+ * the same etag is skipped, so a second pull downloads only what changed.
+ */
+export async function fetchReferencedMedia(
+  client: RemoteClient,
+  options: {
+    environment: string
+    blobs: readonly Pick<BundleBlob, 'key' | 'etag'>[]
+    store: MediaStore
+    siteDir: string
+    onProgress?(done: number, total: number): void
+    http?: typeof fetch
+  },
+): Promise<MediaFetchResult> {
+  const fetched = readFetched(options.siteDir)
+  const result: MediaFetchResult = { fetched: 0, skipped: 0, bytes: 0, failed: [] }
+  const wanted: string[] = []
+  for (const blob of options.blobs) {
+    const have = await options.store.get(blob.key)
+    if (have && blob.etag && fetched[blob.key] === blob.etag) result.skipped++
+    else wanted.push(blob.key)
+  }
+  let done = 0
+  for (let at = 0; at < wanted.length; at += 200) {
+    const batch = wanted.slice(at, at + 200)
+    const { links } = await client.post<{
+      links: Record<string, { url: string; etag: string; size: number } | null>
+    }>(`environments/${options.environment}/media/links`, { keys: batch })
+    const queue = [...batch]
+    const worker = async () => {
+      for (;;) {
+        const key = queue.shift()
+        if (!key) return
+        const link = links[key]
+        try {
+          if (!link) throw new Error('not on the remote')
+          // A link through the API needs the sign-in; a presigned one does not.
+          const response = link.url.startsWith(client.discovery.api)
+            ? await client.call(link.url)
+            : await (options.http ?? fetch)(link.url)
+          if (!response.ok) throw await problemOf(response)
+          const body = await response.blob()
+          await options.store.put(key, body)
+          fetched[key] = link.etag
+          result.fetched++
+          result.bytes += body.size
+        } catch {
+          result.failed.push(key)
+        }
+        options.onProgress?.(++done, wanted.length)
+      }
+    }
+    await Promise.all(Array.from({ length: 6 }, worker))
+  }
+  mkdirSync(join(options.siteDir, '.crombolo'), { recursive: true })
+  writeFileSync(join(options.siteDir, FETCHED_FILE), `${JSON.stringify(fetched, null, 2)}\n`)
+  return result
+}
+
+/** The blobs a file names by reference, read from its manifest. */
+export async function referencedBlobs(file: string): Promise<BundleBlob[]> {
+  const dir = mkdtempSync(join(tmpdir(), 'crombolo-pulled-'))
+  try {
+    await extractTarball(file, dir)
+    const manifest = loadBundle(join(dir, 'content')).set?.manifest
+    return manifest?.media === 'referenced' ? manifest.blobs : []
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+export interface PushCheck {
+  environment: string
+  recycles: Record<string, number>
+  removesFiles: string[]
+  keepsPeople: boolean
+  /** Whether the remote answered it, or it is only what this side is sending. */
+  fromRemote: boolean
+  sending: { counts: Record<string, number>; files: number; media: number }
+}
+
+export interface PushResult {
+  check: PushCheck
+  job: Job
+}
+
+/**
+ * Exports this site and sends it. `confirm` is asked with the check for a
+ * protected environment unless `autoApprove`; `onApproval` hears the job when
+ * the remote holds it for approval. The flags are this side's guard against a
+ * slip, never authority: the remote decides.
+ */
+export async function pushSite(
+  config: CromboloConfig,
+  client: RemoteClient,
+  options: {
+    environment: string
+    force?: boolean
+    autoApprove?: boolean
+    mediaInfoOnly?: boolean
+    confirm?(check: PushCheck): Promise<boolean>
+    onCheck?(check: PushCheck): void
+    onProgress?(job: Job): void
+    onApproval?(job: Job): void | Promise<void>
+    onMedia?(done: number, total: number): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PushResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} has no environment "${options.environment}".`,
+    )
+  if (!env.may.push) throw new RefusedBeforeSending(`You may not push to ${env.name}.`)
+  if (env.protected && !options.force)
+    throw new RefusedBeforeSending(
+      `${env.name} is protected: pushing replaces what people rely on. Pass --force to go on.`,
+    )
+  if (options.mediaInfoOnly && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      `${client.discovery.name} does not take media by reference; push without --media-info-only.`,
+    )
+
+  const work = mkdtempSync(join(tmpdir(), 'crombolo-push-'))
+  try {
+    const out = join(work, 'site.tgz')
+    const passphrase = transferPassphrase()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      await exportSite(db, config, {
+        out,
+        sealContent: true,
+        passphrase,
+        media: options.mediaInfoOnly ? 'referenced' : true,
+      })
+    } finally {
+      await db.close()
+    }
+    const unpacked = join(work, 'unpacked')
+    await extractTarball(out, unpacked)
+    const loaded = loadBundle(join(unpacked, 'content'))
+    const manifest = loaded.set?.manifest
+    if (!manifest) throw new Error('The export could not be read back.')
+    const sending = {
+      counts: { ...manifest.counts } as Record<string, number>,
+      files: loaded.files.size,
+      media: manifest.blobs.length,
+    }
+    const check: PushCheck = client.discovery.capabilities.includes('push-check')
+      ? {
+          ...(await client.post<Omit<PushCheck, 'fromRemote' | 'sending'>>(
+            `environments/${env.id}/pushes/check`,
+            {
+              nodes: (loaded.set?.nodes ?? []).map((node) => node.key),
+              files: [...loaded.files.keys()],
+            },
+          )),
+          fromRemote: true,
+          sending,
+        }
+      : {
+          environment: env.id,
+          recycles: {},
+          removesFiles: [],
+          keepsPeople: true,
+          fromRemote: false,
+          sending,
+        }
+    options.onCheck?.(check)
+    if (env.protected && !options.autoApprove) {
+      const yes = (await options.confirm?.(check)) ?? false
+      if (!yes) throw new RefusedBeforeSending('Nothing was sent.')
+    }
+
+    if (options.mediaInfoOnly && manifest.blobs.length > 0) {
+      const store = await mediaStoreFor(config)
+      const { have } = await client.post<{ have: string[] }>(`environments/${env.id}/media/have`, {
+        blobs: manifest.blobs.map((blob) => ({ key: blob.key, etag: blob.etag ?? undefined })),
+      })
+      const present = new Set(have)
+      const missing = manifest.blobs.filter((blob) => !present.has(blob.key))
+      let done = 0
+      for (const blob of missing) {
+        const file = await store.get(blob.key)
+        if (file) {
+          const bytes = await file.bytes()
+          const response = await client.api(
+            `environments/${env.id}/media/${blob.key.split('/').map(encodeURIComponent).join('/')}`,
+            {
+              method: 'PUT',
+              headers: {
+                'crombolo-sha256': new Bun.CryptoHasher('sha256').update(bytes).digest('hex'),
+              },
+              body: bytes as BodyInit,
+            },
+          )
+          if (!response.ok) throw await problemOf(response)
+        }
+        options.onMedia?.(++done, missing.length)
+      }
+    }
+
+    const response = await client.api(`environments/${env.id}/pushes`, {
+      method: 'PUT',
+      headers: {
+        'content-type': 'application/gzip',
+        'crombolo-sha256': await sha256File(out),
+        'crombolo-passphrase': passphrase,
+      },
+      body: Bun.file(out),
+    })
+    if (!response.ok) throw await problemOf(response)
+    const started = (await response.json()) as Job
+    const job = await client.settle(started.id, {
+      onProgress: options.onProgress,
+      onApproval: options.onApproval,
+      timeoutMs: options.timeoutMs,
+      sleep: options.sleep,
+    })
+    return { check, job }
+  } finally {
+    rmSync(work, { recursive: true, force: true })
+  }
+}

package/templates/basic/bundle/package.json

@@ -6,7 +6,7 @@
     "content"
   ],
   "peerDependencies": {
-    "@crombolo/crombolo": ">=0.14.1"
+    "@crombolo/crombolo": ">=0.14.4"
   },
   "crombolo": {
     "id": "basic",

package/templates/demo/harbourstone/bundle/package.json

@@ -6,7 +6,7 @@
     "content"
   ],
   "peerDependencies": {
-    "@crombolo/crombolo": ">=0.14.1"
+    "@crombolo/crombolo": ">=0.14.4"
   },
   "crombolo": {
     "id": "demo-harbourstone",

package/templates/starter/bundle/package.json

@@ -6,7 +6,7 @@
     "content"
   ],
   "peerDependencies": {
-    "@crombolo/crombolo": ">=0.14.1"
+    "@crombolo/crombolo": ">=0.14.4"
   },
   "crombolo": {
     "id": "starter",