@crombolo/cli 0.13.2 → 0.14.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/cli",
-  "version": "0.13.2",
+  "version": "0.14.4",
   "description": "The crombolo command line: scaffolding a site, schema, views, domains, content transfer and upgrades.",
   "license": "MIT",
   "author": "Crombolo",
@@ -34,11 +34,11 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/auth": "0.13.2",
-    "@crombolo/core": "0.13.2",
-    "@crombolo/data": "0.13.2",
-    "@crombolo/schema": "0.13.2",
-    "@crombolo/server": "0.13.2",
-    "@crombolo/transfer": "0.13.2"
+    "@crombolo/auth": "0.14.4",
+    "@crombolo/core": "0.14.4",
+    "@crombolo/data": "0.14.4",
+    "@crombolo/schema": "0.14.4",
+    "@crombolo/server": "0.14.4",
+    "@crombolo/transfer": "0.14.4"
   }
 }

package/src/operations/commands.ts

@@ -5,7 +5,10 @@
  * a platform's own tooling — gets the same results as data. One implementation,
  * so the two can never disagree about what an import or an upgrade does.
  */
+import { existsSync } from 'node:fs'
+import { rm } from 'node:fs/promises'
 import { userInfo } from 'node:os'
+import { basename, join, resolve } from 'node:path'
 import { ROOT_ACCESS } from '@crombolo/core'
 import {
   type AuditCategory,
@@ -46,6 +49,8 @@ import {
   checkSchemaDirectory,
   createFileIntake,
   createMediaPort,
+  exportSite,
+  extractTarball,
   formTypesFor,
   installBundleContent,
   installDatabase,
@@ -53,11 +58,18 @@ import {
   mediaStoreFor,
   pendingBundleSteps,
   planBundleUpgrades,
+  type ReplaceMode,
+  type ReplaceSummary,
+  RUN_BACKUP_DIR,
   readBlobs,
   readBundle,
   recordAudit,
+  replaceSummary,
+  type SiteExportOptions,
+  type SiteExportResult,
   syncOptionsFor,
   type TransferInput,
+  useSchemaStore,
   VERSION,
 } from '@crombolo/server'
 import {
@@ -95,6 +107,7 @@ export {
   readBundle,
   replacementsNeedingPermission,
   restoreSectionFiles,
+  revertSections,
   type SectionFile,
   type SectionOptions,
   type SectionPlan,
@@ -109,16 +122,39 @@ export {
   type Resolution,
   type TransferCheck,
 } from '@crombolo/transfer'
-export type { BackupResult, WritesPaused }
+export type { BackupResult, ReplaceMode, ReplaceSummary, WritesPaused }
+
+/**
+ * A bundle file (`.tgz`, as `export --site` writes) unpacked under the site's
+ * transfer directory, where an import can read it; a directory or a package
+ * name is a bundle as it is.
+ */
+export async function unpackBundle(target: string, siteDir: string): Promise<string> {
+  if (!/\.(tgz|tar\.gz)$/i.test(target)) return target
+  const file = resolve(siteDir, target)
+  if (!existsSync(file)) throw new CommandError(`${target} does not exist.`)
+  const dir = join(
+    siteDir,
+    RUN_BACKUP_DIR,
+    'incoming',
+    basename(file).replace(/\.(tgz|tar\.gz)$/i, ''),
+  )
+  await rm(dir, { recursive: true, force: true })
+  await extractTarball(file, dir)
+  return dir
+}
 
 /** Read-only: what installing a bundle here would do. Its findings go to the Changes dashboard. */
 export async function contentCheck(
   config: CromboloConfig,
   input: TransferInput,
 ): Promise<ContentCheckResult> {
+  // Against the store's schema, not whatever local directory there is (story 106).
+  await useSchemaStore(config)
+  const bundle = await unpackBundle(input.bundle, config.siteDir)
   const { db } = await bootstrapDatabase(config)
   try {
-    return await checkBundleContent(db, config, input)
+    return await checkBundleContent(db, config, { ...input, bundle })
   } finally {
     await db.close()
   }
@@ -129,12 +165,80 @@ export async function contentImport(
   config: CromboloConfig,
   input: ContentImportInput,
 ): Promise<ContentImportResult> {
+  const bundle = await unpackBundle(input.bundle, config.siteDir)
   // Read before anything is opened or backed up, so a bundle that is not one is
   // refused before a backup is taken on its behalf.
-  readBundle(input.bundle, config.siteDir)
+  readBundle(bundle, config.siteDir)
+  await useSchemaStore(config)
+  const { db } = await bootstrapDatabase(config)
+  try {
+    return await installBundleContent(db, config, { ...input, bundle })
+  } finally {
+    await db.close()
+  }
+}
+
+export interface SiteReplaceCheck {
+  check: ContentCheckResult
+  summary: ReplaceSummary
+}
+
+/**
+ * Read-only: what `crombolo import <file>` would do (story 105) — the content
+ * check a replace runs with, and what it would take out of this site.
+ */
+export async function siteReplaceCheck(
+  config: CromboloConfig,
+  input: TransferInput & { replace: ReplaceMode },
+): Promise<SiteReplaceCheck> {
+  await useSchemaStore(config)
+  const bundle = await unpackBundle(input.bundle, config.siteDir)
+  const resolved = readBundle(bundle, config.siteDir, input.passphrase)
   const { db } = await bootstrapDatabase(config)
   try {
-    return await installBundleContent(db, config, input)
+    const check = await checkBundleContent(db, config, {
+      ...input,
+      bundle,
+      resolveAll: 'take-bundle',
+      replaceFiles: true,
+    })
+    const summary = await replaceSummary(
+      db,
+      resolved.set,
+      new Set(resolved.carried.files.keys()),
+      [
+        { section: 'schema', dir: config.schemaDir },
+        { section: 'components', dir: config.componentsDir },
+        { section: 'styles', dir: config.stylesheetsDir },
+        { section: 'scripts', dir: config.scriptsDir },
+      ],
+      input.replace,
+    )
+    return { check, summary }
+  } finally {
+    await db.close()
+  }
+}
+
+export type SiteExportInput = SiteExportOptions
+
+/**
+ * Writes the whole site to one file (story 104), streaming its media from the
+ * store. Unlike `contentExport` this writes: a site's media does not fit in a
+ * returned value.
+ */
+export async function siteExport(
+  config: CromboloConfig,
+  input: SiteExportInput,
+): Promise<SiteExportResult> {
+  await useSchemaStore(config)
+  const { db } = await bootstrapDatabase(config)
+  try {
+    return await exportSite(db, config, input)
+  } catch (error) {
+    if (error instanceof Error && !(error instanceof CommandError))
+      throw new CommandError(error.message)
+    throw error
   } finally {
     await db.close()
   }

package/src/remote/client.ts

@@ -0,0 +1,288 @@
+/**
+ * The CLI's side of the remote protocol (story 108, `docs/remote-protocol.md`):
+ * discovery, signing in by the device grant, and the API — jobs, downloads,
+ * uploads and media — under a sign-in kept in the keychain and refreshed as it
+ * runs out.
+ */
+import { createHash } from 'node:crypto'
+import { createWriteStream } from 'node:fs'
+import { freshAccessToken, type Keychain, readSignIn, saveSignIn } from '@crombolo/server'
+
+export const PROTOCOL_VERSION = '1'
+
+export interface Discovery {
+  protocol: string
+  versions: string[]
+  name: string
+  api: string
+  auth: { deviceAuthorization: string; token: string; clientId: string; grantType: string }
+  capabilities: string[]
+  environments: Array<{ id: string; name: string; protected: boolean }>
+}
+
+export interface Problem {
+  status: number
+  code: string
+  title: string
+  detail?: string
+  items?: string[]
+}
+
+export class RemoteError extends Error {
+  constructor(readonly problem: Problem) {
+    super(problem.detail ? `${problem.title}: ${problem.detail}` : problem.title)
+  }
+}
+
+export interface Job {
+  id: string
+  kind: 'pull' | 'push'
+  environment: string
+  state: 'queued' | 'running' | 'approval-required' | 'done' | 'failed' | 'refused'
+  stage?: string | null
+  progress?: number | null
+  pollAfter?: number
+  approvalUrl?: string
+  download?: { url: string; expiresAt: string; size: number | null; sha256: string | null }
+  problem?: Problem
+  runId?: string
+}
+
+/** Reads a remote's discovery document; refuses a site that is not a remote, or one speaking only other versions. */
+export async function discover(url: string, http: typeof fetch = fetch): Promise<Discovery> {
+  const origin = new URL(url).origin
+  let response: Response
+  try {
+    response = await http(`${origin}/.well-known/crombolo-remote`)
+  } catch (error) {
+    throw new Error(
+      `${origin} could not be reached: ${error instanceof Error ? error.message : error}`,
+    )
+  }
+  if (response.status === 404) throw new Error(`${origin} is not a crombolo remote.`)
+  if (!response.ok)
+    throw new Error(`${origin} answered ${response.status} for its discovery document.`)
+  const body = (await response.json()) as Discovery
+  if (body.protocol !== 'crombolo-remote') throw new Error(`${origin} is not a crombolo remote.`)
+  if (!body.versions?.includes(PROTOCOL_VERSION))
+    throw new Error(
+      `${origin} speaks protocol ${body.versions?.join(', ') || 'none'}; this crombolo speaks ${PROTOCOL_VERSION}.`,
+    )
+  return body
+}
+
+/**
+ * Signs in by the device grant (RFC 8628): `onCode` shows the person the code
+ * and the page to approve it on, then this polls until they decide. The tokens
+ * go into the keychain.
+ */
+export async function signIn(
+  discovery: Discovery,
+  keychain: Keychain,
+  options: {
+    clientName: string
+    onCode(code: string, url: string): void | Promise<void>
+    http?: typeof fetch
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<void> {
+  const http = options.http ?? fetch
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms))
+  const form = (url: string, fields: Record<string, string>) =>
+    http(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/x-www-form-urlencoded' },
+      body: new URLSearchParams(fields).toString(),
+    })
+  const started = await form(discovery.auth.deviceAuthorization, {
+    client_id: discovery.auth.clientId,
+    client_name: options.clientName,
+  })
+  if (!started.ok) throw new Error(`The remote would not start a sign-in (${started.status}).`)
+  const codes = (await started.json()) as {
+    device_code: string
+    user_code: string
+    verification_uri: string
+    verification_uri_complete?: string
+    expires_in: number
+    interval: number
+  }
+  await options.onCode(codes.user_code, codes.verification_uri_complete ?? codes.verification_uri)
+  let interval = codes.interval * 1000
+  const deadline = Date.now() + codes.expires_in * 1000
+  for (;;) {
+    await sleep(interval)
+    const polled = await form(discovery.auth.token, {
+      grant_type: discovery.auth.grantType,
+      device_code: codes.device_code,
+      client_id: discovery.auth.clientId,
+    })
+    const body = (await polled.json().catch(() => ({}))) as {
+      error?: string
+      access_token?: string
+      refresh_token?: string
+      expires_in?: number
+    }
+    if (polled.ok && body.access_token) {
+      await saveSignIn(keychain, discovery.api, {
+        accessToken: body.access_token,
+        refreshToken: body.refresh_token ?? '',
+        expiresAt: Date.now() + (body.expires_in ?? 0) * 1000,
+        tokenUrl: discovery.auth.token,
+        clientId: discovery.auth.clientId,
+      })
+      return
+    }
+    if (body.error === 'slow_down') interval += 5000
+    else if (body.error === 'access_denied') throw new Error('The sign-in was declined.')
+    else if (body.error === 'expired_token' || Date.now() > deadline)
+      throw new Error('The code expired before it was approved. Sign in again.')
+    else if (body.error !== 'authorization_pending')
+      throw new Error(`The sign-in failed: ${body.error ?? polled.status}`)
+  }
+}
+
+/** The protocol's API for one remote, as the signed-in person. */
+export class RemoteClient {
+  readonly discovery: Discovery
+  #keychain: Keychain
+  #http: typeof fetch
+
+  constructor(discovery: Discovery, keychain: Keychain, http: typeof fetch = fetch) {
+    this.discovery = discovery
+    this.#keychain = keychain
+    this.#http = http
+  }
+
+  async signedIn(): Promise<boolean> {
+    return (await readSignIn(this.#keychain, this.discovery.api)) !== undefined
+  }
+
+  /** One authorised request; a 401 refreshes the sign-in once and tries again. */
+  async call(url: string, init: RequestInit = {}): Promise<Response> {
+    const attempt = async (force: boolean) => {
+      const token = await freshAccessToken(this.#keychain, this.discovery.api, this.#http, force)
+      if (!token)
+        throw new RemoteError({
+          status: 401,
+          code: 'sign-in-required',
+          title: `Not signed in to ${new URL(this.discovery.api).origin}`,
+          detail: 'Run `crombolo login` first.',
+        })
+      const headers = new Headers(init.headers)
+      headers.set('authorization', `Bearer ${token}`)
+      return this.#http(url, { ...init, headers })
+    }
+    const first = await attempt(false)
+    if (first.status !== 401) return first
+    return attempt(true)
+  }
+
+  async api(path: string, init: RequestInit = {}): Promise<Response> {
+    return this.call(`${this.discovery.api}/${path}`, init)
+  }
+
+  async json<T>(path: string, init: RequestInit = {}): Promise<T> {
+    const response = await this.api(path, init)
+    if (!response.ok) throw await problemOf(response)
+    return (await response.json()) as T
+  }
+
+  post<T>(path: string, body: unknown): Promise<T> {
+    return this.json<T>(path, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+    })
+  }
+
+  environments() {
+    return this.json<
+      Array<{
+        id: string
+        name: string
+        protected: boolean
+        may: { pull: boolean; pullFull: boolean; push: boolean }
+      }>
+    >('environments')
+  }
+
+  /** Waits for a job to stop moving, reporting each change; `onApproval` hears it once. */
+  async settle(
+    id: string,
+    options: {
+      onProgress?(job: Job): void
+      onApproval?(job: Job): void | Promise<void>
+      timeoutMs?: number
+      sleep?: (ms: number) => Promise<void>
+    } = {},
+  ): Promise<Job> {
+    const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms))
+    const deadline = Date.now() + (options.timeoutMs ?? 30 * 60_000)
+    let announced = false
+    let last = ''
+    for (;;) {
+      const job = await this.json<Job>(`jobs/${id}`)
+      const seen = `${job.state}|${job.stage ?? ''}|${job.progress ?? ''}`
+      if (seen !== last) options.onProgress?.(job)
+      last = seen
+      if (job.state === 'approval-required' && !announced) {
+        announced = true
+        await options.onApproval?.(job)
+      }
+      if (job.state !== 'queued' && job.state !== 'running' && job.state !== 'approval-required')
+        return job
+      if (Date.now() > deadline) throw new TimedOut(job)
+      await sleep((job.pollAfter ?? 2) * 1000)
+    }
+  }
+
+  /** Streams a finished pull to `path`, checking it against the hash the job stated. */
+  async download(job: Job, path: string): Promise<{ size: number }> {
+    if (!job.download) throw new Error('The job has nothing to download.')
+    const response = await this.call(job.download.url)
+    if (!response.ok || !response.body) throw await problemOf(response)
+    const hasher = createHash('sha256')
+    const sink = createWriteStream(path)
+    let size = 0
+    const reader = response.body.getReader()
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) break
+      hasher.update(value)
+      size += value.length
+      if (!sink.write(value)) await new Promise((r) => sink.once('drain', r))
+    }
+    await new Promise<void>((resolve, reject) =>
+      sink.end((error?: Error | null) => (error ? reject(error) : resolve())),
+    )
+    if (job.download.sha256 && hasher.digest('hex') !== job.download.sha256)
+      throw new Error('The download does not match the hash the remote stated; try again.')
+    return { size }
+  }
+}
+
+/** A job still moving when the client stopped waiting. */
+export class TimedOut extends Error {
+  constructor(readonly job: Job) {
+    super(`Still ${job.state} when the wait ran out.`)
+  }
+}
+
+export async function problemOf(response: Response): Promise<RemoteError> {
+  const body = (await response.json().catch(() => ({}))) as Partial<Problem>
+  return new RemoteError({
+    status: response.status,
+    code: body.code ?? `http-${response.status}`,
+    title: body.title ?? `The remote answered ${response.status}`,
+    ...(body.detail ? { detail: body.detail } : {}),
+    ...(body.items ? { items: body.items } : {}),
+  })
+}
+
+/** SHA-256 of a file, streamed. */
+export async function sha256File(path: string): Promise<string> {
+  const hasher = createHash('sha256')
+  for await (const chunk of Bun.file(path).stream()) hasher.update(chunk)
+  return hasher.digest('hex')
+}

package/src/remote/remotes-file.ts

@@ -0,0 +1,80 @@
+/**
+ * `remotes.toml` at a site's root (story 108): the remotes a team pulls from
+ * and pushes to, by name — URLs only, never credentials, so it is committed.
+ *
+ *   [remotes.production]
+ *   url = "https://example.com"
+ *   environment = "production"
+ */
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+export const REMOTES_FILE = 'remotes.toml'
+
+export interface RemoteEntry {
+  name: string
+  url: string
+  /** The environment a pull or push goes to when `--env` is not given. */
+  environment?: string
+}
+
+const NAME = /^[a-z][a-z0-9-]{0,39}$/
+
+export function readRemotes(siteDir: string): RemoteEntry[] {
+  const file = join(siteDir, REMOTES_FILE)
+  if (!existsSync(file)) return []
+  const parsed = Bun.TOML.parse(readFileSync(file, 'utf8')) as {
+    remotes?: Record<string, { url?: unknown; environment?: unknown }>
+  }
+  return Object.entries(parsed.remotes ?? {}).flatMap(([name, entry]) =>
+    typeof entry?.url === 'string'
+      ? [
+          {
+            name,
+            url: entry.url,
+            ...(typeof entry.environment === 'string' ? { environment: entry.environment } : {}),
+          },
+        ]
+      : [],
+  )
+}
+
+const quote = (value: string) => JSON.stringify(value)
+
+export function writeRemotes(siteDir: string, remotes: readonly RemoteEntry[]): void {
+  const lines = [
+    '# The remotes `crombolo pull` and `push` reach, by name. URLs only: sign-ins',
+    '# are kept in your keychain, never here.',
+  ]
+  for (const remote of remotes) {
+    lines.push('', `[remotes.${remote.name}]`, `url = ${quote(remote.url)}`)
+    if (remote.environment) lines.push(`environment = ${quote(remote.environment)}`)
+  }
+  writeFileSync(join(siteDir, REMOTES_FILE), `${lines.join('\n')}\n`)
+}
+
+/** Adds or replaces one, refusing a name that would not survive as a TOML key. */
+export function addRemote(siteDir: string, remote: RemoteEntry): void {
+  if (!NAME.test(remote.name))
+    throw new Error(`"${remote.name}" is not a remote name: lower case letters, digits and dashes.`)
+  const others = readRemotes(siteDir).filter((r) => r.name !== remote.name)
+  writeRemotes(
+    siteDir,
+    [...others, remote].sort((a, b) => a.name.localeCompare(b.name)),
+  )
+}
+
+export function removeRemote(siteDir: string, name: string): boolean {
+  const remotes = readRemotes(siteDir)
+  const kept = remotes.filter((r) => r.name !== name)
+  if (kept.length === remotes.length) return false
+  writeRemotes(siteDir, kept)
+  return true
+}
+
+/** The remote a command names, or the only one there is. */
+export function pickRemote(siteDir: string, name: string | undefined): RemoteEntry | undefined {
+  const remotes = readRemotes(siteDir)
+  if (name) return remotes.find((r) => r.name === name)
+  return remotes.length === 1 ? remotes[0] : undefined
+}