@crombolo/cli 0.14.1 → 0.15.0

package/src/remote/transfer.ts

@@ -0,0 +1,353 @@
+/**
+ * `crombolo pull` and `crombolo push` as functions (story 108): they take a
+ * signed-in client and options, and return what happened; the binary prints.
+ *
+ * A pull is a job on the remote ending in a sealed file, opened here with a
+ * passphrase made for that one pull and kept in memory. A push exports this
+ * site sealed the same way, checks with the remote what it would replace, and
+ * uploads it as a job — which the remote may hold for approval.
+ */
+import { randomBytes } from 'node:crypto'
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+  bootstrapDatabase,
+  type CromboloConfig,
+  exportSite,
+  extractTarball,
+  type MediaStore,
+  mediaStoreFor,
+  writeFileAtomic,
+} from '@crombolo/server'
+import { type BundleBlob, loadBundle } from '@crombolo/transfer'
+import { CliError, type ErrorKind } from '../errors.ts'
+import { type Job, problemOf, type RemoteClient, RemoteError, sha256File } from './client.ts'
+
+export type MediaMode = 'included' | 'omitted' | 'referenced'
+
+/** A passphrase for one transfer: never shown, never stored. */
+export const transferPassphrase = () => randomBytes(24).toString('base64url')
+
+/** Refused before anything was sent: the CLI's own guard. */
+export class RefusedBeforeSending extends CliError {
+  constructor(kind: ErrorKind, message: string, hint?: string) {
+    super(kind, message, { hint })
+  }
+}
+
+export interface PullResult {
+  job: Job
+  file: string
+  passphrase: string
+  size: number
+}
+
+/** Asks the remote for its site and downloads the file to `out`. */
+export async function pullFile(
+  client: RemoteClient,
+  options: {
+    environment: string
+    out: string
+    full?: boolean
+    media?: MediaMode
+    onProgress?(job: Job): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PullResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      'not_found',
+      `${client.discovery.name} has no environment "${options.environment}": it has ${environments.map((e) => e.id).join(', ')}.`,
+    )
+  if (!env.may.pull) throw new RefusedBeforeSending('refused', `You may not pull from ${env.name}.`)
+  if (options.full && !env.may.pullFull)
+    throw new RefusedBeforeSending(
+      'refused',
+      `You may not pull a full copy of ${env.name}; pull without --full for an anonymised one.`,
+    )
+  if (options.media === 'referenced' && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      'refused',
+      `${client.discovery.name} does not serve media by reference; pull without --media-info-only.`,
+    )
+  const passphrase = transferPassphrase()
+  const started = await client.post<Job>(`environments/${env.id}/pulls`, {
+    passphrase,
+    full: options.full === true,
+    media: options.media ?? 'included',
+  })
+  const job = await client.settle(started.id, {
+    onProgress: options.onProgress,
+    timeoutMs: options.timeoutMs,
+    sleep: options.sleep,
+  })
+  if (job.state !== 'done') throw new RemoteError(job.problem ?? jobProblem(job))
+  const { size } = await client.download(job, options.out)
+  return { job, file: options.out, passphrase, size }
+}
+
+const jobProblem = (job: Job) => ({
+  status: 0,
+  code: job.state,
+  title: `The ${job.kind} ended ${job.state}`,
+})
+
+/** What the local site has already fetched of a remote's media, by key: the remote's etag. */
+const FETCHED_FILE = join('.crombolo', 'remote-media-fetched.json')
+
+function readFetched(siteDir: string): Record<string, string> {
+  const file = join(siteDir, FETCHED_FILE)
+  if (!existsSync(file)) return {}
+  try {
+    return JSON.parse(readFileSync(file, 'utf8')) as Record<string, string>
+  } catch {
+    return {}
+  }
+}
+
+export interface MediaFetchResult {
+  fetched: number
+  skipped: number
+  bytes: number
+  failed: string[]
+}
+
+/**
+ * Fetches the media files a file names by reference into the local store, a
+ * batch of links at a time and a few files at once. A file already fetched at
+ * the same etag is skipped, so a second pull downloads only what changed.
+ */
+export async function fetchReferencedMedia(
+  client: RemoteClient,
+  options: {
+    environment: string
+    blobs: readonly Pick<BundleBlob, 'key' | 'etag'>[]
+    store: MediaStore
+    siteDir: string
+    onProgress?(done: number, total: number): void
+    http?: typeof fetch
+  },
+): Promise<MediaFetchResult> {
+  const fetched = readFetched(options.siteDir)
+  const result: MediaFetchResult = { fetched: 0, skipped: 0, bytes: 0, failed: [] }
+  const wanted: string[] = []
+  for (const blob of options.blobs) {
+    const have = await options.store.get(blob.key)
+    if (have && blob.etag && fetched[blob.key] === blob.etag) result.skipped++
+    else wanted.push(blob.key)
+  }
+  let done = 0
+  for (let at = 0; at < wanted.length; at += 200) {
+    const batch = wanted.slice(at, at + 200)
+    const { links } = await client.post<{
+      links: Record<string, { url: string; etag: string; size: number } | null>
+    }>(`environments/${options.environment}/media/links`, { keys: batch })
+    const queue = [...batch]
+    const worker = async () => {
+      for (;;) {
+        const key = queue.shift()
+        if (!key) return
+        const link = links[key]
+        try {
+          if (!link) throw new Error('not on the remote')
+          // A link through the API needs the sign-in; a presigned one does not.
+          const response = link.url.startsWith(client.discovery.api)
+            ? await client.call(link.url)
+            : await (options.http ?? fetch)(link.url)
+          if (!response.ok) throw await problemOf(response)
+          const body = await response.blob()
+          await options.store.put(key, body)
+          fetched[key] = link.etag
+          result.fetched++
+          result.bytes += body.size
+        } catch {
+          result.failed.push(key)
+        }
+        options.onProgress?.(++done, wanted.length)
+      }
+    }
+    await Promise.all(Array.from({ length: 6 }, worker))
+  }
+  writeFileAtomic(join(options.siteDir, FETCHED_FILE), `${JSON.stringify(fetched, null, 2)}\n`)
+  return result
+}
+
+/** The blobs a file names by reference, read from its manifest. */
+export async function referencedBlobs(file: string): Promise<BundleBlob[]> {
+  const dir = mkdtempSync(join(tmpdir(), 'crombolo-pulled-'))
+  try {
+    await extractTarball(file, dir)
+    const manifest = loadBundle(join(dir, 'content')).set?.manifest
+    return manifest?.media === 'referenced' ? manifest.blobs : []
+  } finally {
+    rmSync(dir, { recursive: true, force: true })
+  }
+}
+
+export interface PushCheck {
+  environment: string
+  recycles: Record<string, number>
+  removesFiles: string[]
+  keepsPeople: boolean
+  /** Whether the remote answered it, or it is only what this side is sending. */
+  fromRemote: boolean
+  sending: { counts: Record<string, number>; files: number; media: number }
+}
+
+export interface PushResult {
+  check: PushCheck
+  /** Absent for a dry run, which stops after the check. */
+  job?: Job
+}
+
+/**
+ * Exports this site and sends it. `confirm` is asked with the check for a
+ * protected environment unless `autoApprove`; `onApproval` hears the job when
+ * the remote holds it for approval. The flags are this side's guard against a
+ * slip, never authority: the remote decides.
+ */
+export async function pushSite(
+  config: CromboloConfig,
+  client: RemoteClient,
+  options: {
+    environment: string
+    force?: boolean
+    autoApprove?: boolean
+    mediaInfoOnly?: boolean
+    /** Export and check, then stop: nothing is uploaded. */
+    dryRun?: boolean
+    confirm?(check: PushCheck): Promise<boolean>
+    onCheck?(check: PushCheck): void
+    onProgress?(job: Job): void
+    onApproval?(job: Job): void | Promise<void>
+    onMedia?(done: number, total: number): void
+    timeoutMs?: number
+    sleep?: (ms: number) => Promise<void>
+  },
+): Promise<PushResult> {
+  const environments = await client.environments()
+  const env = environments.find((e) => e.id === options.environment)
+  if (!env)
+    throw new RefusedBeforeSending(
+      'not_found',
+      `${client.discovery.name} has no environment "${options.environment}".`,
+    )
+  if (!env.may.push) throw new RefusedBeforeSending('refused', `You may not push to ${env.name}.`)
+  if (env.protected && !options.force && !options.dryRun)
+    throw new RefusedBeforeSending(
+      'confirmation_required',
+      `${env.name} is protected: pushing replaces what people rely on.`,
+      'Pass --force to go on.',
+    )
+  if (options.mediaInfoOnly && !client.discovery.capabilities.includes('media-fetch'))
+    throw new RefusedBeforeSending(
+      'refused',
+      `${client.discovery.name} does not take media by reference; push without --media-info-only.`,
+    )
+
+  const work = mkdtempSync(join(tmpdir(), 'crombolo-push-'))
+  try {
+    const out = join(work, 'site.tgz')
+    const passphrase = transferPassphrase()
+    const { db } = await bootstrapDatabase(config)
+    try {
+      await exportSite(db, config, {
+        out,
+        sealContent: true,
+        passphrase,
+        media: options.mediaInfoOnly ? 'referenced' : true,
+      })
+    } finally {
+      await db.close()
+    }
+    const unpacked = join(work, 'unpacked')
+    await extractTarball(out, unpacked)
+    const loaded = loadBundle(join(unpacked, 'content'))
+    const manifest = loaded.set?.manifest
+    if (!manifest) throw new Error('The export could not be read back.')
+    const sending = {
+      counts: { ...manifest.counts } as Record<string, number>,
+      files: loaded.files.size,
+      media: manifest.blobs.length,
+    }
+    const check: PushCheck = client.discovery.capabilities.includes('push-check')
+      ? {
+          ...(await client.post<Omit<PushCheck, 'fromRemote' | 'sending'>>(
+            `environments/${env.id}/pushes/check`,
+            {
+              nodes: (loaded.set?.nodes ?? []).map((node) => node.key),
+              files: [...loaded.files.keys()],
+            },
+          )),
+          fromRemote: true,
+          sending,
+        }
+      : {
+          environment: env.id,
+          recycles: {},
+          removesFiles: [],
+          keepsPeople: true,
+          fromRemote: false,
+          sending,
+        }
+    options.onCheck?.(check)
+    if (options.dryRun) return { check }
+    if (env.protected && !options.autoApprove) {
+      const yes = (await options.confirm?.(check)) ?? false
+      if (!yes) throw new RefusedBeforeSending('refused', 'Nothing was sent.')
+    }
+
+    if (options.mediaInfoOnly && manifest.blobs.length > 0) {
+      const store = await mediaStoreFor(config)
+      const { have } = await client.post<{ have: string[] }>(`environments/${env.id}/media/have`, {
+        blobs: manifest.blobs.map((blob) => ({ key: blob.key, etag: blob.etag ?? undefined })),
+      })
+      const present = new Set(have)
+      const missing = manifest.blobs.filter((blob) => !present.has(blob.key))
+      let done = 0
+      for (const blob of missing) {
+        const file = await store.get(blob.key)
+        if (file) {
+          const bytes = await file.bytes()
+          const response = await client.api(
+            `environments/${env.id}/media/${blob.key.split('/').map(encodeURIComponent).join('/')}`,
+            {
+              method: 'PUT',
+              headers: {
+                'crombolo-sha256': new Bun.CryptoHasher('sha256').update(bytes).digest('hex'),
+              },
+              body: bytes as BodyInit,
+            },
+          )
+          if (!response.ok) throw await problemOf(response)
+        }
+        options.onMedia?.(++done, missing.length)
+      }
+    }
+
+    const response = await client.api(`environments/${env.id}/pushes`, {
+      method: 'PUT',
+      headers: {
+        'content-type': 'application/gzip',
+        'crombolo-sha256': await sha256File(out),
+        'crombolo-passphrase': passphrase,
+      },
+      body: Bun.file(out),
+    })
+    if (!response.ok) throw await problemOf(response)
+    const started = (await response.json()) as Job
+    const job = await client.settle(started.id, {
+      onProgress: options.onProgress,
+      onApproval: options.onApproval,
+      timeoutMs: options.timeoutMs,
+      sleep: options.sleep,
+    })
+    return { check, job }
+  } finally {
+    rmSync(work, { recursive: true, force: true })
+  }
+}

package/src/support/components.ts

@@ -0,0 +1,7 @@
+/** The content types a site declares, which say which components are templates. */
+import { existsSync } from 'node:fs'
+import type { CromboloConfig } from '@crombolo/server'
+import { loadTypes } from './types.ts'
+
+export const declaredTypes = (config: CromboloConfig) =>
+  existsSync(config.schemaDir) ? loadTypes(config).set.contentTypes : []

package/src/support/describe.ts

@@ -0,0 +1,179 @@
+/** How the CLI says what happened: the lines shared between commands. */
+import { relative } from 'node:path'
+import type { Finding } from '@crombolo/data'
+import type { SyncReport } from '@crombolo/schema'
+import { propertyHandle, resolutionsFor, type TransferCheck } from '@crombolo/transfer'
+import type * as ops from '../operations/commands.ts'
+import type { Job } from '../remote/client.ts'
+import { cwd, type Io } from './site.ts'
+
+export function banner(io: Io, lines: string[]): void {
+  const width = Math.max(...lines.map((line) => line.length))
+  io.say(`┌─${'─'.repeat(width)}─┐`)
+  for (const line of lines) io.say(`│ ${line.padEnd(width)} │`)
+  io.say(`└─${'─'.repeat(width)}─┘`)
+}
+
+export const here = (path: string): string => relative(cwd(), path) || '.'
+
+/** Bytes as somebody reads them, since a bundle of media is measured in them. */
+export function describeBytes(bytes: number): string {
+  if (bytes < 1024) return `${bytes} B`
+  const units = ['KB', 'MB', 'GB']
+  let value = bytes / 1024
+  let unit = 0
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024
+    unit += 1
+  }
+  return `${value < 10 ? value.toFixed(1) : Math.round(value)} ${units[unit]}`
+}
+
+export const counted = (counts: Record<string, number>, none = 'nothing'): string => {
+  const entries = Object.entries(counts)
+  return entries.length > 0 ? entries.map(([kind, n]) => `${n} ${kind}`).join(', ') : none
+}
+
+/**
+ * Puts the media files a bundle carries into this environment's store.
+ *
+ * After the content, because the run is what decides whether any of this
+ * happened: a refused import leaves the store alone.
+ */
+export function describePlaced(
+  placed: { placed: number; bytes: number } | null,
+): string | undefined {
+  if (!placed) return undefined
+  return `${placed.placed} media file(s) placed, ${describeBytes(placed.bytes)}`
+}
+
+export function describeSync(report: SyncReport): string {
+  switch (report.action) {
+    case 'applied':
+    case 'would-apply': {
+      const verb = report.action === 'applied' ? 'applied' : 'would apply'
+      const c = report.created
+      const parts = [
+        `${c.types} type(s)`,
+        `${c.properties} propert${c.properties === 1 ? 'y' : 'ies'}`,
+        `${c.dataTypes} data type(s)`,
+        `${c.languages} language(s)`,
+      ]
+      const extra = [
+        report.retired.length > 0 ? `retired ${report.retired.join(', ')}` : '',
+        report.retiredTypes.length > 0 ? `retired types ${report.retiredTypes.join(', ')}` : '',
+        report.revived.length > 0 ? `revived ${report.revived.join(', ')}` : '',
+        report.assignedKeys.length > 0 ? `${report.assignedKeys.length} key(s) assigned` : '',
+      ].filter(Boolean)
+      return `${verb} ${report.state?.version ?? ''}: created ${parts.join(', ')}${extra.length > 0 ? `; ${extra.join('; ')}` : ''}`
+    }
+    case 'skipped-same':
+      return 'up to date'
+    case 'skipped-older':
+      return `compatibility mode: ${report.reason ?? ''}`
+    case 'refused':
+      return `refused: ${report.reason ?? `${report.problems.length} problem(s)`}`
+  }
+}
+
+export const describeProblem = (p: SyncReport['problems'][number]): string =>
+  `${p.file}${p.path ? ` (${p.path})` : ''}: ${p.message}`
+
+export function printProblems(io: Io, problems: SyncReport['problems']): void {
+  for (const p of problems) io.warning(`  ${describeProblem(p)}`)
+}
+
+export function describeBundle(io: Io, bundle: ops.BundleSummary): void {
+  io.say(
+    `  bundle   ${bundle.id} from ${bundle.from || 'elsewhere'} ` +
+      `(schema ${bundle.schemaVersion}+${bundle.schemaRevision}, ` +
+      `${bundle.snapshot} snapshot, ${bundle.nodes} node(s))`,
+  )
+  if (bundle.blobsCarried > 0)
+    io.say(
+      `  blobs    ${bundle.blobsCarried} media file(s) carried, ${describeBytes(bundle.blobBytes)}`,
+    )
+}
+
+/** What the carried sections would do, or did. */
+export function printSections(io: Io, plan: ops.SectionPlan | undefined): void {
+  if (!plan) return
+  io.say(`  carries  ${plan.sections.join(', ')}`)
+  const counts = new Map<string, number>()
+  for (const file of plan.files) counts.set(file.action, (counts.get(file.action) ?? 0) + 1)
+  const summary = [...counts].map(([action, n]) => `${n} ${action}`).join(', ')
+  if (summary) io.say(`  files    ${summary}`)
+  for (const file of plan.files)
+    if (file.action === 'overwrite') io.say(`    overwrites ${here(file.target)}`)
+  if (plan.dictionaryItems > 0) io.say(`  dict     ${plan.dictionaryItems} file(s)`)
+  if (plan.schema) {
+    io.say(`  schema   ${plan.schema.classification}`)
+    for (const finding of plan.schema.findings) io.say(`    ${finding}`)
+  }
+}
+
+export function printTransferFindings(io: Io, check: TransferCheck): void {
+  const byKind = { blocking: 0, person: 0, auto: 0 }
+  for (const f of check.findings) byKind[f.kind] += 1
+  const c = check.counts
+  io.say(
+    `  plan     ${c.create} new, ${c.update} updated, ${c.unchanged} unchanged, ${c.skip} skipped`,
+  )
+  io.say(
+    `  findings ${byKind.blocking} blocking, ${byKind.person} need a person, ${byKind.auto} automatic`,
+  )
+  for (const f of check.findings) {
+    io.say(`  [${f.kind.padEnd(8)}] ${f.message}`)
+    // The handle is the CLI's to name; the dashboard offers the same answers as choices.
+    if (f.code === 'local-edit' && f.propertyAlias)
+      io.say(
+        `             answer it as ${f.subjectKey}/${propertyHandle(f.propertyAlias, f.culture)}`,
+      )
+    for (const suggestion of resolutionsFor(f.code)) io.say(`             → ${suggestion}`)
+    if (f.link) io.say(`             ${f.link}`)
+  }
+}
+
+/** The findings a check reports, as data: what the envelope's `details` carry. */
+export const findingsOf = (
+  findings: readonly { kind: string; message: string; code?: string; link?: string | null }[],
+) =>
+  findings.map((f) => ({
+    kind: f.kind,
+    ...(f.code ? { code: f.code } : {}),
+    message: f.message,
+    ...(f.link ? { link: f.link } : {}),
+  }))
+
+export function printFindings(
+  io: Io,
+  report: { classification: string; findings: Finding[] },
+): void {
+  const byKind = { blocking: 0, person: 0, auto: 0 }
+  for (const f of report.findings) byKind[f.kind] += 1
+  io.say(
+    `change: ${report.classification}; findings: ${byKind.blocking} blocking, ${byKind.person} need a person, ${byKind.auto} automatic`,
+  )
+  for (const f of report.findings) {
+    const where = f.link ? `  ${f.link}` : ''
+    io.say(`  [${f.kind.padEnd(8)}] ${f.message}${where}`)
+  }
+}
+
+export const describeJob = (job: Job): string =>
+  `  ${job.kind.padEnd(5)} ${job.state}${job.stage ? `, ${job.stage}` : ''}${job.progress != null ? ` (${job.progress}%)` : ''}`
+
+export function describeContentMode(
+  io: Io,
+  content: 'omitted' | 'sealed' | 'open',
+  people?: { members: number; users: number },
+): void {
+  if (content === 'omitted')
+    io.say(
+      '  content  not included: pages are placed and named, and no value is written. Pass --include-content to bring the values and media files',
+    )
+  else if (content === 'sealed') io.say('  content  encrypted: importing needs the passphrase')
+  else io.say('  content  unencrypted')
+  if (people && people.members + people.users > 0)
+    io.say(`  people   ${people.members} member(s), ${people.users} user(s), encrypted`)
+}

package/src/support/domains.ts

@@ -0,0 +1,89 @@
+/**
+ * Which hostnames reach which page. `domains.toml` is the truth and the
+ * database follows it, so `set` and `clear` write the file and then apply it —
+ * one command, rather than an edit here and a deploy to make it real.
+ */
+import {
+  bootstrapDatabase,
+  type CromboloConfig,
+  type DomainDeclaration,
+  domainsFilePath,
+  readDomainsFile,
+  resolveDomainNode,
+  syncDomainsFile,
+  withoutNode,
+  writeDomains,
+} from '@crombolo/server'
+import { CliError } from '../errors.ts'
+import { here } from './describe.ts'
+import type { Io } from './site.ts'
+
+/** The file's declarations, saying what is wrong with it as it goes. */
+export function readDeclarations(io: Io, config: CromboloConfig) {
+  const read = readDomainsFile(config.siteDir)
+  for (const problem of read?.problems ?? [])
+    io.warning(`  ${domainsFilePath(config.siteDir)}: ${problem}`)
+  return { read, declarations: read?.declarations ?? [] }
+}
+
+/** Applies the file as a boot would, and reports what it did. */
+export async function applyDomains(io: Io, config: CromboloConfig) {
+  const { db } = await bootstrapDatabase(config)
+  try {
+    const report = await syncDomainsFile(db, config.siteDir)
+    for (const applied of report.applied)
+      io.say(
+        `  applied  ${applied.node.padEnd(24)} ${applied.host}${applied.culture ? `  ${applied.culture}` : ''}`,
+      )
+    if (report.removed > 0) io.say(`  removed  ${report.removed} binding(s)`)
+    if (report.problems.length > 0)
+      throw new CliError('refused', `${report.problems.length} domain(s) could not be applied.`, {
+        details: report.problems,
+      })
+    return {
+      applied: report.applied.map((a) => ({
+        node: a.node,
+        host: a.host,
+        culture: a.culture ?? null,
+      })),
+      removed: report.removed,
+    }
+  } finally {
+    await db.close()
+  }
+}
+
+/**
+ * Stating a node's hostnames says what they are, as the backoffice's dialog
+ * does: whatever that page had is replaced, so running this twice is the same
+ * as running it once. Which entries belong to that page is decided by
+ * resolving them, not by matching the text — `/` and `/Home` can be the same
+ * page, and clearing one while the other stayed would do nothing at all.
+ */
+export async function rewriteDomains(
+  io: Io,
+  config: CromboloConfig,
+  ref: string,
+  add: DomainDeclaration[],
+) {
+  const { declarations } = readDeclarations(io, config)
+  const { db } = await bootstrapDatabase(config)
+  let kept: DomainDeclaration[] = []
+  try {
+    const target = await resolveDomainNode(db, ref)
+    if (!target.ok) throw new CliError('not_found', target.message)
+    kept = await withoutNode(db, declarations, target.node)
+  } finally {
+    await db.close()
+  }
+  const written = await writeDomains(config.siteDir, [...kept, ...add])
+  io.say(
+    `  wrote    ${here(written.file)}${written.backup ? '  (previous kept; `crombolo domains undo` puts it back)' : ''}`,
+  )
+  return { wrote: here(written.file), ...(await applyDomains(io, config)) }
+}
+
+export const appliedFields = [
+  { name: 'applied', type: 'array', items: { type: 'object' } },
+  { name: 'removed', type: 'integer' },
+] as const