@crombolo/server 0.9.1 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crombolo/server",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "description": "The crombolo HTTP server: routing, configuration, media stores and background jobs.",
   "license": "MIT",
   "author": "Crombolo",
@@ -28,16 +28,16 @@
     "access": "public"
   },
   "dependencies": {
-    "@crombolo/api-management": "0.9.1",
-    "@crombolo/assistant": "0.9.1",
-    "@crombolo/auth": "0.9.1",
-    "@crombolo/backoffice-host": "0.9.1",
-    "@crombolo/contracts": "0.9.1",
-    "@crombolo/core": "0.9.1",
-    "@crombolo/data": "0.9.1",
-    "@crombolo/render": "0.9.1",
-    "@crombolo/schema": "0.9.1",
-    "@crombolo/transfer": "0.9.1",
+    "@crombolo/api-management": "0.10.0",
+    "@crombolo/assistant": "0.10.0",
+    "@crombolo/auth": "0.10.0",
+    "@crombolo/backoffice-host": "0.10.0",
+    "@crombolo/contracts": "0.10.0",
+    "@crombolo/core": "0.10.0",
+    "@crombolo/data": "0.10.0",
+    "@crombolo/render": "0.10.0",
+    "@crombolo/schema": "0.10.0",
+    "@crombolo/transfer": "0.10.0",
     "@logtape/logtape": "^2.3.8"
   }
 }

package/src/adapters/bundles.ts

@@ -1,6 +1,6 @@
 /**
- * Bundles: the definitions, and the zip a download builds from one.
- * `docs/17-bundles.md`.
+ * Bundles: the definitions, and the npm package a download builds from one.
+ * `docs/bundles.md`.
  *
  * Nothing new is serialized here. Schema travels as the canonical TOML
  * `@crombolo/schema` already writes, content and media as the bundle
@@ -48,14 +48,27 @@ import {
   type BundleKind,
   type BundleNode,
   type BundleSection,
+  bundleIdFor,
   type ContentSet,
   exportBundle,
+  isPackageName,
+  packageNameFor,
   writeBundle,
+  writeBundlePackage,
 } from '@crombolo/transfer'
+import { VERSION } from '../config.ts'
 import { dictionaryToUdt, writeDictionaryUdt } from '../dictionary-transfer.ts'
 import { readBlobs } from '../media-files.ts'
 import type { MediaStore } from '../media-store.ts'
-import { writeZip, type ZipEntry } from '../zip.ts'
+import { writeNpmTarball } from '../tar.ts'
+
+/** The npm package a definition is built as: what was set on it, or what its name gives. */
+export function bundlePackageOf(row: Bundle): { name: string; version: string } {
+  return {
+    name: row.packageName ?? packageNameFor(row.name),
+    version: row.packageVersion ?? '1.0.0',
+  }
+}
 
 /** The content kinds a bundle's picks can name. */
 const CONTENT_OBJECT_TYPES = [ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Element]
@@ -376,17 +389,19 @@ export function createPackagePort(db: Db, options: PackagePortOptions): PackageP
     },
 
     /**
-     * The definition as one bundle, zipped.
+     * The definition as one bundle, packed as the npm package it is.
      *
      * Everything goes through `writeBundle` so there is a single manifest over
-     * the whole artifact: it declares the sections, carries the hash that covers
-     * them, and names the package — a zip of loose directories would have no
-     * record of what it was or whether it arrived whole.
+     * the whole payload: it declares the sections, carries the hash that covers
+     * them, and names the bundle. The package around it is what gives it a
+     * name, a version and the core it was written for, so the download is
+     * something `bun add ./file.tgz` and `bun publish` take as it is.
      */
     async build(id): Promise<BuiltPackage | undefined> {
       const row = await bundles.byId(id)
       if (!row) return undefined
       const from = definition(row)
+      const pkg = bundlePackageOf(row)
 
       // The schema version the site is actually at, so the files a package
       // carries say where they came from rather than claiming version 0.
@@ -422,11 +437,71 @@ export function createPackagePort(db: Db, options: PackagePortOptions): PackageP
 
       const labelled: ContentSet = { ...set, manifest: { ...set.manifest, label: from.name } }
       const encoder = new TextEncoder()
-      const entries: ZipEntry[] = writeBundle(labelled, content.blobs, sections).map((file) => ({
+      const files = writeBundlePackage(writeBundle(labelled, content.blobs, sections), {
+        name: pkg.name,
+        version: pkg.version,
+        id: bundleIdFor(pkg.name.replace(/^@[^/]+\//, '')),
+        label: from.name,
+        peerRange: `^${VERSION}`,
+      })
+      const entries = files.map((file) => ({
         path: file.path,
         bytes: file.bytes ?? encoder.encode(file.text ?? ''),
       }))
-      return { fileName: packageFileName(from.name), bytes: writeZip(entries) }
+      return {
+        fileName: packageFileName(pkg.name, pkg.version),
+        contentType: 'application/gzip',
+        bytes: writeNpmTarball(entries),
+      }
     },
   }
 }
+
+/** A created bundle's npm package as the Created view edits it. */
+export interface CreatedBundlePackage {
+  name: string
+  version: string
+  /** Whether each was set, rather than derived from the definition's name. */
+  set: { name: boolean; version: boolean }
+}
+
+const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/
+
+/** The npm package a definition builds as, or undefined when it is gone. */
+export async function readCreatedBundlePackage(
+  db: Db,
+  id: string,
+): Promise<CreatedBundlePackage | undefined> {
+  const row = await new BundleRepository(db).byId(id)
+  if (!row) return undefined
+  return {
+    ...bundlePackageOf(row),
+    set: { name: row.packageName !== null, version: row.packageVersion !== null },
+  }
+}
+
+/**
+ * Sets the npm package a definition builds as. An empty value goes back to its
+ * default; anything else must be what npm itself would accept.
+ */
+export async function writeCreatedBundlePackage(
+  db: Db,
+  id: string,
+  input: { name?: unknown; version?: unknown },
+): Promise<{ ok: true; package: CreatedBundlePackage } | { ok: false; message: string }> {
+  const bundles = new BundleRepository(db)
+  if (!(await bundles.byId(id))) return { ok: false, message: 'That bundle no longer exists.' }
+  const text = (value: unknown) => (typeof value === 'string' ? value.trim() : '')
+  const name = text(input.name)
+  const version = text(input.version)
+  if (name && !isPackageName(name))
+    return {
+      ok: false,
+      message:
+        'A package name is lowercase letters, digits, dots, hyphens and underscores, optionally with an @scope/ in front.',
+    }
+  if (version && !SEMVER.test(version))
+    return { ok: false, message: 'A version is three numbers, like 1.0.0.' }
+  await bundles.setPackage(id, { name: name || null, version: version || null })
+  return { ok: true, package: (await readCreatedBundlePackage(db, id)) as CreatedBundlePackage }
+}

package/src/adapters/schema-files.ts

@@ -2,7 +2,7 @@
  * The backoffice writes schema files. A document-type save lands in the
  * database as before and, when the directory is writable, in
  * `schema/content-types/<alias>.toml` — the file the next boot will sync.
- * docs/09-schema-as-code.md, "The backoffice writes files".
+ * docs/schema-as-code.md, "The backoffice writes files".
  */
 
 import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'

package/src/backup.ts

@@ -1,5 +1,5 @@
 /**
- * Rollback is a backup (docs/10-packaging-and-upgrades.md): SQLite is copied
+ * Rollback is a backup (docs/packaging-and-upgrades.md): SQLite is copied
  * beside the database; Postgres runs the operator's `CROMBOLO_PG_DUMP` command
  * or refuses unless the caller vouches with `--backup-taken`.
  */

package/src/bundle-install.ts

@@ -0,0 +1,417 @@
+/**
+ * Checking and installing a bundle's content half, against a database the
+ * caller holds. `docs/bundles.md`, "Installing one".
+ *
+ * A bundle is an npm package; what it installs is the `bundle.json` payload its
+ * `crombolo.content` names. It is found either as a dependency the site
+ * declares, by package name, or as a package directory applied in place, which
+ * is how an environment's export arrives. A bare payload with no package around
+ * it is refused, so there is one kind of bundle and not two.
+ *
+ * This lives in the server rather than the CLI so the backoffice can install
+ * with the connection it already has: the CLI's `content check` and
+ * `content import` open a database and call these.
+ */
+import { existsSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import {
+  type BundlePackage,
+  discoverBundles,
+  isDirectory,
+  PAYLOAD_MANIFEST,
+  peerRangeProblem,
+  readBundlePackage,
+  readPackageJson,
+} from '@crombolo/backoffice-host'
+import { ObjectTypes } from '@crombolo/core'
+import {
+  type Db,
+  describeRefFailure,
+  type NodeRow,
+  recordInLedger,
+  resolveNodeRef,
+  writeReport,
+} from '@crombolo/data'
+import {
+  type CheckOptions,
+  type ContentSet,
+  checkBundle,
+  importBundle,
+  isPackageName,
+  loadBundle,
+  loadResolutions,
+  type Resolution,
+  type TransferCheck,
+} from '@crombolo/transfer'
+import { type BackupResult, backupBefore } from './backup.ts'
+import {
+  applySections,
+  type CarriedSections,
+  carriedSections,
+  carriesSections,
+  planSections,
+  promoteBackups,
+  replacementsNeedingPermission,
+  type SectionFile,
+  type SectionOptions,
+  type SectionPlan,
+  type SectionResult,
+  stageBackups,
+} from './bundle-sections.ts'
+import { type CromboloConfig, VERSION } from './config.ts'
+import { mediaStoreFor, placeBlobs } from './media-files.ts'
+import { componentAliasesIn } from './schema.ts'
+
+/** The trees a content selector or placement may name. */
+export const CONTENT_OBJECT_TYPES = [ObjectTypes.Document, ObjectTypes.Media, ObjectTypes.Element]
+
+/** A refusal the caller should read: nothing was written. */
+export class CommandError extends Error {
+  constructor(
+    message: string,
+    readonly problems: string[] = [],
+  ) {
+    super(message)
+  }
+}
+
+export interface BundleSummary {
+  id: string
+  from: string
+  schemaVersion: string
+  schemaRevision: string
+  snapshot: string
+  nodes: number
+  blobsCarried: number
+  blobBytes: number
+  /** The package the payload came in. */
+  package: { name: string; version: string }
+}
+
+/** A bundle found and read: its package, and the payload inside it. */
+export interface ResolvedBundle {
+  package: BundlePackage
+  /** Where the `bundle.json` payload is. */
+  payloadDir: string
+  /** Whether it was found as one of the site's dependencies rather than a directory. */
+  dependency: boolean
+  set: ContentSet
+  blobs: ReadonlyMap<string, string>
+  /** The sections it carries beyond its content; empty for a content-only bundle. */
+  carried: CarriedSections
+}
+
+/** Where a package directory or a dependency's name points, before anything is read. */
+function locate(target: string, siteDir: string): { directory: string; dependency: boolean } {
+  const path = resolve(target)
+  if (isDirectory(path)) return { directory: path, dependency: false }
+  if (!isPackageName(target))
+    throw new CommandError(`${target} is neither a directory nor a package name.`)
+  const declared = discoverBundles(siteDir)
+  const found = declared.bundles.find((bundle) => bundle.packageName === target)
+  if (found) return { directory: found.directory, dependency: true }
+  const broken = declared.problems.find((problem) => problem.packageName === target)
+  if (broken)
+    throw new CommandError(`${target} is not a valid bundle; nothing was read.`, broken.problems)
+  throw new CommandError(
+    `${target} is not a bundle this site depends on. Add it with \`bun add ${target}\`, or name a package directory.`,
+  )
+}
+
+/**
+ * The bundle a path or a package name names, read whole, or a refusal: a
+ * half-copied bundle must never be checked as though complete.
+ */
+export function readBundle(target: string, siteDir: string): ResolvedBundle {
+  const { directory, dependency } = locate(target, siteDir)
+  const json = readPackageJson(join(directory, 'package.json'))
+  if (!json) {
+    if (existsSync(join(directory, PAYLOAD_MANIFEST)))
+      throw new CommandError(
+        `${target} is a bare ${PAYLOAD_MANIFEST} with no package.json. Every bundle is an npm package: ` +
+          `a package.json whose "crombolo" field names the payload directory, with the payload under content/.`,
+      )
+    throw new CommandError(`${target} has no package.json, so it is not a bundle.`)
+  }
+  const read = readBundlePackage(directory)
+  if (!read.bundle)
+    throw new CommandError(
+      `${target} is not a valid bundle; nothing was read from it.`,
+      read.problems.length > 0 ? read.problems : ['its package.json has no "crombolo" field'],
+    )
+  const pkg = read.bundle
+  if (!pkg.contentDir)
+    throw new CommandError(
+      `${pkg.packageName} carries no content to install: its manifest names no "content".`,
+    )
+  const peer = peerRangeProblem(pkg, VERSION)
+  if (peer) throw new CommandError(`${pkg.packageName} cannot be installed here: ${peer}.`)
+
+  const loaded = loadBundle(pkg.contentDir)
+  const problems = loaded.problems.map((problem) => `${problem.file}: ${problem.message}`)
+  if (!loaded.set || problems.length > 0)
+    throw new CommandError(
+      'The bundle is incomplete or damaged; nothing was read from it.',
+      problems,
+    )
+  return {
+    package: pkg,
+    payloadDir: pkg.contentDir,
+    dependency,
+    set: loaded.set,
+    blobs: loaded.blobs,
+    carried: carriedSections(loaded),
+  }
+}
+
+/**
+ * The check an install that never reached the content reports.
+ *
+ * Nothing was examined, so nothing is outstanding: the refusal is in
+ * `problems`, and a caller reading `outstanding.length` must not be told there
+ * were content findings when there were none.
+ */
+const emptyCheck = (bundleId = ''): TransferCheck => ({
+  bundleId,
+  findings: [],
+  outstanding: [],
+  plan: [],
+  counts: { create: 0, update: 0, unchanged: 0, skip: 0 },
+})
+
+function summarise(bundle: ResolvedBundle): BundleSummary {
+  const set = bundle.set
+  const carried = set.manifest.blobs.filter((blob) => blob.included)
+  return {
+    id: set.manifest.id,
+    from: set.manifest.provenance.siteName || '',
+    schemaVersion: set.manifest.provenance.schemaVersion,
+    schemaRevision: set.manifest.provenance.schemaRevision,
+    snapshot: set.manifest.snapshot,
+    nodes: set.nodes.length,
+    blobsCarried: carried.length,
+    blobBytes: carried.reduce((total, blob) => total + (blob.size ?? 0), 0),
+    package: { name: bundle.package.packageName, version: bundle.package.version },
+  }
+}
+
+export interface TransferInput extends SectionOptions {
+  /** A package directory on this machine, or the name of a bundle the site depends on. */
+  bundle: string
+  /** Where the bundle's roots go: a path or a key. */
+  under?: string
+  /** A decision per node key, on top of any the bundle carries. */
+  resolutions?: Record<string, Resolution>
+  resolveAll?: Resolution
+  allowMissingBlobs?: boolean
+}
+
+/** The options `check` and `import` share: saved decisions, then the caller's, then placement. */
+export async function transferOptions(
+  db: Db,
+  config: CromboloConfig,
+  bundle: ResolvedBundle,
+  input: Omit<TransferInput, 'bundle'>,
+): Promise<{ options: CheckOptions; under: NodeRow | undefined; problems: string[] }> {
+  const saved = loadResolutions(bundle.payloadDir, bundle.set.manifest.id)
+  const resolutions = { ...(saved.file?.byNode ?? {}), ...(input.resolutions ?? {}) }
+  const resolveAll = input.resolveAll ?? saved.file?.all ?? undefined
+  const underRef = input.under ?? saved.file?.under ?? undefined
+  let under: NodeRow | undefined
+  if (underRef) {
+    const resolved = await resolveNodeRef(db, CONTENT_OBJECT_TYPES, underRef)
+    if (!resolved.ok) throw new CommandError(describeRefFailure(underRef, resolved))
+    under = resolved.node
+  }
+  // Whether this environment has a media file is a question only its store can
+  // answer; without it `missing-blob` is a finding the check could never raise.
+  const store = await mediaStoreFor(config)
+  return {
+    options: {
+      under,
+      resolutions,
+      resolveAll,
+      componentAliases: componentAliasesIn(config.componentsDir),
+      allowMissingBlobs: input.allowMissingBlobs ?? false,
+      hasBlob: async (key: string) => Boolean(await store.get(key)),
+      backOfficePath: config.backOfficePath,
+    },
+    under,
+    problems: saved.problems,
+  }
+}
+
+export interface ContentCheckResult {
+  bundle: BundleSummary
+  /** Where the payload was read from, which is where saved decisions are written. */
+  payloadDir: string
+  under: string | null
+  check: TransferCheck
+  /** What the carried sections would do here; undefined for a content-only bundle. */
+  sections: SectionPlan | undefined
+  /** Nothing outstanding: the bundle would import cleanly. */
+  clean: boolean
+  /** The decisions this check ran with — the bundle's saved ones under the caller's. */
+  decisions: { all: Resolution | null; byNode: Record<string, Resolution> }
+  problems: string[]
+}
+
+/** Read-only: what installing a bundle here would do. Its findings go to the Changes dashboard. */
+export async function checkBundleContent(
+  db: Db,
+  config: CromboloConfig,
+  input: TransferInput,
+): Promise<ContentCheckResult> {
+  const bundle = readBundle(input.bundle, config.siteDir)
+  const set = bundle.set
+  const prepared = await transferOptions(db, config, bundle, input)
+  const check = await checkBundle(db, set, prepared.options)
+  // Under this bundle's own scope, so two bundles in flight do not resolve each other's.
+  await writeReport(db, check.findings, { source: 'transfer', scope: set.manifest.id })
+  return {
+    bundle: summarise(bundle),
+    payloadDir: bundle.payloadDir,
+    under: prepared.under ? (prepared.under.text ?? prepared.under.key) : null,
+    check,
+    sections: carriesSections(bundle.carried)
+      ? await planSections(db, config, bundle.carried, input)
+      : undefined,
+    clean: check.outstanding.length === 0,
+    decisions: {
+      all: prepared.options.resolveAll ?? null,
+      byNode: prepared.options.resolutions ?? {},
+    },
+    problems: prepared.problems,
+  }
+}
+
+export interface ContentImportInput extends TransferInput {
+  publish?: boolean
+  label?: string
+  /** Postgres: the caller has taken a backup itself. */
+  backupTaken?: boolean
+}
+
+export interface ContentImportResult {
+  bundle: BundleSummary
+  backup: BackupResult
+  /**
+   * What the carried sections did, when the bundle carried any.
+   *
+   * Applied before the content and outside the content run's ledger entry: the
+   * files are now in `schema/` and the views directory, where the repository
+   * owns them — so a revert takes the content back and git takes the files.
+   */
+  sections: SectionResult | undefined
+  /** Where the originals of any replaced files were kept, for a revert. */
+  replacedFilesKept?: string | undefined
+  /** Undefined when the import was refused; then nothing was written. */
+  runId: string | undefined
+  check: TransferCheck
+  blobs: { placed: number; bytes: number } | null
+  published: number
+  publishFailures: { key: string; reason: string }[]
+  problems: string[]
+}
+
+/** Back up, re-check, then apply in one transaction and record the run. */
+export async function installBundleContent(
+  db: Db,
+  config: CromboloConfig,
+  input: ContentImportInput,
+): Promise<ContentImportResult> {
+  const bundle = readBundle(input.bundle, config.siteDir)
+  const { set, blobs, carried } = bundle
+  // The coarse layer, under the fine-grained revert: in-place conversion is not
+  // reversible by itself, so the same rule as `upgrade` applies here.
+  const backup = await backupBefore(config, {
+    reason: 'content-import',
+    backupTaken: input.backupTaken,
+  })
+  const prepared = await transferOptions(db, config, bundle, input)
+  const started = performance.now()
+
+  /**
+   * Structure first, but planned before anything is written.
+   *
+   * The order is forced: content naming a type this bundle brings cannot pass
+   * a check until that type exists, so the schema has to land before the
+   * content is checked. Which means a blocking schema change has to be caught
+   * *before* the first file is written — otherwise a refused install leaves a
+   * half-applied site behind. `planSections` answers that from an overlay
+   * copy, so the site is untouched until it is known the schema can apply.
+   */
+  const plan = carriesSections(carried) ? await planSections(db, config, carried, input) : undefined
+  const refuse = (problems: string[]): ContentImportResult => ({
+    bundle: summarise(bundle),
+    backup,
+    runId: undefined,
+    check: emptyCheck(set.manifest.id),
+    blobs: null,
+    published: 0,
+    publishFailures: [],
+    problems,
+    sections: undefined,
+  })
+
+  if (plan?.schema?.classification === 'breaking')
+    return refuse([
+      'the schema this bundle carries cannot be applied here without data work',
+      ...plan.schema.findings,
+    ])
+
+  /**
+   * A file this site already has is not replaced without being asked.
+   *
+   * Every section is editable in the backoffice — a stylesheet, a template, a
+   * content type — so an overwrite destroys somebody's work, and the site
+   * may have no git to recover it from. Refused by name, like any other
+   * finding a person has to answer.
+   */
+  const needingPermission = plan ? replacementsNeedingPermission(plan, input) : []
+  if (needingPermission.length > 0)
+    return refuse([
+      `${needingPermission.length} file(s) here would be replaced; pass --replace-files to allow it`,
+      ...needingPermission.map((file: SectionFile) => file.path),
+    ])
+
+  // Staged whenever something *will* be replaced — which is decided by the
+  // plan, not by the permission check: by this point permission has been
+  // given, so the files needing it are exactly the ones to back up.
+  const willReplace = (plan?.files ?? []).some((file) => file.action === 'overwrite')
+  const staging = plan && willReplace ? stageBackups(config) : undefined
+  const sections = plan
+    ? await applySections(db, config, carried, { ...input, backupDir: staging })
+    : undefined
+  const result = await importBundle(db, set, {
+    ...prepared.options,
+    publish: input.publish ?? false,
+    nodeId: config.nodeId,
+    label: input.label,
+    package: { name: bundle.package.packageName, version: bundle.package.version },
+  })
+  await writeReport(db, result.check.findings, { source: 'transfer', scope: set.manifest.id })
+  const outcome: ContentImportResult = {
+    bundle: summarise(bundle),
+    backup,
+    runId: result.runId,
+    check: result.check,
+    blobs: null,
+    published: result.published.length,
+    publishFailures: result.publishFailures,
+    problems: prepared.problems,
+    sections,
+  }
+  if (!result.runId) return outcome
+  // The backups become the run's, so `content revert` can find them.
+  if (staging) outcome.replacedFilesKept = promoteBackups(config, staging, result.runId)
+  if (blobs.size > 0) outcome.blobs = await placeBlobs(await mediaStoreFor(config), blobs)
+  await recordInLedger(db, {
+    name: `content import ${set.manifest.id}`,
+    kind: 'transfer',
+    durationMs: Math.round(performance.now() - started),
+    appliedBy: config.nodeId,
+    note: `run ${result.runId}, ${bundle.package.packageName}@${bundle.package.version}`,
+  })
+  return outcome
+}