@crombolo/server 0.13.2 → 0.14.4

package/src/site-file.ts

@@ -0,0 +1,292 @@
+/**
+ * A whole site in one file (story 104): every content node, media file, schema
+ * file, component, stylesheet, script and dictionary item, as one private npm
+ * package tarball — the bundle format with `scope: 'site'`. `docs/bundles.md`.
+ *
+ * Written as it goes. The media files stream from the store into the gzip, and
+ * the integrity hash is computed as each file passes, in the path order
+ * `bundleIntegrity` defines; only the small files are held. The tarball is
+ * written beside its destination and renamed into place once complete, so a
+ * failed export never leaves a file that looks whole.
+ */
+import { rename, rm } from 'node:fs/promises'
+import { discoverBundles } from '@crombolo/backoffice-host'
+import { type Db, NodeRepository, type NodeRow, ObjectTypes } from '@crombolo/data'
+import {
+  BUNDLE_SECTIONS,
+  type BundleFile,
+  type BundlePackageInfo,
+  blobPath,
+  bundleIdFor,
+  bundlePackageJson,
+  exportBundle,
+  exportPeople,
+  MANIFEST_FILE,
+  newSeal,
+  PAYLOAD_DIR,
+  packageNameFor,
+  planStreamedBundle,
+  sanitise,
+  sealBytes,
+  writeManifest,
+} from '@crombolo/transfer'
+import { createFileSystemPort } from './adapters/file-system.ts'
+import { createTemplateFileStore } from './adapters/template-files.ts'
+import { type CromboloConfig, VERSION } from './config.ts'
+import { mediaStoreFor } from './media-files.ts'
+import { type StoredFile, streamOf } from './media-store.ts'
+import { siteSections } from './site-sections.ts'
+import { TarballWriter } from './tar.ts'
+
+export interface SiteExportOptions {
+  /** The `.tgz` to write. */
+  out: string
+  /**
+   * False for `--no-media`: the media nodes travel, their files do not.
+   * `referenced`: likewise, for a remote and its CLI to move the files one at a
+   * time (story 107).
+   */
+  media?: boolean | 'referenced'
+  /** Carry the latest drafts rather than what is published. */
+  drafts?: boolean
+  withMembers?: boolean
+  withUsers?: boolean
+  /** Replace every personal value before anything is written (`sanitise`). */
+  anonymise?: boolean
+  /**
+   * Seal the content data — property values and media files — as well as any
+   * people, under `passphrase`: the bundle rule, metadata readable and content
+   * not (story 107). Each media file is sealed whole, one at a time.
+   */
+  sealContent?: boolean
+  /** Seals the people, and the content with `sealContent`; needed when either travels sealed. */
+  passphrase?: string
+  package?: { name?: string; version?: string }
+}
+
+export interface SiteExportResult {
+  out: string
+  bundleId: string
+  package: { name: string; version: string }
+  counts: Record<string, number>
+  sections: Partial<Record<keyof typeof BUNDLE_SECTIONS, number>>
+  media: { carried: number; bytes: number; missing: string[]; omitted: boolean }
+  people: { members: number; users: number }
+  /** How many distinct email addresses `--anonymise` replaced. */
+  anonymised?: number
+}
+
+/** The content kinds a whole site carries, each from its tree's top. */
+const SITE_TREES = [
+  ObjectTypes.Document,
+  ObjectTypes.Media,
+  ObjectTypes.Element,
+  ObjectTypes.DocumentBlueprint,
+]
+
+async function topLevelNodes(db: Db): Promise<NodeRow[]> {
+  const nodes = new NodeRepository(db)
+  const roots: NodeRow[] = []
+  for (const tree of SITE_TREES) {
+    for (let skip = 0; ; skip += 500) {
+      const page = await nodes.roots(tree, skip, 500)
+      roots.push(...page.items)
+      if (skip + 500 >= page.total) break
+    }
+  }
+  return roots
+}
+
+/** The server bundles the site wires, by package name and version. */
+function requiredPackages(config: CromboloConfig): Record<string, string> {
+  if (config.bundles.length === 0) return {}
+  const wired = new Set(config.bundles.map((bundle) => bundle.id))
+  const required: Record<string, string> = {}
+  for (const found of discoverBundles(config.siteDir).bundles)
+    if (wired.has(found.id)) required[found.packageName] = found.version
+  return required
+}
+
+export async function exportSite(
+  db: Db,
+  config: CromboloConfig,
+  options: SiteExportOptions,
+): Promise<SiteExportResult> {
+  const withMedia = options.media !== false && options.media !== 'referenced'
+  const carryMembers = Boolean(options.withMembers)
+  // Anonymised users are removed, so there are none to carry.
+  const carryUsers = Boolean(options.withUsers) && !options.anonymise
+  if ((carryMembers || carryUsers) && !options.passphrase)
+    throw new Error('Members and users travel sealed: give a passphrase.')
+  if (options.sealContent && !options.passphrase)
+    throw new Error('Sealed content needs a passphrase.')
+
+  const exported = await exportBundle(db, {
+    roots: await topLevelNodes(db),
+    asGiven: [],
+    descendants: true,
+    snapshot: options.drafts ? 'drafts' : 'published',
+    blueprints: true,
+    withBlobs: withMedia,
+    siteName: config.siteName,
+    nodeId: config.nodeId,
+  })
+
+  let sections: BundleFile[] = await siteSections(
+    {
+      db,
+      componentFiles: createTemplateFileStore(config.componentsDir),
+      components: createFileSystemPort(config.componentsDir, {
+        extension: '.tsx',
+        rewrites: ['.cshtml'],
+      }),
+      stylesheets: createFileSystemPort(config.stylesheetsDir, { extension: '.css' }),
+      scripts: createFileSystemPort(config.scriptsDir, { extension: '.js' }),
+      schemaDir: config.schemaDir,
+    },
+    'all',
+  )
+
+  let set = exported.set
+  let people =
+    carryMembers || carryUsers
+      ? await exportPeople(db, { members: carryMembers, users: carryUsers })
+      : undefined
+  let anonymised: number | undefined
+  if (options.anonymise) {
+    const dictionaryPath = `${BUNDLE_SECTIONS.dictionary}.udt`
+    const dictionary = sections.find((file) => file.path === dictionaryPath)
+    const clean = sanitise({ set, people, dictionary: dictionary?.text })
+    set = clean.set
+    people = clean.people
+    anonymised = clean.emails
+    if (dictionary && clean.dictionary !== undefined)
+      sections = sections.map((file) =>
+        file.path === dictionaryPath ? { ...file, text: clean.dictionary } : file,
+      )
+  }
+
+  // Each media file is looked up once: its size goes in the manifest now, its
+  // bytes stream into the tarball later.
+  const store = await mediaStoreFor(config)
+  const stored = new Map<string, StoredFile>()
+  const missing: string[] = []
+  // Referenced, each file is named with its size and etag, so the far side can
+  // fetch it — or skip it, holding one with the same etag already.
+  const described = new Map<string, { size: number; etag: string }>()
+  if (withMedia || options.media === 'referenced')
+    for (const key of exported.blobKeys) {
+      const file = await store.get(key)
+      if (!file) missing.push(key)
+      else if (withMedia) stored.set(key, file)
+      else described.set(key, { size: file.size, etag: file.etag })
+    }
+
+  const site = {
+    ...set,
+    manifest: {
+      ...set.manifest,
+      blobs: set.manifest.blobs.map((blob) => {
+        const found = described.get(blob.key)
+        return found ? { ...blob, size: found.size, etag: found.etag } : blob
+      }),
+      label: `${config.siteName} (whole site)`,
+      provenance: { ...set.manifest.provenance, cmsVersion: VERSION },
+      scope: 'site' as const,
+      ...(withMedia
+        ? {}
+        : {
+            media: options.media === 'referenced' ? ('referenced' as const) : ('omitted' as const),
+          }),
+      requires: requiredPackages(config),
+    },
+  }
+  const seal =
+    (people || options.sealContent) && options.passphrase ? newSeal(options.passphrase) : undefined
+  const planned = planStreamedBundle(
+    site,
+    new Map([...stored].map(([key, file]) => [key, file.size])),
+    sections,
+    { people, seal, content: options.sealContent ? 'sealed' : 'open' },
+  )
+
+  const name = options.package?.name ?? packageNameFor(`${config.siteName}-site`)
+  const info: BundlePackageInfo = {
+    name,
+    version: options.package?.version ?? '1.0.0',
+    id: bundleIdFor(name.replace(/^@[^/]+\//, '')),
+    label: site.manifest.label,
+    private: true,
+    peerRange: `>=${VERSION}`,
+  }
+
+  const partial = `${options.out}.partial`
+  const writer = TarballWriter.create(partial)
+  const encoder = new TextEncoder()
+  const hasher = new Bun.CryptoHasher('sha256')
+  let bytes = 0
+  try {
+    const small = new Map(planned.files.map((file) => [file.path, file]))
+    const blobs = new Map(planned.blobs.map((key) => [blobPath(key), key]))
+    const paths = [...small.keys(), ...blobs.keys()].sort((a, b) => a.localeCompare(b))
+    for (const path of paths) {
+      hasher.update(path)
+      hasher.update('\0')
+      const file = small.get(path)
+      if (file) {
+        const body = file.bytes ?? encoder.encode(file.text ?? '')
+        hasher.update(body)
+        await writer.add(`${PAYLOAD_DIR}/${path}`, body)
+      } else {
+        const media = stored.get(blobs.get(path) as string) as StoredFile
+        if (planned.sealing && seal) {
+          const sealed = sealBytes(seal.key, path, await media.bytes())
+          hasher.update(sealed)
+          hasher.update('\n')
+          await writer.add(`${PAYLOAD_DIR}/${path}`, sealed)
+          bytes += media.size
+          continue
+        }
+        async function* hashed() {
+          for await (const chunk of streamOf(media)) {
+            hasher.update(chunk)
+            yield chunk
+          }
+        }
+        await writer.addStream(`${PAYLOAD_DIR}/${path}`, media.size, hashed())
+        bytes += media.size
+      }
+      hasher.update('\n')
+    }
+    const manifest = { ...planned.manifest, integrity: `sha256-${hasher.digest('hex')}` }
+    await writer.add(`${PAYLOAD_DIR}/${MANIFEST_FILE}`, encoder.encode(writeManifest(manifest)))
+    await writer.add(
+      'package.json',
+      encoder.encode(`${JSON.stringify(bundlePackageJson(info), null, 2)}\n`),
+    )
+    await writer.close()
+    await rename(partial, options.out)
+  } catch (error) {
+    await writer.abort()
+    await rm(partial, { force: true })
+    throw error
+  }
+
+  const counted: SiteExportResult['sections'] = {}
+  for (const file of sections) {
+    const section = (Object.keys(BUNDLE_SECTIONS) as (keyof typeof BUNDLE_SECTIONS)[]).find(
+      (s) => file.path === `${BUNDLE_SECTIONS[s]}.udt` || file.path.startsWith(`${s}/`),
+    )
+    if (section) counted[section] = (counted[section] ?? 0) + 1
+  }
+  return {
+    out: options.out,
+    bundleId: site.manifest.id,
+    package: { name: info.name, version: info.version },
+    counts: { ...site.manifest.counts },
+    sections: counted,
+    media: { carried: stored.size, bytes, missing, omitted: !withMedia },
+    people: { members: people?.members.length ?? 0, users: people?.users.length ?? 0 },
+    ...(anonymised !== undefined ? { anonymised } : {}),
+  }
+}

package/src/site-replace.ts

@@ -0,0 +1,188 @@
+/**
+ * Replacing a site with a whole-site file (story 105): what has to be true
+ * before it starts, and what the site has that the file does not.
+ *
+ * `--replace` recycles content and media the file does not carry, so a revert
+ * brings them back, and keeps the site's people. `--replace-all` deletes them,
+ * and the members and users the file does not carry too.
+ */
+import { existsSync } from 'node:fs'
+import { join, sep } from 'node:path'
+import { discoverBundles } from '@crombolo/backoffice-host'
+import {
+  currentSchemaState,
+  type Db,
+  DocumentRepository,
+  MemberRepository,
+  ObjectTypes,
+  TransferRunRepository,
+  UserRepository,
+} from '@crombolo/data'
+import type { BundleManifest, BundlePeople, ContentSet } from '@crombolo/transfer'
+
+export type ReplaceMode = 'replace' | 'replace-all'
+
+/**
+ * The server bundles a file needs and this site lacks, or has older, each
+ * named with the version wanted.
+ */
+export function requiresProblems(manifest: BundleManifest, siteDir: string): string[] {
+  const required = Object.entries(manifest.requires ?? {})
+  if (required.length === 0) return []
+  const installed = new Map(
+    discoverBundles(siteDir).bundles.map((found) => [found.packageName, found.version]),
+  )
+  const problems: string[] = []
+  for (const [name, version] of required) {
+    const here = installed.get(name)
+    if (!here) problems.push(`the file needs ${name}@${version}, which this site does not have`)
+    else if (Bun.semver.order(here, version) < 0)
+      problems.push(`the file needs ${name}@${version}, and this site has ${here}`)
+  }
+  return problems
+}
+
+/** Whether the file's people include someone who can administer the back office. */
+export function carriesAdministrator(people: BundlePeople | undefined): boolean {
+  return (people?.users ?? []).some((user) => !user.disabled && user.groupAliases.includes('admin'))
+}
+
+const PRUNED_KINDS = [
+  { objectType: ObjectTypes.Document, kind: 'document' },
+  { objectType: ObjectTypes.Media, kind: 'media' },
+  { objectType: ObjectTypes.Element, kind: 'element' },
+  { objectType: ObjectTypes.DocumentBlueprint, kind: 'blueprint' },
+] as const
+
+export interface PrunedNode {
+  key: string
+  kind: (typeof PRUNED_KINDS)[number]['kind']
+  head: number | undefined
+  published: number | undefined
+}
+
+/**
+ * Takes out what the site has and the file does not: the topmost of each such
+ * branch, recycled or deleted, inside the caller's transaction. Answers what
+ * was recycled, for the run to record so its revert can bring it back.
+ */
+export async function pruneSite(
+  tx: Db,
+  set: ContentSet,
+  mode: ReplaceMode,
+  nodeId: string,
+): Promise<PrunedNode[]> {
+  const inFile = new Set(set.nodes.map((node) => node.key.toLowerCase()))
+  const state = await currentSchemaState(tx)
+  const nodeState = { version: state.version, revision: state.revision }
+  const pruned: PrunedNode[] = []
+  for (const { objectType, kind } of PRUNED_KINDS) {
+    const rows = await tx.query<{ id: number; unique_id: string; parent_id: number }>(
+      'SELECT id, unique_id, parent_id FROM node WHERE node_object_type = ? AND trashed = ?',
+      [objectType, tx.dialect.boolValue(false)],
+    )
+    const leaving = rows.filter((row) => !inFile.has(String(row.unique_id).toLowerCase()))
+    const leavingIds = new Set(leaving.map((row) => Number(row.id)))
+    // The top of each branch: recycling or deleting it takes its children too.
+    const tops = leaving.filter((row) => !leavingIds.has(Number(row.parent_id)))
+    const repo = new DocumentRepository(tx, { kind, nodeState, nodeId, inTreeLock: true })
+    for (const row of tops) {
+      const key = String(row.unique_id)
+      if (mode === 'replace-all') {
+        await repo.delete(key)
+        continue
+      }
+      const markers = await repo.eventMarkers(key)
+      await repo.moveToRecycleBin(key)
+      pruned.push({ key, kind, head: markers?.head, published: markers?.published })
+    }
+  }
+  return pruned
+}
+
+/** Members and users the file does not carry, deleted: `--replace-all` only. */
+export async function prunePeople(tx: Db, people: BundlePeople | undefined): Promise<void> {
+  const members = new Set((people?.members ?? []).map((m) => m.key.toLowerCase()))
+  const users = new Set((people?.users ?? []).map((u) => u.key.toLowerCase()))
+  const state = await currentSchemaState(tx)
+  const memberRepo = new MemberRepository(tx, {
+    nodeState: { version: state.version, revision: state.revision },
+  })
+  const memberRows = await tx.query<{ unique_id: string }>(
+    'SELECT unique_id FROM node WHERE node_object_type = ?',
+    [ObjectTypes.Member],
+  )
+  for (const row of memberRows)
+    if (!members.has(String(row.unique_id).toLowerCase()))
+      await memberRepo.delete(String(row.unique_id))
+  const userRepo = new UserRepository(tx)
+  for (const row of await tx.query<{ key: string }>('SELECT key FROM user_account'))
+    if (!users.has(String(row.key).toLowerCase())) await userRepo.delete(String(row.key))
+}
+
+/** Records what a replace recycled on its run, so `content revert` brings it back. */
+export async function recordPruned(tx: Db, runId: string, pruned: PrunedNode[]): Promise<void> {
+  const runs = new TransferRunRepository(tx)
+  for (const node of pruned)
+    await runs.record({
+      runId,
+      nodeKey: node.key,
+      kind: node.kind,
+      action: 'recycle',
+      beforeEventId: node.head ?? null,
+      beforePublishedEventId: node.published ?? null,
+      eventId: null,
+    })
+}
+
+export interface ReplaceSummary {
+  /** What the site has that the file does not, by kind: recycled, or deleted. */
+  leaving: Record<string, number>
+  /** Files in the site's folders that the file does not carry. */
+  filesRemoved: string[]
+  /** Members and users `--replace-all` would delete. */
+  people: { members: number; users: number }
+}
+
+/** Read-only: what replacing this site with the file would take out. */
+export async function replaceSummary(
+  db: Db,
+  set: ContentSet,
+  carriedPaths: ReadonlySet<string>,
+  folders: ReadonlyArray<{ section: string; dir: string }>,
+  mode: ReplaceMode,
+): Promise<ReplaceSummary> {
+  const inFile = new Set(set.nodes.map((node) => node.key.toLowerCase()))
+  const leaving: Record<string, number> = {}
+  for (const { objectType, kind } of PRUNED_KINDS) {
+    const rows = await db.query<{ unique_id: string }>(
+      'SELECT unique_id FROM node WHERE node_object_type = ? AND trashed = ?',
+      [objectType, db.dialect.boolValue(false)],
+    )
+    const count = rows.filter((row) => !inFile.has(String(row.unique_id).toLowerCase())).length
+    if (count > 0) leaving[kind] = count
+  }
+  const filesRemoved: string[] = []
+  for (const { section, dir } of folders) {
+    if (!existsSync(dir)) continue
+    for (const path of new Bun.Glob('**/*').scanSync({ cwd: dir, onlyFiles: true })) {
+      const bundlePath = `${section}/${path.split(sep).join('/')}`
+      if (!carriedPaths.has(bundlePath)) filesRemoved.push(join(dir, path))
+    }
+  }
+  let people = { members: 0, users: 0 }
+  if (mode === 'replace-all') {
+    const members = new Set((set.people?.members ?? []).map((m) => m.key.toLowerCase()))
+    const users = new Set((set.people?.users ?? []).map((u) => u.key.toLowerCase()))
+    const memberRows = await db.query<{ unique_id: string }>(
+      'SELECT unique_id FROM node WHERE node_object_type = ?',
+      [ObjectTypes.Member],
+    )
+    const userRows = await db.query<{ key: string }>('SELECT key FROM user_account')
+    people = {
+      members: memberRows.filter((r) => !members.has(String(r.unique_id).toLowerCase())).length,
+      users: userRows.filter((r) => !users.has(String(r.key).toLowerCase())).length,
+    }
+  }
+  return { leaving, filesRemoved, people }
+}

package/src/site-search.ts

@@ -0,0 +1,51 @@
+/**
+ * Site search for a render (story 140): the published-only index, narrowed to
+ * what the person asking may open, each hit read back from the published cache
+ * so a page unpublished since it was indexed is never shown. docs/search.md.
+ */
+import { branchAccess } from '@crombolo/core'
+import { type Db, PublicAccessRepository, SiteSearch } from '@crombolo/data'
+import {
+  excerpt,
+  type PublishedCache,
+  type RequestMember,
+  type SearchResult,
+  type SiteSearchPort,
+} from '@crombolo/render'
+
+/** The search `search()` calls during one render, for `member` or for nobody signed in. */
+export function siteSearchFor(
+  db: Db,
+  cache: PublishedCache,
+  member: RequestMember | undefined,
+): SiteSearchPort {
+  return async (query, options) => {
+    const rules = await new PublicAccessRepository(db).allRules()
+    const access = branchAccess(
+      rules,
+      member ? { username: member.username, groups: member.roles } : undefined,
+    )
+    const culture = options.culture ?? (await cache.defaultCulture())
+    const hits = await new SiteSearch(db).query(query, {
+      culture: culture ?? '',
+      types: options.types,
+      skip: options.skip,
+      take: options.take,
+      access,
+    })
+    const nav = await cache.navigation(culture)
+    const items: SearchResult[] = []
+    for (const hit of hits.items) {
+      const content = nav.byKey(hit.key)
+      if (!content) continue
+      items.push({
+        content,
+        url: content.url,
+        name: content.name,
+        score: hit.score,
+        excerpt: excerpt(hit.text, query),
+      })
+    }
+    return { total: hits.total, items }
+  }
+}