@crombolo/server 0.13.2 → 0.14.4

package/src/site-sections.ts

@@ -0,0 +1,234 @@
+/**
+ * The sections a bundle carries beyond its content — schema, components,
+ * styles, scripts and the dictionary — gathered from a site. One place, so the
+ * backoffice's Packages builder (picked items) and `crombolo export --site`
+ * (everything) compose a bundle the same way. `docs/bundles.md`.
+ */
+import { readdir, readFile } from 'node:fs/promises'
+import { join, relative, sep } from 'node:path'
+import type { FileSystemPort } from '@crombolo/api-management'
+import {
+  type ComponentFileStore,
+  ComponentRepository,
+  currentSchemaState,
+  type Db,
+  DictionaryRepository,
+} from '@crombolo/data'
+import {
+  exportSchemaSet,
+  fileNameFor,
+  type SchemaSet,
+  writeContentType,
+  writeDataType,
+  writeLanguages,
+  writeMediaType,
+  writeSchemaVersion,
+} from '@crombolo/schema'
+import { BUNDLE_SECTIONS, type BundleFile, type BundleSection } from '@crombolo/transfer'
+import { dictionaryToUdt, writeDictionaryUdt } from './dictionary-transfer.ts'
+
+export interface SiteSectionSources {
+  db: Db
+  /** Reads a template's view, which lives on disk rather than in the row. */
+  componentFiles: ComponentFileStore
+  components?: FileSystemPort
+  stylesheets?: FileSystemPort
+  scripts?: FileSystemPort
+  /** The schema directory, read whole for `'all'`: the files are the schema. */
+  schemaDir?: string
+}
+
+/** What a Packages definition picks, by key or alias as the backoffice sends them. */
+export interface SectionPicks {
+  contentTypes: readonly string[]
+  mediaTypes: readonly string[]
+  dataTypes: readonly string[]
+  languages: readonly string[]
+  /** Template aliases and paths in the components tree, in one list. */
+  components: readonly string[]
+  stylesheets: readonly string[]
+  scripts: readonly string[]
+  dictionaryItems: readonly string[]
+}
+
+/** Matches a selection against a type's key or its alias, since the UI sends either. */
+const picks = (selected: readonly string[]) => {
+  const wanted = new Set(selected.map((value) => value.trim().toLowerCase()))
+  return (type: { key?: string; alias: string }): boolean =>
+    wanted.has(type.alias.toLowerCase()) ||
+    (type.key !== undefined && wanted.has(type.key.toLowerCase()))
+}
+
+/** The selected slice of the site's schema, as the files `schema/` would hold. */
+function schemaEntries(set: SchemaSet, from: SectionPicks): BundleFile[] {
+  const entries: BundleFile[] = []
+  const put = (path: string, text: string) =>
+    entries.push({ path: `${BUNDLE_SECTIONS.schema}/${path}`, text })
+
+  const contentTypes = set.contentTypes.filter(picks(from.contentTypes))
+  const mediaTypes = (set.mediaTypes ?? []).filter(picks(from.mediaTypes))
+  // Member types have no picker in the builder, and the contract's definition
+  // has no field for them, so a picked bundle never carries one.
+  const dataTypes = set.dataTypes.filter(picks(from.dataTypes))
+  const languages = set.languages.filter((language) =>
+    from.languages.some((pick) => pick.toLowerCase() === language.iso.toLowerCase()),
+  )
+
+  if (
+    contentTypes.length === 0 &&
+    mediaTypes.length === 0 &&
+    dataTypes.length === 0 &&
+    languages.length === 0
+  )
+    return entries
+
+  put('schema.toml', writeSchemaVersion(set.version))
+  for (const type of contentTypes)
+    put(join('content-types', fileNameFor(type.alias)), writeContentType(type))
+  for (const type of mediaTypes)
+    put(join('media-types', fileNameFor(type.alias)), writeMediaType(type))
+  for (const type of dataTypes)
+    put(join('data-types', fileNameFor(type.alias)), writeDataType(type))
+  if (languages.length > 0) put('languages.toml', writeLanguages(languages))
+  return entries
+}
+
+/** Every `.toml` under the schema directory, as it stands: the whole site's schema. */
+async function wholeSchema(dir: string | undefined): Promise<BundleFile[]> {
+  if (!dir) return []
+  const files: BundleFile[] = []
+  const walk = async (at: string): Promise<void> => {
+    let entries: import('node:fs').Dirent[]
+    try {
+      entries = await readdir(at, { withFileTypes: true })
+    } catch {
+      return
+    }
+    for (const entry of entries) {
+      const path = join(at, entry.name)
+      if (entry.isDirectory()) await walk(path)
+      else if (entry.name.toLowerCase().endsWith('.toml'))
+        files.push({
+          path: `${BUNDLE_SECTIONS.schema}/${relative(dir, path).split(sep).join('/')}`,
+          text: await readFile(path, 'utf8'),
+        })
+    }
+  }
+  await walk(dir)
+  return files
+}
+
+/** Every file a file area holds, by its path within the area. */
+async function everyPath(port: FileSystemPort | undefined): Promise<string[]> {
+  if (!port) return []
+  const paths: string[] = []
+  const walk = async (parent: string | null): Promise<void> => {
+    for (const entry of (await port.children(parent)) ?? []) {
+      if (entry.isFolder) await walk(entry.path)
+      else paths.push(entry.path)
+    }
+  }
+  await walk(null)
+  return paths
+}
+
+/**
+ * The files a selection of file-system paths resolves to, under one section.
+ *
+ * The section is a logical name — `styles`, not `css` — because a site holds
+ * these wherever it is configured to, and the importer is what maps a section
+ * onto a directory.
+ */
+async function fileEntries(
+  port: FileSystemPort | undefined,
+  paths: readonly string[],
+  section: BundleSection,
+): Promise<BundleFile[]> {
+  if (!port) return []
+  const files: BundleFile[] = []
+  for (const path of paths) {
+    const file = await port.read(path)
+    if (!file) continue
+    const within = file.path.replace(/^\/+/, '')
+    files.push({ path: `${BUNDLE_SECTIONS[section]}/${within}`, text: file.content })
+  }
+  return files
+}
+
+async function componentEntries(
+  sources: SiteSectionSources,
+  keys: readonly string[],
+): Promise<BundleFile[]> {
+  if (keys.length === 0) return []
+  const templates = new ComponentRepository(sources.db, sources.componentFiles)
+  const wanted = new Set(keys.map((key) => key.trim().toLowerCase()))
+  const files: BundleFile[] = []
+  for (const template of await templates.all()) {
+    if (!wanted.has(template.key.toLowerCase()) && !wanted.has(template.alias.toLowerCase()))
+      continue
+    files.push({
+      path: `${BUNDLE_SECTIONS.components}/${template.alias}.tsx`,
+      text: template.content ?? '',
+    })
+  }
+  return files
+}
+
+/**
+ * The components a bundle carries, from one list that holds two kinds of name.
+ *
+ * The contract has `templates` and `partialViews`; this side has one list, so a
+ * picked component arrives either as a template alias — read through the alias
+ * — or as a path in the components tree. Both are tried and the results are
+ * keyed by path, so a component picked both ways is carried once.
+ */
+async function componentSection(
+  sources: SiteSectionSources,
+  picked: readonly string[],
+): Promise<BundleFile[]> {
+  const byPath = new Map<string, BundleFile>()
+  for (const file of await componentEntries(sources, picked)) byPath.set(file.path, file)
+  for (const file of await fileEntries(sources.components, picked, 'components'))
+    byPath.set(file.path, file)
+  return [...byPath.values()]
+}
+
+/** The dictionary as one `.udt`: the picked items, or every root item when none are named. */
+async function dictionaryEntries(db: Db, keys: readonly string[] | 'all'): Promise<BundleFile[]> {
+  const repo = new DictionaryRepository(db)
+  const items =
+    keys === 'all'
+      ? await dictionaryToUdt(repo)
+      : (await Promise.all(keys.map((key) => dictionaryToUdt(repo, key)))).flat()
+  if (items.length === 0) return []
+  return [{ path: `${BUNDLE_SECTIONS.dictionary}.udt`, text: writeDictionaryUdt(items) }]
+}
+
+/** The picked sections, or with `'all'` every one of them in full. */
+export async function siteSections(
+  sources: SiteSectionSources,
+  selection: SectionPicks | 'all',
+): Promise<BundleFile[]> {
+  if (selection === 'all') {
+    return [
+      ...(await wholeSchema(sources.schemaDir)),
+      ...(await fileEntries(sources.components, await everyPath(sources.components), 'components')),
+      ...(await fileEntries(sources.stylesheets, await everyPath(sources.stylesheets), 'styles')),
+      ...(await fileEntries(sources.scripts, await everyPath(sources.scripts), 'scripts')),
+      ...(await dictionaryEntries(sources.db, 'all')),
+    ]
+  }
+  // The schema version the site is actually at, so the files a package carries
+  // say where they came from rather than claiming version 0.
+  const state = await currentSchemaState(sources.db).catch(() => undefined)
+  const schema = await exportSchemaSet(sources.db, state?.version ?? '0')
+  return [
+    ...schemaEntries(schema, selection),
+    ...(await componentSection(sources, selection.components)),
+    ...(await fileEntries(sources.stylesheets, selection.stylesheets, 'styles')),
+    ...(await fileEntries(sources.scripts, selection.scripts, 'scripts')),
+    ...(selection.dictionaryItems.length > 0
+      ? await dictionaryEntries(sources.db, selection.dictionaryItems)
+      : []),
+  ]
+}

package/src/tar.ts

@@ -7,6 +7,12 @@
  * `Bun.gzipSync` does the compression. Timestamps are fixed, as npm fixes them,
  * so the same selection packs to the same bytes.
  */
+import { once } from 'node:events'
+import { createReadStream, createWriteStream, type WriteStream } from 'node:fs'
+import { mkdir } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
+import { pipeline } from 'node:stream/promises'
+import { createGunzip, createGzip, type Gzip } from 'node:zlib'
 
 export interface TarEntry {
   /** The path inside the package, `/`-separated and never absolute. */
@@ -83,3 +89,161 @@ export function writeNpmTarball(entries: readonly TarEntry[]): Uint8Array<ArrayB
   }
   return Bun.gzipSync(tar) as Uint8Array<ArrayBuffer>
 }
+
+const padding = (size: number) => (BLOCK - (size % BLOCK)) % BLOCK
+
+const checkInside = (path: string) => {
+  if (path.startsWith('/') || path.split('/').includes('..'))
+    throw new Error(`${path} is not a path inside the package`)
+}
+
+/**
+ * An npm package tarball written as it goes (story 104), for a package too big
+ * to hold in memory: entries are gzipped straight to the file, and a file's
+ * bytes may arrive as a stream. The layout is `writeNpmTarball`'s.
+ */
+export class TarballWriter {
+  #gzip: Gzip
+  #done: Promise<void>
+
+  private constructor(file: string) {
+    this.#gzip = createGzip()
+    const out = createWriteStream(file)
+    this.#done = pipeline(this.#gzip, out)
+  }
+
+  static create(file: string): TarballWriter {
+    return new TarballWriter(file)
+  }
+
+  async #write(bytes: Uint8Array): Promise<void> {
+    if (!this.#gzip.write(bytes)) await once(this.#gzip, 'drain')
+  }
+
+  async add(path: string, bytes: Uint8Array): Promise<void> {
+    checkInside(path)
+    await this.#write(header(`package/${path}`, bytes.length))
+    await this.#write(bytes)
+    const pad = padding(bytes.length)
+    if (pad > 0) await this.#write(new Uint8Array(pad))
+  }
+
+  /** A file whose `size` is known and whose bytes arrive in chunks; refused if they come to another size. */
+  async addStream(path: string, size: number, chunks: AsyncIterable<Uint8Array>): Promise<void> {
+    checkInside(path)
+    await this.#write(header(`package/${path}`, size))
+    let written = 0
+    for await (const chunk of chunks) {
+      written += chunk.length
+      if (written > size) throw new Error(`${path} is larger than its declared ${size} bytes`)
+      await this.#write(chunk)
+    }
+    if (written !== size) throw new Error(`${path} was ${written} bytes, not ${size}`)
+    const pad = padding(size)
+    if (pad > 0) await this.#write(new Uint8Array(pad))
+  }
+
+  async close(): Promise<void> {
+    await this.#write(new Uint8Array(BLOCK * 2))
+    this.#gzip.end()
+    await this.#done
+  }
+
+  /**
+   * Gives up on a half-written file: the stream ends now, and its error is
+   * swallowed, so nothing surfaces later after the caller has cleaned up.
+   */
+  async abort(): Promise<void> {
+    this.#gzip.destroy()
+    await this.#done.catch(() => {})
+  }
+}
+
+const decoder = new TextDecoder()
+
+const field = (block: Uint8Array, offset: number, width: number) =>
+  decoder.decode(block.subarray(offset, offset + width)).replace(/\0.*$/s, '')
+
+/**
+ * Unpacks a gzipped tarball into `dir`, streaming, and answers the paths it
+ * wrote. npm's `package/` folder is dropped, so `dir` holds what the package
+ * holds. Only regular files are written; a path that climbs out is refused.
+ */
+export async function extractTarball(file: string, dir: string): Promise<string[]> {
+  const written: string[] = []
+  let pending = new Uint8Array(0)
+  let current: { out: WriteStream; left: number; skip: number } | undefined
+  let skipping = 0
+  let ended = false
+
+  const take = (n: number): Uint8Array | undefined => {
+    if (pending.length < n) return undefined
+    const head = pending.subarray(0, n)
+    pending = pending.subarray(n)
+    return head
+  }
+
+  const writeTo = async (out: WriteStream, bytes: Uint8Array) => {
+    if (!out.write(bytes)) await once(out, 'drain')
+  }
+
+  const finishEntry = async (out: WriteStream) => {
+    out.end()
+    await once(out, 'finish')
+  }
+
+  for await (const chunk of createReadStream(file).pipe(createGunzip())) {
+    const bytes = chunk as Uint8Array
+    const joined = new Uint8Array(pending.length + bytes.length)
+    joined.set(pending)
+    joined.set(bytes, pending.length)
+    pending = joined
+    while (!ended) {
+      if (current) {
+        if (current.left > 0) {
+          if (pending.length === 0) break
+          const part = pending.subarray(0, Math.min(current.left, pending.length))
+          pending = pending.subarray(part.length)
+          current.left -= part.length
+          await writeTo(current.out, part)
+          continue
+        }
+        skipping = current.skip
+        await finishEntry(current.out)
+        current = undefined
+      }
+      if (skipping > 0) {
+        const part = Math.min(skipping, pending.length)
+        pending = pending.subarray(part)
+        skipping -= part
+        if (skipping > 0) break
+      }
+      const block = take(BLOCK)
+      if (!block) break
+      if (block.every((byte) => byte === 0)) {
+        ended = true
+        break
+      }
+      const name = field(block, 0, 100)
+      const prefix = field(block, 345, 155)
+      const size = Number.parseInt(field(block, 124, 12).trim() || '0', 8)
+      const type = String.fromCharCode(block[156] ?? 48)
+      const full = prefix ? `${prefix}/${name}` : name
+      if (type !== '0' && type !== '\0') {
+        skipping = size + padding(size)
+        continue
+      }
+      const inside = full.replace(/^package\//, '')
+      checkInside(inside)
+      const target = join(dir, ...inside.split('/'))
+      await mkdir(dirname(target), { recursive: true })
+      written.push(inside)
+      current = { out: createWriteStream(target), left: size, skip: padding(size) }
+    }
+  }
+  if (current) {
+    if (current.left > 0) throw new Error(`${file} ends part-way through a file`)
+    await finishEntry(current.out)
+  }
+  return written
+}

package/src/two-factor.ts

@@ -0,0 +1,138 @@
+/**
+ * Two-factor sign-in with an authenticator app (story 032), for backoffice users
+ * and members alike: setting it up, turning it on with ten recovery codes,
+ * checking a code at sign-in, and turning it off. The secret is sealed with
+ * `CROMBOLO_SECRET_KEY`, so two-factor sign-in cannot be turned on without the
+ * key. docs/operations.md.
+ *
+ * Setup is stateless, as Umbraco's is: the secret goes to the browser with its
+ * QR code, and comes back with the first code, which proves the app has it.
+ */
+import {
+  generateRecoveryCodes,
+  generateTotpSecret,
+  hashRecoveryCode,
+  otpauthUri,
+  qrCodeDataUrl,
+  verifyTotp,
+} from '@crombolo/auth'
+import { type Db, TwoFactorRepository } from '@crombolo/data'
+import { openSecret, SECRET_KEY_VARIABLE, sealSecret, usableSecretKey } from './secrets.ts'
+
+/** Umbraco's names for the authenticator provider, which the backoffice matches by. */
+export const USER_TWO_FACTOR_PROVIDER = 'UmbracoUserAppAuthenticator'
+export const MEMBER_TWO_FACTOR_PROVIDER = 'UmbracoMemberAppAuthenticator'
+
+export type TwoFactorResult<T = undefined> =
+  | ({ ok: true } & (T extends undefined ? object : { value: T }))
+  | { ok: false; reason: 'unavailable' | 'invalidCode' | 'notEnabled'; message: string }
+
+export class TwoFactorService {
+  #repository: TwoFactorRepository
+  #key: string | undefined
+  #issuer: string
+
+  constructor(db: Db, options: { secretKey: string | undefined; issuer: string }) {
+    this.#repository = new TwoFactorRepository(db)
+    this.#key = usableSecretKey(options.secretKey)
+    this.#issuer = options.issuer
+  }
+
+  /** Whether two-factor sign-in can be turned on here: only with the key to seal secrets. */
+  get available(): boolean {
+    return this.#key !== undefined
+  }
+
+  #unavailable() {
+    return {
+      ok: false as const,
+      reason: 'unavailable' as const,
+      message: `Two-factor sign-in needs ${SECRET_KEY_VARIABLE} set on the server.`,
+    }
+  }
+
+  providers(ownerKey: string): Promise<string[]> {
+    return this.#repository.providers(ownerKey)
+  }
+
+  enabledAmong(ownerKeys: readonly string[]): Promise<Set<string>> {
+    return this.#repository.enabledAmong(ownerKeys)
+  }
+
+  /** A new secret, its QR code and the URI in it, for an account named `account`. */
+  setup(
+    account: string,
+  ): TwoFactorResult<{ secret: string; qrCodeSetupImageUrl: string; otpauthUri: string }> {
+    if (!this.#key) return this.#unavailable()
+    const secret = generateTotpSecret()
+    const uri = otpauthUri({ secret, account, issuer: this.#issuer })
+    return { ok: true, value: { secret, qrCodeSetupImageUrl: qrCodeDataUrl(uri), otpauthUri: uri } }
+  }
+
+  /** Turns the provider on when `code` is right for `secret`; answers with the recovery codes, shown once. */
+  async enable(
+    ownerKey: string,
+    provider: string,
+    secret: string,
+    code: string,
+  ): Promise<TwoFactorResult<{ recoveryCodes: string[] }>> {
+    if (!this.#key) return this.#unavailable()
+    const step = verifyTotp(secret, code)
+    if (step === undefined)
+      return {
+        ok: false,
+        reason: 'invalidCode',
+        message: 'That code is not right. Try the next one.',
+      }
+    const recoveryCodes = generateRecoveryCodes()
+    await this.#repository.enable(
+      ownerKey,
+      provider,
+      sealSecret(secret, this.#key),
+      step,
+      recoveryCodes.map(hashRecoveryCode),
+    )
+    return { ok: true, value: { recoveryCodes } }
+  }
+
+  /**
+   * Whether a code — from the app, or a recovery code — is this owner's, now. An
+   * accepted app code's step is recorded, so it cannot be used again; a recovery
+   * code is spent.
+   */
+  async verify(ownerKey: string, provider: string, code: string): Promise<boolean> {
+    const login = await this.#repository.get(ownerKey, provider)
+    if (!login || !this.#key) return false
+    const digits = code.replace(/\s/g, '')
+    if (/^\d{6}$/.test(digits)) {
+      let secret: string
+      try {
+        secret = openSecret(login.sealedSecret, this.#key)
+      } catch {
+        return false
+      }
+      const step = verifyTotp(secret, digits, { lastUsedStep: login.lastUsedStep })
+      return step !== undefined && (await this.#repository.useStep(ownerKey, provider, step))
+    }
+    return this.#repository.useRecoveryCode(ownerKey, hashRecoveryCode(code))
+  }
+
+  /** Turns a provider off for its owner, who proves it with a code. */
+  async disable(ownerKey: string, provider: string, code: string): Promise<TwoFactorResult> {
+    if (!(await this.#repository.get(ownerKey, provider)))
+      return { ok: false, reason: 'notEnabled', message: 'Two-factor sign-in is not on.' }
+    if (!(await this.verify(ownerKey, provider, code)))
+      return { ok: false, reason: 'invalidCode', message: 'That code is not right.' }
+    await this.#repository.disable(ownerKey, provider)
+    return { ok: true }
+  }
+
+  /** Turns it off for someone else, as an administrator does for a lost phone. */
+  async reset(ownerKey: string, provider?: string): Promise<void> {
+    await this.#repository.disable(ownerKey, provider)
+  }
+
+  recoveryCodesLeft(ownerKey: string): Promise<number> {
+    return this.#repository.recoveryCodesLeft(ownerKey)
+  }
+}