@adea-ai/plugins 4.0.0 → 4.0.6

package/src/catalog-store.ts

@@ -0,0 +1,519 @@
+/**
+ * Catalog output coordination and publication.
+ *
+ * Contract (see docs/architecture.md, "Catalog publication"):
+ *
+ * - Every writer and every coordinated reader of one output directory uses a
+ *   lock directory that sits beside the output, in the output's real parent.
+ *   The lock is keyed by the output, never by `TMPDIR`, so processes with
+ *   different temporary directories, users or working directories still meet.
+ * - Supported filesystems are local filesystems where `mkdir` and same-directory
+ *   `rename` are atomic: Linux (ext4, xfs, btrfs, tmpfs, overlayfs), macOS
+ *   (APFS, HFS+) and Windows (NTFS, ReFS). Network filesystems are not
+ *   supported: their lock leases depend on clocks and caches this module does
+ *   not control.
+ * - A sync reads a base snapshot under the lock, builds without it, and then
+ *   publishes only if the output still has the same fingerprint (compare and
+ *   swap). A writer that replaced the catalog in between makes the later
+ *   publication fail with `CATALOG_BASE_CHANGED` instead of being overwritten.
+ * - Publication stages a complete directory, records a journal, moves the
+ *   current output to `<output>.previous`, then renames the staged directory
+ *   into place. Coordinated readers never observe the window between the two
+ *   renames. A process killed at any point leaves a journal that the next lock
+ *   holder (writer or reader) rolls back or forward before doing anything else.
+ * - A lock whose holder process is gone (same host) or whose heartbeat is older
+ *   than the lease is broken automatically and safely: the stale directory is
+ *   renamed aside first, so only one contender can break it, and a holder that
+ *   lost its lock refuses to swap with `CATALOG_LOCK_LOST`.
+ */
+import { createHash, randomUUID } from 'node:crypto'
+import { promises as fs } from 'node:fs'
+import { hostname } from 'node:os'
+import { basename, dirname, join, sep } from 'node:path'
+import { setTimeout as sleep } from 'node:timers/promises'
+import { writeDurably } from './fs-atomic.js'
+
+export type PublicationStep = 'acquiring' | 'staged' | 'backed-up' | 'swapped' | 'cleaned'
+
+export interface CatalogCoordinationOptions {
+  /** How long to wait for another holder before failing; default 30 s. */
+  readonly waitMs?: number
+  /** Heartbeat age after which a holder is considered dead; default 120 s. */
+  readonly staleAfterMs?: number
+  /** Interval at which a holder refreshes its heartbeat; default 5 s. */
+  readonly heartbeatMs?: number
+  /** Interval between lock attempts while waiting; default 50 ms. */
+  readonly pollMs?: number
+  /** Test seam: called after each publication step while the lock is held. */
+  readonly onStep?: (step: PublicationStep) => void | Promise<void>
+  /** Test seam: called once a lock is held. */
+  readonly onLocked?: (mode: LockMode) => void | Promise<void>
+}
+
+export interface PublishOptions extends CatalogCoordinationOptions {
+  /**
+   * Fingerprint of the output the caller built from, as returned by
+   * {@link catalogFingerprint} inside {@link withCatalogRead}. When set, the
+   * publication fails with `CATALOG_BASE_CHANGED` if the output differs.
+   */
+  readonly expectedBase?: string
+  /** Runs after the swap while the lock is still held, e.g. to publish an index. */
+  readonly afterPublish?: () => Promise<void>
+}
+
+type LockMode = 'read' | 'write'
+
+const LOCK_PREFIX = '.catalog-'
+const ABSENT = 'absent'
+const READ_ONLY_CODES = new Set(['EACCES', 'EPERM', 'EROFS'])
+
+export interface CatalogPaths {
+  readonly output: string
+  readonly parent: string
+  readonly backup: string
+  readonly lock: string
+  readonly owner: string
+  readonly journal: string
+  readonly stagePrefix: string
+}
+
+/** Names the coordination files of one output, beside it in its real parent. */
+export function catalogPathsIn(parent: string, name: string): CatalogPaths {
+  const normalized = name.normalize('NFC').toLowerCase()
+  // Case-insensitive and Unicode-normalized so that `Generated` and
+  // `generated` on macOS or Windows share one lock. On a case-sensitive
+  // filesystem such outputs are merely serialized together, which is harmless.
+  const hash = createHash('sha256').update(normalized).digest('hex').slice(0, 16)
+  const label = normalized.replace(/[^a-z0-9_-]/g, '_').slice(0, 32)
+  const stem = `${LOCK_PREFIX}${label}-${hash}`
+  const output = join(parent, name)
+  const lock = `${stem}.lock`
+  return {
+    output,
+    parent,
+    backup: `${output}.previous`,
+    lock: join(parent, lock),
+    owner: join(parent, lock, 'owner.json'),
+    journal: join(parent, `${stem}.journal.json`),
+    stagePrefix: join(parent, `${stem}.stage-`),
+  }
+}
+
+function isReservedName(name: string): boolean {
+  return name.startsWith(LOCK_PREFIX)
+}
+
+function reserved(component: string): boolean {
+  return isReservedName(component) || component.endsWith('.previous')
+}
+
+/** Resolves the output's real parent, optionally creating it. */
+async function resolvePaths(output: string, create: boolean): Promise<CatalogPaths | undefined> {
+  // Not `resolve()`: it would collapse `link/..` lexically, while the
+  // filesystem resolves the symbolic link first. `realpath` below does that.
+  const name = basename(output)
+  const parent = dirname(output)
+  // Reading a backup or a coordination directory, for example while
+  // inspecting one by hand, is allowed but not coordinated; writing is not.
+  if (create && (name === '' || name === '.' || name === '..'))
+    throw new Error(`CATALOG_OUTPUT_INVALID: ${output}`)
+  if (reserved(name) || parent.split(/[\\/]/).some(reserved)) {
+    if (!create) return undefined
+    throw new Error(`CATALOG_WRITE_LOCK_RESERVED: ${output}`)
+  }
+  if (create) await fs.mkdir(parent, { recursive: true })
+  let actualParent: string
+  try {
+    actualParent = await fs.realpath(parent)
+  } catch (error) {
+    if (!create && (error as NodeJS.ErrnoException).code === 'ENOENT') return undefined
+    throw error
+  }
+  // An output reached through a coordination directory, for example a
+  // symbolic link into another output's lock, would be destroyed when that
+  // lock is released or broken.
+  if (actualParent.split(sep).some(reserved)) {
+    if (!create) return undefined
+    throw new Error(`CATALOG_WRITE_LOCK_RESERVED: ${output}`)
+  }
+  return catalogPathsIn(actualParent, name)
+}
+
+interface Holder {
+  readonly token?: string
+  readonly pid?: number
+  readonly host?: string
+  readonly heartbeatAt: number
+}
+
+async function inspectHolder(lock: string): Promise<Holder | undefined> {
+  const owner = join(lock, 'owner.json')
+  try {
+    const [text, stat] = await Promise.all([fs.readFile(owner, 'utf8'), fs.stat(owner)])
+    const parsed = JSON.parse(text) as { token?: unknown; pid?: unknown; host?: unknown }
+    return {
+      ...(typeof parsed.token === 'string' ? { token: parsed.token } : {}),
+      ...(typeof parsed.pid === 'number' ? { pid: parsed.pid } : {}),
+      ...(typeof parsed.host === 'string' ? { host: parsed.host } : {}),
+      heartbeatAt: stat.mtimeMs,
+    }
+  } catch (error) {
+    const code = (error as NodeJS.ErrnoException).code
+    if (code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error
+  }
+  // The holder created the directory but has not written its owner record yet
+  // (or was killed in between): the directory's own age is the heartbeat.
+  try {
+    return { heartbeatAt: (await fs.stat(lock)).mtimeMs }
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined
+    throw error
+  }
+}
+
+function processIsAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0)
+    return true
+  } catch (error) {
+    return (error as NodeJS.ErrnoException).code === 'EPERM'
+  }
+}
+
+function holderIsStale(holder: Holder, staleAfterMs: number): boolean {
+  if (holder.pid !== undefined && holder.host === hostname() && !processIsAlive(holder.pid))
+    return true
+  return Date.now() - holder.heartbeatAt > staleAfterMs
+}
+
+/**
+ * Breaks a stale lock. Renaming first means two contenders cannot both remove
+ * it; re-inspecting the renamed directory catches the case where the stale
+ * lock was replaced by a live one between inspection and rename.
+ */
+async function breakLock(paths: CatalogPaths, staleAfterMs: number): Promise<void> {
+  const grave = `${paths.lock}.stale-${randomUUID()}`
+  try {
+    await fs.rename(paths.lock, grave)
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return
+    throw error
+  }
+  const taken = await inspectHolder(grave)
+  if (taken && !holderIsStale(taken, staleAfterMs)) {
+    // A live holder: give the lock back. If someone already took the name,
+    // the live holder will see CATALOG_LOCK_LOST before it swaps anything.
+    await fs.rename(grave, paths.lock).catch(() => undefined)
+    return
+  }
+  await fs.rm(grave, { recursive: true, force: true })
+}
+
+export class CatalogLock {
+  #timer: ReturnType<typeof setInterval> | undefined
+  #released = false
+
+  constructor(
+    readonly paths: CatalogPaths,
+    readonly token: string,
+    heartbeatMs: number
+  ) {
+    this.#timer = setInterval(() => {
+      const now = new Date()
+      fs.utimes(paths.owner, now, now).catch(() => undefined)
+    }, heartbeatMs)
+    this.#timer.unref?.()
+  }
+
+  /** Fails with `CATALOG_LOCK_LOST` if another process broke this lock. */
+  async assertHeld(): Promise<void> {
+    const holder = await inspectHolder(this.paths.lock)
+    if (holder?.token !== this.token) throw new Error(`CATALOG_LOCK_LOST: ${this.paths.output}`)
+  }
+
+  async release(): Promise<void> {
+    if (this.#released) return
+    this.#released = true
+    clearInterval(this.#timer)
+    const holder = await inspectHolder(this.paths.lock)
+    if (holder?.token !== this.token) return
+    await fs.rm(this.paths.lock, { recursive: true, force: true })
+  }
+}
+
+async function acquire(
+  paths: CatalogPaths,
+  mode: LockMode,
+  options: CatalogCoordinationOptions
+): Promise<CatalogLock | undefined> {
+  const waitMs = options.waitMs ?? 30_000
+  const staleAfterMs = options.staleAfterMs ?? 120_000
+  const pollMs = options.pollMs ?? 50
+  const deadline = Date.now() + waitMs
+  for (;;) {
+    try {
+      await fs.mkdir(paths.lock)
+    } catch (error) {
+      const code = (error as NodeJS.ErrnoException).code
+      // A read-only output cannot be published by anyone using this mount, so
+      // a reader proceeds without coordination there.
+      if (mode === 'read' && code !== undefined && READ_ONLY_CODES.has(code)) return undefined
+      if (code !== 'EEXIST') throw error
+      const holder = await inspectHolder(paths.lock)
+      if (holder === undefined) continue
+      if (holderIsStale(holder, staleAfterMs)) {
+        await breakLock(paths, staleAfterMs)
+        continue
+      }
+      if (Date.now() >= deadline) {
+        const who = holder.pid === undefined ? '' : ` (held by pid ${holder.pid} on ${holder.host})`
+        throw new Error(`CATALOG_WRITE_IN_PROGRESS: ${paths.output}${who}`, { cause: error })
+      }
+      await sleep(pollMs)
+      continue
+    }
+    const token = randomUUID()
+    try {
+      await fs.writeFile(
+        paths.owner,
+        `${JSON.stringify({ token, pid: process.pid, host: hostname(), mode, acquiredAt: new Date().toISOString() })}\n`,
+        { flag: 'wx' }
+      )
+    } catch (error) {
+      // A contender that misjudged a live lock as stale restored it over the
+      // directory just created here (see breakLock): the lock is not ours.
+      if ((error as NodeJS.ErrnoException).code === 'EEXIST') continue
+      throw error
+    }
+    const lock = new CatalogLock(paths, token, options.heartbeatMs ?? 5_000)
+    try {
+      await options.onLocked?.(mode)
+    } catch (error) {
+      await lock.release()
+      throw error
+    }
+    return lock
+  }
+}
+
+async function exists(path: string): Promise<boolean> {
+  try {
+    await fs.lstat(path)
+    return true
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false
+    throw error
+  }
+}
+
+interface Journal {
+  readonly version: 1
+  readonly stage: string
+}
+
+async function readJournal(paths: CatalogPaths): Promise<Journal | undefined> {
+  let text: string
+  try {
+    text = await fs.readFile(paths.journal, 'utf8')
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined
+    throw error
+  }
+  try {
+    const parsed = JSON.parse(text) as Partial<Journal>
+    if (
+      parsed.version === 1 &&
+      typeof parsed.stage === 'string' &&
+      join(paths.parent, parsed.stage).startsWith(paths.stagePrefix) &&
+      !parsed.stage.includes(sep) &&
+      !parsed.stage.includes('/')
+    )
+      return parsed as Journal
+  } catch {
+    // An empty or torn journal: the writer died while writing it, before it
+    // moved anything, so the output and backup have not been touched.
+    return { version: 1, stage: '' }
+  }
+  throw new Error(`CATALOG_RECOVERY_REQUIRED: unrecognized journal ${paths.journal}`)
+}
+
+/** Best effort: directory fsync is unavailable on Windows and some filesystems. */
+async function syncDirectory(path: string): Promise<void> {
+  let handle
+  try {
+    handle = await fs.open(path, 'r')
+    await handle.sync()
+  } catch {
+    // Process-crash safety does not depend on it; power-loss durability does.
+  } finally {
+    await handle?.close().catch(() => undefined)
+  }
+}
+
+async function removeOrphanStages(paths: CatalogPaths, keep?: string): Promise<void> {
+  const prefix = basename(paths.stagePrefix)
+  for (const name of await fs.readdir(paths.parent))
+    if (name.startsWith(prefix) && name !== keep)
+      await fs.rm(join(paths.parent, name), { recursive: true, force: true })
+}
+
+/**
+ * Completes or undoes an interrupted publication. Must run under the lock.
+ *
+ * | output | backup | staged dir | meaning                          | action         |
+ * | ------ | ------ | ---------- | -------------------------------- | -------------- |
+ * | yes    | no     | any        | not swapped, or already cleaned  | drop staging   |
+ * | no     | yes    | any        | died between the two renames     | roll back      |
+ * | yes    | yes    | no         | swapped, backup not yet removed  | roll forward   |
+ * | no     | no     | any        | first publication never swapped  | drop staging   |
+ * | yes    | yes    | yes        | inconsistent                     | stop, preserve |
+ */
+export async function recoverCatalogOutput(paths: CatalogPaths): Promise<void> {
+  const journal = await readJournal(paths)
+  if (journal === undefined) {
+    // Staging directories without a journal belong to a writer that died
+    // before it recorded anything; nothing else references them.
+    await removeOrphanStages(paths)
+    return
+  }
+  const stage = journal.stage === '' ? undefined : join(paths.parent, journal.stage)
+  const [hasOutput, hasBackup, hasStage] = await Promise.all([
+    exists(paths.output),
+    exists(paths.backup),
+    stage ? exists(stage) : Promise.resolve(false),
+  ])
+  if (stage === undefined && hasBackup)
+    // A torn journal is written before anything moves, so a backup next to it
+    // was not made by that writer.
+    throw new Error(`CATALOG_RECOVERY_REQUIRED: preserve and inspect ${paths.backup}`)
+  if (!hasOutput && hasBackup) await fs.rename(paths.backup, paths.output)
+  else if (hasOutput && hasBackup && !hasStage)
+    await fs.rm(paths.backup, { recursive: true, force: true })
+  else if (hasOutput && hasBackup)
+    throw new Error(`CATALOG_RECOVERY_REQUIRED: preserve and inspect ${paths.backup}`)
+  await removeOrphanStages(paths)
+  await syncDirectory(paths.parent)
+  await fs.rm(paths.journal, { force: true })
+}
+
+/**
+ * A digest of the output's entries and bytes, or `absent`. Run it under the
+ * lock (inside {@link withCatalogRead}) so it describes a published state.
+ */
+export async function catalogFingerprint(output: string): Promise<string> {
+  let names: string[]
+  try {
+    names = (await fs.readdir(output)).toSorted()
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return ABSENT
+    throw error
+  }
+  const hash = createHash('sha256')
+  for (const name of names) {
+    const path = join(output, name)
+    const stat = await fs.lstat(path)
+    hash.update(`${name}\0`)
+    if (stat.isFile()) hash.update('f\0').update(await fs.readFile(path))
+    else if (stat.isSymbolicLink()) hash.update('l\0').update(await fs.readlink(path))
+    else hash.update(`o\0${stat.mode}\0${stat.size}`)
+    hash.update('\0')
+  }
+  return `sha256:${hash.digest('hex')}`
+}
+
+/**
+ * Runs `read` while no writer can publish to `output`, after finishing any
+ * interrupted publication. Readers are serialized with writers and with each
+ * other; the lock is held only for the read itself.
+ */
+export async function withCatalogRead<T>(
+  output: string,
+  read: () => Promise<T>,
+  options: CatalogCoordinationOptions = {}
+): Promise<T> {
+  const paths = await resolvePaths(output, false)
+  if (paths === undefined) return read()
+  const lock = await acquire(paths, 'read', options)
+  try {
+    if (lock) await recoverCatalogOutput(paths)
+    return await read()
+  } finally {
+    await lock?.release()
+  }
+}
+
+/**
+ * Replaces `output` with exactly `files`. The previous output stays in place
+ * until the new one is complete, and a coordinated reader sees either one or
+ * the other.
+ */
+export async function publishCatalog(
+  output: string,
+  files: Readonly<Record<string, string | Uint8Array>>,
+  options: PublishOptions = {}
+): Promise<void> {
+  const paths = await resolvePaths(output, true)
+  if (paths === undefined) throw new Error(`CATALOG_OUTPUT_UNRESOLVED: ${output}`)
+  await options.onStep?.('acquiring')
+  const lock = await acquire(paths, 'write', options)
+  if (lock === undefined) throw new Error(`CATALOG_OUTPUT_UNRESOLVED: ${output}`)
+  try {
+    await recoverCatalogOutput(paths)
+    // A backup without a journal predates coordinated publication or was made
+    // by hand. Its provenance is unknown, so it is never deleted or restored.
+    if (await exists(paths.backup))
+      throw new Error(`CATALOG_RECOVERY_REQUIRED: preserve and inspect ${paths.backup}`)
+    if (options.expectedBase !== undefined) {
+      const current = await catalogFingerprint(paths.output)
+      if (current !== options.expectedBase)
+        throw new Error(
+          `CATALOG_BASE_CHANGED: ${paths.output} was replaced after this sync read it; run the sync again`
+        )
+    }
+    const stage = await fs.mkdtemp(paths.stagePrefix)
+    try {
+      await writeDurably(
+        paths.journal,
+        `${JSON.stringify({ version: 1, stage: basename(stage) } satisfies Journal)}\n`
+      )
+      for (const [name, content] of Object.entries(files)) {
+        if (
+          name === '' ||
+          name.includes('/') ||
+          name.includes('\\') ||
+          name === '.' ||
+          name === '..'
+        )
+          throw new Error(`CATALOG_ARTIFACT_NAME_INVALID: ${name}`)
+        await writeDurably(join(stage, name), content)
+      }
+      await syncDirectory(stage)
+      await options.onStep?.('staged')
+      await lock.assertHeld()
+      try {
+        await fs.rename(paths.output, paths.backup)
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
+      }
+      await options.onStep?.('backed-up')
+      await fs.rename(stage, paths.output)
+      await syncDirectory(paths.parent)
+      await options.onStep?.('swapped')
+      await fs.rm(paths.backup, { recursive: true, force: true })
+      await fs.rm(paths.journal, { force: true })
+      await options.onStep?.('cleaned')
+    } catch (error) {
+      // Same state machine a later process would run after a crash, unless
+      // another process now holds the lock: then it owns the recovery.
+      const held = await lock.assertHeld().then(
+        () => true,
+        () => false
+      )
+      if (held) await recoverCatalogOutput(paths)
+      throw error
+    }
+    await options.afterPublish?.()
+  } finally {
+    await lock.release()
+  }
+}

package/src/concurrency.ts

@@ -0,0 +1,19 @@
+export async function mapWithConcurrency<T, U>(
+  values: readonly T[],
+  concurrency: number,
+  mapper: (value: T, index: number) => Promise<U>
+): Promise<U[]> {
+  const results = Array.from<U>({ length: values.length })
+  let nextIndex = 0
+  const worker = async (): Promise<void> => {
+    while (true) {
+      const index = nextIndex++
+      if (index >= values.length) return
+      results[index] = await mapper(values[index]!, index)
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(Math.max(concurrency, 1), values.length) }, () => worker())
+  )
+  return results
+}