@huynle/hub 0.7.1 → 0.7.3

package/packages/cli/src/bootstrap/transaction/state.ts

@@ -0,0 +1,88 @@
+// File states as the journal records them: existence, digest and mode, never contents. A symlink's digest is the
+// digest of its target text; a harness install root (a directory tree) can be digested as a whole, so recovery can
+// tell "exactly as we left it" from "the user changed something inside". Files are digested in chunks (fileDigest):
+// a harness binary is never read whole.
+import { digestOf, sha256Hex, type FileState } from "../contracts/index.ts"
+import type { TreeFile } from "./executor.ts"
+import type { TxFs } from "./fs.ts"
+import { fileDigest } from "./stream.ts"
+
+export const sha256Of = (data: Uint8Array | string): string => `sha256:${sha256Hex(data)}`
+
+export function fileState(fs: TxFs, path: string, options: { readonly tree?: boolean } = {}): FileState {
+  const info = fs.lstat(path)
+  if (!info) return { exists: false }
+  switch (info.kind) {
+    case "file":
+      return { exists: true, sha256: fileDigest(fs, path), mode: info.mode }
+    case "symlink":
+      return { exists: true, symlink: true, sha256: sha256Of(fs.readlink(path)) }
+    case "directory":
+      return options.tree ? { exists: true, sha256: treeDigest(fs, path), mode: info.mode } : { exists: true, mode: info.mode }
+    default:
+      return { exists: true }
+  }
+}
+
+/** Digest over every entry under `dir` (relative path, kind, mode, content or link digest), in sorted order. */
+export function treeDigest(fs: TxFs, dir: string): string {
+  const entries: unknown[] = []
+  const walk = (abs: string, rel: string) => {
+    for (const name of fs.readdir(abs).sort()) {
+      const childAbs = `${abs}/${name}`
+      const childRel = rel ? `${rel}/${name}` : name
+      const info = fs.lstat(childAbs)
+      if (!info) continue
+      if (info.kind === "directory") {
+        entries.push({ rel: childRel, kind: "directory", mode: info.mode })
+        walk(childAbs, childRel)
+      } else if (info.kind === "file") entries.push({ rel: childRel, kind: "file", mode: info.mode, sha256: fileDigest(fs, childAbs) })
+      else if (info.kind === "symlink") entries.push({ rel: childRel, kind: "symlink", sha256: sha256Of(fs.readlink(childAbs)) })
+      else entries.push({ rel: childRel, kind: "other" })
+    }
+  }
+  walk(dir, "")
+  return digestOf({ kind: "hub.tree", entries })
+}
+
+/**
+ * The digest treeDigest would compute for a directory holding exactly `files` (directories implied by their paths,
+ * all with `dirMode`), so the journal can record a tree's state before it is staged.
+ */
+export function treeDigestOfFiles(files: readonly TreeFile[], dirMode: number): string {
+  type Dir = Map<string, Dir | { readonly sha256: string; readonly mode: number }>
+  const root: Dir = new Map()
+  for (const f of files) {
+    const parts = f.rel.split("/")
+    let dir = root
+    for (const part of parts.slice(0, -1)) {
+      let next = dir.get(part)
+      if (!(next instanceof Map)) {
+        next = new Map()
+        dir.set(part, next)
+      }
+      dir = next
+    }
+    dir.set(parts[parts.length - 1]!, { sha256: "bytes" in f ? sha256Of(f.bytes) : f.staged.sha256, mode: f.mode })
+  }
+  const entries: unknown[] = []
+  const walk = (dir: Dir, rel: string) => {
+    for (const name of [...dir.keys()].sort()) {
+      const child = dir.get(name)!
+      const childRel = rel ? `${rel}/${name}` : name
+      if (child instanceof Map) {
+        entries.push({ rel: childRel, kind: "directory", mode: dirMode })
+        walk(child, childRel)
+      } else entries.push({ rel: childRel, kind: "file", mode: child.mode, sha256: child.sha256 })
+    }
+  }
+  walk(root, "")
+  return digestOf({ kind: "hub.tree", entries })
+}
+
+/** The journal's notion of "the same file" (recoveryDecision uses the same rule). */
+export const sameState = (a: FileState, b: FileState): boolean =>
+  a.exists === b.exists &&
+  (a.sha256 ?? null) === (b.sha256 ?? null) &&
+  (a.mode === undefined || b.mode === undefined || a.mode === b.mode) &&
+  Boolean(a.symlink) === Boolean(b.symlink)

package/packages/cli/src/bootstrap/transaction/store.ts

@@ -0,0 +1,189 @@
+// Where the applier keeps its own state, under the state root next to the lock (contract LOCK_PATH):
+//
+//   state:hub/bootstrap/apply.lock               the apply lock
+//   state:hub/bootstrap/journal/<txID>.json      the transaction journal (digests and modes only)
+//   state:hub/bootstrap/journal/<txID>.plan.json the plan it applies (secret-free by contract), for resume/rollback
+//   state:hub/bootstrap/backups/<txID>/<bk_id>   owner-only copies of files before they changed (never credentials)
+//   state:hub/bootstrap/staging/<txID>/          a step's scratch space for large downloads and what they unpack to
+//                                                (stream.ts); removed after each step, swept under the lock
+//
+// Every document is validated before it is written and after it is read, written atomically (temporary file,
+// fsync, rename) with owner-only permissions, and reached only through symlink-checked paths.
+import {
+  canonicalJSON,
+  LOCK_PATH,
+  planDigest,
+  validateJournal,
+  validateSetupPlan,
+  type ExecutionContext,
+  type RootMapping,
+  type SetupPlan,
+  type TransactionJournal,
+} from "../contracts/index.ts"
+import type { TxFs } from "./fs.ts"
+import { resolveTarget } from "./resolve.ts"
+import { errnoOf, TxError } from "./result.ts"
+import { copyFile, fileDigest } from "./stream.ts"
+
+const decode = (bytes: Uint8Array) => new TextDecoder().decode(bytes)
+const STALE_TEMP = /\.tmp$|\.broken-/
+
+export class TxStore {
+  readonly base: string
+  readonly lockPath: string
+  private readonly journalDir: string
+  private readonly backupDir: string
+  private readonly stagingRoot: string
+
+  constructor(
+    private readonly fs: TxFs,
+    roots: readonly RootMapping[],
+    context: ExecutionContext = "host",
+  ) {
+    const lock = resolveTarget(LOCK_PATH, roots, fs, { context })
+    if (!lock.ok) throw new TxError(lock.code === "conflict" ? "conflict" : "invalid", `the bootstrap state directory is unusable: ${lock.message}`)
+    this.lockPath = lock.value
+    this.base = lock.value.slice(0, lock.value.lastIndexOf("/"))
+    this.journalDir = `${this.base}/journal`
+    this.backupDir = `${this.base}/backups`
+    this.stagingRoot = `${this.base}/staging`
+  }
+
+  /** The scratch directory of one transaction (created on first use by fsStaging). */
+  stagingDir(txID: string): string {
+    return `${this.stagingRoot}/${txID}`
+  }
+
+  /** Creates the state directories (owner-only) and refuses if any of them is not a plain directory. */
+  init(): void {
+    for (const dir of [this.base, this.journalDir, this.backupDir]) {
+      const info = this.fs.lstat(dir)
+      if (!info) this.fs.mkdirp(dir, 0o700)
+      else if (info.kind !== "directory") throw new TxError("conflict", `${dir} is not a directory`)
+    }
+  }
+
+  /**
+   * Removes temporary files an interrupted writer left behind, and every staging directory (only while holding the
+   * lock: no step is running, and nothing in staging is ever needed to recover or resume).
+   */
+  sweep(): void {
+    for (const dir of [this.base, this.journalDir]) {
+      for (const name of this.list(dir)) if (STALE_TEMP.test(name)) this.removeQuietly(`${dir}/${name}`)
+    }
+    const info = this.fs.lstat(this.stagingRoot)
+    if (info?.kind === "directory") for (const name of this.list(this.stagingRoot)) this.removeTree(`${this.stagingRoot}/${name}`)
+    else if (info) this.removeQuietly(this.stagingRoot)
+  }
+
+  private removeTree(path: string): void {
+    const info = this.fs.lstat(path)
+    if (!info) return
+    if (info.kind !== "directory") return this.removeQuietly(path)
+    for (const name of this.list(path)) this.removeTree(`${path}/${name}`)
+    this.fs.rmdir(path)
+  }
+
+  journals(): TransactionJournal[] {
+    const out: TransactionJournal[] = []
+    for (const name of this.list(this.journalDir).sort()) {
+      if (!name.endsWith(".json") || name.endsWith(".plan.json")) continue
+      const path = `${this.journalDir}/${name}`
+      let parsed: unknown
+      try {
+        parsed = JSON.parse(decode(this.fs.readFile(path)))
+      } catch {
+        throw new TxError("recovery_required", `the journal ${path} cannot be read; inspect it before applying again`)
+      }
+      const checked = validateJournal(parsed)
+      if (!checked.ok) throw new TxError("recovery_required", `the journal ${path} is not valid (${checked.issues[0]!.message}); inspect it before applying again`)
+      out.push(checked.value)
+    }
+    return out
+  }
+
+  writeJournal(journal: TransactionJournal): void {
+    const checked = validateJournal(journal)
+    if (!checked.ok) throw new TxError("internal", `refusing to write an invalid journal: ${checked.issues[0]!.path} ${checked.issues[0]!.message}`)
+    this.writeAtomic(`${this.journalDir}/${journal.txID}.json`, `${canonicalJSON(journal)}\n`)
+  }
+
+  writePlan(txID: string, plan: SetupPlan): void {
+    const checked = validateSetupPlan(plan)
+    if (!checked.ok) throw new TxError("invalid", `refusing to store an invalid plan: ${checked.issues[0]!.message}`)
+    this.writeAtomic(`${this.journalDir}/${txID}.plan.json`, `${canonicalJSON(plan)}\n`)
+  }
+
+  /** The stored plan of a transaction, if it is valid and is the one the journal pins. */
+  readPlan(journal: TransactionJournal): SetupPlan | undefined {
+    const path = `${this.journalDir}/${journal.txID}.plan.json`
+    if (!this.fs.lstat(path)) return undefined
+    try {
+      const checked = validateSetupPlan(JSON.parse(decode(this.fs.readFile(path))))
+      return checked.ok && planDigest(checked.value) === journal.planDigest ? checked.value : undefined
+    } catch {
+      return undefined
+    }
+  }
+
+  writeBackup(txID: string, id: string, bytes: Uint8Array | string): void {
+    const dir = `${this.backupDir}/${txID}`
+    if (!this.fs.lstat(dir)) this.fs.mkdir(dir, 0o700)
+    const path = `${dir}/${id}`
+    this.removeQuietly(path)
+    this.fs.writeExclusive(path, bytes, 0o600)
+  }
+
+  /** Backs up the regular file `from` by a streaming copy (a large file is never read whole). */
+  writeBackupFrom(txID: string, id: string, from: string): void {
+    const dir = `${this.backupDir}/${txID}`
+    if (!this.fs.lstat(dir)) this.fs.mkdir(dir, 0o700)
+    const path = `${dir}/${id}`
+    this.removeQuietly(path)
+    copyFile(this.fs, from, path, 0o600)
+  }
+
+  /** A file backup's path and the digest of what it holds now (undefined when it is missing). */
+  backupFile(txID: string, id: string): { readonly path: string; readonly sha256: string } | undefined {
+    const path = `${this.backupDir}/${txID}/${id}`
+    return this.fs.lstat(path)?.kind === "file" ? { path, sha256: fileDigest(this.fs, path) } : undefined
+  }
+
+  readBackup(txID: string, id: string): Uint8Array | undefined {
+    const path = `${this.backupDir}/${txID}/${id}`
+    const info = this.fs.lstat(path)
+    return info?.kind === "file" ? this.fs.readFile(path) : undefined
+  }
+
+  dropBackups(txID: string): void {
+    const dir = `${this.backupDir}/${txID}`
+    if (!this.fs.lstat(dir)) return
+    for (const name of this.list(dir)) this.fs.unlink(`${dir}/${name}`)
+    this.fs.rmdir(dir)
+  }
+
+  private writeAtomic(path: string, text: string): void {
+    const tmp = `${path}.tmp`
+    this.removeQuietly(tmp)
+    try {
+      this.fs.writeExclusive(tmp, text, 0o600)
+      this.fs.rename(tmp, path)
+    } catch (error) {
+      this.removeQuietly(tmp)
+      throw error
+    }
+    this.fs.fsyncDir(this.journalDir)
+  }
+
+  private list(dir: string): string[] {
+    return this.fs.lstat(dir)?.kind === "directory" ? this.fs.readdir(dir) : []
+  }
+
+  private removeQuietly(path: string): void {
+    try {
+      if (this.fs.lstat(path)) this.fs.unlink(path)
+    } catch (error) {
+      if (!errnoOf(error)) throw error
+    }
+  }
+}

package/packages/cli/src/bootstrap/transaction/stream.ts

@@ -0,0 +1,254 @@
+// Bounded-memory plumbing for large artifacts (a harness archive, what it unpacks to, hubctl itself). Nothing here
+// ever holds a whole file: bytes arrive as chunks (ByteSource), are hashed while they are written to disk, and are
+// read back in chunks (ByteReader). A 4 GB host installs a 350 MB harness with a few MiB of buffers.
+//
+//   ByteSource    what a download yields: a Uint8Array (small things, test fakes) or an async iterable of chunks
+//   StagingArea   the transaction's private scratch space for one step (state:hub/bootstrap/staging/<txID>, owner
+//                 only): write() streams a source into a new file (O_EXCL, no symlinks) and records the sha256 and
+//                 size of what it wrote, so the executor checks the pin against the engine's own digest, never one a
+//                 source reports; open() reads a staged file back by offset (the zip central directory) or as a
+//                 stream (gunzip). The engine moves staged files into the install tree and removes the rest.
+//   fileDigest    the sha256 of a file on disk, read in chunks
+//   copyFile      a streaming copy (exclusive create, fsync) that returns the digest of what it copied
+import { createHash } from "node:crypto"
+import type { TxFs } from "./fs.ts"
+
+/** The largest piece of a file the transaction layer reads or writes at once. */
+export const CHUNK_BYTES = 1024 * 1024
+
+export type ByteSource = Uint8Array | AsyncIterable<Uint8Array>
+
+/** A source yielded more than allowed. */
+export class TooLarge extends Error {
+  constructor(readonly maxBytes: number) {
+    super(`more than ${maxBytes} bytes`)
+    this.name = "TooLarge"
+  }
+}
+
+/** The chunks of a source, none larger than CHUNK_BYTES. */
+export async function* chunksOf(source: ByteSource): AsyncGenerator<Uint8Array> {
+  if (source instanceof Uint8Array) {
+    for (let at = 0; at < source.byteLength; at += CHUNK_BYTES) yield source.subarray(at, Math.min(source.byteLength, at + CHUNK_BYTES))
+    return
+  }
+  for await (const chunk of source) {
+    for (let at = 0; at < chunk.byteLength; at += CHUNK_BYTES) yield chunk.subarray(at, Math.min(chunk.byteLength, at + CHUNK_BYTES))
+  }
+}
+
+/** A small source, whole (a signature, a manifest); TooLarge beyond `maxBytes`. */
+export async function collect(source: ByteSource, maxBytes: number): Promise<Uint8Array> {
+  if (source instanceof Uint8Array) {
+    if (source.byteLength > maxBytes) throw new TooLarge(maxBytes)
+    return source
+  }
+  const parts: Uint8Array[] = []
+  let total = 0
+  for await (const chunk of source) {
+    total += chunk.byteLength
+    if (total > maxBytes) throw new TooLarge(maxBytes)
+    parts.push(chunk)
+  }
+  const out = new Uint8Array(total)
+  let at = 0
+  for (const p of parts) {
+    out.set(p, at)
+    at += p.byteLength
+  }
+  return out
+}
+
+/**
+ * A response body as chunks, cancelled (and failing with tooLarge()) as soon as it runs past `maxBytes`: what the
+ * https downloaders hand to a staging area.
+ */
+export async function* responseChunks(response: Response, maxBytes: number, tooLarge: () => Error): AsyncGenerator<Uint8Array> {
+  const reader = response.body?.getReader()
+  if (!reader) return
+  let total = 0
+  try {
+    for (;;) {
+      const { done, value } = await reader.read()
+      if (done) return
+      total += value.byteLength
+      if (total > maxBytes) {
+        await reader.cancel().catch(() => {})
+        throw tooLarge()
+      }
+      yield value
+    }
+  } finally {
+    reader.releaseLock()
+  }
+}
+
+/** A file in a staging area (or, for hubctl's own executable, a file elsewhere), with its digest and size. */
+export interface StagedFile {
+  readonly path: string
+  readonly sha256: string
+  readonly size: number
+}
+
+/** Random and sequential access to a staged file, in bounded pieces. */
+export interface ByteReader {
+  readonly size: number
+  /** At most `length` bytes from `offset`. */
+  read(offset: number, length: number): Uint8Array
+  /** The bytes from `start` (default 0) to `end` (default the size), in chunks of at most CHUNK_BYTES. */
+  stream(start?: number, end?: number): AsyncGenerator<Uint8Array>
+}
+
+export function bytesReader(bytes: Uint8Array): ByteReader {
+  return {
+    size: bytes.byteLength,
+    read: (offset, length) => bytes.subarray(Math.max(0, offset), Math.max(0, Math.min(bytes.byteLength, offset + length))),
+    async *stream(start = 0, end = bytes.byteLength) {
+      yield* chunksOf(bytes.subarray(start, Math.min(end, bytes.byteLength)))
+    },
+  }
+}
+
+export function fileReader(fs: TxFs, path: string, size: number): ByteReader {
+  return {
+    size,
+    read: (offset, length) => (offset >= size || length <= 0 ? new Uint8Array() : fs.readRange(path, offset, Math.min(length, size - offset))),
+    async *stream(start = 0, end = size) {
+      for (let at = start; at < Math.min(end, size); ) {
+        const chunk = fs.readRange(path, at, Math.min(CHUNK_BYTES, end - at))
+        if (chunk.byteLength === 0) return
+        at += chunk.byteLength
+        yield chunk
+      }
+    },
+  }
+}
+
+export interface StagingArea {
+  /**
+   * Streams `source` into a new private file named after `name` (its last path segment), hashing it on the way;
+   * TooLarge (and nothing left behind) beyond `maxBytes`. Errors of the source propagate the same way.
+   */
+  write(name: string, source: ByteSource, options: { readonly maxBytes: number }): Promise<StagedFile>
+  /** Reads back a file this area wrote. */
+  open(file: StagedFile): ByteReader
+  /** The digest and size this area recorded while writing `file`; undefined for any file it did not write. */
+  written(file: StagedFile): { readonly sha256: string; readonly size: number } | undefined
+}
+
+const safeName = (name: string) => (name.slice(name.lastIndexOf("/") + 1).replace(/[^A-Za-z0-9._-]/g, "_") || "file").slice(0, 100)
+const hex = (hash: ReturnType<typeof createHash>) => `sha256:${hash.digest("hex")}`
+
+/** A staging area in `dir` (created owner-only on first use, below an existing, symlink-checked parent). */
+export function fsStaging(fs: TxFs, dir: string): StagingArea & { dispose(): void } {
+  const files = new Map<string, { sha256: string; size: number }>()
+  let count = 0
+  const ensure = () => {
+    const info = fs.lstat(dir)
+    if (!info) {
+      const parent = dir.slice(0, dir.lastIndexOf("/"))
+      if (!fs.lstat(parent)) fs.mkdir(parent, 0o700)
+      else if (fs.lstat(parent)?.kind !== "directory") throw new Error(`${parent} is not a directory`)
+      fs.mkdir(dir, 0o700)
+    } else if (info.kind !== "directory") throw new Error(`${dir} is not a directory`)
+  }
+  const remove = (path: string) => {
+    const info = fs.lstat(path)
+    if (!info) return
+    if (info.kind === "directory") {
+      for (const name of fs.readdir(path)) remove(`${path}/${name}`)
+      fs.rmdir(path)
+    } else fs.unlink(path)
+  }
+  return {
+    async write(name, source, { maxBytes }) {
+      ensure()
+      const path = `${dir}/${++count}-${safeName(name)}`
+      const out = fs.createExclusive(path, 0o600)
+      const hash = createHash("sha256")
+      let size = 0
+      try {
+        for await (const chunk of chunksOf(source)) {
+          size += chunk.byteLength
+          if (size > maxBytes) throw new TooLarge(maxBytes)
+          hash.update(chunk)
+          out.write(chunk)
+        }
+        out.close()
+      } catch (error) {
+        out.abort()
+        throw error
+      }
+      const file = { path, sha256: hex(hash), size }
+      files.set(path, { sha256: file.sha256, size })
+      return file
+    },
+    open: (file) => {
+      const known = files.get(file.path)
+      if (!known) throw new Error(`${file.path} is not a file of this staging area`)
+      return fileReader(fs, file.path, known.size)
+    },
+    written: (file) => files.get(file.path),
+    dispose() {
+      files.clear()
+      remove(dir)
+    },
+  }
+}
+
+/** An in-memory staging area (tests, and the pure in-memory unpack helpers); `bytes` reads a file back whole. */
+export function memoryStaging(): StagingArea & { bytes(file: StagedFile): Uint8Array } {
+  const files = new Map<string, Uint8Array>()
+  let count = 0
+  const get = (file: StagedFile) => {
+    const bytes = files.get(file.path)
+    if (!bytes) throw new Error(`${file.path} is not a file of this staging area`)
+    return bytes
+  }
+  return {
+    async write(name, source, { maxBytes }) {
+      const path = `memory:/${++count}-${safeName(name)}`
+      const bytes = await collect(chunksOf(source), maxBytes)
+      files.set(path, bytes)
+      return { path, sha256: hex(createHash("sha256").update(bytes)), size: bytes.byteLength }
+    },
+    open: (file) => bytesReader(get(file)),
+    written: (file) => {
+      const bytes = files.get(file.path)
+      return bytes ? { sha256: hex(createHash("sha256").update(bytes)), size: bytes.byteLength } : undefined
+    },
+    bytes: get,
+  }
+}
+
+/** The sha256 of the regular file at `path`, read in chunks. */
+export function fileDigest(fs: TxFs, path: string): string {
+  const hash = createHash("sha256")
+  for (let at = 0; ; ) {
+    const chunk = fs.readRange(path, at, CHUNK_BYTES)
+    if (chunk.byteLength === 0) break
+    hash.update(chunk)
+    at += chunk.byteLength
+  }
+  return hex(hash)
+}
+
+/** Copies `from` into a new file `to` (exclusive, `mode`, fsynced) in chunks; returns the digest of what it copied. */
+export function copyFile(fs: TxFs, from: string, to: string, mode: number): string {
+  const out = fs.createExclusive(to, mode)
+  const hash = createHash("sha256")
+  try {
+    for (let at = 0; ; ) {
+      const chunk = fs.readRange(from, at, CHUNK_BYTES)
+      if (chunk.byteLength === 0) break
+      hash.update(chunk)
+      out.write(chunk)
+      at += chunk.byteLength
+    }
+    out.close()
+  } catch (error) {
+    out.abort()
+    throw error
+  }
+  return hex(hash)
+}