@hoardodile/host 0.0.0

package/src/hoard/sanitize.ts

@@ -0,0 +1,230 @@
+/**
+ * Filename hygiene for bare-file resources: map a user-supplied name
+ * (upload filename or zip entry name) onto a safe relative path inside
+ * a resource folder.
+ *
+ * Resource folders hold user entries next to metadata dotfiles
+ * (`.cover.*`, `.deleted`), so every rule here also guarantees the result
+ * can never collide with those: leading dots are stripped, and a
+ * trailing-dot/space rule mirrors Windows normalization.
+ *
+ * Cross-platform guarantees: names are NFC-normalized (macOS stores
+ * NFD on disk — one canonical form keeps names stable when a library
+ * moves between systems), segments are capped at 240 UTF-8 bytes
+ * (ext4's 255-byte component limit, minus suffix headroom), and the
+ * Windows reserved-name set includes the device aliases (`CONIN$`,
+ * superscript COM/LPT variants).
+ *
+ * The migration script (`scripts/migrate-hoard-to-files.mjs`) mirrors
+ * these rules verbatim — keep both in sync when changing them.
+ */
+
+const FORBIDDEN_VISIBLE_CHARS = '<>:"|?*'
+
+/** Max UTF-8 bytes per path segment (ext4 limit 255, suffix headroom). */
+const MAX_SEGMENT_BYTES = 240
+
+/** Max characters for the whole relative path. */
+const MAX_REL_PATH_CHARS = 700
+
+/** Safety net for the collision-suffix loop in {@link uniqueEntryName}. */
+const MAX_UNIQUE_RETRIES = 10_000
+
+/** Windows reserved base names (case-insensitive), incl. device aliases. */
+const WINDOWS_RESERVED = new Set([
+	"CON",
+	"PRN",
+	"AUX",
+	"NUL",
+	"COM0",
+	"COM1",
+	"COM2",
+	"COM3",
+	"COM4",
+	"COM5",
+	"COM6",
+	"COM7",
+	"COM8",
+	"COM9",
+	"LPT0",
+	"LPT1",
+	"LPT2",
+	"LPT3",
+	"LPT4",
+	"LPT5",
+	"LPT6",
+	"LPT7",
+	"LPT8",
+	"LPT9",
+	"CONIN$",
+	"CONOUT$",
+	"COM\u00b9",
+	"COM\u00b2",
+	"COM\u00b3",
+	"LPT\u00b9",
+	"LPT\u00b2",
+	"LPT\u00b3",
+])
+
+/**
+ * Sanitize a single entry name into a safe relative path. Separators
+ * (`/` and `\`) become subdirectory boundaries; `..` and absolute paths
+ * are rejected. Returns `undefined` when the name is unusable (empty,
+ * escapes the folder, or cleans to nothing) — callers drop or reject
+ * such entries.
+ */
+export function sanitizeEntryName(name: string): string | undefined {
+	if (name.length === 0 || name.includes("\0")) return undefined
+	// One canonical Unicode form so names survive filesystem
+	// normalization (macOS NFD) when a library moves between hosts.
+	const normalized = name.normalize("NFC")
+	const rawSegments = normalized.replace(/\\/g, "/").split("/")
+	const segments: string[] = []
+	for (const raw of rawSegments) {
+		const cleaned = cleanSegment(raw)
+		if (cleaned === undefined) continue
+		segments.push(cleaned)
+	}
+	if (segments.length === 0) return undefined
+	const relPath = segments.join("/")
+	if (relPath.length > MAX_REL_PATH_CHARS) return undefined
+	return relPath
+}
+
+/** Clean one path segment; `undefined` when it must be dropped entirely. */
+function cleanSegment(raw: string): string | undefined {
+	if (raw.length === 0 || raw === "." || raw === "..") return undefined
+	let out = ""
+	for (const ch of raw) {
+		const code = ch.codePointAt(0)
+		if (code !== undefined && code < 32) continue
+		if (FORBIDDEN_VISIBLE_CHARS.includes(ch)) {
+			out += "_"
+			continue
+		}
+		out += ch
+	}
+	// Leading dots would make the entry a dotfile (colliding with the
+	// metadata namespace `.cover.*` / `.deleted`); trailing dots and
+	// spaces vanish on Windows and would break path equality.
+	let trimmed = out.replace(/^\.+/, "").replace(/[. ]+$/, "")
+	if (trimmed.length === 0) return undefined
+	trimmed = truncateUtf8(trimmed, MAX_SEGMENT_BYTES)
+	if (trimmed.length === 0) return undefined
+	const base = trimmed.split(".")[0]?.toUpperCase()
+	if (base !== undefined && WINDOWS_RESERVED.has(base)) {
+		trimmed = `_${trimmed}`
+	}
+	return trimmed
+}
+
+/** Truncate `value` so its UTF-8 length stays within `maxBytes`. */
+function truncateUtf8(value: string, maxBytes: number): string {
+	let bytes = 0
+	let out = ""
+	for (const ch of value) {
+		const len = Buffer.byteLength(ch, "utf8")
+		if (bytes + len > maxBytes) break
+		bytes += len
+		out += ch
+	}
+	return out
+}
+
+/**
+ * Occupancy bookkeeping for name resolution: which paths are already
+ * taken by files (blocking everything) and which are directories
+ * (blocking only exact-path installs — a file cannot replace a folder).
+ */
+export type OccupiedNames = {
+	readonly files: Set<string>
+	readonly dirs: Set<string>
+}
+
+/** Build occupancy from optional existing on-disk names (with types). */
+export function createOccupiedNames(existing?: {
+	readonly files?: Iterable<string>
+	readonly dirs?: Iterable<string>
+}): OccupiedNames {
+	return {
+		files: new Set(existing?.files ?? []),
+		dirs: new Set(existing?.dirs ?? []),
+	}
+}
+
+/**
+ * Record `relPath` as an installed file; every ancestor prefix becomes
+ * a directory. A later entry colliding with one of the ancestors (a
+ * file that sits on a directory path) is resolved by
+ * {@link uniqueEntryName}.
+ */
+export function occupyEntryName(
+	occupied: OccupiedNames,
+	relPath: string,
+): void {
+	occupied.files.add(relPath)
+	let prefix = ""
+	for (const segment of relPath.split("/").slice(0, -1)) {
+		prefix = prefix.length > 0 ? `${prefix}/${segment}` : segment
+		occupied.dirs.add(prefix)
+	}
+}
+
+/**
+ * Resolve a sanitized relative path against the names already present
+ * in the target folder by suffixing the colliding segment (`-1`, `-2`,
+ * …). The comparison is case-insensitive (Windows and macOS filesystems
+ * are). Files block their path and every ancestor prefix; directories
+ * block only their exact path. The suffix lands on the segment whose
+ * prefix collides, so `x/y.txt` against an occupied file `x` becomes
+ * `x-1/y.txt`, not `x/y-1.txt` (which would collide forever).
+ */
+export function uniqueEntryName(
+	occupied: OccupiedNames,
+	relPath: string,
+): string {
+	const lowerFiles = new Set([...occupied.files].map((p) => p.toLowerCase()))
+	const lowerDirs = new Set([...occupied.dirs].map((p) => p.toLowerCase()))
+	let candidate = relPath
+	for (let guard = 0; guard < MAX_UNIQUE_RETRIES; guard += 1) {
+		const segIdx = collidingSegment(lowerFiles, lowerDirs, candidate)
+		if (segIdx < 0) return candidate
+		const segments = candidate.split("/")
+		const target = segments[segIdx]!
+		const dot = target.lastIndexOf(".")
+		const stem = dot > 0 ? target.slice(0, dot) : target
+		const ext = dot > 0 ? target.slice(dot) : ""
+		for (let i = 1; ; i += 1) {
+			const next = `${stem}-${i}${ext}`
+			const rebuilt = segments
+				.map((seg, idx) => (idx === segIdx ? next : seg))
+				.join("/")
+			if (collidingSegment(lowerFiles, lowerDirs, rebuilt) < 0) {
+				candidate = rebuilt
+				break
+			}
+		}
+	}
+	throw new Error(`unable to resolve a unique name for ${relPath}`)
+}
+
+/**
+ * Index of the segment whose path prefix is taken by a FILE, or whose
+ * exact path is taken by a file or directory, or `-1`. Directory
+ * ancestors are not collisions (nested entries need them).
+ */
+function collidingSegment(
+	lowerFiles: ReadonlySet<string>,
+	lowerDirs: ReadonlySet<string>,
+	candidate: string,
+): number {
+	const segments = candidate.split("/")
+	const lower = candidate.toLowerCase()
+	if (lowerFiles.has(lower) || lowerDirs.has(lower)) return segments.length - 1
+	let prefix = ""
+	for (let i = 0; i < segments.length - 1; i += 1) {
+		prefix = prefix.length > 0 ? `${prefix}/${segments[i]}` : segments[i]!
+		if (lowerFiles.has(prefix.toLowerCase())) return i
+	}
+	return -1
+}

package/src/hoard/staging-dir.ts

@@ -0,0 +1,133 @@
+import { createWriteStream } from "node:fs"
+import { mkdir, readdir, rename, rm, stat } from "node:fs/promises"
+import { basename, extname, join } from "node:path"
+import { pipeline } from "node:stream/promises"
+import type { StoragePaths } from "./paths.ts"
+
+/**
+ * Extension used for archive (zip) uploads staged in the global pool.
+ * The original filename is ignored — archive uploads are always stored
+ * as `<fileId>.zip` so commit / extraction can locate them by `fileId`
+ * alone.
+ */
+const STAGED_ARCHIVE_EXT = ".zip"
+
+/**
+ * Resolve the on-disk path of a file staged in the global pool by
+ * `fileId`. The pool is flat: each file is stored as `<fileId><ext>`
+ * where `ext` is the lower-cased extension of the original filename
+ * (or `.zip` for archive uploads). Because the extension is part of the
+ * filename we scan the directory for a sibling whose stem matches
+ * `fileId`.
+ *
+ * Returns `undefined` when no staged file matches `fileId`.
+ */
+export async function findStagedPoolFile(
+	paths: StoragePaths,
+	fileId: string,
+): Promise<string | undefined> {
+	const poolDir = paths.local.stagingPoolRoot()
+	const names = await readdir(poolDir).catch(() => [])
+	for (const name of names) {
+		const ext = extname(name)
+		const stem = basename(name, ext)
+		if (stem === fileId) return join(poolDir, name)
+	}
+	return undefined
+}
+
+/**
+ * Resolve multiple pool files by `fileId` with a SINGLE directory scan.
+ * Committing an ordered upload resolves and then consumes every staged
+ * file; the per-file {@link findStagedPoolFile} scan would cost one full
+ * `readdir` per file, and the pool can hold hundreds of entries under
+ * concurrent uploads. Returns a map keyed by `fileId`; ids without a
+ * staged match are absent.
+ */
+export async function resolveStagedPoolFiles(
+	paths: StoragePaths,
+	fileIds: readonly string[],
+): Promise<Map<string, string>> {
+	const poolDir = paths.local.stagingPoolRoot()
+	const names = await readdir(poolDir).catch(() => [])
+	const byStem = new Map<string, string>()
+	for (const name of names) {
+		const ext = extname(name)
+		const stem = basename(name, ext)
+		if (!byStem.has(stem)) byStem.set(stem, join(poolDir, name))
+	}
+	const out = new Map<string, string>()
+	for (const fileId of fileIds) {
+		const hit = byStem.get(fileId)
+		if (hit !== undefined) out.set(fileId, hit)
+	}
+	return out
+}
+
+/**
+ * Stream a single file into the global staging pool as
+ * `<fileId><ext>`. The file is first written to a `.tmp-*` sibling and
+ * atomically renamed so concurrent readers (preview) never observe a
+ * partial file. Returns the final staged path.
+ */
+export async function writeStagedPoolFile(
+	paths: StoragePaths,
+	fileId: string,
+	filename: string,
+	stream: NodeJS.ReadableStream,
+): Promise<string> {
+	await mkdir(paths.local.stagingPoolRoot(), { recursive: true })
+	const ext = extname(filename).toLowerCase()
+	const dest = paths.local.stagingPoolFile(fileId, ext)
+	const tmp = `${dest}.tmp-${Date.now()}`
+	try {
+		await pipeline(stream, createWriteStream(tmp))
+		await rename(tmp, dest)
+		return dest
+	} catch (err) {
+		await rm(tmp, { force: true }).catch(() => {})
+		throw err
+	}
+}
+
+/**
+ * Stream an archive (zip) upload into the global staging pool as
+ * `<fileId>.zip`. Same atomic write semantics as
+ * {@link writeStagedPoolFile}. Returns the final staged path.
+ */
+export async function writeStagedArchiveFile(
+	paths: StoragePaths,
+	fileId: string,
+	stream: NodeJS.ReadableStream,
+): Promise<string> {
+	return writeStagedPoolFile(paths, fileId, "archive.zip", stream)
+}
+
+/** Resolve the staged path of an archive upload by `fileId`. */
+export async function findStagedArchiveFile(
+	paths: StoragePaths,
+	fileId: string,
+): Promise<string | undefined> {
+	const existing = await findStagedPoolFile(paths, fileId)
+	if (existing === undefined) return undefined
+	// Archives are always stored with the `.zip` extension.
+	return extname(existing).toLowerCase() === STAGED_ARCHIVE_EXT
+		? existing
+		: undefined
+}
+
+/**
+ * Remove a single file from the global staging pool. Returns `true` when
+ * a file was removed, `false` when `fileId` was not present.
+ */
+export async function removeStagedPoolFile(
+	paths: StoragePaths,
+	fileId: string,
+): Promise<boolean> {
+	const existing = await findStagedPoolFile(paths, fileId)
+	if (existing === undefined) return false
+	await rm(existing, { force: true }).catch(() => {})
+	// Best-effort: the file may have been removed concurrently.
+	const stillThere = await stat(existing).catch(() => undefined)
+	return stillThere === undefined
+}

package/src/hoard/version.test.ts

@@ -0,0 +1,285 @@
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { afterEach, beforeEach, describe, expect, test } from "vitest"
+import { DomainError } from "../errors.ts"
+import {
+	createNextVersion,
+	currentVersion,
+	ensureBootstrapVersion,
+	listVersions,
+	readActiveVersion,
+	writeActiveVersion,
+} from "./version.ts"
+
+describe("listVersions", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-list-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("empty directory returns empty array", () => {
+		expect(listVersions(root)).toEqual([])
+	})
+
+	test("ignores non-numeric directory names", () => {
+		mkdirSync(join(root, "versions", "1"), { recursive: true })
+		mkdirSync(join(root, "versions", "foo"), { recursive: true })
+		mkdirSync(join(root, "versions", ".hidden"), { recursive: true })
+		mkdirSync(join(root, "versions", "01"), { recursive: true })
+		expect(listVersions(root)).toEqual([1])
+	})
+
+	test("returns sorted ascending", () => {
+		for (const v of [3, 1, 10, 2]) {
+			mkdirSync(join(root, "versions", String(v)), { recursive: true })
+		}
+		expect(listVersions(root)).toEqual([1, 2, 3, 10])
+	})
+})
+
+describe("currentVersion", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-cur-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("returns 0 when no versions exist", () => {
+		expect(currentVersion(root)).toBe(0)
+	})
+
+	test("returns max version number", () => {
+		mkdirSync(join(root, "versions", "1"), { recursive: true })
+		mkdirSync(join(root, "versions", "5"), { recursive: true })
+		expect(currentVersion(root)).toBe(5)
+	})
+})
+
+describe("ensureBootstrapVersion", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-boot-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("creates versions/1 on empty storage", () => {
+		const v = ensureBootstrapVersion(root)
+		expect(v).toBe(1)
+		expect(existsSync(join(root, "versions", "1"))).toBe(true)
+	})
+
+	test("is a no-op when any version already exists", () => {
+		mkdirSync(join(root, "versions", "3"), { recursive: true })
+		const v = ensureBootstrapVersion(root)
+		expect(v).toBe(3)
+		expect(existsSync(join(root, "versions", "1"))).toBe(false)
+	})
+})
+
+describe("readActiveVersion", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-read-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("falls back to current when state file is missing", () => {
+		mkdirSync(join(root, "versions", "2"), { recursive: true })
+		expect(readActiveVersion(root)).toBe(2)
+	})
+
+	test("returns persisted active when it exists", () => {
+		mkdirSync(join(root, "versions", "1"), { recursive: true })
+		mkdirSync(join(root, "versions", "2"), { recursive: true })
+		mkdirSync(join(root, "local"), { recursive: true })
+		writeFileSync(
+			join(root, "local", "version-state.json"),
+			JSON.stringify({ active: 1 }),
+		)
+		expect(readActiveVersion(root)).toBe(1)
+	})
+
+	test("falls back to current when state points at missing version", () => {
+		mkdirSync(join(root, "versions", "2"), { recursive: true })
+		mkdirSync(join(root, "local"), { recursive: true })
+		writeFileSync(
+			join(root, "local", "version-state.json"),
+			JSON.stringify({ active: 99 }),
+		)
+		expect(readActiveVersion(root)).toBe(2)
+	})
+
+	test("falls back to current when state file is malformed", () => {
+		mkdirSync(join(root, "versions", "3"), { recursive: true })
+		mkdirSync(join(root, "local"), { recursive: true })
+		writeFileSync(join(root, "local", "version-state.json"), "not json")
+		expect(readActiveVersion(root)).toBe(3)
+	})
+})
+
+describe("writeActiveVersion", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-write-"))
+		mkdirSync(join(root, "versions", "1"), { recursive: true })
+		mkdirSync(join(root, "versions", "2"), { recursive: true })
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("persists valid active version", () => {
+		writeActiveVersion(root, 1)
+		const raw = readFileSync(join(root, "local", "version-state.json"), "utf8")
+		expect(JSON.parse(raw)).toEqual({ active: 1 })
+	})
+
+	test("rejects unknown version", () => {
+		try {
+			writeActiveVersion(root, 99)
+			expect.unreachable("should have thrown")
+		} catch (err) {
+			expect(err).toBeInstanceOf(DomainError)
+			expect((err as DomainError).kind).toBe("version.not_found")
+		}
+	})
+})
+
+describe("createNextVersion", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "ver-next-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("throws when no version exists yet", () => {
+		try {
+			createNextVersion(root, () => {})
+			expect.unreachable("should have thrown")
+		} catch (err) {
+			expect(err).toBeInstanceOf(DomainError)
+			expect((err as DomainError).kind).toBe("version.bootstrap_required")
+		}
+	})
+
+	test("snapshots the live DB into versions/<prev>/app.sqlite", () => {
+		// Bootstrap version 1, then have the injected vacuum write a marker
+		// file — the primitive's job is the destination, not the sqlite
+		// snapshot (the server's DbHandles.vacuumInto owns that).
+		ensureBootstrapVersion(root)
+		const result = createNextVersion(root, (dest) => {
+			writeFileSync(dest, "snapshot")
+		})
+
+		expect(result.previous).toBe(1)
+		expect(result.created).toBe(2)
+
+		const snapshotPath = join(root, "versions", "1", "app.sqlite")
+		expect(existsSync(snapshotPath)).toBe(true)
+		expect(readFileSync(snapshotPath, "utf8")).toBe("snapshot")
+	})
+
+	test("throws when snapshot already exists", () => {
+		ensureBootstrapVersion(root)
+
+		// First call succeeds
+		createNextVersion(root, (dest) => {
+			writeFileSync(dest, "snapshot")
+		})
+
+		// After first publish, current version becomes 2; the next snapshot
+		// target would be versions/2/app.sqlite. Pre-create it to trigger
+		// the already_exists guard.
+		mkdirSync(join(root, "versions", "2"), { recursive: true })
+		writeFileSync(join(root, "versions", "2", "app.sqlite"), "")
+
+		try {
+			createNextVersion(root, () => {
+				expect.unreachable("vacuumInto should not be called")
+			})
+			expect.unreachable("should have thrown")
+		} catch (err) {
+			expect(err).toBeInstanceOf(DomainError)
+			expect((err as DomainError).kind).toBe("version.already_exists")
+		}
+		expect(existsSync(join(root, "versions", "3"))).toBe(false)
+		expect(currentVersion(root)).toBe(2)
+	})
+
+	test("copies installed plugins into the next version", () => {
+		ensureBootstrapVersion(root)
+		const pluginId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
+		const src = join(root, "versions", "1", "plugins", pluginId)
+		mkdirSync(src, { recursive: true })
+		writeFileSync(join(src, "manifest.json"), JSON.stringify({ id: pluginId }))
+		writeFileSync(join(src, "main.js"), "export default {}\n")
+		mkdirSync(join(root, "versions", "1", "plugins", ".staging-x"), {
+			recursive: true,
+		})
+		mkdirSync(join(root, "versions", "1", "plugins", "not-a-plugin"), {
+			recursive: true,
+		})
+
+		createNextVersion(root, (dest) => {
+			writeFileSync(dest, "snapshot")
+		})
+
+		expect(
+			existsSync(join(root, "versions", "2", "plugins", pluginId, "main.js")),
+		).toBe(true)
+		expect(
+			existsSync(join(root, "versions", "1", "plugins", pluginId, "main.js")),
+		).toBe(true)
+		expect(
+			existsSync(join(root, "versions", "2", "plugins", ".staging-x")),
+		).toBe(false)
+		expect(
+			existsSync(join(root, "versions", "2", "plugins", "not-a-plugin")),
+		).toBe(false)
+	})
+
+	test("removes the next version directory when vacuumInto throws", () => {
+		ensureBootstrapVersion(root)
+		try {
+			createNextVersion(root, () => {
+				throw new Error("vacuum failed")
+			})
+			expect.unreachable("should have thrown")
+		} catch (err) {
+			expect((err as Error).message).toBe("vacuum failed")
+		}
+		expect(existsSync(join(root, "versions", "2"))).toBe(false)
+		expect(currentVersion(root)).toBe(1)
+	})
+})