@hoardodile/host 0.0.0

package/src/archive/format.ts

@@ -0,0 +1,49 @@
+/**
+ * Container format identification from magic bytes. The single source of
+ * truth for "what archive format is this" — the engine matrix in
+ * `./index.ts` maps each format to its list/extract/address
+ * capabilities, and every dispatch (yauzl vs 7-Zip, nested resolver vs
+ * whole-archive) keys off {@link sniffContainerFormat}.
+ */
+
+/** Byte window read from an entry to decide container-ness. */
+export const SNIFF_WINDOW_BYTES = 4100
+
+export type ContainerFormat = "zip" | "tar" | "7z" | "rar" | "xz" | "gzip"
+
+/** True when `head` starts with the given magic byte sequence. */
+function hasMagic(head: Uint8Array, magic: readonly number[]): boolean {
+	if (head.byteLength < magic.length) return false
+	for (let i = 0; i < magic.length; i++) {
+		if (head[i] !== magic[i]) return false
+	}
+	return true
+}
+
+/**
+ * Identify a container format from its leading bytes. Zip is addressable
+ * as a nested container (`outer!inner`); the rest (tar/7z/rar/xz/gzip)
+ * are only extractable as whole archives (see `extract-archive.ts` and
+ * `extract.ts`).
+ */
+export function sniffContainerFormat(
+	head: Uint8Array,
+): ContainerFormat | undefined {
+	if (hasMagic(head, [0x50, 0x4b, 0x03, 0x04])) return "zip"
+	if (hasMagic(head, [0x50, 0x4b, 0x05, 0x06])) return "zip"
+	if (head.byteLength >= 262) {
+		const magic = String.fromCharCode(
+			head[257]!,
+			head[258]!,
+			head[259]!,
+			head[260]!,
+			head[261]!,
+		)
+		if (magic === "ustar") return "tar"
+	}
+	if (hasMagic(head, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])) return "7z"
+	if (hasMagic(head, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07])) return "rar"
+	if (hasMagic(head, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00])) return "xz"
+	if (hasMagic(head, [0x1f, 0x8b, 0x08])) return "gzip"
+	return undefined
+}

package/src/archive/index.ts

@@ -0,0 +1,116 @@
+/**
+ * Archive runtime internal to the host: zip reading over random-access
+ * sources, `outer!inner` virtual-path resolution, whole-archive
+ * extraction through 7-Zip, and zip packing. This barrel only
+ * re-exports what the host itself consumes — the public host surface
+ * (`../index.ts`) picks its own much smaller set.
+ *
+ * ── Engine matrix ────────────────────────────────────────────────────
+ *
+ * Two engines, two jobs:
+ *   - **yauzl** is the zip *random-access* engine: central-directory
+ *     listing, byte-range entry streams, ZIP64. It cannot read other
+ *     formats and is the only engine that can — the read path
+ *     (nested addressing, thumbnails, source view) is inseparable from
+ *     it. It also serves as the no-7-Zip fallback for zip extraction.
+ *   - **7-Zip** is the *whole-archive* engine: `l -slt -ba` listing and
+ *     `x` extraction for every supported format. It cannot express
+ *     random access (no entry offsets, no byte ranges), so listing
+ *     through it is a whole-file operation.
+ *
+ * Capability matrix (list = file listing, extract = materialize to a
+ * directory, address = `outer!inner` single-entry reads):
+ *
+ *   format   list        extract     address     notes
+ *   zip      yauzl       7-Zip       virtual     virtual addressing reads
+ *                                                 the CD, never materializes
+ *   tar      7-Zip       7-Zip       materialized extracted files are
+ *   7z       "           "           "           served from disk via the
+ *   rar      "           "           "           manifest whitelist
+ *   xz       "           "           (none)      whole-archive only
+ *   gzip     7-Zip       7-Zip       (none)      lists as its single
+ *                                                 decompressed stream
+ *                                                 (e.g. `t.tar`); never
+ *                                                 addressable
+ *
+ *   Degradation without the optional `@hoardodile/7z-bin` binary: zip
+ *   keeps full functionality (yauzl everywhere, streaming extraction
+ *   with per-byte budget counting); every non-zip list/extract rejects
+ *   with "7-Zip is not installed".
+ *
+ *   Budget semantics differ by engine: with 7-Zip the listing
+ *   pre-checks sizes and the extracted tree is re-verified afterwards
+ *   (sizes are advisory until measured); the yauzl streaming fallback
+ *   counts bytes mid-stream, which also catches lying headers. Both map
+ *   to `resource.archive_too_large`.
+ *
+ *   Encrypted archives reject with different kinds depending on when
+ *   the encryption is seen: the 7-Zip listing pre-check reports
+ *   `resource.archive_invalid_entry`, the yauzl entry-stream open
+ *   reports `resource.archive_open_failed`.
+ *
+ * Legacy zip entry names (no UTF-8 name flag) decode as codepage 437
+ * (`-mcp=437`) on the 7-Zip path; the switch is applied automatically
+ * after sniffing, so callers never branch on format.
+ */
+
+export type {
+	ExtractArchiveOptions,
+	ZipExtractReport,
+	ZipExtractReporter,
+} from "./extract.ts"
+export { assertExtractedTree, extractArchiveInto } from "./extract.ts"
+export type {
+	ArchiveExtractor,
+	ExtractedEntry,
+	ExtractionResult,
+	ExtractProgress,
+} from "./extract-archive.ts"
+export {
+	createArchiveExtractor,
+	EXTRACT_INDEX_VERSION,
+	PROBE_HEADER_BYTES,
+	sanitizeExtractPath,
+} from "./extract-archive.ts"
+export type {
+	ContainerFormat,
+	SNIFF_WINDOW_BYTES,
+} from "./format.ts"
+export { sniffContainerFormat } from "./format.ts"
+export {
+	listArchiveEntries,
+	listingBudgetExceeded,
+	validateArchiveBudget,
+} from "./listing.ts"
+export { materializeFile } from "./materialize.ts"
+export type {
+	NestedCdCache,
+	NestedCdCacheOptions,
+} from "./nested-cd-cache.ts"
+export { createNestedCdCache } from "./nested-cd-cache.ts"
+export type {
+	ArchiveEntry,
+	NestedResolver,
+	OpenableArchiveEntry,
+	OuterEntrySource,
+	PathResolution,
+} from "./nested-entry.ts"
+export {
+	createNestedResolver,
+	createOuterArchiveSource,
+	splitVirtualPath,
+	VIRTUAL_PATH_SEPARATOR,
+} from "./nested-entry.ts"
+export type { ZipStreamEntry } from "./pack.ts"
+export { streamStoredZip } from "./pack.ts"
+export type { ArchiveSource, ZipEntry } from "./zip-entries.ts"
+export {
+	createFileArchiveSource,
+	listZipEntries,
+	listZipEntriesFromSource,
+	normalizeZipError,
+	openZipEntryStream,
+	readFileRange,
+	readZipRange,
+	streamRange,
+} from "./zip-entries.ts"

package/src/archive/listing.ts

@@ -0,0 +1,121 @@
+import { invalid } from "../errors.ts"
+import { listSevenZipEntries, resolveSevenZipPath } from "./7z.ts"
+import { SNIFF_WINDOW_BYTES, sniffContainerFormat } from "./format.ts"
+import type { ArchiveEntry } from "./nested-entry.ts"
+import { listZipEntries, readFileRange } from "./zip-entries.ts"
+
+/**
+ * Cross-engine archive listing and budget validation. Zip lists through
+ * yauzl, tar/7z/rar/xz/gzip through the 7-Zip binary; the engine choice
+ * (and the zip-only legacy-cp437 name decoding, handled inside the 7-Zip
+ * wrapper) lives here so callers never branch on format.
+ */
+
+/**
+ * List the file entries of an on-disk archive across engines: zip lists
+ * through yauzl, tar/7z/rar/xz/gzip through the 7-Zip binary (gzip
+ * reports its single decompressed stream).
+ *
+ * @throws DomainError `resource.archive_open_failed` when the file is
+ *   not a readable archive, or 7-Zip is missing for a non-zip format.
+ */
+export async function listArchiveEntries(
+	archivePath: string,
+): Promise<readonly ArchiveEntry[]> {
+	const head = await readFileRange(archivePath, 0, SNIFF_WINDOW_BYTES - 1)
+	const format = sniffContainerFormat(head)
+	if (format === "zip") {
+		const records = await listZipEntries(archivePath)
+		return records.map((r) => ({ name: r.name, sizeBytes: r.uncompressedSize }))
+	}
+	if (format === undefined) {
+		throw invalid(
+			"resource.archive_open_failed",
+			"not a supported archive (zip/tar/7z/rar/xz/gzip)",
+			{},
+		)
+	}
+	if (resolveSevenZipPath() === undefined) {
+		throw invalid(
+			"resource.archive_open_failed",
+			"7-Zip is not installed — cannot list tar/7z/rar/xz/gzip archives",
+			{},
+		)
+	}
+	const entries = await listSevenZipEntries(archivePath)
+	return entries
+		.filter((e) => !e.folder)
+		.map((e) => ({ name: e.name, sizeBytes: e.sizeBytes }))
+}
+
+/**
+ * Which listing budget is exceeded, or `undefined` when the listing
+ * fits. Pure computation — callers decide the error kind and message
+ * (the plugin extractor and the whole-archive extractor use different
+ * domain kinds), so one budget rule serves both.
+ */
+export function listingBudgetExceeded(
+	entries: readonly { readonly sizeBytes: number }[],
+	opts: { readonly maxBytes: number; readonly maxEntries?: number },
+):
+	| {
+			readonly kind: "bytes"
+			readonly totalBytes: number
+			readonly maxBytes: number
+	  }
+	| {
+			readonly kind: "entries"
+			readonly count: number
+			readonly maxEntries: number
+	  }
+	| undefined {
+	if (opts.maxEntries !== undefined && entries.length > opts.maxEntries) {
+		return {
+			kind: "entries",
+			count: entries.length,
+			maxEntries: opts.maxEntries,
+		}
+	}
+	let totalBytes = 0
+	for (const entry of entries) {
+		totalBytes += entry.sizeBytes
+		if (totalBytes > opts.maxBytes) {
+			return {
+				kind: "bytes",
+				totalBytes,
+				maxBytes: opts.maxBytes,
+			}
+		}
+	}
+	return undefined
+}
+
+/**
+ * Validate an on-disk archive's cumulative uncompressed size against
+ * `maxBytes` without inflating any entry bytes (metadata-only — the
+ * listing never decompresses). Format-agnostic via
+ * {@link listArchiveEntries}; unknown content and non-zip formats
+ * without the 7-Zip binary are rejected outright.
+ *
+ * Used at commit time for uploaded archives so every supported format
+ * passes the same bomb gate.
+ *
+ * @throws DomainError `resource.archive_open_failed` when the file is
+ *   not a readable archive (or 7-Zip is missing for non-zip formats).
+ * @throws DomainError `resource.archive_too_large` when the cumulative
+ *   uncompressed size exceeds `maxBytes`.
+ */
+export async function validateArchiveBudget(
+	archivePath: string,
+	maxBytes: number,
+): Promise<void> {
+	const entries = await listArchiveEntries(archivePath)
+	const exceeded = listingBudgetExceeded(entries, { maxBytes })
+	if (exceeded?.kind === "bytes") {
+		throw invalid(
+			"resource.archive_too_large",
+			`archive uncompressed size exceeds ${maxBytes} bytes`,
+			{ maxBytes },
+		)
+	}
+}

package/src/archive/materialize.ts

@@ -0,0 +1,88 @@
+import { createWriteStream } from "node:fs"
+import { mkdir, rename, rm, stat, writeFile } from "node:fs/promises"
+import { dirname } from "node:path"
+import type { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+
+/**
+ * Materialization primitives shared by every code path that writes
+ * derived files into the extraction cache: the host's container
+ * extractor and the server's artifact view (thumb/probe materialization
+ * for both literal and virtual entries). One partial+rename+size-check
+ * recipe and one single-flight guard, instead of a copy per caller.
+ */
+
+/**
+ * Write `openStream`'s bytes to `target` atomically: stream to a
+ * `.partial` sibling, verify the written size matches `expectedSize`,
+ * then rename into place. On any failure the partial is removed and the
+ * error rethrown. An already-correct `target` is left untouched.
+ *
+ * With `key` set, concurrent calls for the same key share one in-flight
+ * write instead of racing it (via {@link globalMaterializeInflight}).
+ */
+export async function materializeFile(opts: {
+	readonly openStream: () => Readable | Promise<Readable>
+	readonly target: string
+	readonly expectedSize: number
+	readonly key?: string
+}): Promise<void> {
+	const { key } = opts
+	if (key !== undefined) {
+		return withSingleFlight(key, () => materializeFileWithoutFlight(opts))
+	}
+	return materializeFileWithoutFlight(opts)
+}
+
+async function materializeFileWithoutFlight(opts: {
+	readonly openStream: () => Readable | Promise<Readable>
+	readonly target: string
+	readonly expectedSize: number
+}): Promise<void> {
+	const { openStream, target, expectedSize } = opts
+	await mkdir(dirname(target), { recursive: true })
+	const partial = `${target}.partial-${process.pid}-${Date.now()}`
+	try {
+		if (expectedSize === 0) {
+			await writeFile(partial, Buffer.alloc(0))
+		} else {
+			await pipeline(await openStream(), createWriteStream(partial))
+		}
+		const written = await stat(partial)
+		if (written.size !== expectedSize) {
+			throw new Error(
+				`materialized ${target} size mismatch: expected ${expectedSize}, got ${written.size}`,
+			)
+		}
+		await rename(partial, target)
+	} catch (err) {
+		await rm(partial, { force: true }).catch(() => {})
+		throw err
+	}
+}
+
+/**
+ * Deduplicate concurrent materialization of the same `key`: the second
+ * caller awaits the first's in-flight work instead of racing it. The
+ * inflight map is shared via {@link globalMaterializeInflight} unless a
+ * custom one is passed, matching how the server's artifact views share
+ * one guard process-wide.
+ */
+export function withSingleFlight<T>(
+	key: string,
+	run: () => Promise<T>,
+	inflight: Map<string, Promise<unknown>> = globalMaterializeInflight,
+): Promise<T> {
+	const pending = inflight.get(key) as Promise<T> | undefined
+	if (pending !== undefined) return pending
+	const work = run()
+	inflight.set(key, work)
+	void work.then(
+		() => inflight.delete(key),
+		() => inflight.delete(key),
+	)
+	return work
+}
+
+/** Process-wide single-flight guard for materialization (host + server). */
+export const globalMaterializeInflight = new Map<string, Promise<unknown>>()

package/src/archive/name-decode.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest"
+import { decodeLegacyZipName } from "./name-decode.ts"
+
+describe("decodeLegacyZipName", () => {
+	it("passes ASCII through unchanged", () => {
+		expect(decodeLegacyZipName(Buffer.from("a/b.txt"))).toBe("a/b.txt")
+		expect(decodeLegacyZipName(Buffer.from(""))).toBe("")
+	})
+
+	it("keeps valid UTF-8 names untouched", () => {
+		const utf8E = Buffer.from("café.jpg", "utf8")
+		expect(decodeLegacyZipName(utf8E)).toBe("café.jpg")
+	})
+
+	it("decodes legacy bytes as cp437 instead of losing them", () => {
+		// 0x82 is é in cp437 but an invalid standalone UTF-8 byte.
+		const legacy = Buffer.from("caf\x82.jpg", "latin1")
+		expect(decodeLegacyZipName(legacy)).toBe("caf\u00e9.jpg")
+	})
+
+	it("maps a spread of high bytes through the cp437 table", () => {
+		// 0xe9 -> Θ, 0xff -> nbsp, 0xa4 -> ñ
+		expect(decodeLegacyZipName(Buffer.from([0xe9]))).toBe("\u0398")
+		expect(decodeLegacyZipName(Buffer.from([0xff]))).toBe("\u00a0")
+		expect(decodeLegacyZipName(Buffer.from([0xa4]))).toBe("\u00f1")
+	})
+
+	it("decodes every byte losslessly (no U+FFFD replacement)", () => {
+		for (let byte = 0x80; byte <= 0xff; byte++) {
+			const decoded = decodeLegacyZipName(Buffer.from([byte]))
+			expect(decoded).not.toBe("\ufffd")
+		}
+	})
+})

package/src/archive/name-decode.ts

@@ -0,0 +1,287 @@
+/**
+ * Zip entry name decoding for the 7-Zip paths. The zip APPNOTE says
+ * names without the UTF-8 flag (general-purpose bit 11) are encoded in
+ * the OEM code page; in practice that means cp437 (the same assumption
+ * yauzl makes on the native zip path). 7-Zip cannot be told to decode
+ * them this way — its `-mcp` switch does not exist in 26.02 and its
+ * POSIX builds write the raw name bytes verbatim — so the host decodes
+ * the bytes itself, from the listing output and again from the on-disk
+ * names after extraction, keeping the two consistent on every platform.
+ *
+ * ASCII and valid UTF-8 names pass through untouched; anything else is
+ * re-interpreted as cp437. The table matches yauzl's, so the native and
+ * 7-Zip paths agree on legacy names.
+ */
+
+const CP437 = String.fromCharCode(
+	0,
+	9786,
+	9787,
+	9829,
+	9830,
+	9827,
+	9824,
+	8226,
+	9688,
+	9675,
+	9689,
+	9794,
+	9792,
+	9834,
+	9835,
+	9788,
+	9658,
+	9668,
+	8597,
+	8252,
+	182,
+	167,
+	9644,
+	8616,
+	8593,
+	8595,
+	8594,
+	8592,
+	8735,
+	8596,
+	9650,
+	9660,
+	32,
+	33,
+	34,
+	35,
+	36,
+	37,
+	38,
+	39,
+	40,
+	41,
+	42,
+	43,
+	44,
+	45,
+	46,
+	47,
+	48,
+	49,
+	50,
+	51,
+	52,
+	53,
+	54,
+	55,
+	56,
+	57,
+	58,
+	59,
+	60,
+	61,
+	62,
+	63,
+	64,
+	65,
+	66,
+	67,
+	68,
+	69,
+	70,
+	71,
+	72,
+	73,
+	74,
+	75,
+	76,
+	77,
+	78,
+	79,
+	80,
+	81,
+	82,
+	83,
+	84,
+	85,
+	86,
+	87,
+	88,
+	89,
+	90,
+	91,
+	92,
+	93,
+	94,
+	95,
+	96,
+	97,
+	98,
+	99,
+	100,
+	101,
+	102,
+	103,
+	104,
+	105,
+	106,
+	107,
+	108,
+	109,
+	110,
+	111,
+	112,
+	113,
+	114,
+	115,
+	116,
+	117,
+	118,
+	119,
+	120,
+	121,
+	122,
+	123,
+	124,
+	125,
+	126,
+	8962,
+	199,
+	252,
+	233,
+	226,
+	228,
+	224,
+	229,
+	231,
+	234,
+	235,
+	232,
+	239,
+	238,
+	236,
+	196,
+	197,
+	201,
+	230,
+	198,
+	244,
+	246,
+	242,
+	251,
+	249,
+	255,
+	214,
+	220,
+	162,
+	163,
+	165,
+	8359,
+	402,
+	225,
+	237,
+	243,
+	250,
+	241,
+	209,
+	170,
+	186,
+	191,
+	8976,
+	172,
+	189,
+	188,
+	161,
+	171,
+	187,
+	9617,
+	9618,
+	9619,
+	9474,
+	9508,
+	9569,
+	9570,
+	9558,
+	9557,
+	9571,
+	9553,
+	9559,
+	9565,
+	9564,
+	9563,
+	9488,
+	9492,
+	9524,
+	9516,
+	9500,
+	9472,
+	9532,
+	9566,
+	9567,
+	9562,
+	9556,
+	9577,
+	9574,
+	9568,
+	9552,
+	9580,
+	9575,
+	9576,
+	9572,
+	9573,
+	9561,
+	9560,
+	9554,
+	9555,
+	9579,
+	9578,
+	9496,
+	9484,
+	9608,
+	9604,
+	9612,
+	9616,
+	9600,
+	945,
+	223,
+	915,
+	960,
+	931,
+	963,
+	181,
+	964,
+	934,
+	920,
+	937,
+	948,
+	8734,
+	966,
+	949,
+	8745,
+	8801,
+	177,
+	8805,
+	8804,
+	8992,
+	8993,
+	247,
+	8776,
+	176,
+	8729,
+	183,
+	8730,
+	8319,
+	178,
+	9632,
+	160,
+)
+
+/**
+ * Decode one name byte string: valid UTF-8 stays as-is, anything else
+ * falls back to cp437 (the zip legacy-encoding convention).
+ */
+export function decodeLegacyZipName(bytes: Uint8Array): string {
+	try {
+		return new TextDecoder("utf-8", { fatal: true }).decode(bytes)
+	} catch {
+		let out = ""
+		for (const b of bytes) out += CP437[b] ?? "\uFFFD"
+		return out
+	}
+}