@hoardodile/host 0.0.0

package/src/archive/nested-cd-cache.ts

@@ -0,0 +1,55 @@
+/**
+ * Process-wide cache of nested container listings (central directories
+ * of zip/tar entries addressed via virtual paths). Resolving
+ * `outer!inner` re-parses the outer archive's CD on every hook call;
+ * this cache keeps the parsed listing resident so a multi-hook pass
+ * (sourceMeta → cover → hashes → listFiles) parses each nested archive
+ * once.
+ *
+ * Keyed by the outer entry name only — callers scope the cache instance
+ * to a fixed (resId, fileVersion), and archives are immutable per
+ * version, so a key never outlives its content.
+ */
+export type NestedCdCache = {
+	readonly get: (key: string) => Promise<unknown> | undefined
+	readonly set: (key: string, value: Promise<unknown>) => void
+	readonly clear: () => void
+}
+
+export type NestedCdCacheOptions = {
+	/** Max number of nested archives kept resident. */
+	readonly maxEntries: number
+}
+
+const DEFAULT_MAX_ENTRIES = 512
+
+/** Create an LRU-bounded nested listing cache. */
+export function createNestedCdCache(
+	opts?: Partial<NestedCdCacheOptions>,
+): NestedCdCache {
+	const maxEntries = opts?.maxEntries ?? DEFAULT_MAX_ENTRIES
+	const lru = new Map<string, Promise<unknown>>()
+
+	function get(key: string): Promise<unknown> | undefined {
+		const value = lru.get(key)
+		if (value === undefined) return undefined
+		lru.delete(key)
+		lru.set(key, value)
+		return value
+	}
+
+	function set(key: string, value: Promise<unknown>): void {
+		lru.set(key, value)
+		while (lru.size > maxEntries) {
+			const oldest = lru.keys().next().value
+			if (oldest === undefined) break
+			lru.delete(oldest)
+		}
+	}
+
+	function clear(): void {
+		lru.clear()
+	}
+
+	return { get, set, clear }
+}

package/src/archive/nested-entry.test.ts

@@ -0,0 +1,194 @@
+// @vitest-environment node
+
+import { describe, expect, it } from "vitest"
+import { makeTar, makeZip } from "../__testutils__/zip-fixtures.ts"
+import { sniffContainerFormat } from "./format.ts"
+import {
+	createNestedResolver,
+	type OuterEntrySource,
+	splitVirtualPath,
+} from "./nested-entry.ts"
+
+// ── Fixture helpers ──────────────────────────────────────────────────────────
+
+function outerFrom(files: Readonly<Record<string, Buffer>>): OuterEntrySource {
+	return {
+		sizeOf: async (rel) => {
+			const buf = files[rel]
+			return buf === undefined ? undefined : buf.length
+		},
+		readSlice: async (rel, start, end) => {
+			const buf = files[rel]
+			if (buf === undefined) throw new Error(`missing fixture entry ${rel}`)
+			return buf.subarray(start, Math.min(end, buf.length))
+		},
+	}
+}
+
+// ── splitVirtualPath ─────────────────────────────────────────────────────────
+
+describe("splitVirtualPath", () => {
+	it("splits at the first separator", () => {
+		expect(splitVirtualPath("a.cbz!Ch1/001.jpg")).toEqual({
+			outer: "a.cbz",
+			inner: "Ch1/001.jpg",
+		})
+	})
+
+	it("handles separators inside the inner path", () => {
+		expect(splitVirtualPath("a.cbz!b!c.png")).toEqual({
+			outer: "a.cbz",
+			inner: "b!c.png",
+		})
+	})
+
+	it("returns undefined without a separator or with an empty inner", () => {
+		expect(splitVirtualPath("plain.jpg")).toBeUndefined()
+		expect(splitVirtualPath("a.cbz!")).toBeUndefined()
+		expect(splitVirtualPath("!a")).toBeUndefined()
+	})
+})
+
+// ── sniffContainerFormat ─────────────────────────────────────────────────────
+
+describe("sniffContainerFormat", () => {
+	it("detects zip and tar magic", () => {
+		expect(
+			sniffContainerFormat(Uint8Array.from([0x50, 0x4b, 0x03, 0x04])),
+		).toBe("zip")
+		const tarHead = new Uint8Array(300)
+		tarHead.set(Buffer.from("ustar"), 257)
+		expect(sniffContainerFormat(tarHead)).toBe("tar")
+	})
+
+	it("detects gzip magic (extractable, not a nested container)", () => {
+		expect(
+			sniffContainerFormat(Uint8Array.from([0x1f, 0x8b, 0x08, 0x00])),
+		).toBe("gzip")
+	})
+
+	it("returns undefined for other content", () => {
+		expect(sniffContainerFormat(new Uint8Array([1, 2, 3, 4]))).toBeUndefined()
+		expect(sniffContainerFormat(new Uint8Array(0))).toBeUndefined()
+	})
+})
+
+// ── createNestedResolver ─────────────────────────────────────────────────────
+
+describe("createNestedResolver", () => {
+	const cbz = makeZip([
+		{ name: "Ch1/001.jpg", data: Uint8Array.from([1, 2, 3]) },
+		{ name: "Ch1/002.jpg", data: Uint8Array.from([4, 5]) },
+		{ name: "Ch2/001.jpg", data: Uint8Array.from([6]) },
+	])
+	const deflated = makeZip([
+		{
+			name: "page.png",
+			data: Uint8Array.from([9, 9, 9, 9, 9, 9, 9, 9, 9, 9]),
+			method: 8,
+		},
+	])
+	const tar = makeTar([
+		{ name: "Ch1/001.jpg", data: Uint8Array.from([7, 7]) },
+		{ name: "Ch1/002.jpg", data: Uint8Array.from([8, 8, 8]) },
+	])
+	const outer = outerFrom({
+		"book.cbz": cbz,
+		"flat.png": Buffer.from([1, 2]),
+		"tome.tar": tar,
+		"weird!name.txt": Buffer.from("literal"),
+	})
+
+	it("resolves zip entries with sizes", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("book.cbz!Ch1/001.jpg")
+		expect(resolved.kind).toBe("container")
+		if (resolved.kind !== "container") return
+		expect(resolved.outer).toBe("book.cbz")
+		expect(resolved.entry).toEqual({ name: "Ch1/001.jpg", sizeBytes: 3 })
+	})
+
+	it("streams STORED entry bytes", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("book.cbz!Ch1/002.jpg")
+		if (resolved.kind !== "container") throw new Error("expected container")
+		const chunks: Buffer[] = []
+		for await (const chunk of resolved.openStream()) {
+			chunks.push(Buffer.from(chunk))
+		}
+		expect(Buffer.concat(chunks)).toEqual(Buffer.from([4, 5]))
+	})
+
+	it("streams DEFLATE entry bytes", async () => {
+		const resolver = createNestedResolver(outerFrom({ "d.zip": deflated }))
+		const resolved = await resolver.resolve("d.zip!page.png")
+		if (resolved.kind !== "container") throw new Error("expected container")
+		const chunks: Buffer[] = []
+		for await (const chunk of resolved.openStream()) {
+			chunks.push(Buffer.from(chunk))
+		}
+		expect(Buffer.concat(chunks)).toEqual(
+			Buffer.from([9, 9, 9, 9, 9, 9, 9, 9, 9, 9]),
+		)
+		expect(resolved.entry.sizeBytes).toBe(10)
+	})
+
+	it("does not address tar archives as nested containers", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("tome.tar!Ch1/001.jpg")
+		expect(resolved.kind).toBe("literal")
+		expect(await resolver.list("tome.tar")).toBeUndefined()
+	})
+
+	it("falls back to literal when the inner entry is missing", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("book.cbz!missing.jpg")
+		expect(resolved.kind).toBe("literal")
+	})
+
+	it("falls back to literal when the outer is not a container", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("flat.png!x")
+		expect(resolved.kind).toBe("literal")
+	})
+
+	it("keeps literal files whose names contain the separator", async () => {
+		const resolver = createNestedResolver(outer)
+		const resolved = await resolver.resolve("weird!name.txt")
+		expect(resolved.kind).toBe("literal")
+	})
+
+	it("lists container entries memoized per outer", async () => {
+		const resolver = createNestedResolver(outer)
+		const first = await resolver.list("book.cbz")
+		const second = await resolver.list("book.cbz")
+		expect(first?.map((e) => e.name)).toEqual([
+			"Ch1/001.jpg",
+			"Ch1/002.jpg",
+			"Ch2/001.jpg",
+		])
+		expect(second).toBe(first)
+		expect(await resolver.list("flat.png")).toBeUndefined()
+	})
+
+	it("rejects encrypted entries with a clear error", async () => {
+		// DEFLATE, not STORED: for stored files yauzl rejects the
+		// inconsistent sizes at central-directory parse time, so the
+		// per-entry encrypted rejection could never fire.
+		const encrypted = makeZip([
+			{
+				name: "page.png",
+				data: Uint8Array.from([1, 2, 3]),
+				method: 8,
+				encrypted: true,
+			},
+		])
+		const resolver = createNestedResolver(
+			outerFrom({ "locked.zip": encrypted }),
+		)
+		const resolved = await resolver.resolve("locked.zip!page.png")
+		expect(resolved.kind).toBe("container")
+		if (resolved.kind !== "container") return
+		expect(() => resolved.openStream()).toThrow(/encrypted/)
+	})
+})

package/src/archive/nested-entry.ts

@@ -0,0 +1,254 @@
+import type { Readable } from "node:stream"
+import { invalid } from "../errors.ts"
+import { SNIFF_WINDOW_BYTES, sniffContainerFormat } from "./format.ts"
+import type { NestedCdCache } from "./nested-cd-cache.ts"
+import {
+	type ArchiveSource,
+	listZipEntriesFromSource,
+	openZipEntryStream,
+} from "./zip-entries.ts"
+
+/**
+ * Container-entry addressing: plugin hooks (and the server view) may
+ * reference a file *inside* a container entry with the virtual path
+ * `outer!inner` — e.g. `manga.cbz!Chapter 1/001.jpg`. This module owns
+ * the resolution rules and the zip listing (format identification
+ * lives in `format.ts`); nested containers reuse `zip-entries.ts` over
+ * a range source. Zip is the only addressable nested format —
+ * tar/7z/rar/xz/gzip are whole-archive formats (extractable via 7-Zip,
+ * not addressable by single entry).
+ *
+ * Ambiguity rule: `outer` is treated as a container only when its magic
+ * bytes say so (zip) AND the archive actually contains `inner`;
+ * otherwise the whole path is the literal container entry. A file whose
+ * name contains `!` therefore stays addressable.
+ */
+
+/** The virtual-path separator between an outer entry and an inner one. */
+export const VIRTUAL_PATH_SEPARATOR = "!"
+
+/**
+ * Read interface an outer container must satisfy. Both the plugin-side
+ * `ResourceContainer` and the server's artifact view provide these — the
+ * virtual resolver is deliberately minimal so it composes with either.
+ */
+export type OuterEntrySource = {
+	/** Byte length of a literal entry; `undefined` when absent. */
+	readonly sizeOf: (rel: string) => Promise<number | undefined>
+	/** Read `[start, end)` of a literal entry (`end` exclusive). */
+	readonly readSlice: (
+		rel: string,
+		start: number,
+		end: number,
+	) => Promise<Buffer>
+}
+
+/** A unified, format-agnostic inner entry. */
+export type ArchiveEntry = {
+	readonly name: string
+	readonly sizeBytes: number
+}
+
+/** An inner entry plus its stream opener (format handles attached). */
+export type OpenableArchiveEntry = ArchiveEntry & {
+	readonly openStream: () => Readable
+}
+
+export type PathResolution =
+	| { readonly kind: "literal" }
+	| {
+			readonly kind: "container"
+			readonly outer: string
+			readonly entry: ArchiveEntry
+			/** Decompressed bytes of the inner entry. */
+			readonly openStream: () => Readable
+	  }
+
+export type NestedResolver = {
+	/**
+	 * Resolve `path` against the outer container. `kind: "container"`
+	 * means the path addressed a file inside a nested zip/tar; anything
+	 * else keeps the literal semantics (a plain entry, possibly one whose
+	 * name happens to contain `!`).
+	 */
+	readonly resolve: (path: string) => Promise<PathResolution>
+	/**
+	 * List every file entry of the container `outer`. Resolves to
+	 * `undefined` when `outer` is not a container (or does not exist).
+	 * Memoized per outer name for the resolver's lifetime.
+	 */
+	readonly list: (
+		outer: string,
+	) => Promise<readonly OpenableArchiveEntry[] | undefined>
+}
+
+/** Split `outer!inner` at the first separator. */
+export function splitVirtualPath(path: string):
+	| {
+			readonly outer: string
+			readonly inner: string
+	  }
+	| undefined {
+	const sep = path.indexOf(VIRTUAL_PATH_SEPARATOR)
+	if (sep <= 0) return undefined
+	const inner = path.slice(sep + 1)
+	if (inner.length === 0) return undefined
+	return { outer: path.slice(0, sep), inner }
+}
+
+/** Wrap an outer container entry as a {@link ArchiveSource} (real size). */
+export function createOuterArchiveSource(
+	outer: OuterEntrySource,
+	outerName: string,
+	size: number,
+): ArchiveSource {
+	return {
+		size,
+		readRange: (start, end) => outer.readSlice(outerName, start, end + 1),
+	}
+}
+
+/**
+ * Build a resolver over one outer container. Archive listings are
+ * memoized per outer entry name for the resolver's lifetime (a
+ * resource-level operation parses each nested archive at most once),
+ * and — when a {@link NestedCdCache} is provided — across resolver
+ * instances, so a multi-hook pass parses each nested archive once.
+ *
+ * `scope` disambiguates cache keys when one shared cache serves many
+ * containers (e.g. every resource of a server): keys become
+ * `<scope>:<outerName>`, so two resources holding same-named archives
+ * never serve each other's listings.
+ */
+export function createNestedResolver(
+	outer: OuterEntrySource,
+	opts: { readonly cdCache?: NestedCdCache; readonly scope?: string } = {},
+): NestedResolver {
+	const cdCache = opts.cdCache
+	const scope = opts.scope
+	const cacheKey = (outerName: string): string =>
+		scope === undefined ? outerName : `${scope}:${outerName}`
+	const listingMemo = new Map<
+		string,
+		Promise<readonly ListedEntry[] | undefined>
+	>()
+	const listedMemo = new Map<
+		string,
+		Promise<readonly OpenableArchiveEntry[] | undefined>
+	>()
+
+	function listOuter(
+		outerName: string,
+	): Promise<readonly ListedEntry[] | undefined> {
+		const key = cacheKey(outerName)
+		const pending = listingMemo.get(key)
+		if (pending !== undefined) return pending
+		const shared = cdCache?.get(key) as
+			| Promise<readonly ListedEntry[] | undefined>
+			| undefined
+		if (shared !== undefined) {
+			listingMemo.set(key, shared)
+			return shared
+		}
+		const work = listOuterOnce(outerName)
+		listingMemo.set(key, work)
+		cdCache?.set(key, work)
+		void work.then(
+			() => undefined,
+			() => undefined,
+		)
+		return work
+	}
+
+	async function listOuterOnce(
+		outerName: string,
+	): Promise<readonly ListedEntry[] | undefined> {
+		const size = await outer.sizeOf(outerName)
+		if (size === undefined) return undefined
+		if (size === 0) return undefined
+		const head = await outer.readSlice(
+			outerName,
+			0,
+			Math.min(size, SNIFF_WINDOW_BYTES),
+		)
+		const format = sniffContainerFormat(head)
+		// Zip is the only addressable nested format; tar/7z/rar/xz/gzip
+		// are whole-archive formats and report "not a container".
+		if (format !== "zip") return undefined
+		const source = createOuterArchiveSource(outer, outerName, size)
+		const records = await listZipEntriesFromSource(source)
+		return records.map((r) => ({
+			name: r.name,
+			sizeBytes: r.uncompressedSize,
+			outerSize: size,
+			zip: r,
+		}))
+	}
+
+	async function resolve(path: string): Promise<PathResolution> {
+		const parts = splitVirtualPath(path)
+		if (parts === undefined) return { kind: "literal" }
+		const { outer: outerName, inner } = parts
+		const entries = await listOuter(outerName)
+		const found = entries?.find((e) => e.name === inner)
+		if (found === undefined) return { kind: "literal" }
+		return {
+			kind: "container",
+			outer: outerName,
+			entry: { name: found.name, sizeBytes: found.sizeBytes },
+			openStream: () => openListedEntry(outer, outerName, found),
+		}
+	}
+
+	async function list(
+		outerName: string,
+	): Promise<readonly OpenableArchiveEntry[] | undefined> {
+		const pending = listedMemo.get(outerName)
+		if (pending !== undefined) return pending
+		const work = listOuter(outerName).then((entries) =>
+			entries === undefined
+				? undefined
+				: entries.map((e) => listedWithStream(outer, outerName, e)),
+		)
+		listedMemo.set(outerName, work)
+		void work.then(
+			() => undefined,
+			() => undefined,
+		)
+		return work
+	}
+
+	return { resolve, list }
+}
+
+type ListedEntry = ArchiveEntry & {
+	/** Byte size of the outer container entry. */
+	readonly outerSize: number
+	readonly zip?: import("./zip-entries.ts").ZipEntry
+}
+
+function listedWithStream(
+	outer: OuterEntrySource,
+	outerName: string,
+	entry: ListedEntry,
+): OpenableArchiveEntry {
+	return {
+		name: entry.name,
+		sizeBytes: entry.sizeBytes,
+		openStream: () => openListedEntry(outer, outerName, entry),
+	}
+}
+
+function openListedEntry(
+	outer: OuterEntrySource,
+	outerName: string,
+	entry: ListedEntry,
+): Readable {
+	const source = createOuterArchiveSource(outer, outerName, entry.outerSize)
+	if (entry.zip !== undefined) return openZipEntryStream(source, entry.zip)
+	throw invalid(
+		"resource.archive_open_failed",
+		`entry "${entry.name}" has no stream backend`,
+		{ name: entry.name },
+	)
+}

package/src/archive/pack.ts

@@ -0,0 +1,40 @@
+import type { Readable } from "node:stream"
+import yazl from "yazl"
+
+/**
+ * On-the-fly zip exports. Resource sources are stored as bare files on
+ * disk — nothing is packed at commit time; this module streams STORED
+ * zip bytes straight to the consumer without a staging file.
+ */
+
+/**
+ * A logical zip entry whose bytes are produced on demand. Streams are
+ * read once, in order, when the output is consumed.
+ */
+export type ZipStreamEntry = {
+	readonly name: string
+	readonly size: number
+	readonly openStream: () => Readable
+}
+
+/**
+ * Stream a STORED zip from an ordered list of logical entries. Used by
+ * the HTTP layer for exports (resource source packs, bulk downloads)
+ * without a staging file on disk. Zero-length entries are packed as
+ * empty buffers.
+ */
+export function streamStoredZip(entries: readonly ZipStreamEntry[]) {
+	const zip = new yazl.ZipFile()
+	for (const entry of entries) {
+		if (entry.size === 0) {
+			zip.addBuffer(Buffer.alloc(0), entry.name, { compress: false })
+			continue
+		}
+		zip.addReadStream(entry.openStream(), entry.name, {
+			compress: false,
+			size: entry.size,
+		})
+	}
+	zip.end()
+	return zip.outputStream
+}

package/src/archive/zip-entries.test.ts

@@ -0,0 +1,56 @@
+import { execSync } from "node:child_process"
+import {
+	closeSync,
+	openSync,
+	truncateSync,
+	unlinkSync,
+	writeSync,
+} from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { afterEach, describe, expect, test } from "vitest"
+import { readFileRange } from "./zip-entries.ts"
+
+const INT32_MAX = 2_147_483_647
+
+describe("readFileRange", () => {
+	const paths: string[] = []
+
+	afterEach(() => {
+		for (const path of paths.splice(0)) {
+			try {
+				unlinkSync(path)
+			} catch {
+				// already removed
+			}
+		}
+	})
+
+	// Generous timeout: without the sparse flag, the positional write past
+	// the 2 GiB mark makes NTFS zero-fill the gap synchronously.
+	test("reads bytes beyond the Int32 position limit", {
+		timeout: 120_000,
+	}, async () => {
+		const path = join(tmpdir(), `hoard-read-range-${Date.now()}.bin`)
+		paths.push(path)
+		const fd = openSync(path, "w")
+		closeSync(fd)
+		if (process.platform === "win32") {
+			// Skip the ~2 GiB zero-fill NTFS would otherwise write on the
+			// positional write below. Best effort — fsutil needs elevation,
+			// and the plain write still works (slowly) without it.
+			try {
+				execSync(`fsutil sparse setflag "${path}"`)
+			} catch {
+				// fall back to the slow fill
+			}
+		}
+		truncateSync(path, INT32_MAX + 64)
+		const rw = openSync(path, "r+")
+		writeSync(rw, Buffer.from("0123456789"), 0, 10, INT32_MAX + 1)
+		closeSync(rw)
+
+		const buf = await readFileRange(path, INT32_MAX + 1, INT32_MAX + 10)
+		expect(buf.toString("utf8")).toBe("0123456789")
+	})
+})