@hoardodile/host 0.0.0

package/src/media/seekable.ts

@@ -0,0 +1,35 @@
+import type { Readable } from "node:stream"
+import { ffmpegThumbSource, type ThumbInput } from "./thumb-input.ts"
+
+/**
+ * Run a media pipeline over an entry, materializing it to a seekable
+ * path first when the container format needs seeking — ISO-BMFF
+ * (.mp4/.mov/.m4v/.m4a) keeps its moov index at the end of the file,
+ * unreachable from a forward-only pipe — or when the stream attempt
+ * fails for another reason.
+ *
+ * One implementation shared by the thumbnail pipeline and the cover-meta
+ * probes, so the "streamable vs materialize" split can never diverge
+ * between the two paths again.
+ */
+export async function withMediaSource<T>(opts: {
+	readonly relPath: string
+	readonly ext: string
+	readonly input: ThumbInput
+	/** Extensions whose container is seekable from a forward-only pipe. */
+	readonly streamableExts: ReadonlySet<string>
+	/** Materialize `relPath` to a seekable on-disk path. */
+	readonly materialize: (relPath: string) => Promise<string>
+	readonly run: (source: string | Readable) => Promise<T>
+}): Promise<T> {
+	const { ext, input, streamableExts, materialize, relPath, run } = opts
+	if (input.kind === "stream" && !streamableExts.has(ext)) {
+		return run(await materialize(relPath))
+	}
+	try {
+		return await run(await ffmpegThumbSource(input))
+	} catch (streamErr) {
+		if (input.kind !== "stream") throw streamErr
+		return run(await materialize(relPath))
+	}
+}

package/src/media/thumb-input.test.ts

@@ -0,0 +1,109 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { buffer } from "node:stream/consumers"
+import sharp from "sharp"
+import { afterEach, beforeEach, describe, expect, test } from "vitest"
+import yazl from "yazl"
+import { createDirectoryContainer } from "../directory-container.ts"
+import { createNestedAwareContainer } from "../nested-view.ts"
+import { THUMB_BUFFER_MAX_BYTES } from "../probe/index.ts"
+import { withThumbInput } from "./thumb-input.ts"
+
+describe("withThumbInput", () => {
+	let root: string
+
+	beforeEach(() => {
+		root = mkdtempSync(join(tmpdir(), "thumb-input-"))
+	})
+
+	afterEach(() => {
+		rmSync(root, { recursive: true, force: true })
+	})
+
+	test("small image entries are read into memory without extraction", async () => {
+		const png = await sharp({
+			create: {
+				width: 24,
+				height: 24,
+				channels: 3,
+				background: { r: 4, g: 5, b: 6 },
+			},
+		})
+			.png()
+			.toBuffer()
+		writeFileSync(join(root, "a.png"), png)
+
+		const container = createDirectoryContainer(root)
+
+		let sawBuffer = false
+		await withThumbInput(container, "a.png", "image", async (input) => {
+			expect(input.kind).toBe("buffer")
+			if (input.kind === "buffer") {
+				sawBuffer = true
+				expect(input.buffer.equals(png)).toBe(true)
+			}
+			return "ok"
+		})
+		expect(sawBuffer).toBe(true)
+		expect(png.length).toBeLessThan(THUMB_BUFFER_MAX_BYTES)
+	})
+
+	test("literal video entries resolve to the seekable file path", async () => {
+		const mp4Bytes = Buffer.from("fake-mp4-bytes")
+		writeFileSync(join(root, "clip.mp4"), mp4Bytes)
+
+		const container = createDirectoryContainer(root)
+
+		await withThumbInput(container, "clip.mp4", "video", async (input) => {
+			expect(input.kind).toBe("path")
+			if (input.kind === "path") {
+				expect(input.path).toBe(join(root, "clip.mp4"))
+			}
+			return "ok"
+		})
+	})
+
+	test("large literal images resolve to the seekable file path", async () => {
+		const big = Buffer.alloc(THUMB_BUFFER_MAX_BYTES + 1, 0xa5)
+		writeFileSync(join(root, "big.png"), big)
+
+		const container = createDirectoryContainer(root)
+
+		await withThumbInput(container, "big.png", "image", async (input) => {
+			expect(input.kind).toBe("path")
+			if (input.kind === "path") {
+				expect(input.path).toBe(join(root, "big.png"))
+			}
+			return "ok"
+		})
+	})
+
+	test("virtual archive entries keep streaming (no seekable path)", async () => {
+		const zip = new yazl.ZipFile()
+		zip.addBuffer(Buffer.from("fake-mp4-bytes"), "clip.mp4")
+		zip.end()
+		const chunks: Buffer[] = []
+		for await (const chunk of zip.outputStream) {
+			chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk)
+		}
+		writeFileSync(join(root, "book.cbz"), Buffer.concat(chunks))
+
+		const container = createNestedAwareContainer(createDirectoryContainer(root))
+
+		await withThumbInput(
+			container,
+			"book.cbz!clip.mp4",
+			"video",
+			async (input) => {
+				expect(input.kind).toBe("stream")
+				if (input.kind === "stream") {
+					expect(input.size).toBe(Buffer.byteLength("fake-mp4-bytes"))
+					const read = await buffer(await input.openStream())
+					expect(read.toString()).toBe("fake-mp4-bytes")
+				}
+				return "ok"
+			},
+		)
+	})
+})

package/src/media/thumb-input.ts

@@ -0,0 +1,194 @@
+import { extname } from "node:path"
+import type { Readable } from "node:stream"
+import type { FileType } from "@hoardodile/sdk-types"
+import type { MediaKind } from "@hoardodile/sdk-types/media-exts"
+import type { ResourceContainer } from "../container.ts"
+import { notFound } from "../errors.ts"
+import {
+	SNIFF_HEADER_BYTES,
+	sniffBytes,
+	THUMB_BUFFER_MAX_BYTES,
+} from "../probe/index.ts"
+import type { PluginProbeCache } from "../probe-cache.ts"
+import type { ImageThumbInput } from "../render/index.ts"
+
+export type ThumbInput =
+	| { readonly kind: "path"; readonly path: string }
+	| { readonly kind: "buffer"; readonly buffer: Buffer }
+	| {
+			readonly kind: "stream"
+			readonly openStream: () => Promise<Readable>
+			readonly size: number
+			/**
+			 * Byte-range read into the entry (zip slice). Lets the render
+			 * pipeline's metadata step read a small header window instead
+			 * of streaming the whole entry — libvips full-reads
+			 * non-seekable input for metadata.
+			 */
+			readonly readRange?: (start: number, end: number) => Promise<Buffer>
+	  }
+
+export function imageThumbSource(input: ThumbInput): ImageThumbInput {
+	if (input.kind === "path") return input.path
+	if (input.kind === "buffer") return input.buffer
+	if (input.kind === "stream") {
+		return input.readRange === undefined
+			? { openStream: input.openStream }
+			: { openStream: input.openStream, readRange: input.readRange }
+	}
+	throw new Error("unsupported image thumb input")
+}
+
+/**
+ * Resolve an ffmpeg-bound entry (video, or audio whose embedded artwork
+ * ffmpeg demuxes as a still video stream) into a path or stream. ffmpeg
+ * reads both from `pipe:0`, so the two share one shape.
+ */
+export async function ffmpegThumbSource(
+	input: ThumbInput,
+): Promise<string | Readable> {
+	if (input.kind === "path") return input.path
+	if (input.kind === "stream") return input.openStream()
+	throw new Error("unsupported ffmpeg thumb input")
+}
+
+function streamThumbInput(
+	container: ResourceContainer,
+	relPath: string,
+	size: number,
+): ThumbInput {
+	return {
+		kind: "stream",
+		openStream: () =>
+			container.openEntryStream(relPath).then((entry) => entry.stream),
+		readRange: (start, end) => container.readEntrySlice(relPath, start, end),
+		size,
+	}
+}
+
+/**
+ * Prefer a seekable on-disk path over a stream when the container can
+ * provide one (bare-file backends): libvips/ffmpeg open the file
+ * directly — mmap + shrink-on-load for images, no pipe for video —
+ * with none of the pipe-copy overhead. Containers without the
+ * capability (in-memory fixtures, virtual `outer!inner` entries) fall
+ * back to streaming.
+ */
+async function pathOrStreamThumbInput(
+	container: ResourceContainer,
+	relPath: string,
+	size: number,
+): Promise<ThumbInput> {
+	const path = await container.resolveSeekablePath?.(relPath)
+	if (path !== undefined) return { kind: "path", path }
+	return streamThumbInput(container, relPath, size)
+}
+
+/**
+ * Identify a container entry from its bytes, falling back to its name.
+ * The thumbnail pipeline and the plugins that pick cover files must
+ * agree on what a file is, so both go through the same sniffer — an
+ * entry the plugin selected as an image never lands in the ffmpeg branch
+ * because of a misleading extension.
+ */
+async function sniffEntry(
+	container: ResourceContainer,
+	relPath: string,
+	opts: WithThumbInputOptions | undefined,
+): Promise<FileType | undefined> {
+	const readHead = async (): Promise<Buffer | undefined> =>
+		container
+			.readEntrySlice(relPath, 0, SNIFF_HEADER_BYTES)
+			.catch(() => undefined)
+	const head =
+		opts?.probeCache !== undefined && opts.cacheScope !== undefined
+			? await opts.probeCache.getOrCompute(
+					`${opts.cacheScope}:sniff:${relPath}`,
+					readHead,
+				)
+			: await readHead()
+	if (head === undefined) return undefined
+	return sniffBytes(head, relPath)
+}
+
+export type WithThumbInputOptions = {
+	/**
+	 * Identifier used in the missing-entry error message (e.g. the
+	 * resource id).
+	 */
+	readonly label?: string
+	/**
+	 * Shared probe cache, keyed by `cacheScope` — pass the same instance
+	 * and scope the plugin API uses so sniff results are reused across
+	 * hook runs and thumb jobs for one resource version.
+	 */
+	readonly probeCache?: PluginProbeCache
+	readonly cacheScope?: string
+}
+
+/**
+ * Resolve a container entry into the cheapest readable form for thumb
+ * synthesis: small images are read into memory; larger images, videos
+ * and audio stream directly from the source file without extracted-cache
+ * writes.
+ *
+ * `expected` guards the caller's assumption — `"any"` accepts whatever
+ * the entry turns out to be. The callback receives the **sniffed**
+ * extension, so sharp and ffmpeg get the container hint the content
+ * actually warrants.
+ */
+export async function withThumbInput<T>(
+	container: ResourceContainer,
+	relPath: string,
+	expected: MediaKind | "any",
+	fn: (input: ThumbInput, ext: string, kind: MediaKind) => Promise<T>,
+	opts?: WithThumbInputOptions,
+): Promise<T> {
+	const type = await sniffEntry(container, relPath, opts)
+	const kind = type?.kind ?? "other"
+	// An unidentified entry keeps the caller's expectation so the ffmpeg
+	// path still gets its chance, exactly as the extension fallback did.
+	const ext = type?.ext ?? extname(relPath).toLowerCase()
+	const effective: MediaKind =
+		type === undefined && expected !== "any" ? expected : kind
+	if (expected !== "any" && effective !== expected) {
+		throw new Error(`unsupported thumb media kind ${effective} for ${relPath}`)
+	}
+	if (effective === "image") {
+		const range = await requireEntryRange(container, relPath, opts?.label)
+		if (range.size <= THUMB_BUFFER_MAX_BYTES) {
+			const buffer = await container.readEntry(relPath)
+			return fn({ kind: "buffer", buffer }, ext, effective)
+		}
+		return fn(
+			await pathOrStreamThumbInput(container, relPath, range.size),
+			ext,
+			effective,
+		)
+	}
+	if (effective === "video" || effective === "audio") {
+		const range = await requireEntryRange(container, relPath, opts?.label)
+		return fn(
+			await pathOrStreamThumbInput(container, relPath, range.size),
+			ext,
+			effective,
+		)
+	}
+	throw new Error(`unsupported thumb media kind ${effective} for ${relPath}`)
+}
+
+async function requireEntryRange(
+	container: ResourceContainer,
+	relPath: string,
+	label: string | undefined,
+): Promise<{ readonly size: number }> {
+	const range = await container.resolveByteRange(relPath)
+	if (range === undefined) {
+		throw notFound(
+			"resource.file_not_found",
+			`resource ${label ?? "?"} has no entry ${relPath}`,
+			{ resId: label, relPath },
+		)
+	}
+	return range
+}

package/src/nested-view.test.ts

@@ -0,0 +1,223 @@
+// @vitest-environment node
+
+import { execFileSync } from "node:child_process"
+import {
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs"
+import { tmpdir } from "node:os"
+import { dirname, join } from "node:path"
+import { Readable } from "node:stream"
+import { afterEach, describe, expect, it } from "vitest"
+import { makeZip } from "./__testutils__/zip-fixtures.ts"
+import { resolveSevenZipPath } from "./archive/7z.ts"
+import { createArchiveExtractor } from "./archive/extract-archive.ts"
+import type { ResourceContainer } from "./container.ts"
+import { createNestedAwareContainer } from "./nested-view.ts"
+
+const sevenZipAvailable = resolveSevenZipPath() !== undefined
+
+function memoryContainer(
+	files: Readonly<Record<string, Buffer>>,
+): ResourceContainer {
+	return {
+		listEntries: async () => Object.keys(files).sort(),
+		readEntry: async (rel) => {
+			const buf = files[rel]
+			if (buf === undefined) throw new Error(`no entry ${rel}`)
+			return buf
+		},
+		readEntrySlice: async (rel, start, end) => {
+			const buf = files[rel]
+			if (buf === undefined) throw new Error(`no entry ${rel}`)
+			return buf.subarray(start, Math.min(end, buf.length))
+		},
+		openEntryStream: async (rel) => {
+			const buf = files[rel]
+			if (buf === undefined) throw new Error(`no entry ${rel}`)
+			return { stream: Readable.from([buf]), size: buf.length }
+		},
+		resolveByteRange: async (rel) => {
+			const buf = files[rel]
+			return buf === undefined ? undefined : { size: buf.length }
+		},
+	}
+}
+
+const DEFLATED = makeZip([
+	{
+		name: "Ch1/001.jpg",
+		data: Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+		method: 8,
+	},
+	{ name: "Ch1/002.jpg", data: Uint8Array.from([11, 12, 13]) },
+])
+
+describe("createNestedAwareContainer", () => {
+	const container = createNestedAwareContainer(
+		memoryContainer({
+			"book.cbz": DEFLATED,
+			"plain.txt": Buffer.from("hello world"),
+		}),
+	)
+
+	it("passes literal entries through unchanged", async () => {
+		expect(await container.resolveByteRange("plain.txt")).toEqual({ size: 11 })
+		expect(await container.readEntrySlice("plain.txt", 0, 5)).toEqual(
+			Buffer.from("hello"),
+		)
+	})
+
+	it("resolves virtual path sizes", async () => {
+		expect(await container.resolveByteRange("book.cbz!Ch1/002.jpg")).toEqual({
+			size: 3,
+		})
+		expect(
+			await container.resolveByteRange("book.cbz!missing.jpg"),
+		).toBeUndefined()
+	})
+
+	it("streams virtual entries with their decompressed size", async () => {
+		const { stream, size } = await container.openEntryStream(
+			"book.cbz!Ch1/001.jpg",
+		)
+		const chunks: Buffer[] = []
+		for await (const chunk of stream) chunks.push(Buffer.from(chunk))
+		expect(size).toBe(10)
+		expect(Buffer.concat(chunks)).toEqual(
+			Buffer.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
+		)
+	})
+
+	it("slices virtual entries without inflating the whole entry", async () => {
+		const head = await container.readEntrySlice("book.cbz!Ch1/001.jpg", 0, 4)
+		expect(head).toEqual(Buffer.from([1, 2, 3, 4]))
+	})
+
+	it("clamps out-of-range slices", async () => {
+		const tail = await container.readEntrySlice("book.cbz!Ch1/001.jpg", 8, 100)
+		expect(tail).toEqual(Buffer.from([9, 10]))
+	})
+
+	it("keeps listEntries flat", async () => {
+		expect(await container.listEntries()).toEqual(["book.cbz", "plain.txt"])
+	})
+})
+
+// ── Materialized addressing (extracted non-zip containers) ──────────────────
+
+function makeSevenZipArchive(
+	dir: string,
+	files: Readonly<Record<string, string>>,
+): Buffer {
+	const payload = join(dir, "payload")
+	mkdirSync(payload, { recursive: true })
+	for (const [name, content] of Object.entries(files)) {
+		const filePath = join(payload, name)
+		mkdirSync(dirname(filePath), { recursive: true })
+		writeFileSync(filePath, content)
+	}
+	const archivePath = join(dir, "book.cb7")
+	execFileSync(resolveSevenZipPath()!, ["a", "-t7z", archivePath, "."], {
+		cwd: payload,
+		stdio: "ignore",
+	})
+	return readFileSync(archivePath)
+}
+
+describe.skipIf(!sevenZipAvailable)(
+	"createNestedAwareContainer materialized addressing",
+	() => {
+		let root: string
+		let cacheDir: string
+
+		afterEach(() => {
+			rmSync(root, { recursive: true, force: true })
+		})
+
+		function extractorFor(base: ResourceContainer) {
+			return createArchiveExtractor({
+				outer: {
+					sizeOf: (rel) => base.resolveByteRange(rel).then((r) => r?.size),
+					readSlice: (rel, start, end) => base.readEntrySlice(rel, start, end),
+				},
+				cacheDir,
+				maxBytes: 1024 * 1024,
+				maxEntries: 100,
+			})
+		}
+
+		it("serves extracted entries from disk with seekable paths", async () => {
+			root = mkdtempSync(join(tmpdir(), "nested-view-"))
+			cacheDir = join(root, "cache")
+			const archive = makeSevenZipArchive(root, {
+				"Ch1/001.jpg": "first",
+				"Ch1/002.jpg": "second",
+			})
+			const base = memoryContainer({ "book.cb7": archive })
+			await extractorFor(base).extract("book.cb7")
+
+			const view = createNestedAwareContainer(
+				base,
+				undefined,
+				undefined,
+				cacheDir,
+			)
+			expect(await view.resolveByteRange("book.cb7!Ch1/001.jpg")).toEqual({
+				size: 5,
+			})
+			const { stream, size, path } = await view.openEntryStream(
+				"book.cb7!Ch1/002.jpg",
+			)
+			const chunks: Buffer[] = []
+			for await (const chunk of stream) chunks.push(Buffer.from(chunk))
+			expect(size).toBe(6)
+			expect(Buffer.concat(chunks).toString("utf8")).toBe("second")
+			expect(path).toBe(join(cacheDir, "book.cb7", "Ch1", "002.jpg"))
+			expect(await view.readEntrySlice("book.cb7!Ch1/001.jpg", 0, 3)).toEqual(
+				Buffer.from("fir"),
+			)
+			expect(await view.resolveSeekablePath?.("book.cb7!Ch1/001.jpg")).toBe(
+				join(cacheDir, "book.cb7", "Ch1", "001.jpg"),
+			)
+		})
+
+		it("does not address non-extracted containers", async () => {
+			root = mkdtempSync(join(tmpdir(), "nested-view-"))
+			cacheDir = join(root, "cache")
+			const archive = makeSevenZipArchive(root, { "a.txt": "x" })
+			const base = memoryContainer({ "book.cb7": archive })
+			const view = createNestedAwareContainer(
+				base,
+				undefined,
+				undefined,
+				cacheDir,
+			)
+			expect(await view.resolveByteRange("book.cb7!a.txt")).toBeUndefined()
+			expect(await view.resolveSeekablePath?.("book.cb7!a.txt")).toBeUndefined()
+		})
+
+		it("ignores paths outside the manifest whitelist", async () => {
+			root = mkdtempSync(join(tmpdir(), "nested-view-"))
+			cacheDir = join(root, "cache")
+			const archive = makeSevenZipArchive(root, { "a.txt": "x" })
+			const base = memoryContainer({ "book.cb7": archive })
+			await extractorFor(base).extract("book.cb7")
+			const view = createNestedAwareContainer(
+				base,
+				undefined,
+				undefined,
+				cacheDir,
+			)
+			expect(
+				await view.resolveByteRange("book.cb7!missing.txt"),
+			).toBeUndefined()
+			expect(
+				await view.resolveByteRange("book.cb7!../escape.txt"),
+			).toBeUndefined()
+		})
+	},
+)