@hoardodile/host 0.0.0

package/src/hash.test.ts

@@ -0,0 +1,301 @@
+import { createHash } from "node:crypto"
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { Readable } from "node:stream"
+import sharp from "sharp"
+import { describe, expect, test } from "vitest"
+import { createDirectoryResourceAPI } from "./directory-api.ts"
+import {
+	computeDHash,
+	computePHash,
+	grayStddev,
+	hashStream,
+	PHASH_GRID,
+} from "./hash.ts"
+
+function grayGrid(values: readonly number[], grid = PHASH_GRID): Uint8Array {
+	return Uint8Array.from({ length: grid * grid }, (_, i) => values[i] ?? 0)
+}
+
+describe("computeDHash", () => {
+	test("is deterministic and returns 16 lowercase hex chars", () => {
+		const gray = grayGrid(
+			Array.from({ length: PHASH_GRID * PHASH_GRID }, (_, i) => i % 256),
+		)
+		const first = computeDHash(gray)
+		expect(first).toBe(computeDHash(gray))
+		expect(first).toMatch(/^[0-9a-f]{16}$/)
+	})
+
+	test("constant image hashes to all-zero bits", () => {
+		expect(computeDHash(grayGrid([]))).toBe("0000000000000000")
+		// Regression: the old implementation compared the last sample
+		// against out-of-bounds black, pinning the final bit of every row
+		// to 1 even for a full-bright image.
+		expect(computeDHash(grayGrid(Array(1024).fill(255)))).toBe(
+			"0000000000000000",
+		)
+	})
+
+	test("left-to-right gradient differs from right-to-left", () => {
+		// Boundary at 14 sits between adjacent samples (12 and 16) so the
+		// comparison grid can see the direction of the transition.
+		const leftDark = grayGrid(
+			Array.from({ length: PHASH_GRID * PHASH_GRID }, (_, i) => {
+				const x = i % PHASH_GRID
+				return x < 14 ? 0 : 255
+			}),
+		)
+		const rightDark = grayGrid(
+			Array.from({ length: PHASH_GRID * PHASH_GRID }, (_, i) => {
+				const x = i % PHASH_GRID
+				return x < 14 ? 255 : 0
+			}),
+		)
+		expect(computeDHash(leftDark)).not.toBe(computeDHash(rightDark))
+	})
+})
+
+describe("computePHash", () => {
+	test("is deterministic and returns 16 lowercase hex chars", () => {
+		const gray = grayGrid(
+			Array.from({ length: PHASH_GRID * PHASH_GRID }, (_, i) => (i * 7) % 256),
+		)
+		const first = computePHash(gray)
+		expect(first).toBe(computePHash(gray))
+		expect(first).toMatch(/^[0-9a-f]{16}$/)
+	})
+
+	test("constant image hashes to all-zero bits", () => {
+		const gray = grayGrid([])
+		expect(computePHash(gray)).toBe("0000000000000000")
+	})
+
+	test("nearly identical images differ in only a few bits", () => {
+		const base = Array.from(
+			{ length: PHASH_GRID * PHASH_GRID },
+			(_, i) => (i * 13 + (i % 7)) % 256,
+		)
+		const baseHash = computePHash(grayGrid(base))
+		const noisy = base.map((v, i) => (i % 97 === 0 ? (v + 3) % 256 : v))
+		const noisyHash = computePHash(grayGrid(noisy))
+		const distance = popcount(
+			BigInt(`0x${baseHash}`) ^ BigInt(`0x${noisyHash}`),
+		)
+		expect(distance).toBeLessThan(8)
+	})
+})
+
+describe("grayStddev", () => {
+	test("is zero for constant buffers", () => {
+		expect(grayStddev(grayGrid([]))).toBe(0)
+		expect(grayStddev(grayGrid(Array(1024).fill(255)))).toBe(0)
+	})
+
+	test("measures spread around the mean", () => {
+		const half = grayGrid([...Array(512).fill(0), ...Array(512).fill(255)])
+		expect(grayStddev(half)).toBeCloseTo(127.5, 5)
+	})
+
+	test("low-contrast ramps stay below the flat threshold", () => {
+		const ramp = grayGrid(
+			Array.from({ length: PHASH_GRID * PHASH_GRID }, (_, i) => {
+				const x = i % PHASH_GRID
+				return 100 + Math.floor((x * 20) / PHASH_GRID)
+			}),
+		)
+		expect(grayStddev(ramp)).toBeLessThan(8)
+	})
+})
+
+describe("hashStream", () => {
+	test("hashes streamed bytes like node:crypto", async () => {
+		const stream = Readable.from([Buffer.from("hello "), Buffer.from("world")])
+		expect(await hashStream(stream, "sha256")).toBe(
+			"b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9",
+		)
+	})
+})
+
+function popcount(v: bigint): number {
+	let count = 0
+	while (v > 0n) {
+		v &= v - 1n
+		count++
+	}
+	return count
+}
+
+// ── End-to-end through the ResourceAPI decode pipeline ──────────────
+
+/**
+ * Deterministic luminance structure: smooth R/G ramps plus a slow sine
+ * ripple (period scales with the image, so it stays coherent through
+ * the 32×32 hash-grid downsample — a fixed-pixel checkerboard would
+ * alias differently across resize paths). The horizontal comparison
+ * bits of dHash track the R ramp direction, so `flipX` inverts nearly
+ * every bit — a clean "different content" pair.
+ */
+function makeGradient(
+	width: number,
+	height: number,
+	opts: { readonly flipX?: boolean } = {},
+): Buffer {
+	const bytes = Buffer.alloc(width * height * 3)
+	const period = width / 16
+	for (let y = 0; y < height; y++) {
+		for (let x = 0; x < width; x++) {
+			const i = (y * width + x) * 3
+			const ramp =
+				opts.flipX === true ? 255 - (x * 255) / width : (x * 255) / width
+			bytes[i] = Math.round(ramp)
+			bytes[i + 1] = Math.round((y * 255) / height)
+			bytes[i + 2] = Math.round(128 + 96 * Math.sin((x + y) / period))
+		}
+	}
+	return bytes
+}
+
+async function writePng(
+	dir: string,
+	name: string,
+	width: number,
+	height: number,
+	opts: { readonly flipX?: boolean } = {},
+): Promise<string> {
+	const path = join(dir, name)
+	await sharp(makeGradient(width, height, opts), {
+		raw: { width, height, channels: 3 },
+	})
+		.png()
+		.toFile(path)
+	return path
+}
+
+/** A solid-color PNG — low-information content the hash path must skip. */
+async function flatPng(): Promise<Buffer> {
+	return sharp(Buffer.alloc(64 * 64 * 3, 200), {
+		raw: { width: 64, height: 64, channels: 3 },
+	})
+		.png()
+		.toBuffer()
+}
+
+function hamming(a: string, b: string): number {
+	return popcount(BigInt(`0x${a}`) ^ BigInt(`0x${b}`))
+}
+
+describe("computeImageHashes end-to-end (real sharp decode)", () => {
+	let dir: string
+
+	test.beforeEach(() => {
+		dir = mkdtempSync(join(tmpdir(), "hash-e2e-"))
+	})
+
+	test.afterEach(() => {
+		rmSync(dir, { recursive: true, force: true })
+	})
+
+	test("hashes a real PNG deterministically with the expected formats", async () => {
+		await writePng(dir, "a.png", 400, 300)
+		const api = createDirectoryResourceAPI(dir)
+
+		const first = await api.computeImageHashes("a.png", [
+			"sha256",
+			"dhash",
+			"phash",
+		])
+		const second = await api.computeImageHashes("a.png", [
+			"sha256",
+			"dhash",
+			"phash",
+		])
+
+		expect(first).toEqual(second)
+		expect(first?.sha256).toMatch(/^[0-9a-f]{64}$/)
+		expect(first?.dhash).toMatch(/^[0-9a-f]{16}$/)
+		expect(first?.phash).toMatch(/^[0-9a-f]{16}$/)
+	})
+
+	test("scaled + recompressed copies stay perceptually close", async () => {
+		await writePng(dir, "a.png", 400, 300)
+		// Downscaled, recompressed JPEG of the same image.
+		const recompressed = await sharp(makeGradient(400, 300), {
+			raw: { width: 400, height: 300, channels: 3 },
+		})
+			.resize(200, 150)
+			.jpeg({ quality: 60 })
+			.toBuffer()
+		writeFileSync(join(dir, "b.jpg"), recompressed)
+		const api = createDirectoryResourceAPI(dir)
+
+		const a = await api.computeImageHashes("a.png", [
+			"sha256",
+			"dhash",
+			"phash",
+		])
+		const b = await api.computeImageHashes("b.jpg", [
+			"sha256",
+			"dhash",
+			"phash",
+		])
+
+		// Byte-exact hash must differ (re-encoding changed the bytes)…
+		expect(a?.sha256).not.toBe(b?.sha256)
+		// …but both perceptual hashes stay within the similarity bounds
+		// (mirrors SIMILAR_MAX_DISTANCE_BY_TYPE in the server hash service).
+		expect(hamming(a?.dhash ?? "", b?.dhash ?? "")).toBeLessThanOrEqual(8)
+		expect(hamming(a?.phash ?? "", b?.phash ?? "")).toBeLessThanOrEqual(6)
+	})
+
+	test("flat images yield no perceptual hashes but keep sha256", async () => {
+		writeFileSync(join(dir, "flat.png"), await flatPng())
+		const api = createDirectoryResourceAPI(dir)
+
+		await expect(
+			api.computeImageHashes("flat.png", ["dhash", "phash"]),
+		).resolves.toBeUndefined()
+
+		const withSha = await api.computeImageHashes("flat.png", [
+			"sha256",
+			"dhash",
+			"phash",
+		])
+		expect(withSha?.sha256).toMatch(/^[0-9a-f]{64}$/)
+		expect(withSha?.dhash).toBeUndefined()
+		expect(withSha?.phash).toBeUndefined()
+	})
+
+	test("different content lands far away in perceptual space", async () => {
+		await writePng(dir, "a.png", 400, 300)
+		await writePng(dir, "b.png", 400, 300, { flipX: true })
+		const api = createDirectoryResourceAPI(dir)
+
+		const a = await api.computeImageHashes("a.png", ["dhash"])
+		const b = await api.computeImageHashes("b.png", ["dhash"])
+
+		expect(hamming(a?.dhash ?? "", b?.dhash ?? "")).toBeGreaterThan(8)
+	})
+
+	test("non-image files resolve to undefined for any kinds", async () => {
+		writeFileSync(join(dir, "note.txt"), "hello")
+		const api = createDirectoryResourceAPI(dir)
+
+		await expect(
+			api.computeImageHashes("note.txt", ["dhash"]),
+		).resolves.toBeUndefined()
+		await expect(
+			api.computeImageHashes("note.txt", ["sha256", "dhash"]),
+		).resolves.toBeUndefined()
+	})
+
+	test("hashBytes matches node:crypto on the raw bytes", async () => {
+		await writePng(dir, "a.png", 64, 64)
+		const api = createDirectoryResourceAPI(dir)
+		const expected = createHash("sha256")
+			.update(readFileSync(join(dir, "a.png")))
+			.digest("hex")
+		await expect(api.hashBytes("a.png", "sha256")).resolves.toBe(expected)
+	})
+})

package/src/hash.ts

@@ -0,0 +1,167 @@
+import { createHash } from "node:crypto"
+import type { Readable } from "node:stream"
+import type { ImageHashKind } from "@hoardodile/sdk-types"
+
+/**
+ * Content hashing for the `imageHashes` plugin hook. All perceptual
+ * hashes derive from one 32×32 grayscale rendition (single sharp
+ * decode), so requesting several kinds costs one decode:
+ * - `sha256`: exact digest of the raw entry bytes (streamed).
+ * - `dhash`: difference hash — 9 samples per row of the 32×32 grid
+ *   (nearest-neighbour spread), adjacent pairs compared, equivalent to
+ *   the standard 9×8 dHash.
+ * - `phash`: standard DCT-based perceptual hash — the 32×32 grid is
+ *   mean-pooled to 8×8, then the 8×8 DCT coefficients are thresholded
+ *   at the median.
+ *
+ * The 64-bit values are hex strings (16 chars), compared by Hamming
+ * distance in `@hoardodile/sdk-server` / the server's hash service.
+ */
+
+/** Side of the grayscale grid every perceptual hash is derived from. */
+export const PHASH_GRID = 32
+
+/**
+ * Minimum grayscale standard deviation (0–255 scale) for an image to
+ * yield perceptual hashes. Near-flat images (solid fills, blank pages)
+ * hash to distance-0 neighbours of every other flat image, so they are
+ * skipped — `sha256` exact hashing is unaffected.
+ */
+export const MIN_PERCEPTUAL_STDDEV = 8
+
+/** Stream `stream` through a digest of `algo` and resolve its lowercase hex. */
+export function hashStream(
+	stream: Readable,
+	algo: "md5" | "sha256",
+): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const hash = createHash(algo)
+		stream.on("data", (chunk: string | Buffer) => hash.update(chunk))
+		stream.on("end", () => resolve(hash.digest("hex")))
+		stream.on("error", reject)
+	})
+}
+
+/**
+ * dHash over a `PHASH_GRID × PHASH_GRID` grayscale buffer: 9 samples per
+ * row (nearest-neighbour spread across the width), each of 8×8 bits
+ * comparing a sample against its right neighbour, packed MSB-first into
+ * a 16-char hex string.
+ */
+export function computeDHash(gray: Uint8Array, grid = PHASH_GRID): string {
+	let bits = 0n
+	for (let row = 0; row < 8; row++) {
+		for (let col = 0; col < 8; col++) {
+			const left = sampleAt(gray, grid, row, col)
+			const right = sampleAt(gray, grid, row, col + 1)
+			bits = (bits << 1n) | BigInt(left > right ? 1 : 0)
+		}
+	}
+	return bits.toString(16).padStart(16, "0")
+}
+
+/** Grayscale value at grid position (row, col), clamped to the grid. */
+function sampleAt(
+	gray: Uint8Array,
+	grid: number,
+	row: number,
+	col: number,
+): number {
+	const x = Math.round((col * (grid - 1)) / 8)
+	return gray[row * grid + x] ?? 0
+}
+
+/**
+ * pHash over a `PHASH_GRID × PHASH_GRID` grayscale buffer: the grid is
+ * mean-pooled to 8×8, then the 8×8 type-II DCT coefficients (precomputed
+ * cosine tables) are thresholded at their median, each bit MSB-first
+ * into a 16-char hex string.
+ */
+export function computePHash(gray: Uint8Array, grid = PHASH_GRID): string {
+	const n = 8
+	const small = meanPool(gray, grid, n)
+	const cos = precomputeCosines(n, n)
+	const coefficients: number[] = []
+	for (let v = 0; v < n; v++) {
+		for (let u = 0; u < n; u++) {
+			let sum = 0
+			for (let y = 0; y < n; y++) {
+				const cosYv = cos[v]?.[y] ?? 0
+				const cosXu = cos[u] ?? []
+				let rowSum = 0
+				for (let x = 0; x < n; x++) {
+					rowSum += (small[y * n + x] ?? 0) * (cosXu[x] ?? 0)
+				}
+				sum += rowSum * cosYv
+			}
+			const cu = u === 0 ? 1 / Math.SQRT2 : 1
+			const cv = v === 0 ? 1 / Math.SQRT2 : 1
+			coefficients.push(0.25 * cu * cv * sum)
+		}
+	}
+	const median = medianOf(coefficients)
+	let bits = 0n
+	for (const coefficient of coefficients) {
+		bits = (bits << 1n) | BigInt(coefficient > median ? 1 : 0)
+	}
+	return bits.toString(16).padStart(16, "0")
+}
+
+/** Mean-pool a `grid × grid` buffer into an `n × n` grid of averages. */
+function meanPool(gray: Uint8Array, grid: number, n: number): number[] {
+	const step = grid / n
+	const pooled: number[] = []
+	for (let by = 0; by < n; by++) {
+		for (let bx = 0; bx < n; bx++) {
+			let sum = 0
+			for (let y = 0; y < step; y++) {
+				for (let x = 0; x < step; x++) {
+					sum += gray[(by * step + y) * grid + bx * step + x] ?? 0
+				}
+			}
+			pooled.push(sum / (step * step))
+		}
+	}
+	return pooled
+}
+
+/**
+ * Population standard deviation of a grayscale buffer. Perceptual hashes
+ * of near-flat images (solid fills, blank pages) cluster at distance 0
+ * regardless of content — callers gate hash emission on this measure.
+ */
+export function grayStddev(gray: Uint8Array): number {
+	if (gray.length === 0) return 0
+	let mean = 0
+	for (const value of gray) mean += value
+	mean /= gray.length
+	let squared = 0
+	for (const value of gray) {
+		const delta = value - mean
+		squared += delta * delta
+	}
+	return Math.sqrt(squared / gray.length)
+}
+
+function precomputeCosines(grid: number, n: number): number[][] {
+	const cosX: number[][] = []
+	for (let k = 0; k < n; k++) {
+		const row: number[] = []
+		for (let x = 0; x < grid; x++) {
+			row.push(Math.cos(((2 * x + 1) * k * Math.PI) / (2 * grid)))
+		}
+		cosX.push(row)
+	}
+	return cosX
+}
+
+function medianOf(values: readonly number[]): number {
+	const sorted = [...values].sort((a, b) => a - b)
+	return sorted[Math.floor(sorted.length / 2)] ?? 0
+}
+
+/** Kinds whose values come from the single grayscale decode. */
+export const PERCEPTUAL_HASH_KINDS: readonly ImageHashKind[] = [
+	"dhash",
+	"phash",
+]

package/src/hoard/dir-size.ts

@@ -0,0 +1,58 @@
+/**
+ * Shared directory-size walk: the one implementation of "recursive byte
+ * size of a directory tree" for host storage. Consumers opt into
+ * different file policies via options:
+ *
+ * - vault totals skip dot-prefixed files (host temp staging), while
+ *   storage totals count everything — see {@link vaultTotalSize} and
+ *   `apps/server/src/domain/storage/service.ts`.
+ *
+ * Symbolic links are skipped so cycles cannot hang the walk and
+ * linked-out files are not double counted; missing/transient entries
+ * are ignored.
+ */
+
+import type { Dirent } from "node:fs"
+import { readdir, stat } from "node:fs/promises"
+import { join } from "node:path"
+
+export type DirSizeOptions = {
+	/** Skip file names starting with a dot (host temp staging). */
+	readonly excludeDotPrefix?: boolean
+}
+
+export async function sumDirSizes(
+	root: string,
+	opts: DirSizeOptions = {},
+): Promise<number> {
+	let total = 0
+	const stack = [root]
+	while (stack.length > 0) {
+		const dir = stack.pop()
+		if (dir === undefined) continue
+		let entries: Dirent[]
+		try {
+			entries = await readdir(dir, { withFileTypes: true })
+		} catch {
+			continue
+		}
+		for (const entry of entries) {
+			if (entry.isSymbolicLink()) continue
+			const full = join(dir, entry.name)
+			if (entry.isDirectory()) {
+				stack.push(full)
+			} else if (entry.isFile()) {
+				if (opts.excludeDotPrefix === true && entry.name.startsWith(".")) {
+					continue
+				}
+				try {
+					const info = await stat(full)
+					total += info.size
+				} catch {
+					// File disappeared between readdir and stat.
+				}
+			}
+		}
+	}
+	return total
+}

package/src/hoard/index.ts

@@ -0,0 +1,84 @@
+/**
+ * The hoard storage runtime: layout authority, versioned-write gate,
+ * version directory mechanics, zip extract + bulk export helpers, and
+ * the upload staging pool. Shared by the server, the CLI and future
+ * shells so every consumer works against one storage implementation.
+ */
+
+export type { ZipStreamEntry } from "../archive/index.ts"
+export {
+	extractArchiveInto,
+	listArchiveEntries,
+	streamStoredZip,
+	validateArchiveBudget,
+} from "../archive/index.ts"
+export type { DirSizeOptions } from "./dir-size.ts"
+export { sumDirSizes } from "./dir-size.ts"
+export {
+	naturalSort,
+	ORDER_MANIFEST_NAME,
+	orderEntries,
+	orderManifestPath,
+	parseOrderManifest,
+	readOrderManifest,
+	writeOrderManifest,
+} from "./order-manifest.ts"
+export type {
+	CreateStoragePathsOptions,
+	LocalPaths,
+	StoragePaths,
+	VersionPaths,
+} from "./paths.ts"
+export {
+	assertInside,
+	assertSafeSegment,
+	createStoragePaths,
+	imageVariantKey,
+	RESOURCE_DATA_DIR_NAME,
+} from "./paths.ts"
+export {
+	commitVaultFile,
+	discardVaultTempFile,
+	PluginVaultPathError,
+	parsePluginVaultDest,
+	type VaultCommitResult,
+	vaultFileSha256,
+	vaultReadFile,
+	vaultRemoveFile,
+	vaultStatFile,
+	vaultTempFile,
+	vaultTotalSize,
+} from "./plugin-vault.ts"
+export type { OccupiedNames } from "./sanitize.ts"
+export {
+	createOccupiedNames,
+	occupyEntryName,
+	sanitizeEntryName,
+	uniqueEntryName,
+} from "./sanitize.ts"
+export {
+	findStagedArchiveFile,
+	findStagedPoolFile,
+	removeStagedPoolFile,
+	resolveStagedPoolFiles,
+	writeStagedArchiveFile,
+	writeStagedPoolFile,
+} from "./staging-dir.ts"
+export type { CreateNextVersionResult } from "./version.ts"
+export {
+	createNextVersion,
+	currentVersion,
+	ensureBootstrapVersion,
+	listVersions,
+	readActiveVersion,
+	versionedDbFile,
+	versionedPath,
+	writeActiveVersion,
+} from "./version.ts"
+export type { VersionedFolderOps } from "./versioned-folder-ops.ts"
+export {
+	archiveStaleFiles,
+	buildVersionedFolderOps,
+} from "./versioned-folder-ops.ts"
+export type { VersionedWriteCommand } from "./write-versioned.ts"
+export { writeVersioned } from "./write-versioned.ts"