@prosopo/datasets-fs 3.0.138 → 3.0.140

package/src/tests/mocked.unit.test.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 // Copyright 2021-2026 Prosopo (UK) Ltd.
 //
@@ -15,9 +16,8 @@ import path from "node:path";
 // limitations under the License.
 import { CaptchasContainerSchema, DataSchema } from "@prosopo/types";
 import { hexHash } from "@prosopo/util-crypto";
-import { getRootDir, getTestResultsDir } from "@prosopo/workspace";
 import sharp from "sharp";
-import { afterAll, beforeAll, describe, test } from "vitest";
+import { afterAll, beforeAll, describe, expect, test } from "vitest";
 import { Flatten } from "../commands/flatten.js";
 import { GenerateV1 } from "../commands/generateV1.js";
 import { GenerateV2 } from "../commands/generateV2.js";
@@ -25,35 +25,59 @@ import { Labels } from "../commands/labels.js";
 import { Relocate } from "../commands/relocate.js";
 import { Resize } from "../commands/resize.js";
 import {
+	REPO_DIR,
+	REPO_PLACEHOLDER,
 	captchasEqFs,
 	fsEq,
 	fsWalk,
-	restoreRepoDir,
+	readJson,
 	substituteRepoDir,
 } from "./utils.js";
 
+const IMAGE_PROCESSING_TIMEOUT_MS = 120_000;
+
 describe("dataset commands", () => {
-	const pkgDir = path.relative(getRootDir(), __dirname);
-	const pkgInternalPath = pkgDir.replace(`${__dirname}/`, "");
-	const testResultsDir = `${getTestResultsDir()}/${pkgInternalPath}`;
+	const fixturesDir = `${__dirname}/data`;
+	// Where the fixtures' `${repo}` placeholder resolves to, i.e. how a
+	// command's output paths must read to match the checked-in fixture.
+	const fixturePathInRepo = (fixture: string): string =>
+		path.relative(REPO_DIR, `${fixturesDir}/${fixture}`);
+	let tmpDir: string;
+	let testResultsDir: string;
+	let resolvedFixturesDir: string;
+
+	const resolvedFixture = (fixture: string): string =>
+		substituteRepoDir(
+			`${fixturesDir}/${fixture}`,
+			`${resolvedFixturesDir}/${fixture}`,
+		);
+
+	// The command wrote absolute paths under `outputDir`; the fixture holds the
+	// same files under `${repo}/<fixture dir>`.
+	const expectDataJsonMatchesFixture = (
+		outputDir: string,
+		fixture: string,
+	): void => {
+		const actual = readJson(`${outputDir}/data.json`, [
+			fs.realpathSync(outputDir),
+			`${REPO_PLACEHOLDER}/${fixturePathInRepo(path.dirname(fixture))}`,
+		]);
+		expect(actual).toEqual(readJson(`${fixturesDir}/${fixture}`));
+	};
 
 	beforeAll(() => {
-		// substitute the repo path in the data for tests
-		substituteRepoDir();
-		fs.mkdirSync(`${testResultsDir}`, { recursive: true });
+		tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "datasets-fs-test-"));
+		testResultsDir = `${tmpDir}/results`;
+		resolvedFixturesDir = `${tmpDir}/fixtures`;
+		fs.mkdirSync(testResultsDir, { recursive: true });
 	});
 
 	afterAll(() => {
-		// restore repo path back to placeholder
-		restoreRepoDir();
-		// remove test results
-		if (fs.existsSync(`${testResultsDir}`)) {
-			fs.rmdirSync(`${testResultsDir}`, { recursive: true });
-		}
+		fs.rmSync(tmpDir, { recursive: true, force: true });
 	});
 
 	test("labels", async () => {
-		const input = `${__dirname}/data/flat_resized/data.json`;
+		const input = resolvedFixture("flat_resized/data.json");
 		const output = `${testResultsDir}/labels.json`;
 		const labels = new Labels();
 		labels.logger.setLogLevel("error");
@@ -61,101 +85,106 @@ describe("dataset commands", () => {
 			input,
 			output,
 		});
-		// make sure the results are the same as the expected results
-		fsEq(output, `${__dirname}/data/flat_resized/labels.json`);
+		expect(readJson(output)).toEqual(
+			readJson(`${fixturesDir}/flat_resized/labels.json`),
+		);
 
 		// make sure there's one class per hierarchical directory
 		const content = fs.readFileSync(output).toString();
 		const labelsJson = JSON.parse(content.toString());
 		const foundLabels = labelsJson.labels as string[];
-		const categories = fs.readdirSync(`${__dirname}/data/hierarchical`);
+		const categories = fs.readdirSync(`${fixturesDir}/hierarchical`);
 		if (JSON.stringify(foundLabels) !== JSON.stringify(categories)) {
 			throw new Error(`expected ${categories} but found ${foundLabels}`);
 		}
 	});
 
-	test("generate v2", async () => {
-		const input = `${__dirname}/data/flat_resized/data.json`;
-		const output = `${testResultsDir}/captchas_v2.json`;
-		const minIncorrect = 1;
-		const minCorrect = 1;
-		const minLabelled = 2;
-		const maxLabelled = 7;
-		const generate = new GenerateV2();
-		generate.logger.setLogLevel("error");
-		await generate.exec({
-			labelled: input,
-			unlabelled: input,
-			overwrite: true,
-			output,
-			count: 100,
-			seed: 0,
-			minIncorrect,
-			minCorrect,
-			minLabelled,
-			maxLabelled,
-			allowDuplicates: true,
-		});
-		// make sure the results are the same as the expected results
-		if (
-			!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v2.json`)
-		) {
-			throw new Error("captchas not equal");
-		}
-
-		// test that the solutions array and unlabelled array per captcha never conflict
-		const content = fs.readFileSync(output).toString();
-		const captchasJson = JSON.parse(content.toString());
-		const captchas = CaptchasContainerSchema.parse(captchasJson);
-		for (const captcha of captchas.captchas) {
-			const solutions = captcha.solution;
-			const unlabelled = captcha.unlabelled;
-			// both should not be undefined
-			if (solutions === undefined || unlabelled === undefined) {
-				console.log(captcha);
-				throw new Error("solutions or unlabelled array is undefined");
+	test(
+		"generate v2",
+		async () => {
+			const input = resolvedFixture("flat_resized/data.json");
+			const output = `${testResultsDir}/captchas_v2.json`;
+			const minIncorrect = 1;
+			const minCorrect = 1;
+			const minLabelled = 2;
+			const maxLabelled = 7;
+			const generate = new GenerateV2();
+			generate.logger.setLogLevel("error");
+			await generate.exec({
+				labelled: input,
+				unlabelled: input,
+				overwrite: true,
+				output,
+				count: 100,
+				seed: 0,
+				minIncorrect,
+				minCorrect,
+				minLabelled,
+				maxLabelled,
+				allowDuplicates: true,
+			});
+			// make sure the results are the same as the expected results
+			if (
+				!captchasEqFs(output, resolvedFixture("flat_resized/captchas_v2.json"))
+			) {
+				throw new Error("captchas not equal");
 			}
-			for (const solution of solutions) {
-				// solution should not be in unlabelled
-				if (unlabelled.includes(solution)) {
-					throw new Error(`solution ${solution} is also in unlabelled array`);
+
+			// test that the solutions array and unlabelled array per captcha never conflict
+			const content = fs.readFileSync(output).toString();
+			const captchasJson = JSON.parse(content.toString());
+			const captchas = CaptchasContainerSchema.parse(captchasJson);
+			for (const captcha of captchas.captchas) {
+				const solutions = captcha.solution;
+				const unlabelled = captcha.unlabelled;
+				// both should not be undefined
+				if (solutions === undefined || unlabelled === undefined) {
+					console.log(captcha);
+					throw new Error("solutions or unlabelled array is undefined");
+				}
+				for (const solution of solutions) {
+					// solution should not be in unlabelled
+					if (unlabelled.includes(solution)) {
+						throw new Error(`solution ${solution} is also in unlabelled array`);
+					}
+				}
+				// check the correct, incorrect, labelled and unlabelled distribution
+				const nCorrect = solutions.length;
+				const nUnlabelled = unlabelled.length;
+				const nIncorrect = 9 - nCorrect - nUnlabelled;
+				const nLabelled = nCorrect + nIncorrect;
+				if (nCorrect < minCorrect) {
+					throw new Error(
+						`expected at least ${minCorrect} correct but found ${nCorrect}`,
+					);
+				}
+				if (nIncorrect < minIncorrect) {
+					throw new Error(
+						`expected at least ${minIncorrect} incorrect but found ${nIncorrect}`,
+					);
+				}
+				if (nLabelled < minLabelled) {
+					throw new Error(
+						`expected at least ${minLabelled} labelled but found ${nLabelled}`,
+					);
+				}
+				if (nLabelled > maxLabelled) {
+					throw new Error(
+						`expected at most ${maxLabelled} labelled but found ${nLabelled}`,
+					);
+				}
+				if (nUnlabelled !== 9 - nLabelled) {
+					throw new Error(
+						`expected ${9 - nLabelled} unlabelled but found ${nUnlabelled}`,
+					);
 				}
 			}
-			// check the correct, incorrect, labelled and unlabelled distribution
-			const nCorrect = solutions.length;
-			const nUnlabelled = unlabelled.length;
-			const nIncorrect = 9 - nCorrect - nUnlabelled;
-			const nLabelled = nCorrect + nIncorrect;
-			if (nCorrect < minCorrect) {
-				throw new Error(
-					`expected at least ${minCorrect} correct but found ${nCorrect}`,
-				);
-			}
-			if (nIncorrect < minIncorrect) {
-				throw new Error(
-					`expected at least ${minIncorrect} incorrect but found ${nIncorrect}`,
-				);
-			}
-			if (nLabelled < minLabelled) {
-				throw new Error(
-					`expected at least ${minLabelled} labelled but found ${nLabelled}`,
-				);
-			}
-			if (nLabelled > maxLabelled) {
-				throw new Error(
-					`expected at most ${maxLabelled} labelled but found ${nLabelled}`,
-				);
-			}
-			if (nUnlabelled !== 9 - nLabelled) {
-				throw new Error(
-					`expected ${9 - nLabelled} unlabelled but found ${nUnlabelled}`,
-				);
-			}
-		}
-	});
+		},
+		IMAGE_PROCESSING_TIMEOUT_MS,
+	);
 
 	test("generate v1", async () => {
-		const input = `${__dirname}/data/flat_resized/data.json`;
+		const input = resolvedFixture("flat_resized/data.json");
 		const output = `${testResultsDir}/captchas_v1.json`;
 		const generate = new GenerateV1();
 		generate.logger.setLogLevel("error");
@@ -173,49 +202,54 @@ describe("dataset commands", () => {
 		});
 		// make sure the results are the same as the expected results
 		if (
-			!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v1.json`)
+			!captchasEqFs(output, resolvedFixture("flat_resized/captchas_v1.json"))
 		) {
 			throw new Error("captchas not equal");
 		}
 	});
 
-	test("resizes data", async () => {
-		const input = `${__dirname}/data/flat/data.json`;
-		const output = `${testResultsDir}/flat_resized`;
-		const resize = new Resize();
-		resize.logger.setLogLevel("error");
-		await resize.exec({
-			input,
-			output,
-			overwrite: true,
-			square: true,
-			size: 128,
-		});
+	test(
+		"resizes data",
+		async () => {
+			const input = resolvedFixture("flat/data.json");
+			const output = `${testResultsDir}/flat_resized`;
+			const resize = new Resize();
+			resize.logger.setLogLevel("error");
+			await resize.exec({
+				input,
+				output,
+				overwrite: true,
+				square: true,
+				size: 128,
+			});
 
-		// make sure the results are the same as the expected results
-		const expected = `${__dirname}/data/flat_resized`;
-		fsEq(output, expected);
+			expect(
+				fsEq(`${output}/images`, `${fixturesDir}/flat_resized/images`),
+			).toBe(true);
+			expectDataJsonMatchesFixture(output, "flat_resized/data.json");
 
-		// check data in resized dir is all 128x128
-		for (const pth of fsWalk(output)) {
-			// if pth is not img, ignore
-			if (
-				!pth.endsWith(".jpg") &&
-				!pth.endsWith(".jpeg") &&
-				!pth.endsWith(".png")
-			) {
-				continue;
-			}
-			const image = sharp(pth);
-			const metadata = await image.metadata();
-			if (metadata.width !== 128 || metadata.height !== 128) {
-				throw new Error(`image ${pth} is not 128x128`);
+			// check data in resized dir is all 128x128
+			for (const pth of fsWalk(output)) {
+				// if pth is not img, ignore
+				if (
+					!pth.endsWith(".jpg") &&
+					!pth.endsWith(".jpeg") &&
+					!pth.endsWith(".png")
+				) {
+					continue;
+				}
+				const image = sharp(pth);
+				const metadata = await image.metadata();
+				if (metadata.width !== 128 || metadata.height !== 128) {
+					throw new Error(`image ${pth} is not 128x128`);
+				}
 			}
-		}
-	});
+		},
+		IMAGE_PROCESSING_TIMEOUT_MS,
+	);
 
 	test("relocates data", async () => {
-		const input = `${__dirname}/data/flat_resized/data.json`;
+		const input = `${fixturesDir}/flat_resized/data.json`;
 		const output = `${testResultsDir}/relocated_data.json`;
 		const relocate = new Relocate();
 		relocate.logger.setLogLevel("error");
@@ -224,118 +258,121 @@ describe("dataset commands", () => {
 			output,
 			overwrite: true,
 			to: "newwebsite.com",
-			from: "${repo}",
+			from: REPO_PLACEHOLDER,
 		});
 
-		// make sure the results are the same as the expected results
-		const expected = `${__dirname}/data/flat_resized/relocated_data.json`;
-		fsEq(output, expected);
+		expect(readJson(output)).toEqual(
+			readJson(`${fixturesDir}/flat_resized/relocated_data.json`),
+		);
 	});
 
-	test("flattens hierarchical data", async () => {
-		const input = `${__dirname}/data/hierarchical`;
-		const output = `${testResultsDir}/flat`;
-		const flatten = new Flatten();
-		flatten.logger.setLogLevel("error");
-		await flatten.exec({
-			input,
-			output,
-			overwrite: true,
-		});
+	test(
+		"flattens hierarchical data",
+		async () => {
+			const input = `${fixturesDir}/hierarchical`;
+			const output = `${testResultsDir}/flat`;
+			const flatten = new Flatten();
+			flatten.logger.setLogLevel("error");
+			await flatten.exec({
+				input,
+				output,
+				overwrite: true,
+			});
 
-		// make sure the results are the same as the expected results
-		const expected = `${__dirname}/data/flat`;
-		fsEq(output, expected);
+			expect(fsEq(`${output}/images`, `${fixturesDir}/flat/images`)).toBe(true);
+			expectDataJsonMatchesFixture(output, "flat/data.json");
 
-		// read data json
-		const content = fs.readFileSync(`${output}/data.json`).toString();
-		const dataJson = JSON.parse(content.toString());
-		const data = DataSchema.parse(dataJson);
+			// read data json
+			const content = fs.readFileSync(`${output}/data.json`).toString();
+			const dataJson = JSON.parse(content.toString());
+			const data = DataSchema.parse(dataJson);
 
-		// check each image from the hierarchical data is in the flat data
-		let hierCount = 0;
-		for (const pth of fsWalk(input)) {
-			// if pth is not img, ignore
-			if (
-				!pth.endsWith(".jpg") &&
-				!pth.endsWith(".jpeg") &&
-				!pth.endsWith(".png")
-			) {
-				continue;
-			}
-			hierCount++;
-			// get the category
-			const category = pth.split("/").slice(-2)[0];
-			const content = fs.readFileSync(pth);
-			// check the image is in the flat data
-			let name = "";
-			for (const pth2 of fsWalk(output)) {
-				// if pth2 is not file, ignore
-				if (!fs.statSync(pth2).isFile()) {
+			// check each image from the hierarchical data is in the flat data
+			let hierCount = 0;
+			for (const pth of fsWalk(input)) {
+				// if pth is not img, ignore
+				if (
+					!pth.endsWith(".jpg") &&
+					!pth.endsWith(".jpeg") &&
+					!pth.endsWith(".png")
+				) {
 					continue;
 				}
-				const content2 = fs.readFileSync(pth2);
-				if (content.equals(content2)) {
-					const str = pth2.split("/").slice(-1)[0];
-					if (str === undefined) {
-						throw new Error(`unable to parse ${pth2}`);
+				hierCount++;
+				// get the category
+				const category = pth.split("/").slice(-2)[0];
+				const content = fs.readFileSync(pth);
+				// check the image is in the flat data
+				let name = "";
+				for (const pth2 of fsWalk(output)) {
+					// if pth2 is not file, ignore
+					if (!fs.statSync(pth2).isFile()) {
+						continue;
+					}
+					const content2 = fs.readFileSync(pth2);
+					if (content.equals(content2)) {
+						const str = pth2.split("/").slice(-1)[0];
+						if (str === undefined) {
+							throw new Error(`unable to parse ${pth2}`);
+						}
+						name = str;
+						break;
 					}
-					name = str;
-					break;
 				}
-			}
 
-			if (!name) {
-				throw new Error(`unable to find image ${pth} in output`);
-			}
-			// check the image is in the data json
-			const item = data.items.find((item) => {
-				const name2 = item.data.split("/").slice(-1)[0];
-				if (name2 !== name) {
-					return false;
+				if (!name) {
+					throw new Error(`unable to find image ${pth} in output`);
+				}
+				// check the image is in the data json
+				const item = data.items.find((item) => {
+					const name2 = item.data.split("/").slice(-1)[0];
+					if (name2 !== name) {
+						return false;
+					}
+					return true;
+				});
+				if (item === undefined) {
+					throw new Error(`unable to find image ${pth} in data.json`);
+				}
+				// correct name, so check category
+				if (item.label !== category) {
+					throw new Error(`expected ${category} but found ${item.label}`);
+				}
+				// type should be image
+				if (item.type !== "image") {
+					throw new Error(`expected image type but found ${item.type}`);
+				}
+				// file name should be the hash
+				if (item.hash !== name.split(".")[0]) {
+					throw new Error(
+						`expected ${name.split(".")[0]} hash but found ${item.hash}`,
+					);
+				}
+				// hash should be the hash of the image content
+				const hash = hexHash(content);
+				if (item.hash !== hash) {
+					throw new Error(`expected ${hash} hash but found ${item.hash}`);
 				}
-				return true;
-			});
-			if (item === undefined) {
-				throw new Error(`unable to find image ${pth} in data.json`);
-			}
-			// correct name, so check category
-			if (item.label !== category) {
-				throw new Error(`expected ${category} but found ${item.label}`);
-			}
-			// type should be image
-			if (item.type !== "image") {
-				throw new Error(`expected image type but found ${item.type}`);
-			}
-			// file name should be the hash
-			if (item.hash !== name.split(".")[0]) {
-				throw new Error(
-					`expected ${name.split(".")[0]} hash but found ${item.hash}`,
-				);
-			}
-			// hash should be the hash of the image content
-			const hash = hexHash(content);
-			if (item.hash !== hash) {
-				throw new Error(`expected ${hash} hash but found ${item.hash}`);
 			}
-		}
 
-		// count the number of images in the flat data
-		let flatCount = 0;
-		for (const pth of fsWalk(output)) {
-			if (
-				!pth.endsWith(".jpg") &&
-				!pth.endsWith(".jpeg") &&
-				!pth.endsWith(".png")
-			) {
-				continue;
+			// count the number of images in the flat data
+			let flatCount = 0;
+			for (const pth of fsWalk(output)) {
+				if (
+					!pth.endsWith(".jpg") &&
+					!pth.endsWith(".jpeg") &&
+					!pth.endsWith(".png")
+				) {
+					continue;
+				}
+				flatCount++;
 			}
-			flatCount++;
-		}
 
-		// check same number of images in flat data
-		if (hierCount !== flatCount) {
-			throw new Error(`expected ${hierCount} images but found ${flatCount}`);
-		}
-	});
+			// check same number of images in flat data
+			if (hierCount !== flatCount) {
+				throw new Error(`expected ${hierCount} images but found ${flatCount}`);
+			}
+		},
+		IMAGE_PROCESSING_TIMEOUT_MS,
+	);
 });

package/src/tests/utils.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs";
+import path from "node:path";
 // Copyright 2021-2026 Prosopo (UK) Ltd.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -79,8 +80,7 @@ export const fsEq = (pth1: string, pth2: string) => {
 
 export const readDataJson = (pth: string) => {
 	let content = fs.readFileSync(pth).toString();
-	// TODO use getPaths() here to find the repo dir
-	content = content.replaceAll("${repo}", `${__dirname}/../../../..`);
+	content = content.replaceAll(REPO_PLACEHOLDER, REPO_DIR);
 	const dataJson = JSON.parse(content.toString());
 	const data = DataSchema.parse(dataJson);
 	return data;
@@ -123,30 +123,23 @@ export const captchasEq = (
 	return true;
 };
 
-export const substituteRepoDir = () => {
-	// read all json files in the test data dir
-	for (const pth of fsWalk(`${__dirname}/data`)) {
-		if (!pth.endsWith(".json")) {
-			continue;
-		}
-		// make a backup of each file
-		fs.copyFileSync(pth, `${pth}.bak`);
-		// replace ${repo} with the path to the repo
-		let content = fs.readFileSync(pth).toString();
-		// TODO use getPaths() here to find the repo dir
-		content = content.replaceAll("${repo}", `${__dirname}/../../../..`);
-		// rewrite the file
-		fs.writeFileSync(pth, content);
-	}
+export const REPO_PLACEHOLDER = "${repo}";
+// TODO use getPaths() here to find the repo dir
+export const REPO_DIR = path.resolve(__dirname, "../../../..");
+
+// Fixture JSON refers to images as `${repo}/...`. Writes a copy of the fixture
+// with the placeholder resolved, leaving the checked-in file untouched.
+export const substituteRepoDir = (fixture: string, dest: string): string => {
+	const content = fs.readFileSync(fixture).toString();
+	fs.mkdirSync(path.dirname(dest), { recursive: true });
+	fs.writeFileSync(dest, content.replaceAll(REPO_PLACEHOLDER, REPO_DIR));
+	return dest;
 };
 
-export const restoreRepoDir = () => {
-	// read all json files in the test data dir
-	for (const pth of fsWalk(`${__dirname}/data`)) {
-		if (!pth.endsWith(".json")) {
-			continue;
-		}
-		// restore the backup of each file
-		fs.renameSync(`${pth}.bak`, pth);
+export const readJson = (pth: string, replace?: [string, string]): unknown => {
+	let content = fs.readFileSync(pth).toString();
+	if (replace) {
+		content = content.replaceAll(replace[0], replace[1]);
 	}
+	return JSON.parse(content);
 };

package/tsconfig.cjs.json

@@ -28,9 +28,6 @@
 		},
 		{
 			"path": "../util-crypto/tsconfig.cjs.json"
-		},
-		{
-			"path": "../../dev/workspace/tsconfig.cjs.json"
 		}
 	]
 }

package/tsconfig.json

@@ -29,9 +29,6 @@
 		},
 		{
 			"path": "../util-crypto"
-		},
-		{
-			"path": "../../dev/workspace"
 		}
 	]
 }