@prosopo/datasets-fs 3.0.142 → 3.0.144

package/src/tests/mocked.unit.test.ts

@@ -1,378 +0,0 @@
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-import { CaptchasContainerSchema, DataSchema } from "@prosopo/types";
-import { hexHash } from "@prosopo/util-crypto";
-import sharp from "sharp";
-import { afterAll, beforeAll, describe, expect, test } from "vitest";
-import { Flatten } from "../commands/flatten.js";
-import { GenerateV1 } from "../commands/generateV1.js";
-import { GenerateV2 } from "../commands/generateV2.js";
-import { Labels } from "../commands/labels.js";
-import { Relocate } from "../commands/relocate.js";
-import { Resize } from "../commands/resize.js";
-import {
-	REPO_DIR,
-	REPO_PLACEHOLDER,
-	captchasEqFs,
-	fsEq,
-	fsWalk,
-	readJson,
-	substituteRepoDir,
-} from "./utils.js";
-
-const IMAGE_PROCESSING_TIMEOUT_MS = 120_000;
-
-describe("dataset commands", () => {
-	const fixturesDir = `${__dirname}/data`;
-	// Where the fixtures' `${repo}` placeholder resolves to, i.e. how a
-	// command's output paths must read to match the checked-in fixture.
-	const fixturePathInRepo = (fixture: string): string =>
-		path.relative(REPO_DIR, `${fixturesDir}/${fixture}`);
-	let tmpDir: string;
-	let testResultsDir: string;
-	let resolvedFixturesDir: string;
-
-	const resolvedFixture = (fixture: string): string =>
-		substituteRepoDir(
-			`${fixturesDir}/${fixture}`,
-			`${resolvedFixturesDir}/${fixture}`,
-		);
-
-	// The command wrote absolute paths under `outputDir`; the fixture holds the
-	// same files under `${repo}/<fixture dir>`.
-	const expectDataJsonMatchesFixture = (
-		outputDir: string,
-		fixture: string,
-	): void => {
-		const actual = readJson(`${outputDir}/data.json`, [
-			fs.realpathSync(outputDir),
-			`${REPO_PLACEHOLDER}/${fixturePathInRepo(path.dirname(fixture))}`,
-		]);
-		expect(actual).toEqual(readJson(`${fixturesDir}/${fixture}`));
-	};
-
-	beforeAll(() => {
-		tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "datasets-fs-test-"));
-		testResultsDir = `${tmpDir}/results`;
-		resolvedFixturesDir = `${tmpDir}/fixtures`;
-		fs.mkdirSync(testResultsDir, { recursive: true });
-	});
-
-	afterAll(() => {
-		fs.rmSync(tmpDir, { recursive: true, force: true });
-	});
-
-	test("labels", async () => {
-		const input = resolvedFixture("flat_resized/data.json");
-		const output = `${testResultsDir}/labels.json`;
-		const labels = new Labels();
-		labels.logger.setLogLevel("error");
-		await labels.exec({
-			input,
-			output,
-		});
-		expect(readJson(output)).toEqual(
-			readJson(`${fixturesDir}/flat_resized/labels.json`),
-		);
-
-		// make sure there's one class per hierarchical directory
-		const content = fs.readFileSync(output).toString();
-		const labelsJson = JSON.parse(content.toString());
-		const foundLabels = labelsJson.labels as string[];
-		const categories = fs.readdirSync(`${fixturesDir}/hierarchical`);
-		if (JSON.stringify(foundLabels) !== JSON.stringify(categories)) {
-			throw new Error(`expected ${categories} but found ${foundLabels}`);
-		}
-	});
-
-	test(
-		"generate v2",
-		async () => {
-			const input = resolvedFixture("flat_resized/data.json");
-			const output = `${testResultsDir}/captchas_v2.json`;
-			const minIncorrect = 1;
-			const minCorrect = 1;
-			const minLabelled = 2;
-			const maxLabelled = 7;
-			const generate = new GenerateV2();
-			generate.logger.setLogLevel("error");
-			await generate.exec({
-				labelled: input,
-				unlabelled: input,
-				overwrite: true,
-				output,
-				count: 100,
-				seed: 0,
-				minIncorrect,
-				minCorrect,
-				minLabelled,
-				maxLabelled,
-				allowDuplicates: true,
-			});
-			// make sure the results are the same as the expected results
-			if (
-				!captchasEqFs(output, resolvedFixture("flat_resized/captchas_v2.json"))
-			) {
-				throw new Error("captchas not equal");
-			}
-
-			// test that the solutions array and unlabelled array per captcha never conflict
-			const content = fs.readFileSync(output).toString();
-			const captchasJson = JSON.parse(content.toString());
-			const captchas = CaptchasContainerSchema.parse(captchasJson);
-			for (const captcha of captchas.captchas) {
-				const solutions = captcha.solution;
-				const unlabelled = captcha.unlabelled;
-				// both should not be undefined
-				if (solutions === undefined || unlabelled === undefined) {
-					console.log(captcha);
-					throw new Error("solutions or unlabelled array is undefined");
-				}
-				for (const solution of solutions) {
-					// solution should not be in unlabelled
-					if (unlabelled.includes(solution)) {
-						throw new Error(`solution ${solution} is also in unlabelled array`);
-					}
-				}
-				// check the correct, incorrect, labelled and unlabelled distribution
-				const nCorrect = solutions.length;
-				const nUnlabelled = unlabelled.length;
-				const nIncorrect = 9 - nCorrect - nUnlabelled;
-				const nLabelled = nCorrect + nIncorrect;
-				if (nCorrect < minCorrect) {
-					throw new Error(
-						`expected at least ${minCorrect} correct but found ${nCorrect}`,
-					);
-				}
-				if (nIncorrect < minIncorrect) {
-					throw new Error(
-						`expected at least ${minIncorrect} incorrect but found ${nIncorrect}`,
-					);
-				}
-				if (nLabelled < minLabelled) {
-					throw new Error(
-						`expected at least ${minLabelled} labelled but found ${nLabelled}`,
-					);
-				}
-				if (nLabelled > maxLabelled) {
-					throw new Error(
-						`expected at most ${maxLabelled} labelled but found ${nLabelled}`,
-					);
-				}
-				if (nUnlabelled !== 9 - nLabelled) {
-					throw new Error(
-						`expected ${9 - nLabelled} unlabelled but found ${nUnlabelled}`,
-					);
-				}
-			}
-		},
-		IMAGE_PROCESSING_TIMEOUT_MS,
-	);
-
-	test("generate v1", async () => {
-		const input = resolvedFixture("flat_resized/data.json");
-		const output = `${testResultsDir}/captchas_v1.json`;
-		const generate = new GenerateV1();
-		generate.logger.setLogLevel("error");
-		await generate.exec({
-			labelled: input,
-			unlabelled: input,
-			overwrite: true,
-			output,
-			solved: 50,
-			unsolved: 50,
-			seed: 0,
-			minCorrect: 1,
-			maxCorrect: 6,
-			allowDuplicates: true,
-		});
-		// make sure the results are the same as the expected results
-		if (
-			!captchasEqFs(output, resolvedFixture("flat_resized/captchas_v1.json"))
-		) {
-			throw new Error("captchas not equal");
-		}
-	});
-
-	test(
-		"resizes data",
-		async () => {
-			const input = resolvedFixture("flat/data.json");
-			const output = `${testResultsDir}/flat_resized`;
-			const resize = new Resize();
-			resize.logger.setLogLevel("error");
-			await resize.exec({
-				input,
-				output,
-				overwrite: true,
-				square: true,
-				size: 128,
-			});
-
-			expect(
-				fsEq(`${output}/images`, `${fixturesDir}/flat_resized/images`),
-			).toBe(true);
-			expectDataJsonMatchesFixture(output, "flat_resized/data.json");
-
-			// check data in resized dir is all 128x128
-			for (const pth of fsWalk(output)) {
-				// if pth is not img, ignore
-				if (
-					!pth.endsWith(".jpg") &&
-					!pth.endsWith(".jpeg") &&
-					!pth.endsWith(".png")
-				) {
-					continue;
-				}
-				const image = sharp(pth);
-				const metadata = await image.metadata();
-				if (metadata.width !== 128 || metadata.height !== 128) {
-					throw new Error(`image ${pth} is not 128x128`);
-				}
-			}
-		},
-		IMAGE_PROCESSING_TIMEOUT_MS,
-	);
-
-	test("relocates data", async () => {
-		const input = `${fixturesDir}/flat_resized/data.json`;
-		const output = `${testResultsDir}/relocated_data.json`;
-		const relocate = new Relocate();
-		relocate.logger.setLogLevel("error");
-		await relocate.exec({
-			input,
-			output,
-			overwrite: true,
-			to: "newwebsite.com",
-			from: REPO_PLACEHOLDER,
-		});
-
-		expect(readJson(output)).toEqual(
-			readJson(`${fixturesDir}/flat_resized/relocated_data.json`),
-		);
-	});
-
-	test(
-		"flattens hierarchical data",
-		async () => {
-			const input = `${fixturesDir}/hierarchical`;
-			const output = `${testResultsDir}/flat`;
-			const flatten = new Flatten();
-			flatten.logger.setLogLevel("error");
-			await flatten.exec({
-				input,
-				output,
-				overwrite: true,
-			});
-
-			expect(fsEq(`${output}/images`, `${fixturesDir}/flat/images`)).toBe(true);
-			expectDataJsonMatchesFixture(output, "flat/data.json");
-
-			// read data json
-			const content = fs.readFileSync(`${output}/data.json`).toString();
-			const dataJson = JSON.parse(content.toString());
-			const data = DataSchema.parse(dataJson);
-
-			// check each image from the hierarchical data is in the flat data
-			let hierCount = 0;
-			for (const pth of fsWalk(input)) {
-				// if pth is not img, ignore
-				if (
-					!pth.endsWith(".jpg") &&
-					!pth.endsWith(".jpeg") &&
-					!pth.endsWith(".png")
-				) {
-					continue;
-				}
-				hierCount++;
-				// get the category
-				const category = pth.split("/").slice(-2)[0];
-				const content = fs.readFileSync(pth);
-				// check the image is in the flat data
-				let name = "";
-				for (const pth2 of fsWalk(output)) {
-					// if pth2 is not file, ignore
-					if (!fs.statSync(pth2).isFile()) {
-						continue;
-					}
-					const content2 = fs.readFileSync(pth2);
-					if (content.equals(content2)) {
-						const str = pth2.split("/").slice(-1)[0];
-						if (str === undefined) {
-							throw new Error(`unable to parse ${pth2}`);
-						}
-						name = str;
-						break;
-					}
-				}
-
-				if (!name) {
-					throw new Error(`unable to find image ${pth} in output`);
-				}
-				// check the image is in the data json
-				const item = data.items.find((item) => {
-					const name2 = item.data.split("/").slice(-1)[0];
-					if (name2 !== name) {
-						return false;
-					}
-					return true;
-				});
-				if (item === undefined) {
-					throw new Error(`unable to find image ${pth} in data.json`);
-				}
-				// correct name, so check category
-				if (item.label !== category) {
-					throw new Error(`expected ${category} but found ${item.label}`);
-				}
-				// type should be image
-				if (item.type !== "image") {
-					throw new Error(`expected image type but found ${item.type}`);
-				}
-				// file name should be the hash
-				if (item.hash !== name.split(".")[0]) {
-					throw new Error(
-						`expected ${name.split(".")[0]} hash but found ${item.hash}`,
-					);
-				}
-				// hash should be the hash of the image content
-				const hash = hexHash(content);
-				if (item.hash !== hash) {
-					throw new Error(`expected ${hash} hash but found ${item.hash}`);
-				}
-			}
-
-			// count the number of images in the flat data
-			let flatCount = 0;
-			for (const pth of fsWalk(output)) {
-				if (
-					!pth.endsWith(".jpg") &&
-					!pth.endsWith(".jpeg") &&
-					!pth.endsWith(".png")
-				) {
-					continue;
-				}
-				flatCount++;
-			}
-
-			// check same number of images in flat data
-			if (hierCount !== flatCount) {
-				throw new Error(`expected ${hierCount} images but found ${flatCount}`);
-			}
-		},
-		IMAGE_PROCESSING_TIMEOUT_MS,
-	);
-});

package/src/tests/utils.ts

@@ -1,145 +0,0 @@
-import fs from "node:fs";
-import path from "node:path";
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-import {
-	type CaptchasContainer,
-	CaptchasContainerSchema,
-	DataSchema,
-} from "@prosopo/types";
-import { at } from "@prosopo/util";
-
-// recursively list files in a directory
-export function* fsWalk(
-	pth: string,
-	options?: {
-		filesFirst?: boolean; // if true, depth first (i.e. yield files inside a dir before the dir itself, recursively), else yield in encounter order (i.e. dir first, then files inside, recursively)
-	},
-): Generator<string> {
-	// if path is a directory, recurse
-	const isDir = fs.existsSync(pth) && fs.statSync(pth).isDirectory();
-	let subpaths = isDir ? fs.readdirSync(pth) : [];
-	subpaths = subpaths.map((f) => `${pth}/${f}`);
-	if (!options?.filesFirst) {
-		yield pth;
-	}
-	for (const subpath of subpaths) {
-		yield* fsWalk(subpath, options);
-	}
-	if (options?.filesFirst) {
-		yield pth;
-	}
-}
-
-// test whether two files/dirs are equal, recursively comparing contents as required
-export const fsEq = (pth1: string, pth2: string) => {
-	const it1 = fsWalk(pth1);
-	const it2 = fsWalk(pth2);
-	// drop the root path
-	const files1 = [...it1].map((f) => f.slice(pth1.length));
-	const files2 = [...it2].map((f) => f.slice(pth2.length));
-	if (files1.length !== files2.length) {
-		return false;
-	}
-	// sort files so in same order
-	files1.sort();
-	files2.sort();
-	if (!files1.every((f, i) => f === files2[i])) {
-		// the two lists of files are not the same
-		return false;
-	}
-	// compare contents of each file
-	for (let i = 0; i < files1.length; i++) {
-		const file1 = pth1 + at(files1, i);
-		const file2 = pth2 + at(files2, i);
-		const stat1 = fs.statSync(file1);
-		if (stat1.isDirectory()) {
-			// already checked above
-			continue;
-		}
-		// files, so compare contents
-		const content1 = fs.readFileSync(file1);
-		const content2 = fs.readFileSync(file2);
-		if (!content1.equals(content2)) {
-			return false;
-		}
-	}
-	return true;
-};
-
-export const readDataJson = (pth: string) => {
-	let content = fs.readFileSync(pth).toString();
-	content = content.replaceAll(REPO_PLACEHOLDER, REPO_DIR);
-	const dataJson = JSON.parse(content.toString());
-	const data = DataSchema.parse(dataJson);
-	return data;
-};
-
-export const readCaptchasJson = (pth: string) => {
-	const content = fs.readFileSync(pth).toString();
-	const dataJson = JSON.parse(content);
-	const captchas = CaptchasContainerSchema.parse(dataJson);
-	return captchas;
-};
-
-export const captchasEqFs = (pth1: string, pth2: string) => {
-	const data1 = readCaptchasJson(pth1);
-	const data2 = readCaptchasJson(pth2);
-	return captchasEq(data1, data2);
-};
-
-export const captchasEq = (
-	first: CaptchasContainer,
-	second: CaptchasContainer,
-) => {
-	if (first.captchas.length !== second.captchas.length) {
-		return false;
-	}
-	for (let i = 0; i < first.captchas.length; i++) {
-		const captcha1 = at(first.captchas, i);
-		const captcha2 = at(second.captchas, i);
-		// salts should not be the same as generated on the fly and not stored!
-		if (captcha1.salt === captcha2.salt) {
-			return false;
-		}
-		// everything else should be the same
-		const { salt: salt1, ...rest1 } = captcha1;
-		const { salt: salt2, ...rest2 } = captcha2;
-		if (JSON.stringify(rest1) !== JSON.stringify(rest2)) {
-			return false;
-		}
-	}
-	return true;
-};
-
-export const REPO_PLACEHOLDER = "${repo}";
-// TODO use getPaths() here to find the repo dir
-export const REPO_DIR = path.resolve(__dirname, "../../../..");
-
-// Fixture JSON refers to images as `${repo}/...`. Writes a copy of the fixture
-// with the placeholder resolved, leaving the checked-in file untouched.
-export const substituteRepoDir = (fixture: string, dest: string): string => {
-	const content = fs.readFileSync(fixture).toString();
-	fs.mkdirSync(path.dirname(dest), { recursive: true });
-	fs.writeFileSync(dest, content.replaceAll(REPO_PLACEHOLDER, REPO_DIR));
-	return dest;
-};
-
-export const readJson = (pth: string, replace?: [string, string]): unknown => {
-	let content = fs.readFileSync(pth).toString();
-	if (replace) {
-		content = content.replaceAll(replace[0], replace[1]);
-	}
-	return JSON.parse(content);
-};

package/src/utils/input.ts

@@ -1,58 +0,0 @@
-import fs from "node:fs";
-import { ProsopoDatasetError, ProsopoError } from "@prosopo/common";
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-import * as z from "zod";
-import { CliCommand } from "../cli/cliCommand.js";
-
-export const InputArgsSchema = z.object({
-	input: z.string(),
-});
-
-export type InputArgs = z.infer<typeof InputArgsSchema>;
-
-export class InputCliCommand<
-	T extends typeof InputArgsSchema,
-> extends CliCommand<T> {
-	public override getArgSchema(): T {
-		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
-	}
-	public override getDescription(): string {
-		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
-	}
-	public override getOptions() {
-		return {
-			input: {
-				string: true,
-				alias: "in",
-				demand: true,
-				description: "The input path",
-			},
-		};
-	}
-
-	public override async _check(args: InputArgs) {
-		this.logger.debug(() => ({ msg: "input _check" }));
-		await super._check(args);
-		// input must exist
-		if (!fs.existsSync(args.input)) {
-			throw new ProsopoDatasetError(
-				new Error(`input path does not exist: ${args.input}`),
-				{
-					translationKey: "FS.FILE_NOT_FOUND",
-				},
-			);
-		}
-	}
-}

package/src/utils/inputOutput.ts

@@ -1,29 +0,0 @@
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-import type * as z from "zod";
-import { CliCommandComposite } from "../cli/cliCommandComposite.js";
-import { InputArgsSchema, InputCliCommand } from "./input.js";
-import { OutputArgsSchema, OutputCliCommand } from "./output.js";
-
-export const InputOutputArgsSchema = InputArgsSchema.merge(OutputArgsSchema);
-
-export type InputOutputArgs = z.infer<typeof InputOutputArgsSchema>;
-
-export abstract class InputOutputCliCommand<
-	T extends typeof InputOutputArgsSchema,
-> extends CliCommandComposite<T> {
-	constructor() {
-		super([new InputCliCommand<T>(), new OutputCliCommand<T>()]);
-	}
-}

package/src/utils/output.ts

@@ -1,86 +0,0 @@
-import fs from "node:fs";
-import { ProsopoEnvError, ProsopoError } from "@prosopo/common";
-import { lodash } from "@prosopo/util/lodash";
-import { boolean, object, string, type infer as zInfer } from "zod";
-// Copyright 2021-2026 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-import { CliCommand } from "../cli/cliCommand.js";
-
-export const OutputArgsSchema = object({
-	output: string(),
-	overwrite: boolean().optional(),
-});
-
-export type OutputArgs = zInfer<typeof OutputArgsSchema>;
-
-export class OutputCliCommand<
-	T extends typeof OutputArgsSchema,
-> extends CliCommand<T> {
-	#outputExists = false;
-
-	public outputExists() {
-		return this.#outputExists;
-	}
-
-	public override getArgSchema(): T {
-		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
-	}
-	public override getDescription(): string {
-		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
-	}
-	public override getOptions() {
-		return lodash().merge(super.getOptions(), {
-			output: {
-				alias: "out",
-				string: true,
-				demand: true,
-				description: "The output path",
-			},
-			overwrite: {
-				boolean: true,
-				description: "Overwrite files in the output path if they already exist",
-			},
-		});
-	}
-
-	public override async _check(args: OutputArgs) {
-		this.logger.debug(() => ({ msg: "Output _check" }));
-		await super._check(args);
-
-		// record whether output already exists
-		this.#outputExists = fs.existsSync(args.output);
-
-		// output must not exist, unless overwrite is true
-		if (this.outputExists()) {
-			if (!args.overwrite) {
-				throw new ProsopoEnvError(
-					new Error(`output path already exists: ${args.output}`),
-					{
-						translationKey: "FS.FILE_ALREADY_EXISTS",
-					},
-				);
-			}
-		}
-	}
-
-	public override async _run(args: OutputArgs) {
-		this.logger.debug(() => ({ msg: "Output run" }));
-		await super._run(args);
-		if (args.overwrite && this.outputExists()) {
-			// if overwrite is true, delete the output directory
-			this.logger.info(() => ({ msg: "cleaning output directory..." }));
-			fs.rmSync(args.output, { recursive: true });
-		}
-	}
-}