@prosopo/datasets-fs 3.0.43 → 3.0.77

package/src/tests/mocked.unit.test.ts

@@ -0,0 +1,341 @@
+import fs from "node:fs";
+import path from "node:path";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { CaptchasContainerSchema, DataSchema } from "@prosopo/types";
+import { hexHash } from "@prosopo/util-crypto";
+import { getRootDir, getTestResultsDir } from "@prosopo/workspace";
+import sharp from "sharp";
+import { afterAll, beforeAll, describe, test } from "vitest";
+import { Flatten } from "../commands/flatten.js";
+import { GenerateV1 } from "../commands/generateV1.js";
+import { GenerateV2 } from "../commands/generateV2.js";
+import { Labels } from "../commands/labels.js";
+import { Relocate } from "../commands/relocate.js";
+import { Resize } from "../commands/resize.js";
+import {
+	captchasEqFs,
+	fsEq,
+	fsWalk,
+	restoreRepoDir,
+	substituteRepoDir,
+} from "./utils.js";
+
+describe("dataset commands", () => {
+	const pkgDir = path.relative(getRootDir(), __dirname);
+	const pkgInternalPath = pkgDir.replace(`${__dirname}/`, "");
+	const testResultsDir = `${getTestResultsDir()}/${pkgInternalPath}`;
+
+	beforeAll(() => {
+		// substitute the repo path in the data for tests
+		substituteRepoDir();
+		fs.mkdirSync(`${testResultsDir}`, { recursive: true });
+	});
+
+	afterAll(() => {
+		// restore repo path back to placeholder
+		restoreRepoDir();
+		// remove test results
+		if (fs.existsSync(`${testResultsDir}`)) {
+			fs.rmdirSync(`${testResultsDir}`, { recursive: true });
+		}
+	});
+
+	test("labels", async () => {
+		const input = `${__dirname}/data/flat_resized/data.json`;
+		const output = `${testResultsDir}/labels.json`;
+		const labels = new Labels();
+		labels.logger.setLogLevel("error");
+		await labels.exec({
+			input,
+			output,
+		});
+		// make sure the results are the same as the expected results
+		fsEq(output, `${__dirname}/data/flat_resized/labels.json`);
+
+		// make sure there's one class per hierarchical directory
+		const content = fs.readFileSync(output).toString();
+		const labelsJson = JSON.parse(content.toString());
+		const foundLabels = labelsJson.labels as string[];
+		const categories = fs.readdirSync(`${__dirname}/data/hierarchical`);
+		if (JSON.stringify(foundLabels) !== JSON.stringify(categories)) {
+			throw new Error(`expected ${categories} but found ${foundLabels}`);
+		}
+	});
+
+	test("generate v2", async () => {
+		const input = `${__dirname}/data/flat_resized/data.json`;
+		const output = `${testResultsDir}/captchas_v2.json`;
+		const minIncorrect = 1;
+		const minCorrect = 1;
+		const minLabelled = 2;
+		const maxLabelled = 7;
+		const generate = new GenerateV2();
+		generate.logger.setLogLevel("error");
+		await generate.exec({
+			labelled: input,
+			unlabelled: input,
+			overwrite: true,
+			output,
+			count: 100,
+			seed: 0,
+			minIncorrect,
+			minCorrect,
+			minLabelled,
+			maxLabelled,
+			allowDuplicates: true,
+		});
+		// make sure the results are the same as the expected results
+		if (
+			!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v2.json`)
+		) {
+			throw new Error("captchas not equal");
+		}
+
+		// test that the solutions array and unlabelled array per captcha never conflict
+		const content = fs.readFileSync(output).toString();
+		const captchasJson = JSON.parse(content.toString());
+		const captchas = CaptchasContainerSchema.parse(captchasJson);
+		for (const captcha of captchas.captchas) {
+			const solutions = captcha.solution;
+			const unlabelled = captcha.unlabelled;
+			// both should not be undefined
+			if (solutions === undefined || unlabelled === undefined) {
+				console.log(captcha);
+				throw new Error("solutions or unlabelled array is undefined");
+			}
+			for (const solution of solutions) {
+				// solution should not be in unlabelled
+				if (unlabelled.includes(solution)) {
+					throw new Error(`solution ${solution} is also in unlabelled array`);
+				}
+			}
+			// check the correct, incorrect, labelled and unlabelled distribution
+			const nCorrect = solutions.length;
+			const nUnlabelled = unlabelled.length;
+			const nIncorrect = 9 - nCorrect - nUnlabelled;
+			const nLabelled = nCorrect + nIncorrect;
+			if (nCorrect < minCorrect) {
+				throw new Error(
+					`expected at least ${minCorrect} correct but found ${nCorrect}`,
+				);
+			}
+			if (nIncorrect < minIncorrect) {
+				throw new Error(
+					`expected at least ${minIncorrect} incorrect but found ${nIncorrect}`,
+				);
+			}
+			if (nLabelled < minLabelled) {
+				throw new Error(
+					`expected at least ${minLabelled} labelled but found ${nLabelled}`,
+				);
+			}
+			if (nLabelled > maxLabelled) {
+				throw new Error(
+					`expected at most ${maxLabelled} labelled but found ${nLabelled}`,
+				);
+			}
+			if (nUnlabelled !== 9 - nLabelled) {
+				throw new Error(
+					`expected ${9 - nLabelled} unlabelled but found ${nUnlabelled}`,
+				);
+			}
+		}
+	});
+
+	test("generate v1", async () => {
+		const input = `${__dirname}/data/flat_resized/data.json`;
+		const output = `${testResultsDir}/captchas_v1.json`;
+		const generate = new GenerateV1();
+		generate.logger.setLogLevel("error");
+		await generate.exec({
+			labelled: input,
+			unlabelled: input,
+			overwrite: true,
+			output,
+			solved: 50,
+			unsolved: 50,
+			seed: 0,
+			minCorrect: 1,
+			maxCorrect: 6,
+			allowDuplicates: true,
+		});
+		// make sure the results are the same as the expected results
+		if (
+			!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v1.json`)
+		) {
+			throw new Error("captchas not equal");
+		}
+	});
+
+	test("resizes data", async () => {
+		const input = `${__dirname}/data/flat/data.json`;
+		const output = `${testResultsDir}/flat_resized`;
+		const resize = new Resize();
+		resize.logger.setLogLevel("error");
+		await resize.exec({
+			input,
+			output,
+			overwrite: true,
+			square: true,
+			size: 128,
+		});
+
+		// make sure the results are the same as the expected results
+		const expected = `${__dirname}/data/flat_resized`;
+		fsEq(output, expected);
+
+		// check data in resized dir is all 128x128
+		for (const pth of fsWalk(output)) {
+			// if pth is not img, ignore
+			if (
+				!pth.endsWith(".jpg") &&
+				!pth.endsWith(".jpeg") &&
+				!pth.endsWith(".png")
+			) {
+				continue;
+			}
+			const image = sharp(pth);
+			const metadata = await image.metadata();
+			if (metadata.width !== 128 || metadata.height !== 128) {
+				throw new Error(`image ${pth} is not 128x128`);
+			}
+		}
+	});
+
+	test("relocates data", async () => {
+		const input = `${__dirname}/data/flat_resized/data.json`;
+		const output = `${testResultsDir}/relocated_data.json`;
+		const relocate = new Relocate();
+		relocate.logger.setLogLevel("error");
+		await relocate.exec({
+			input,
+			output,
+			overwrite: true,
+			to: "newwebsite.com",
+			from: "${repo}",
+		});
+
+		// make sure the results are the same as the expected results
+		const expected = `${__dirname}/data/flat_resized/relocated_data.json`;
+		fsEq(output, expected);
+	});
+
+	test("flattens hierarchical data", async () => {
+		const input = `${__dirname}/data/hierarchical`;
+		const output = `${testResultsDir}/flat`;
+		const flatten = new Flatten();
+		flatten.logger.setLogLevel("error");
+		await flatten.exec({
+			input,
+			output,
+			overwrite: true,
+		});
+
+		// make sure the results are the same as the expected results
+		const expected = `${__dirname}/data/flat`;
+		fsEq(output, expected);
+
+		// read data json
+		const content = fs.readFileSync(`${output}/data.json`).toString();
+		const dataJson = JSON.parse(content.toString());
+		const data = DataSchema.parse(dataJson);
+
+		// check each image from the hierarchical data is in the flat data
+		let hierCount = 0;
+		for (const pth of fsWalk(input)) {
+			// if pth is not img, ignore
+			if (
+				!pth.endsWith(".jpg") &&
+				!pth.endsWith(".jpeg") &&
+				!pth.endsWith(".png")
+			) {
+				continue;
+			}
+			hierCount++;
+			// get the category
+			const category = pth.split("/").slice(-2)[0];
+			const content = fs.readFileSync(pth);
+			// check the image is in the flat data
+			let name = "";
+			for (const pth2 of fsWalk(output)) {
+				// if pth2 is not file, ignore
+				if (!fs.statSync(pth2).isFile()) {
+					continue;
+				}
+				const content2 = fs.readFileSync(pth2);
+				if (content.equals(content2)) {
+					const str = pth2.split("/").slice(-1)[0];
+					if (str === undefined) {
+						throw new Error(`unable to parse ${pth2}`);
+					}
+					name = str;
+					break;
+				}
+			}
+
+			if (!name) {
+				throw new Error(`unable to find image ${pth} in output`);
+			}
+			// check the image is in the data json
+			const item = data.items.find((item) => {
+				const name2 = item.data.split("/").slice(-1)[0];
+				if (name2 !== name) {
+					return false;
+				}
+				return true;
+			});
+			if (item === undefined) {
+				throw new Error(`unable to find image ${pth} in data.json`);
+			}
+			// correct name, so check category
+			if (item.label !== category) {
+				throw new Error(`expected ${category} but found ${item.label}`);
+			}
+			// type should be image
+			if (item.type !== "image") {
+				throw new Error(`expected image type but found ${item.type}`);
+			}
+			// file name should be the hash
+			if (item.hash !== name.split(".")[0]) {
+				throw new Error(
+					`expected ${name.split(".")[0]} hash but found ${item.hash}`,
+				);
+			}
+			// hash should be the hash of the image content
+			const hash = hexHash(content);
+			if (item.hash !== hash) {
+				throw new Error(`expected ${hash} hash but found ${item.hash}`);
+			}
+		}
+
+		// count the number of images in the flat data
+		let flatCount = 0;
+		for (const pth of fsWalk(output)) {
+			if (
+				!pth.endsWith(".jpg") &&
+				!pth.endsWith(".jpeg") &&
+				!pth.endsWith(".png")
+			) {
+				continue;
+			}
+			flatCount++;
+		}
+
+		// check same number of images in flat data
+		if (hierCount !== flatCount) {
+			throw new Error(`expected ${hierCount} images but found ${flatCount}`);
+		}
+	});
+});

package/src/tests/utils.ts

@@ -0,0 +1,152 @@
+import fs from "node:fs";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import {
+	type CaptchasContainer,
+	CaptchasContainerSchema,
+	DataSchema,
+} from "@prosopo/types";
+import { at } from "@prosopo/util";
+
+// recursively list files in a directory
+export function* fsWalk(
+	pth: string,
+	options?: {
+		filesFirst?: boolean; // if true, depth first (i.e. yield files inside a dir before the dir itself, recursively), else yield in encounter order (i.e. dir first, then files inside, recursively)
+	},
+): Generator<string> {
+	// if path is a directory, recurse
+	const isDir = fs.existsSync(pth) && fs.statSync(pth).isDirectory();
+	let subpaths = isDir ? fs.readdirSync(pth) : [];
+	subpaths = subpaths.map((f) => `${pth}/${f}`);
+	if (!options?.filesFirst) {
+		yield pth;
+	}
+	for (const subpath of subpaths) {
+		yield* fsWalk(subpath, options);
+	}
+	if (options?.filesFirst) {
+		yield pth;
+	}
+}
+
+// test whether two files/dirs are equal, recursively comparing contents as required
+export const fsEq = (pth1: string, pth2: string) => {
+	const it1 = fsWalk(pth1);
+	const it2 = fsWalk(pth2);
+	// drop the root path
+	const files1 = [...it1].map((f) => f.slice(pth1.length));
+	const files2 = [...it2].map((f) => f.slice(pth2.length));
+	if (files1.length !== files2.length) {
+		return false;
+	}
+	// sort files so in same order
+	files1.sort();
+	files2.sort();
+	if (!files1.every((f, i) => f === files2[i])) {
+		// the two lists of files are not the same
+		return false;
+	}
+	// compare contents of each file
+	for (let i = 0; i < files1.length; i++) {
+		const file1 = pth1 + at(files1, i);
+		const file2 = pth2 + at(files2, i);
+		const stat1 = fs.statSync(file1);
+		if (stat1.isDirectory()) {
+			// already checked above
+			continue;
+		}
+		// files, so compare contents
+		const content1 = fs.readFileSync(file1);
+		const content2 = fs.readFileSync(file2);
+		if (!content1.equals(content2)) {
+			return false;
+		}
+	}
+	return true;
+};
+
+export const readDataJson = (pth: string) => {
+	let content = fs.readFileSync(pth).toString();
+	// TODO use getPaths() here to find the repo dir
+	content = content.replaceAll("${repo}", `${__dirname}/../../../..`);
+	const dataJson = JSON.parse(content.toString());
+	const data = DataSchema.parse(dataJson);
+	return data;
+};
+
+export const readCaptchasJson = (pth: string) => {
+	const content = fs.readFileSync(pth).toString();
+	const dataJson = JSON.parse(content);
+	const captchas = CaptchasContainerSchema.parse(dataJson);
+	return captchas;
+};
+
+export const captchasEqFs = (pth1: string, pth2: string) => {
+	const data1 = readCaptchasJson(pth1);
+	const data2 = readCaptchasJson(pth2);
+	return captchasEq(data1, data2);
+};
+
+export const captchasEq = (
+	first: CaptchasContainer,
+	second: CaptchasContainer,
+) => {
+	if (first.captchas.length !== second.captchas.length) {
+		return false;
+	}
+	for (let i = 0; i < first.captchas.length; i++) {
+		const captcha1 = at(first.captchas, i);
+		const captcha2 = at(second.captchas, i);
+		// salts should not be the same as generated on the fly and not stored!
+		if (captcha1.salt === captcha2.salt) {
+			return false;
+		}
+		// everything else should be the same
+		const { salt: salt1, ...rest1 } = captcha1;
+		const { salt: salt2, ...rest2 } = captcha2;
+		if (JSON.stringify(rest1) !== JSON.stringify(rest2)) {
+			return false;
+		}
+	}
+	return true;
+};
+
+export const substituteRepoDir = () => {
+	// read all json files in the test data dir
+	for (const pth of fsWalk(`${__dirname}/data`)) {
+		if (!pth.endsWith(".json")) {
+			continue;
+		}
+		// make a backup of each file
+		fs.copyFileSync(pth, `${pth}.bak`);
+		// replace ${repo} with the path to the repo
+		let content = fs.readFileSync(pth).toString();
+		// TODO use getPaths() here to find the repo dir
+		content = content.replaceAll("${repo}", `${__dirname}/../../../..`);
+		// rewrite the file
+		fs.writeFileSync(pth, content);
+	}
+};
+
+export const restoreRepoDir = () => {
+	// read all json files in the test data dir
+	for (const pth of fsWalk(`${__dirname}/data`)) {
+		if (!pth.endsWith(".json")) {
+			continue;
+		}
+		// restore the backup of each file
+		fs.renameSync(`${pth}.bak`, pth);
+	}
+};

package/src/utils/input.ts

@@ -0,0 +1,58 @@
+import fs from "node:fs";
+import { ProsopoDatasetError, ProsopoError } from "@prosopo/common";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as z from "zod";
+import { CliCommand } from "../cli/cliCommand.js";
+
+export const InputArgsSchema = z.object({
+	input: z.string(),
+});
+
+export type InputArgs = z.infer<typeof InputArgsSchema>;
+
+export class InputCliCommand<
+	T extends typeof InputArgsSchema,
+> extends CliCommand<T> {
+	public override getArgSchema(): T {
+		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
+	}
+	public override getDescription(): string {
+		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
+	}
+	public override getOptions() {
+		return {
+			input: {
+				string: true,
+				alias: "in",
+				demand: true,
+				description: "The input path",
+			},
+		};
+	}
+
+	public override async _check(args: InputArgs) {
+		this.logger.debug(() => ({ msg: "input _check" }));
+		await super._check(args);
+		// input must exist
+		if (!fs.existsSync(args.input)) {
+			throw new ProsopoDatasetError(
+				new Error(`input path does not exist: ${args.input}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+	}
+}

package/src/utils/inputOutput.ts

@@ -0,0 +1,29 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import type * as z from "zod";
+import { CliCommandComposite } from "../cli/cliCommandComposite.js";
+import { InputArgsSchema, InputCliCommand } from "./input.js";
+import { OutputArgsSchema, OutputCliCommand } from "./output.js";
+
+export const InputOutputArgsSchema = InputArgsSchema.merge(OutputArgsSchema);
+
+export type InputOutputArgs = z.infer<typeof InputOutputArgsSchema>;
+
+export abstract class InputOutputCliCommand<
+	T extends typeof InputOutputArgsSchema,
+> extends CliCommandComposite<T> {
+	constructor() {
+		super([new InputCliCommand<T>(), new OutputCliCommand<T>()]);
+	}
+}

package/src/utils/output.ts

@@ -0,0 +1,86 @@
+import fs from "node:fs";
+import { ProsopoEnvError, ProsopoError } from "@prosopo/common";
+import { lodash } from "@prosopo/util/lodash";
+import { boolean, object, string, type infer as zInfer } from "zod";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { CliCommand } from "../cli/cliCommand.js";
+
+export const OutputArgsSchema = object({
+	output: string(),
+	overwrite: boolean().optional(),
+});
+
+export type OutputArgs = zInfer<typeof OutputArgsSchema>;
+
+export class OutputCliCommand<
+	T extends typeof OutputArgsSchema,
+> extends CliCommand<T> {
+	#outputExists = false;
+
+	public outputExists() {
+		return this.#outputExists;
+	}
+
+	public override getArgSchema(): T {
+		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
+	}
+	public override getDescription(): string {
+		throw new ProsopoError("DEVELOPER.METHOD_NOT_IMPLEMENTED");
+	}
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			output: {
+				alias: "out",
+				string: true,
+				demand: true,
+				description: "The output path",
+			},
+			overwrite: {
+				boolean: true,
+				description: "Overwrite files in the output path if they already exist",
+			},
+		});
+	}
+
+	public override async _check(args: OutputArgs) {
+		this.logger.debug(() => ({ msg: "Output _check" }));
+		await super._check(args);
+
+		// record whether output already exists
+		this.#outputExists = fs.existsSync(args.output);
+
+		// output must not exist, unless overwrite is true
+		if (this.outputExists()) {
+			if (!args.overwrite) {
+				throw new ProsopoEnvError(
+					new Error(`output path already exists: ${args.output}`),
+					{
+						translationKey: "FS.FILE_ALREADY_EXISTS",
+					},
+				);
+			}
+		}
+	}
+
+	public override async _run(args: OutputArgs) {
+		this.logger.debug(() => ({ msg: "Output run" }));
+		await super._run(args);
+		if (args.overwrite && this.outputExists()) {
+			// if overwrite is true, delete the output directory
+			this.logger.info(() => ({ msg: "cleaning output directory..." }));
+			fs.rmSync(args.output, { recursive: true });
+		}
+	}
+}

package/tsconfig.cjs.json

@@ -0,0 +1,36 @@
+{
+	"extends": "../../tsconfig.cjs.json",
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist/cjs"
+	},
+	"include": [
+		"./src/**/*.ts",
+		"./src/**/*.json",
+		"./src/**/*.d.ts",
+		"./src/**/*.tsx"
+	],
+	"references": [
+		{
+			"path": "../../dev/config/tsconfig.cjs.json"
+		},
+		{
+			"path": "../common/tsconfig.cjs.json"
+		},
+		{
+			"path": "../logger/tsconfig.cjs.json"
+		},
+		{
+			"path": "../util/tsconfig.cjs.json"
+		},
+		{
+			"path": "../types/tsconfig.cjs.json"
+		},
+		{
+			"path": "../util-crypto/tsconfig.cjs.json"
+		},
+		{
+			"path": "../../dev/workspace/tsconfig.cjs.json"
+		}
+	]
+}