@prosopo/datasets-fs 3.0.39 → 3.0.77

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@prosopo/datasets-fs",
-	"version": "3.0.39",
+	"version": "3.0.77",
 	"author": "PROSOPO LIMITED <info@prosopo.io>",
 	"license": "Apache-2.0",
 	"private": false,
@@ -10,7 +10,8 @@
 	},
 	"scripts": {
 		"clean": "del-cli --verbose dist tsconfig.tsbuildinfo",
-		"build": "NODE_ENV=${NODE_ENV:-development}; vite build --config vite.esm.config.ts --mode $NODE_ENV",
+		"build": "npm run build:cross-env -- --mode ${NODE_ENV:-development}",
+		"build:cross-env": "vite build --config vite.esm.config.ts",
 		"build:tsc": "tsc --build --verbose",
 		"build:cjs": "NODE_ENV=${NODE_ENV:-development}; vite build --config vite.cjs.config.ts --mode $NODE_ENV",
 		"typecheck": "tsc --project tsconfig.types.json",
@@ -29,11 +30,12 @@
 	},
 	"dependencies": {
 		"@noble/hashes": "1.8.0",
-		"@prosopo/common": "3.1.26",
-		"@prosopo/types": "3.6.4",
-		"@prosopo/util": "3.2.4",
-		"@prosopo/util-crypto": "13.5.28",
-		"@prosopo/workspace": "3.1.19",
+		"@prosopo/common": "3.1.38",
+		"@prosopo/logger": "1.0.2",
+		"@prosopo/types": "4.3.0",
+		"@prosopo/util": "3.2.15",
+		"@prosopo/util-crypto": "13.5.29",
+		"@prosopo/workspace": "3.1.20",
 		"bcrypt": "6.0.0",
 		"cli-progress": "3.12.0",
 		"sharp": "0.33.5",
@@ -41,7 +43,7 @@
 		"zod": "3.23.8"
 	},
 	"devDependencies": {
-		"@prosopo/config": "3.1.26",
+		"@prosopo/config": "3.3.1",
 		"@types/bcrypt": "5.0.2",
 		"@types/cli-progress": "3.11.6",
 		"@types/node": "22.10.2",
@@ -58,7 +60,8 @@
 	},
 	"repository": {
 		"type": "git",
-		"url": "git+https://github.com/prosopo/captcha.git"
+		"url": "git+https://github.com/prosopo/captcha.git",
+		"directory": "packages/datasets-fs"
 	},
 	"bugs": {
 		"url": "https://github.com/prosopo/captcha/issues"

package/src/cli/cli.ts

@@ -0,0 +1,84 @@
+import { ProsopoCliError } from "@prosopo/common";
+import { LogLevel, getLogger } from "@prosopo/logger";
+import yargs, { type Argv } from "yargs";
+import { hideBin } from "yargs/helpers";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import type { CliCommandAny } from "./cliCommand.js";
+
+const dirname = process.cwd();
+
+export class Cli {
+	#commands: CliCommandAny[];
+	logger = getLogger("info", import.meta.url);
+
+	constructor(commands: CliCommandAny[]) {
+		this.#commands = commands;
+	}
+
+	private config() {
+		let y = yargs(hideBin(process.argv))
+			.option("log-level", {
+				type: "string",
+				choices: Object.values(LogLevel.options),
+				default: LogLevel.enum.info,
+				description: "The log level",
+			})
+			// biome-ignore lint/suspicious/noExplicitAny: TODO fix
+			.middleware((argv: any) => {
+				this.logger.setLogLevel(argv.logLevel);
+			}, true);
+
+		for (const command of this.#commands) {
+			y = y.command({
+				command: command.getCommandName(),
+				describe: command.getDescription(),
+				builder: command.getOptions(),
+				// biome-ignore lint/suspicious/noExplicitAny: TODO fix
+				handler: async (argv: any) => {
+					this.logger.debug(() => ({
+						msg: `running ${command.getCommandName()}`,
+					}));
+					const args = await command.parse(argv);
+					await command.exec(args);
+				},
+			});
+		}
+
+		if (!this.#commands.find((c) => c.getCommandName() === "$0")) {
+			// no default command
+			y = y.command(
+				"$0",
+				"default command",
+				(y: Argv) => y,
+				() => {
+					throw new ProsopoCliError("CLI.PARAMETER_ERROR", {
+						context: { error: "no command specified" },
+					});
+				},
+			);
+		}
+		y = y
+			.demandCommand()
+			.strict()
+			.showHelpOnFail(false, "Specify --help for available options");
+		return y;
+	}
+
+	public async exec(args: string[] = process.argv.slice(2)) {
+		const config = this.config();
+		this.logger.debug(() => ({ data: { args }, msg: "parsing" }));
+		await config.parse(args);
+	}
+}

package/src/cli/cliCommand.ts

@@ -0,0 +1,68 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { getLogger } from "@prosopo/logger";
+import { kebabCase } from "@prosopo/util";
+import type { Options } from "yargs";
+import type * as z from "zod";
+
+export abstract class CliCommand<T extends z.ZodTypeAny> {
+	logger = getLogger("info", import.meta.url);
+
+	// get the options for the CLI
+	public getOptions(): {
+		[key: string]: Options;
+	} {
+		return {};
+	}
+
+	// get the schema for the args to this command
+	public abstract getArgSchema(): T;
+
+	// parse args using the schema
+	// use this when you have an args object but not sure if it's in the right format / contains the correct fields, e.g. args from cmdline
+	public async parse(args: unknown): Promise<z.infer<T>> {
+		const argsSchema = this.getArgSchema();
+		const parsed = await argsSchema.parse(args);
+		this.logger.debug(() => ({ data: { parsed }, msg: "parsed args:" }));
+		return parsed;
+	}
+
+	// run any checks before the main run function. This function should have no side effects, i.e. not write to disk. It's purely for conducting checks / setting up
+	public async _check(args: z.infer<T>) {
+		this.logger.debug(() => ({ msg: "runChecks", args }));
+	}
+
+	// the main run function. This should be the function that does the work
+	public async _run(args: z.infer<T>) {
+		this.logger.debug(() => ({ msg: "run", args }));
+	}
+
+	// exec is a public facing function that should be called by the CLI. It will run the preRun, run, and postRun functions in order
+	public async exec(args: z.infer<T>) {
+		this.logger.debug(() => ({ msg: "exec", args }));
+		// run any checks
+		await this._check(args);
+		// then run the command
+		await this._run(args);
+	}
+
+	public getCommandName() {
+		return kebabCase(this.constructor.name);
+	}
+
+	public abstract getDescription(): string;
+}
+
+export type CliCommandAny = CliCommand<z.ZodTypeAny>;

package/src/cli/cliCommandComposite.ts

@@ -0,0 +1,54 @@
+import type { Options } from "yargs";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import type * as z from "zod";
+import { CliCommand, type CliCommandAny } from "./cliCommand.js";
+
+export abstract class CliCommandComposite<
+	T extends z.ZodTypeAny,
+> extends CliCommand<T> {
+	#commands: CliCommandAny[] = [];
+
+	public getCommands() {
+		return this.#commands;
+	}
+
+	constructor(commands: CliCommandAny[]) {
+		super();
+		// take a copy of the array to avoid leakage
+		this.#commands = [...commands];
+	}
+
+	public override getOptions(): { [key: string]: Options } {
+		// merge options in turn from each command. Command order matters, latter commands will overwrite earlier ones
+		return this.#commands.reduce((prev, command) => {
+			// biome-ignore lint/performance/noAccumulatingSpread: TODO fix
+			return { ...prev, ...command.getOptions() };
+		}, {});
+	}
+
+	public override _run(args: z.TypeOf<T>): Promise<void> {
+		return this.#commands.reduce(async (prev, command) => {
+			await prev;
+			await command._run(args);
+		}, Promise.resolve());
+	}
+
+	public override _check(args: z.TypeOf<T>): Promise<void> {
+		return this.#commands.reduce(async (prev, command) => {
+			await prev;
+			await command._check(args);
+		}, Promise.resolve());
+	}
+}

package/src/cli.ts

@@ -0,0 +1,50 @@
+import { isMain } from "@prosopo/util";
+import type { CliCommandAny } from "./cli/cliCommand.js";
+import { Flatten } from "./commands/flatten.js";
+import { GenerateV1 } from "./commands/generateV1.js";
+import { GenerateV2 } from "./commands/generateV2.js";
+import { Get } from "./commands/get.js";
+import { Labels } from "./commands/labels.js";
+import { Relocate } from "./commands/relocate.js";
+import { Resize } from "./commands/resize.js";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { Cli } from "./index.js";
+
+const main = async () => {
+	const commands: CliCommandAny[] = [
+		new Flatten(),
+		new GenerateV1(),
+		new GenerateV2(),
+		new Get(),
+		new Labels(),
+		new Relocate(),
+		new Resize(),
+	];
+	const cli = new Cli(commands);
+	cli.logger.setLogLevel("debug");
+	await cli.exec();
+};
+
+//if main process
+if (isMain(import.meta.url)) {
+	main()
+		.then(() => {
+			process.exit(0);
+		})
+		.catch((err) => {
+			console.log("error", err);
+			process.exit(1);
+		});
+}

package/src/commands/flatten.ts

@@ -0,0 +1,152 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import {
+	CaptchaItemTypes,
+	type Data,
+	DataSchema,
+	type LabelledItem,
+} from "@prosopo/types";
+import { at } from "@prosopo/util";
+import { hexHash } from "@prosopo/util-crypto";
+import { lodash } from "@prosopo/util/lodash";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import {
+	InputOutputArgsSchema,
+	InputOutputCliCommand,
+} from "../utils/inputOutput.js";
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+	allowDuplicates: z.boolean().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getDescription(): string {
+		return "Restructure a directory containing directories for each image classification into a single directory with a file containing the labels";
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			input: {
+				description:
+					"Path to the data directory containing subdirectories for each image classification",
+			},
+			output: {
+				description:
+					"Where to put the output file containing the labels and single directory of images",
+			},
+			allowDuplicates: {
+				boolean: true,
+				description: "If true, allow duplicates in the data",
+			},
+		});
+	}
+
+	public override async _run(args: Args) {
+		this.logger.debug(() => ({ msg: "flatten run" }));
+		await super._run(args);
+
+		const dataDir = args.input;
+		const outDir = args.output;
+
+		// find the labels (these should be subdirectories of the data directory)
+		this.logger.info(() => ({ msg: "reading data" }));
+		const labels: string[] = fs
+			.readdirSync(dataDir, { withFileTypes: true })
+			.filter((dirent) => dirent.isDirectory())
+			.map((dirent) => dirent.name)
+			.sort();
+		const imagesByLabel: string[][] = labels.map((label) =>
+			fs.readdirSync(`${dataDir}/${label}`),
+		);
+
+		// create the output directory
+		const imageDir = `${outDir}/images`;
+		fs.mkdirSync(imageDir, { recursive: true });
+
+		const bar = new cliProgress.SingleBar(
+			{},
+			cliProgress.Presets.shades_classic,
+		);
+
+		// for each label
+		const items: LabelledItem[] = [];
+		bar.start(
+			imagesByLabel.reduce((acc, images) => acc + images.length, 0),
+			0,
+		);
+		labels.forEach((label, i) => {
+			const images: string[] = at(imagesByLabel, i);
+			// for each image
+			for (const image of images) {
+				bar.increment();
+				// this.logger.log(`flattening ${label}/${image}`)
+				// copy the image to the output directory
+				const extension = image.split(".").pop();
+				// read file to bytes
+				const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+				// hash based on the content of the image
+				const hex = hexHash(content);
+				const name = `${hex}.${extension}`;
+				if (fs.existsSync(`${imageDir}/${name}`)) {
+					for (const item of items) {
+						if (item.hash === hex) {
+							this.logger.info(() => ({
+								msg: `\ndupe: ${label}/${image}`,
+								data: { item },
+							}));
+						}
+					}
+					if (!args.allowDuplicates) {
+						throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+							context: { image: `${label}/${image}` },
+						});
+					}
+				}
+				fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+				const filePath = fs.realpathSync(`${imageDir}/${name}`);
+				// add the image to the map file
+				const entry: LabelledItem = {
+					data: filePath,
+					type: CaptchaItemTypes.Image,
+					label,
+					hash: hex,
+				};
+				items.push(entry);
+			}
+		});
+		bar.stop();
+
+		const data: Data = {
+			items,
+		};
+
+		// verify data
+		this.logger.info(() => ({ msg: "verifying data", data: { ...data } }));
+		DataSchema.parse(data);
+
+		// write map file
+		this.logger.info(() => ({ msg: "writing data" }));
+		fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+	}
+}

package/src/commands/generate.ts

@@ -0,0 +1,218 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import {
+	DataSchema,
+	type Item,
+	LabelledDataSchema,
+	type LabelledItem,
+	LabelsContainerSchema,
+} from "@prosopo/types";
+import { lodash, setSeedGlobal } from "@prosopo/util/lodash";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as z from "zod";
+import { OutputArgsSchema, OutputCliCommand } from "../utils/output.js";
+
+export const ArgsSchema = OutputArgsSchema.extend({
+	labels: z.string().optional(),
+	labelled: z.string().optional(),
+	unlabelled: z.string().optional(),
+	seed: z.number(),
+	size: z.number().optional(),
+	overwrite: z.boolean().optional(),
+	allowDuplicates: z.boolean().optional(),
+	allowDuplicatesLabelled: z.boolean().optional(),
+	allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export abstract class Generate<
+	T extends ArgsSchemaType,
+> extends OutputCliCommand<T> {
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			output: {
+				description: "Where to write the captchas JSON file",
+			},
+			labelled: {
+				string: true,
+				demand: true,
+				description: "Path to JSON file containing labelled data",
+			},
+			unlabelled: {
+				string: true,
+				demand: false,
+				description: "Path to JSON file containing unlabelled data",
+			},
+			seed: {
+				number: true,
+				demand: true,
+				description: "Seed for random number generator",
+			},
+			size: {
+				number: true,
+				description: "Number of images in each captcha",
+			},
+			labels: {
+				string: true,
+				description:
+					"Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.",
+			},
+			allowDuplicates: {
+				boolean: true,
+				description:
+					"If true, allow duplicates in the data (labelled and unlabelled)",
+			},
+			allowDuplicatesLabelled: {
+				boolean: true,
+				description: "If true, allow duplicates in the labelled data",
+			},
+			allowDuplicatesUnlabelled: {
+				boolean: true,
+				description: "If true, allow duplicates in the unlabelled data",
+			},
+		});
+	}
+
+	public override async _check(args: Args) {
+		// if specified, check files exist
+		const labelledMapFile: string | undefined = args.labelled;
+		if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+			throw new ProsopoDatasetError(
+				new Error(`labelled map file does not exist: ${labelledMapFile}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+		const unlabelledMapFile: string | undefined = args.unlabelled;
+		if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+			throw new ProsopoDatasetError(
+				new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+		this.labelledMapFile = labelledMapFile || "";
+		this.unlabelledMapFile = unlabelledMapFile || "";
+	}
+
+	labelled: LabelledItem[] = [];
+	unlabelled: Item[] = [];
+	labels: string[] = [];
+	labelledMapFile = "";
+	unlabelledMapFile = "";
+	labelToImages: { [label: string]: Item[] } = {};
+	targets: string[] = [];
+	saltRounds = 10;
+
+	private loadData(args: Args) {
+		const allowDuplicatesLabelled =
+			args.allowDuplicatesLabelled || args.allowDuplicates || false;
+		const allowDuplicatesUnlabelled =
+			args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+
+		// load the map to get the labelled and unlabelled data
+		this.labelled = this.labelledMapFile
+			? LabelledDataSchema.parse(
+					JSON.parse(fs.readFileSync(this.labelledMapFile, "utf8")),
+				).items
+			: [];
+		this.unlabelled = this.unlabelledMapFile
+			? DataSchema.parse(
+					JSON.parse(fs.readFileSync(this.unlabelledMapFile, "utf8")),
+				).items
+			: [];
+
+		// check for duplicates
+		checkDuplicates(this.labelled, this.unlabelled, {
+			allowDuplicatesLabelled,
+			allowDuplicatesUnlabelled,
+		});
+
+		// split the labelled data by label
+		this.labelToImages = {};
+		for (const entry of this.labelled) {
+			const arr = this.labelToImages[entry.label] || [];
+			arr.push(entry);
+			this.labelToImages[entry.label] = arr;
+		}
+		this.targets = Object.keys(this.labelToImages);
+	}
+
+	private loadLabels(args: Args) {
+		// load the labels from file
+		// these are the labels that unlabelled data will be assigned to
+		// note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+		this.labels = [];
+		if (args.labels && fs.existsSync(args.labels)) {
+			this.labels.push(
+				...[
+					...LabelsContainerSchema.parse(
+						JSON.parse(fs.readFileSync(args.labels, "utf8")),
+					).labels,
+				],
+			);
+		} else {
+			// else default to the labels in the labelled data
+			this.labels.push(...[...this.targets]);
+		}
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+		// set the seed
+		setSeedGlobal(args.seed || 0);
+		// get lodash (with seeded rng)
+		const _ = lodash();
+
+		this.loadData(args);
+
+		this.loadLabels(args);
+	}
+}
+
+export const checkDuplicates = (
+	labelled: LabelledItem[],
+	unlabelled: Item[],
+	options: {
+		allowDuplicatesLabelled?: boolean;
+		allowDuplicatesUnlabelled?: boolean;
+	},
+) => {
+	// check for duplicates
+	const all = new Set<string>();
+	if (!options.allowDuplicatesLabelled) addAllUnique(all, labelled, "labelled");
+	if (!options.allowDuplicatesUnlabelled)
+		addAllUnique(all, unlabelled, "unlabelled");
+};
+
+const addAllUnique = (all: Set<string>, entries: Item[], dataType: string) => {
+	for (const entry of entries) {
+		addUnique(all, entry, dataType);
+	}
+};
+
+const addUnique = (all: Set<string>, entry: Item, dataType: string) => {
+	if (all.has(entry.data)) {
+		throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+			context: {
+				error: `Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`,
+			},
+		});
+	}
+	all.add(entry.data);
+};