@prosopo/datasets-fs 3.0.43 → 3.0.77

package/src/cli.ts

@@ -0,0 +1,50 @@
+import { isMain } from "@prosopo/util";
+import type { CliCommandAny } from "./cli/cliCommand.js";
+import { Flatten } from "./commands/flatten.js";
+import { GenerateV1 } from "./commands/generateV1.js";
+import { GenerateV2 } from "./commands/generateV2.js";
+import { Get } from "./commands/get.js";
+import { Labels } from "./commands/labels.js";
+import { Relocate } from "./commands/relocate.js";
+import { Resize } from "./commands/resize.js";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import { Cli } from "./index.js";
+
+const main = async () => {
+	const commands: CliCommandAny[] = [
+		new Flatten(),
+		new GenerateV1(),
+		new GenerateV2(),
+		new Get(),
+		new Labels(),
+		new Relocate(),
+		new Resize(),
+	];
+	const cli = new Cli(commands);
+	cli.logger.setLogLevel("debug");
+	await cli.exec();
+};
+
+//if main process
+if (isMain(import.meta.url)) {
+	main()
+		.then(() => {
+			process.exit(0);
+		})
+		.catch((err) => {
+			console.log("error", err);
+			process.exit(1);
+		});
+}

package/src/commands/flatten.ts

@@ -0,0 +1,152 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import {
+	CaptchaItemTypes,
+	type Data,
+	DataSchema,
+	type LabelledItem,
+} from "@prosopo/types";
+import { at } from "@prosopo/util";
+import { hexHash } from "@prosopo/util-crypto";
+import { lodash } from "@prosopo/util/lodash";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import {
+	InputOutputArgsSchema,
+	InputOutputCliCommand,
+} from "../utils/inputOutput.js";
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+	allowDuplicates: z.boolean().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getDescription(): string {
+		return "Restructure a directory containing directories for each image classification into a single directory with a file containing the labels";
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			input: {
+				description:
+					"Path to the data directory containing subdirectories for each image classification",
+			},
+			output: {
+				description:
+					"Where to put the output file containing the labels and single directory of images",
+			},
+			allowDuplicates: {
+				boolean: true,
+				description: "If true, allow duplicates in the data",
+			},
+		});
+	}
+
+	public override async _run(args: Args) {
+		this.logger.debug(() => ({ msg: "flatten run" }));
+		await super._run(args);
+
+		const dataDir = args.input;
+		const outDir = args.output;
+
+		// find the labels (these should be subdirectories of the data directory)
+		this.logger.info(() => ({ msg: "reading data" }));
+		const labels: string[] = fs
+			.readdirSync(dataDir, { withFileTypes: true })
+			.filter((dirent) => dirent.isDirectory())
+			.map((dirent) => dirent.name)
+			.sort();
+		const imagesByLabel: string[][] = labels.map((label) =>
+			fs.readdirSync(`${dataDir}/${label}`),
+		);
+
+		// create the output directory
+		const imageDir = `${outDir}/images`;
+		fs.mkdirSync(imageDir, { recursive: true });
+
+		const bar = new cliProgress.SingleBar(
+			{},
+			cliProgress.Presets.shades_classic,
+		);
+
+		// for each label
+		const items: LabelledItem[] = [];
+		bar.start(
+			imagesByLabel.reduce((acc, images) => acc + images.length, 0),
+			0,
+		);
+		labels.forEach((label, i) => {
+			const images: string[] = at(imagesByLabel, i);
+			// for each image
+			for (const image of images) {
+				bar.increment();
+				// this.logger.log(`flattening ${label}/${image}`)
+				// copy the image to the output directory
+				const extension = image.split(".").pop();
+				// read file to bytes
+				const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+				// hash based on the content of the image
+				const hex = hexHash(content);
+				const name = `${hex}.${extension}`;
+				if (fs.existsSync(`${imageDir}/${name}`)) {
+					for (const item of items) {
+						if (item.hash === hex) {
+							this.logger.info(() => ({
+								msg: `\ndupe: ${label}/${image}`,
+								data: { item },
+							}));
+						}
+					}
+					if (!args.allowDuplicates) {
+						throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+							context: { image: `${label}/${image}` },
+						});
+					}
+				}
+				fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+				const filePath = fs.realpathSync(`${imageDir}/${name}`);
+				// add the image to the map file
+				const entry: LabelledItem = {
+					data: filePath,
+					type: CaptchaItemTypes.Image,
+					label,
+					hash: hex,
+				};
+				items.push(entry);
+			}
+		});
+		bar.stop();
+
+		const data: Data = {
+			items,
+		};
+
+		// verify data
+		this.logger.info(() => ({ msg: "verifying data", data: { ...data } }));
+		DataSchema.parse(data);
+
+		// write map file
+		this.logger.info(() => ({ msg: "writing data" }));
+		fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+	}
+}

package/src/commands/generate.ts

@@ -0,0 +1,218 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import {
+	DataSchema,
+	type Item,
+	LabelledDataSchema,
+	type LabelledItem,
+	LabelsContainerSchema,
+} from "@prosopo/types";
+import { lodash, setSeedGlobal } from "@prosopo/util/lodash";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as z from "zod";
+import { OutputArgsSchema, OutputCliCommand } from "../utils/output.js";
+
+export const ArgsSchema = OutputArgsSchema.extend({
+	labels: z.string().optional(),
+	labelled: z.string().optional(),
+	unlabelled: z.string().optional(),
+	seed: z.number(),
+	size: z.number().optional(),
+	overwrite: z.boolean().optional(),
+	allowDuplicates: z.boolean().optional(),
+	allowDuplicatesLabelled: z.boolean().optional(),
+	allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export abstract class Generate<
+	T extends ArgsSchemaType,
+> extends OutputCliCommand<T> {
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			output: {
+				description: "Where to write the captchas JSON file",
+			},
+			labelled: {
+				string: true,
+				demand: true,
+				description: "Path to JSON file containing labelled data",
+			},
+			unlabelled: {
+				string: true,
+				demand: false,
+				description: "Path to JSON file containing unlabelled data",
+			},
+			seed: {
+				number: true,
+				demand: true,
+				description: "Seed for random number generator",
+			},
+			size: {
+				number: true,
+				description: "Number of images in each captcha",
+			},
+			labels: {
+				string: true,
+				description:
+					"Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.",
+			},
+			allowDuplicates: {
+				boolean: true,
+				description:
+					"If true, allow duplicates in the data (labelled and unlabelled)",
+			},
+			allowDuplicatesLabelled: {
+				boolean: true,
+				description: "If true, allow duplicates in the labelled data",
+			},
+			allowDuplicatesUnlabelled: {
+				boolean: true,
+				description: "If true, allow duplicates in the unlabelled data",
+			},
+		});
+	}
+
+	public override async _check(args: Args) {
+		// if specified, check files exist
+		const labelledMapFile: string | undefined = args.labelled;
+		if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+			throw new ProsopoDatasetError(
+				new Error(`labelled map file does not exist: ${labelledMapFile}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+		const unlabelledMapFile: string | undefined = args.unlabelled;
+		if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+			throw new ProsopoDatasetError(
+				new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+		this.labelledMapFile = labelledMapFile || "";
+		this.unlabelledMapFile = unlabelledMapFile || "";
+	}
+
+	labelled: LabelledItem[] = [];
+	unlabelled: Item[] = [];
+	labels: string[] = [];
+	labelledMapFile = "";
+	unlabelledMapFile = "";
+	labelToImages: { [label: string]: Item[] } = {};
+	targets: string[] = [];
+	saltRounds = 10;
+
+	private loadData(args: Args) {
+		const allowDuplicatesLabelled =
+			args.allowDuplicatesLabelled || args.allowDuplicates || false;
+		const allowDuplicatesUnlabelled =
+			args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+
+		// load the map to get the labelled and unlabelled data
+		this.labelled = this.labelledMapFile
+			? LabelledDataSchema.parse(
+					JSON.parse(fs.readFileSync(this.labelledMapFile, "utf8")),
+				).items
+			: [];
+		this.unlabelled = this.unlabelledMapFile
+			? DataSchema.parse(
+					JSON.parse(fs.readFileSync(this.unlabelledMapFile, "utf8")),
+				).items
+			: [];
+
+		// check for duplicates
+		checkDuplicates(this.labelled, this.unlabelled, {
+			allowDuplicatesLabelled,
+			allowDuplicatesUnlabelled,
+		});
+
+		// split the labelled data by label
+		this.labelToImages = {};
+		for (const entry of this.labelled) {
+			const arr = this.labelToImages[entry.label] || [];
+			arr.push(entry);
+			this.labelToImages[entry.label] = arr;
+		}
+		this.targets = Object.keys(this.labelToImages);
+	}
+
+	private loadLabels(args: Args) {
+		// load the labels from file
+		// these are the labels that unlabelled data will be assigned to
+		// note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+		this.labels = [];
+		if (args.labels && fs.existsSync(args.labels)) {
+			this.labels.push(
+				...[
+					...LabelsContainerSchema.parse(
+						JSON.parse(fs.readFileSync(args.labels, "utf8")),
+					).labels,
+				],
+			);
+		} else {
+			// else default to the labels in the labelled data
+			this.labels.push(...[...this.targets]);
+		}
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+		// set the seed
+		setSeedGlobal(args.seed || 0);
+		// get lodash (with seeded rng)
+		const _ = lodash();
+
+		this.loadData(args);
+
+		this.loadLabels(args);
+	}
+}
+
+export const checkDuplicates = (
+	labelled: LabelledItem[],
+	unlabelled: Item[],
+	options: {
+		allowDuplicatesLabelled?: boolean;
+		allowDuplicatesUnlabelled?: boolean;
+	},
+) => {
+	// check for duplicates
+	const all = new Set<string>();
+	if (!options.allowDuplicatesLabelled) addAllUnique(all, labelled, "labelled");
+	if (!options.allowDuplicatesUnlabelled)
+		addAllUnique(all, unlabelled, "unlabelled");
+};
+
+const addAllUnique = (all: Set<string>, entries: Item[], dataType: string) => {
+	for (const entry of entries) {
+		addUnique(all, entry, dataType);
+	}
+};
+
+const addUnique = (all: Set<string>, entry: Item, dataType: string) => {
+	if (all.has(entry.data)) {
+		throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+			context: {
+				error: `Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`,
+			},
+		});
+	}
+	all.add(entry.data);
+};

package/src/commands/generateV1.ts

@@ -0,0 +1,273 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import fs from "node:fs";
+import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
+import {
+	CaptchaTypes,
+	type CaptchaWithoutId,
+	type Captchas,
+	CaptchasContainerSchema,
+	type Item,
+	type RawSolution,
+} from "@prosopo/types";
+import { at, get } from "@prosopo/util";
+import { blake2AsHex } from "@prosopo/util-crypto";
+import { lodash } from "@prosopo/util/lodash";
+import bcrypt from "bcrypt";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import { Generate, ArgsSchema as GenerateArgsSchema } from "./generate.js";
+
+export const ArgsSchema = GenerateArgsSchema.extend({
+	solved: z.number().optional(),
+	unsolved: z.number().optional(),
+	minCorrect: z.number().optional(),
+	maxCorrect: z.number().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class GenerateV1 extends Generate<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getDescription(): string {
+		return "Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled";
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			solved: {
+				description: "Number of captchas to generate that are solved",
+				number: true,
+			},
+			unsolved: {
+				description: "Number of captchas to generate that are unsolved",
+				number: true,
+			},
+			minCorrect: {
+				description: "Minimum number of target images in each captcha",
+				number: true,
+			},
+			maxCorrect: {
+				description: "Maximum number of target images in each captcha",
+				number: true,
+			},
+		});
+	}
+
+	private generateSolved(
+		solved: number,
+		size: number,
+		minCorrect: number,
+		maxCorrect: number,
+		bar: cliProgress.SingleBar,
+	) {
+		const _ = lodash();
+		// generate n solved captchas
+		const solvedCaptchas: CaptchaWithoutId[] = [];
+		for (let i = 0; i < solved; i++) {
+			// update the current value in your application..
+			bar.increment();
+
+			if (this.targets.length <= 1) {
+				throw new ProsopoDatasetError(
+					new Error("not enough different labels in labelled data"),
+					{
+						translationKey: "DATASET.NOT_ENOUGH_LABELS",
+					},
+				);
+			}
+
+			// uniformly sample targets
+			const target = at(this.targets, i % this.targets.length);
+			const notTargets = this.targets.filter((t) => t !== target);
+
+			// how many correct items should be in the captcha?
+			const nCorrect = _.random(minCorrect, maxCorrect);
+			// how many incorrect items should be in the captcha?
+			const nIncorrect = size - nCorrect;
+
+			const targetItems: Item[] = get(this.labelToImages, target);
+			const notTargetItems: Item[] = notTargets.flatMap((notTarget) =>
+				get(this.labelToImages, notTarget),
+			);
+
+			if (targetItems.length < nCorrect) {
+				throw new ProsopoEnvError(
+					new Error(`not enough images for target (${target})`),
+					{
+						translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+					},
+				);
+			}
+			if (notTargetItems.length < nIncorrect) {
+				throw new ProsopoDatasetError(
+					new Error(`not enough non-matching images for target (${target})`),
+					{
+						translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+					},
+				);
+			}
+
+			// get the correct items
+			const correctItems: Item[] = _.sampleSize(targetItems, nCorrect);
+
+			// get the incorrect items
+			const incorrectItems: Item[] = _.sampleSize(notTargetItems, nIncorrect);
+
+			let items: Item[] = [...correctItems, ...incorrectItems];
+			let indices: number[] = [...Array(items.length).keys()];
+			indices = _.shuffle(indices);
+			items = indices.map((i) => at(items, i));
+			items = items.map((item) => {
+				return {
+					data: item.data,
+					hash: item.hash,
+					type: item.type,
+				};
+			});
+
+			// the first n indices are the correct items
+			const solution: RawSolution[] = indices
+				.map((index, i) => {
+					return {
+						pre: index, // the index of the item in the items array before shuffle
+						post: i, // the index of the item in the shuffled array
+					};
+				})
+				.filter((item) => item.pre < correctItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
+				.map((item) => {
+					return item.post; // return the index in the shuffled array
+				});
+
+			const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds));
+			// create the captcha
+			const captcha: CaptchaWithoutId = {
+				salt,
+				target,
+				items,
+				solution,
+			};
+			solvedCaptchas.push(captcha);
+		}
+		return solvedCaptchas;
+	}
+
+	private generateUnsolved(
+		unsolved: number,
+		size: number,
+		bar: cliProgress.SingleBar,
+	) {
+		const _ = lodash();
+		// this.logger.info(`Generating ${unsolved} unsolved captchas...`)
+		// create a new progress bar instance and use shades_classic theme
+		// generate n unsolved captchas
+		const unsolvedCaptchas: CaptchaWithoutId[] = [];
+		for (let i = 0; i < unsolved; i++) {
+			bar.increment();
+			if (this.unlabelled.length <= size) {
+				throw new ProsopoDatasetError(
+					new Error("unlabelled map file does not contain enough data"),
+					{
+						translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+					},
+				);
+			}
+			// pick a random label to be the target
+			// note that these are potentially different to the labelled data labels
+			if (this.labels.length <= 0) {
+				throw new ProsopoDatasetError(
+					new Error("no labels found for unlabelled data"),
+					{
+						translationKey: "DATASET.NOT_ENOUGH_LABELS",
+					},
+				);
+			}
+			const index = _.random(0, this.labels.length - 1);
+			const target = at(this.labels, index);
+			// randomly pick images from the unlabelled data
+			const itemSet: Item[] = _.sampleSize(this.unlabelled, size);
+			// shuffle the items
+			let items: Item[] = [...itemSet];
+			let indices: number[] = [...Array(items.length).keys()];
+			indices = _.shuffle(indices);
+			items = indices.map((i) => at(items, i));
+			items = items.map((item) => {
+				return {
+					data: item.data,
+					hash: item.hash,
+					type: item.type,
+				};
+			});
+			const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds));
+			// create the captcha
+			const captcha: CaptchaWithoutId = {
+				salt,
+				target,
+				items,
+			};
+			unsolvedCaptchas.push(captcha);
+		}
+		return unsolvedCaptchas;
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+
+		const outFile: string = args.output;
+
+		// get lodash (with seeded rng)
+		const _ = lodash();
+
+		const size: number = args.size || 9;
+		const minCorrect: number = args.minCorrect || 1;
+		const maxCorrect: number = args.maxCorrect || size - 1;
+		const solved: number = args.solved || 0;
+		const unsolved: number = args.unsolved || 0;
+
+		// create a new progress bar instance and use shades_classic theme
+		const bar = new cliProgress.SingleBar(
+			{},
+			cliProgress.Presets.shades_classic,
+		);
+
+		// this.logger.info(`Generating ${solved} solved captchas...`)
+		bar.start(solved + unsolved, 0);
+		const solvedCaptchas = this.generateSolved(
+			solved,
+			size,
+			minCorrect,
+			maxCorrect,
+			bar,
+		);
+		const unsolvedCaptchas = this.generateUnsolved(unsolved, size, bar);
+		bar.stop();
+		// write to file
+		const output: Captchas = {
+			captchas: [...solvedCaptchas, ...unsolvedCaptchas],
+			format: CaptchaTypes.SelectAll,
+		};
+
+		// verify the output
+		CaptchasContainerSchema.parse(output);
+
+		fs.mkdirSync(args.output.split("/").slice(0, -1).join("/"), {
+			recursive: true,
+		});
+		fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+	}
+}