@prosopo/datasets-fs 3.0.39 → 3.0.77

package/src/commands/labels.ts

@@ -0,0 +1,78 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import { LabelledDataSchema, type LabelledItem } from "@prosopo/types";
+import { lodash } from "@prosopo/util/lodash";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import type * as z from "zod";
+import {
+	InputOutputArgsSchema,
+	InputOutputCliCommand,
+} from "../utils/inputOutput.js";
+
+export const ArgsSchema = InputOutputArgsSchema.extend({});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class Labels extends InputOutputCliCommand<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			input: {
+				description: "JSON file containing labelled data",
+			},
+			output: {
+				description: "Where to put the JSON file containing labels",
+			},
+		});
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+
+		const file = args.input;
+		if (!fs.existsSync(file)) {
+			throw new ProsopoDatasetError(new Error(`file does not exist: ${file}`), {
+				translationKey: "FS.FILE_NOT_FOUND",
+			});
+		}
+
+		const labelled: LabelledItem[] = file
+			? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(file, "utf8")))
+					.items
+			: [];
+
+		const labels = new Set<string>();
+		for (const item of labelled) {
+			labels.add(item.label);
+		}
+		const labelArray = Array.from(labels);
+		labelArray.sort();
+
+		fs.mkdirSync(args.output.split("/").slice(0, -1).join("/"), {
+			recursive: true,
+		});
+		fs.writeFileSync(
+			args.output,
+			JSON.stringify({ labels: labelArray }, null, 4),
+		);
+	}
+
+	public override getDescription(): string {
+		return "get all labels from some data";
+	}
+}

package/src/commands/relocate.ts

@@ -0,0 +1,109 @@
+import fs from "node:fs";
+import { get } from "@prosopo/util";
+import { lodash } from "@prosopo/util/lodash";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as z from "zod";
+import {
+	InputOutputArgsSchema,
+	InputOutputCliCommand,
+} from "../utils/inputOutput.js";
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+	from: z.string(),
+	to: z.string(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class Relocate extends InputOutputCliCommand<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			input: {
+				description:
+					"A json file containing a list of objects with (at least) a url",
+			},
+			output: {
+				description: "Where to write the new json file containing the new urls",
+			},
+			from: {
+				string: true,
+				demand: true,
+				description: "The string to replace in the urls",
+			},
+			to: {
+				description: "The string to substitute in the urls",
+				string: true,
+				demand: true,
+			},
+		});
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+
+		const replace = (data: unknown, from: string, to: string) => {
+			if (Array.isArray(data)) {
+				for (let i = 0; i < data.length; i++) {
+					data[i] = replace(data[i], from, to);
+				}
+			} else if (typeof data === "object") {
+				const obj = data as {
+					[key: string]: unknown;
+				};
+				for (const key of Object.keys(obj)) {
+					if (key === "data") {
+						const value = get(obj, key);
+						if (typeof value === "string") {
+							if (value.startsWith(from)) {
+								this.logger.debug(() => ({
+									msg: "replacing",
+									data: { value },
+								}));
+								obj[key] = to + value.slice(from.length);
+								this.logger.debug(() => ({
+									msg: "replaced",
+									data: { value: obj[key] },
+								}));
+							}
+						}
+					} else {
+						obj[key] = replace(obj[key], from, to);
+					}
+				}
+			}
+			return data;
+		};
+
+		const file: string = args.input;
+		this.logger.info(() => ({ data: args, msg: "relocate" }));
+		// read the file
+		let data = JSON.parse(fs.readFileSync(file, "utf8"));
+		// replace the urls by recursively traversing the data
+		data = replace(data, args.from, args.to);
+		// write the file
+		fs.mkdirSync(args.output.split("/").slice(0, -1).join("/"), {
+			recursive: true,
+		});
+		fs.writeFileSync(args.output, JSON.stringify(data, null, 4));
+	}
+
+	public override getDescription(): string {
+		return "Relocate urls in a JSON file using string substitution";
+	}
+}

package/src/commands/resize.ts

@@ -0,0 +1,171 @@
+import fs from "node:fs";
+import { blake2b } from "@noble/hashes/blake2b";
+import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
+import { type Data, DataSchema, type Item } from "@prosopo/types";
+import { u8aToHex } from "@prosopo/util";
+import { lodash } from "@prosopo/util/lodash";
+import cliProgress from "cli-progress";
+import sharp from "sharp";
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+import * as z from "zod";
+import {
+	InputOutputArgsSchema,
+	InputOutputCliCommand,
+} from "../utils/inputOutput.js";
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+	size: z.number(),
+	square: z.boolean().optional(),
+});
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+
+export class Resize extends InputOutputCliCommand<ArgsSchemaType> {
+	public override getArgSchema() {
+		return ArgsSchema;
+	}
+
+	public override getOptions() {
+		return lodash().merge(super.getOptions(), {
+			input: {
+				description:
+					"JSON file containing a list of objects with (at least) a url",
+			},
+			output: {
+				description:
+					"Where to put the output directory containing the map file and single directory of images. The map file will contain the new urls of the scaled images, new hashes and pass through any other information, e.g. labels.",
+			},
+			size: {
+				number: true,
+				description:
+					"The dimension (height/width) of the scaled image. If the image is not square, the other dimension will be scaled to maintain the aspect ratio.",
+			},
+			square: {
+				boolean: true,
+				description:
+					"If true, the image will be cropped to a square before scaling. If false, the image will be scaled to the given size, maintaining the aspect ratio.",
+			},
+		});
+	}
+
+	public override async _run(args: Args) {
+		await super._run(args);
+
+		const size = args.size;
+		const square = args.square ?? false;
+
+		const mapFile: string = args.input;
+		if (!fs.existsSync(mapFile)) {
+			throw new ProsopoDatasetError(
+				new Error(`Map file does not exist: ${mapFile}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+		const outDir: string = args.output;
+		const overwrite = args.overwrite || false;
+		if (!overwrite && fs.existsSync(outDir)) {
+			throw new ProsopoEnvError(
+				new Error(`Output directory already exists: ${outDir}`),
+				{
+					translationKey: "FS.FILE_NOT_FOUND",
+				},
+			);
+		}
+
+		// create the output directory
+		const imgDir = `${outDir}/images`;
+		fs.mkdirSync(imgDir, { recursive: true });
+
+		// read the map file
+		const inputItems: Item[] = DataSchema.parse(
+			JSON.parse(fs.readFileSync(mapFile, "utf8")),
+		).items;
+
+		// for each item
+		const bar = new cliProgress.SingleBar(
+			{},
+			cliProgress.Presets.shades_classic,
+		);
+		bar.start(inputItems.length, 0);
+		const outputItems: Item[] = [];
+		for (const inputItem of inputItems) {
+			bar.increment();
+			// this.logger.log(`resizing ${inputItem.data}`)
+			// read the file
+			const img = fs.readFileSync(inputItem.data);
+			// resize the image
+			const resized = await sharp(img)
+				.resize({
+					width: size,
+					height: size,
+					fit: square ? "cover" : "inside",
+				})
+				.removeAlpha() // remove the alpha channel
+				.toColorspace("srgb") // 8-bits per channel
+				.jpeg({
+					quality: 75,
+					chromaSubsampling: "4:2:0",
+					optimiseCoding: true,
+					trellisQuantisation: true,
+					overshootDeringing: true,
+					optimizeScans: true,
+					quantisationTable: 3,
+					force: true,
+				});
+			const tmpFilePath = `${imgDir}/tmp.jpg`;
+			await resized.toFile(tmpFilePath);
+			// read the resized image
+			const resizedImg = fs.readFileSync(tmpFilePath);
+			// hash the image
+			const hash = blake2b(resizedImg);
+			const hex = u8aToHex(hash);
+			// move the image
+			const finalFilePath = `${imgDir}/${hex}.jpg`;
+			fs.renameSync(tmpFilePath, finalFilePath);
+
+			// add the item to the output
+			const outputItem: Item = {
+				...inputItem,
+				hash: hex,
+				data: fs.realpathSync(finalFilePath),
+			};
+			outputItems.push(outputItem);
+		}
+		bar.stop();
+
+		// write the map file
+		const outputMapFile = `${outDir}/data.json`;
+
+		const data: Data = {
+			items: outputItems,
+		};
+
+		// verify the output
+		this.logger.info(() => ({ msg: "verifying data" }));
+		DataSchema.parse(data);
+
+		this.logger.info(() => ({ msg: "writing data" }));
+		fs.mkdirSync(args.output.split("/").slice(0, -1).join("/"), {
+			recursive: true,
+		});
+		fs.writeFileSync(outputMapFile, JSON.stringify(data, null, 4));
+	}
+
+	public override getDescription(): string {
+		return "Resize images to a given size";
+	}
+}

package/src/dummy.ts

@@ -0,0 +1,52 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// generate some dummy data for testing the generate fns
+
+import fs from "node:fs";
+import { CaptchaItemTypes, type Item, type LabelledItem } from "@prosopo/types";
+
+const nLabels = 4;
+const nLabelledImages = 100;
+const nUnlabelledImages = 100;
+
+const labelledImages: LabelledItem[] = [];
+const unlabelledImages: Item[] = [];
+
+for (let i = 0; i < nLabelledImages; i++) {
+	const label = i % nLabels;
+	labelledImages.push({
+		label: label.toString(),
+		data: `abc${i}`,
+		type: CaptchaItemTypes.Image,
+		hash: `abc${i}hash`,
+	});
+}
+
+for (let i = 0; i < nUnlabelledImages; i++) {
+	unlabelledImages.push({
+		data: `def${i}`,
+		type: CaptchaItemTypes.Image,
+		hash: `def${i}hash`,
+	});
+}
+
+fs.writeFileSync("labelled.json", JSON.stringify(labelledImages));
+fs.writeFileSync("unlabelled.json", JSON.stringify(unlabelledImages));
+fs.writeFileSync(
+	"labels.json",
+	JSON.stringify(Array.from(Array(nLabels).keys()).map((x) => x.toString())),
+);
+
+process.exit(0);

package/src/index.ts

@@ -0,0 +1,14 @@
+// Copyright 2021-2026 Prosopo (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+export * from "./cli/cli.js";