@prosopo/datasets-fs 3.0.39 → 3.0.77

package/dist/commands/flatten.d.ts

@@ -0,0 +1,59 @@
+import * as z from "zod";
+import { InputOutputCliCommand } from "../utils/inputOutput.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+    input: z.ZodString;
+}, {
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}>, {
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+}>, "strip", z.ZodTypeAny, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+    getArgSchema(): z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+        input: z.ZodString;
+    }, {
+        output: z.ZodString;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+    }>, {
+        allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    }>, "strip", z.ZodTypeAny, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }>;
+    getDescription(): string;
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        input: {
+            description: string;
+        };
+        output: {
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+    };
+    _run(args: Args): Promise<void>;
+}
+//# sourceMappingURL=flatten.d.ts.map

package/dist/commands/flatten.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.d.ts","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEN,qBAAqB,EACrB,MAAM,yBAAyB,CAAC;AAEjC,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;EAErB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,qBAAa,OAAQ,SAAQ,qBAAqB,CAAC,cAAc,CAAC;IACjD,YAAY;;;;;;;;;;;;;;;;;;IAIZ,cAAc,IAAI,MAAM;IAIxB,UAAU;;;;;;;;;;;;;;IAiBJ,IAAI,CAAC,IAAI,EAAE,IAAI;CAuFrC"}

package/dist/commands/flatten.js

@@ -0,0 +1,95 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import { CaptchaItemTypes, DataSchema, } from "@prosopo/types";
+import { at } from "@prosopo/util";
+import { hexHash } from "@prosopo/util-crypto";
+import { lodash } from "@prosopo/util/lodash";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import { InputOutputArgsSchema, InputOutputCliCommand, } from "../utils/inputOutput.js";
+export const ArgsSchema = InputOutputArgsSchema.extend({
+    allowDuplicates: z.boolean().optional(),
+});
+export class Flatten extends InputOutputCliCommand {
+    getArgSchema() {
+        return ArgsSchema;
+    }
+    getDescription() {
+        return "Restructure a directory containing directories for each image classification into a single directory with a file containing the labels";
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            input: {
+                description: "Path to the data directory containing subdirectories for each image classification",
+            },
+            output: {
+                description: "Where to put the output file containing the labels and single directory of images",
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: "If true, allow duplicates in the data",
+            },
+        });
+    }
+    async _run(args) {
+        this.logger.debug(() => ({ msg: "flatten run" }));
+        await super._run(args);
+        const dataDir = args.input;
+        const outDir = args.output;
+        this.logger.info(() => ({ msg: "reading data" }));
+        const labels = fs
+            .readdirSync(dataDir, { withFileTypes: true })
+            .filter((dirent) => dirent.isDirectory())
+            .map((dirent) => dirent.name)
+            .sort();
+        const imagesByLabel = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`));
+        const imageDir = `${outDir}/images`;
+        fs.mkdirSync(imageDir, { recursive: true });
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+        const items = [];
+        bar.start(imagesByLabel.reduce((acc, images) => acc + images.length, 0), 0);
+        labels.forEach((label, i) => {
+            const images = at(imagesByLabel, i);
+            for (const image of images) {
+                bar.increment();
+                const extension = image.split(".").pop();
+                const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+                const hex = hexHash(content);
+                const name = `${hex}.${extension}`;
+                if (fs.existsSync(`${imageDir}/${name}`)) {
+                    for (const item of items) {
+                        if (item.hash === hex) {
+                            this.logger.info(() => ({
+                                msg: `\ndupe: ${label}/${image}`,
+                                data: { item },
+                            }));
+                        }
+                    }
+                    if (!args.allowDuplicates) {
+                        throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+                            context: { image: `${label}/${image}` },
+                        });
+                    }
+                }
+                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+                const filePath = fs.realpathSync(`${imageDir}/${name}`);
+                const entry = {
+                    data: filePath,
+                    type: CaptchaItemTypes.Image,
+                    label,
+                    hash: hex,
+                };
+                items.push(entry);
+            }
+        });
+        bar.stop();
+        const data = {
+            items,
+        };
+        this.logger.info(() => ({ msg: "verifying data", data: { ...data } }));
+        DataSchema.parse(data);
+        this.logger.info(() => ({ msg: "writing data" }));
+        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+    }
+}
+//# sourceMappingURL=flatten.js.map

package/dist/commands/flatten.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EACN,gBAAgB,EAEhB,UAAU,GAEV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,MAAM,eAAe,CAAC;AACnC,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACN,qBAAqB,EACrB,qBAAqB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACtD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAIH,MAAM,OAAO,OAAQ,SAAQ,qBAAqC;IACjD,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,cAAc;QAC7B,OAAO,wIAAwI,CAAC;IACjJ,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,KAAK,EAAE;gBACN,WAAW,EACV,oFAAoF;aACrF;YACD,MAAM,EAAE;gBACP,WAAW,EACV,mFAAmF;aACpF;YACD,eAAe,EAAE;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,uCAAuC;aACpD;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAG3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,MAAM,GAAa,EAAE;aACzB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;aACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;aAC5B,IAAI,EAAE,CAAC;QACT,MAAM,aAAa,GAAe,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACtD,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CACrC,CAAC;QAGF,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAC;QACpC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QAGF,MAAM,KAAK,GAAmB,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,CACR,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAC7D,CAAC,CACD,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAa,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;YAE9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC5B,GAAG,CAAC,SAAS,EAAE,CAAC;gBAGhB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;gBAEzC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAEhE,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;gBACnC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;oBAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC;4BACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gCACvB,GAAG,EAAE,WAAW,KAAK,IAAI,KAAK,EAAE;gCAChC,IAAI,EAAE,EAAE,IAAI,EAAE;6BACd,CAAC,CAAC,CAAC;wBACL,CAAC;oBACF,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;wBAC3B,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,EAAE;4BACxD,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,EAAE,EAAE;yBACvC,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;gBAExD,MAAM,KAAK,GAAiB;oBAC3B,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;oBAC5B,KAAK;oBACL,IAAI,EAAE,GAAG;iBACT,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACF,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,EAAE,CAAC;QAEX,MAAM,IAAI,GAAS;YAClB,KAAK;SACL,CAAC;QAGF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAGvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;QAClD,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;CACD"}

package/dist/commands/generate.d.ts

@@ -0,0 +1,115 @@
+import { type Item, type LabelledItem } from "@prosopo/types";
+import * as z from "zod";
+import { OutputCliCommand } from "../utils/output.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<{
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, {
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodNumber;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}>, "strip", z.ZodTypeAny, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare abstract class Generate<T extends ArgsSchemaType> extends OutputCliCommand<T> {
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        output: {
+            alias: string;
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        overwrite: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        output: {
+            description: string;
+        };
+        labelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        unlabelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        seed: {
+            number: boolean;
+            demand: boolean;
+            description: string;
+        };
+        size: {
+            number: boolean;
+            description: string;
+        };
+        labels: {
+            string: boolean;
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesLabelled: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesUnlabelled: {
+            boolean: boolean;
+            description: string;
+        };
+    };
+    _check(args: Args): Promise<void>;
+    labelled: LabelledItem[];
+    unlabelled: Item[];
+    labels: string[];
+    labelledMapFile: string;
+    unlabelledMapFile: string;
+    labelToImages: {
+        [label: string]: Item[];
+    };
+    targets: string[];
+    saltRounds: number;
+    private loadData;
+    private loadLabels;
+    _run(args: Args): Promise<void>;
+}
+export declare const checkDuplicates: (labelled: LabelledItem[], unlabelled: Item[], options: {
+    allowDuplicatesLabelled?: boolean;
+    allowDuplicatesUnlabelled?: boolean;
+}) => void;
+//# sourceMappingURL=generate.d.ts.map

package/dist/commands/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":"AAEA,OAAO,EAEN,KAAK,IAAI,EAET,KAAK,YAAY,EAEjB,MAAM,gBAAgB,CAAC;AAexB,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAoB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAExE,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAUrB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,8BAAsB,QAAQ,CAC7B,CAAC,SAAS,cAAc,CACvB,SAAQ,gBAAgB,CAAC,CAAC,CAAC;IACZ,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA6CJ,MAAM,CAAC,IAAI,EAAE,IAAI;IAwBvC,QAAQ,EAAE,YAAY,EAAE,CAAM;IAC9B,UAAU,EAAE,IAAI,EAAE,CAAM;IACxB,MAAM,EAAE,MAAM,EAAE,CAAM;IACtB,eAAe,SAAM;IACrB,iBAAiB,SAAM;IACvB,aAAa,EAAE;QAAE,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,CAAA;KAAE,CAAM;IAChD,OAAO,EAAE,MAAM,EAAE,CAAM;IACvB,UAAU,SAAM;IAEhB,OAAO,CAAC,QAAQ;IAkChB,OAAO,CAAC,UAAU;IAmBI,IAAI,CAAC,IAAI,EAAE,IAAI;CAWrC;AAED,eAAO,MAAM,eAAe,aACjB,YAAY,EAAE,cACZ,IAAI,EAAE,WACT;IACR,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,yBAAyB,CAAC,EAAE,OAAO,CAAC;CACpC,SAOD,CAAC"}

package/dist/commands/generate.js

@@ -0,0 +1,150 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import { DataSchema, LabelledDataSchema, LabelsContainerSchema, } from "@prosopo/types";
+import { lodash, setSeedGlobal } from "@prosopo/util/lodash";
+import * as z from "zod";
+import { OutputArgsSchema, OutputCliCommand } from "../utils/output.js";
+export const ArgsSchema = OutputArgsSchema.extend({
+    labels: z.string().optional(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+export class Generate extends OutputCliCommand {
+    constructor() {
+        super(...arguments);
+        this.labelled = [];
+        this.unlabelled = [];
+        this.labels = [];
+        this.labelledMapFile = "";
+        this.unlabelledMapFile = "";
+        this.labelToImages = {};
+        this.targets = [];
+        this.saltRounds = 10;
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            output: {
+                description: "Where to write the captchas JSON file",
+            },
+            labelled: {
+                string: true,
+                demand: true,
+                description: "Path to JSON file containing labelled data",
+            },
+            unlabelled: {
+                string: true,
+                demand: false,
+                description: "Path to JSON file containing unlabelled data",
+            },
+            seed: {
+                number: true,
+                demand: true,
+                description: "Seed for random number generator",
+            },
+            size: {
+                number: true,
+                description: "Number of images in each captcha",
+            },
+            labels: {
+                string: true,
+                description: "Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.",
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: "If true, allow duplicates in the data (labelled and unlabelled)",
+            },
+            allowDuplicatesLabelled: {
+                boolean: true,
+                description: "If true, allow duplicates in the labelled data",
+            },
+            allowDuplicatesUnlabelled: {
+                boolean: true,
+                description: "If true, allow duplicates in the unlabelled data",
+            },
+        });
+    }
+    async _check(args) {
+        const labelledMapFile = args.labelled;
+        if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+            throw new ProsopoDatasetError(new Error(`labelled map file does not exist: ${labelledMapFile}`), {
+                translationKey: "FS.FILE_NOT_FOUND",
+            });
+        }
+        const unlabelledMapFile = args.unlabelled;
+        if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+            throw new ProsopoDatasetError(new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`), {
+                translationKey: "FS.FILE_NOT_FOUND",
+            });
+        }
+        this.labelledMapFile = labelledMapFile || "";
+        this.unlabelledMapFile = unlabelledMapFile || "";
+    }
+    loadData(args) {
+        const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+        const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+        this.labelled = this.labelledMapFile
+            ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(this.labelledMapFile, "utf8"))).items
+            : [];
+        this.unlabelled = this.unlabelledMapFile
+            ? DataSchema.parse(JSON.parse(fs.readFileSync(this.unlabelledMapFile, "utf8"))).items
+            : [];
+        checkDuplicates(this.labelled, this.unlabelled, {
+            allowDuplicatesLabelled,
+            allowDuplicatesUnlabelled,
+        });
+        this.labelToImages = {};
+        for (const entry of this.labelled) {
+            const arr = this.labelToImages[entry.label] || [];
+            arr.push(entry);
+            this.labelToImages[entry.label] = arr;
+        }
+        this.targets = Object.keys(this.labelToImages);
+    }
+    loadLabels(args) {
+        this.labels = [];
+        if (args.labels && fs.existsSync(args.labels)) {
+            this.labels.push(...[
+                ...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(args.labels, "utf8"))).labels,
+            ]);
+        }
+        else {
+            this.labels.push(...[...this.targets]);
+        }
+    }
+    async _run(args) {
+        await super._run(args);
+        setSeedGlobal(args.seed || 0);
+        const _ = lodash();
+        this.loadData(args);
+        this.loadLabels(args);
+    }
+}
+export const checkDuplicates = (labelled, unlabelled, options) => {
+    const all = new Set();
+    if (!options.allowDuplicatesLabelled)
+        addAllUnique(all, labelled, "labelled");
+    if (!options.allowDuplicatesUnlabelled)
+        addAllUnique(all, unlabelled, "unlabelled");
+};
+const addAllUnique = (all, entries, dataType) => {
+    for (const entry of entries) {
+        addUnique(all, entry, dataType);
+    }
+};
+const addUnique = (all, entry, dataType) => {
+    if (all.has(entry.data)) {
+        throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+            context: {
+                error: `Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`,
+            },
+        });
+    }
+    all.add(entry.data);
+};
+//# sourceMappingURL=generate.js.map

package/dist/commands/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EACN,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAc7D,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC;IACjD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/C,yBAAyB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAIH,MAAM,OAAgB,QAEpB,SAAQ,gBAAmB;IAF7B;;QAwEC,aAAQ,GAAmB,EAAE,CAAC;QAC9B,eAAU,GAAW,EAAE,CAAC;QACxB,WAAM,GAAa,EAAE,CAAC;QACtB,oBAAe,GAAG,EAAE,CAAC;QACrB,sBAAiB,GAAG,EAAE,CAAC;QACvB,kBAAa,GAAgC,EAAE,CAAC;QAChD,YAAO,GAAa,EAAE,CAAC;QACvB,eAAU,GAAG,EAAE,CAAC;IAkEjB,CAAC;IA9IgB,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,MAAM,EAAE;gBACP,WAAW,EAAE,uCAAuC;aACpD;YACD,QAAQ,EAAE;gBACT,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,4CAA4C;aACzD;YACD,UAAU,EAAE;gBACX,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,8CAA8C;aAC3D;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,kCAAkC;aAC/C;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,kCAAkC;aAC/C;YACD,MAAM,EAAE;gBACP,MAAM,EAAE,IAAI;gBACZ,WAAW,EACV,6IAA6I;aAC9I;YACD,eAAe,EAAE;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EACV,iEAAiE;aAClE;YACD,uBAAuB,EAAE;gBACxB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,gDAAgD;aAC7D;YACD,yBAAyB,EAAE;gBAC1B,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,kDAAkD;aAC/D;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,MAAM,CAAC,IAAU;QAEtC,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAC;QAC1D,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAC;QAC9D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,IAAI,EAAE,CAAC;IAClD,CAAC;IAWO,QAAQ,CAAC,IAAU;QAC1B,MAAM,uBAAuB,GAC5B,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC;QAC/D,MAAM,yBAAyB,GAC9B,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC;QAGjE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,eAAe;YACnC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CACxB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CACzD,CAAC,KAAK;YACR,CAAC,CAAC,EAAE,CAAC;QACN,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB;YACvC,CAAC,CAAC,UAAU,CAAC,KAAK,CAChB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAC3D,CAAC,KAAK;YACR,CAAC,CAAC,EAAE,CAAC;QAGN,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;YAC/C,uBAAuB;YACvB,yBAAyB;SACzB,CAAC,CAAC;QAGH,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAEO,UAAU,CAAC,IAAU;QAI5B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,GAAG;gBACF,GAAG,qBAAqB,CAAC,KAAK,CAC7B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAChD,CAAC,MAAM;aACR,CACD,CAAC;QACH,CAAC;aAAM,CAAC;YAEP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QAE9B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAEnB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEpB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;CACD;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,CAC9B,QAAwB,EACxB,UAAkB,EAClB,OAGC,EACA,EAAE;IAEH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,CAAC,OAAO,CAAC,uBAAuB;QAAE,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9E,IAAI,CAAC,OAAO,CAAC,yBAAyB;QACrC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAgB,EAAE,OAAe,EAAE,QAAgB,EAAE,EAAE;IAC5E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,GAAgB,EAAE,KAAW,EAAE,QAAgB,EAAE,EAAE;IACrE,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,EAAE;YACxD,OAAO,EAAE;gBACR,KAAK,EAAE,2BAA2B,QAAQ,UAAU,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;aAC3E;SACD,CAAC,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC"}

package/dist/commands/generateV1.d.ts

@@ -0,0 +1,179 @@
+import * as z from "zod";
+import { Generate } from "./generate.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, {
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodNumber;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}>, {
+    solved: z.ZodOptional<z.ZodNumber>;
+    unsolved: z.ZodOptional<z.ZodNumber>;
+    minCorrect: z.ZodOptional<z.ZodNumber>;
+    maxCorrect: z.ZodOptional<z.ZodNumber>;
+}>, "strip", z.ZodTypeAny, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare class GenerateV1 extends Generate<ArgsSchemaType> {
+    getArgSchema(): z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+        output: z.ZodString;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+    }, {
+        labels: z.ZodOptional<z.ZodString>;
+        labelled: z.ZodOptional<z.ZodString>;
+        unlabelled: z.ZodOptional<z.ZodString>;
+        seed: z.ZodNumber;
+        size: z.ZodOptional<z.ZodNumber>;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+    }>, {
+        solved: z.ZodOptional<z.ZodNumber>;
+        unsolved: z.ZodOptional<z.ZodNumber>;
+        minCorrect: z.ZodOptional<z.ZodNumber>;
+        maxCorrect: z.ZodOptional<z.ZodNumber>;
+    }>, "strip", z.ZodTypeAny, {
+        output: string;
+        seed: number;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+        labels?: string | undefined;
+        labelled?: string | undefined;
+        unlabelled?: string | undefined;
+        size?: number | undefined;
+        allowDuplicatesLabelled?: boolean | undefined;
+        allowDuplicatesUnlabelled?: boolean | undefined;
+        solved?: number | undefined;
+        unsolved?: number | undefined;
+        minCorrect?: number | undefined;
+        maxCorrect?: number | undefined;
+    }, {
+        output: string;
+        seed: number;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+        labels?: string | undefined;
+        labelled?: string | undefined;
+        unlabelled?: string | undefined;
+        size?: number | undefined;
+        allowDuplicatesLabelled?: boolean | undefined;
+        allowDuplicatesUnlabelled?: boolean | undefined;
+        solved?: number | undefined;
+        unsolved?: number | undefined;
+        minCorrect?: number | undefined;
+        maxCorrect?: number | undefined;
+    }>;
+    getDescription(): string;
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        output: {
+            alias: string;
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        overwrite: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        output: {
+            description: string;
+        };
+        labelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        unlabelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        seed: {
+            number: boolean;
+            demand: boolean;
+            description: string;
+        };
+        size: {
+            number: boolean;
+            description: string;
+        };
+        labels: {
+            string: boolean;
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesLabelled: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesUnlabelled: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        solved: {
+            description: string;
+            number: boolean;
+        };
+        unsolved: {
+            description: string;
+            number: boolean;
+        };
+        minCorrect: {
+            description: string;
+            number: boolean;
+        };
+        maxCorrect: {
+            description: string;
+            number: boolean;
+        };
+    };
+    private generateSolved;
+    private generateUnsolved;
+    _run(args: Args): Promise<void>;
+}
+//# sourceMappingURL=generateV1.d.ts.map

package/dist/commands/generateV1.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateV1.d.ts","sourceRoot":"","sources":["../../src/commands/generateV1.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAoC,MAAM,eAAe,CAAC;AAE3E,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKrB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,qBAAa,UAAW,SAAQ,QAAQ,CAAC,cAAc,CAAC;IACvC,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAIZ,cAAc,IAAI,MAAM;IAIxB,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAqB1B,OAAO,CAAC,cAAc;IAkGtB,OAAO,CAAC,gBAAgB;IA0DF,IAAI,CAAC,IAAI,EAAE,IAAI;CA6CrC"}