@prosopo/datasets-fs 3.0.39 → 3.0.43

package/dist/commands/generate.js

@@ -0,0 +1,150 @@
+import fs from "node:fs";
+import { ProsopoDatasetError } from "@prosopo/common";
+import { DataSchema, LabelledDataSchema, LabelsContainerSchema, } from "@prosopo/types";
+import { lodash, setSeedGlobal } from "@prosopo/util/lodash";
+import * as z from "zod";
+import { OutputArgsSchema, OutputCliCommand } from "../utils/output.js";
+export const ArgsSchema = OutputArgsSchema.extend({
+    labels: z.string().optional(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+export class Generate extends OutputCliCommand {
+    constructor() {
+        super(...arguments);
+        this.labelled = [];
+        this.unlabelled = [];
+        this.labels = [];
+        this.labelledMapFile = "";
+        this.unlabelledMapFile = "";
+        this.labelToImages = {};
+        this.targets = [];
+        this.saltRounds = 10;
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            output: {
+                description: "Where to write the captchas JSON file",
+            },
+            labelled: {
+                string: true,
+                demand: true,
+                description: "Path to JSON file containing labelled data",
+            },
+            unlabelled: {
+                string: true,
+                demand: false,
+                description: "Path to JSON file containing unlabelled data",
+            },
+            seed: {
+                number: true,
+                demand: true,
+                description: "Seed for random number generator",
+            },
+            size: {
+                number: true,
+                description: "Number of images in each captcha",
+            },
+            labels: {
+                string: true,
+                description: "Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.",
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: "If true, allow duplicates in the data (labelled and unlabelled)",
+            },
+            allowDuplicatesLabelled: {
+                boolean: true,
+                description: "If true, allow duplicates in the labelled data",
+            },
+            allowDuplicatesUnlabelled: {
+                boolean: true,
+                description: "If true, allow duplicates in the unlabelled data",
+            },
+        });
+    }
+    async _check(args) {
+        const labelledMapFile = args.labelled;
+        if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+            throw new ProsopoDatasetError(new Error(`labelled map file does not exist: ${labelledMapFile}`), {
+                translationKey: "FS.FILE_NOT_FOUND",
+            });
+        }
+        const unlabelledMapFile = args.unlabelled;
+        if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+            throw new ProsopoDatasetError(new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`), {
+                translationKey: "FS.FILE_NOT_FOUND",
+            });
+        }
+        this.labelledMapFile = labelledMapFile || "";
+        this.unlabelledMapFile = unlabelledMapFile || "";
+    }
+    loadData(args) {
+        const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+        const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+        this.labelled = this.labelledMapFile
+            ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(this.labelledMapFile, "utf8"))).items
+            : [];
+        this.unlabelled = this.unlabelledMapFile
+            ? DataSchema.parse(JSON.parse(fs.readFileSync(this.unlabelledMapFile, "utf8"))).items
+            : [];
+        checkDuplicates(this.labelled, this.unlabelled, {
+            allowDuplicatesLabelled,
+            allowDuplicatesUnlabelled,
+        });
+        this.labelToImages = {};
+        for (const entry of this.labelled) {
+            const arr = this.labelToImages[entry.label] || [];
+            arr.push(entry);
+            this.labelToImages[entry.label] = arr;
+        }
+        this.targets = Object.keys(this.labelToImages);
+    }
+    loadLabels(args) {
+        this.labels = [];
+        if (args.labels && fs.existsSync(args.labels)) {
+            this.labels.push(...[
+                ...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(args.labels, "utf8"))).labels,
+            ]);
+        }
+        else {
+            this.labels.push(...[...this.targets]);
+        }
+    }
+    async _run(args) {
+        await super._run(args);
+        setSeedGlobal(args.seed || 0);
+        const _ = lodash();
+        this.loadData(args);
+        this.loadLabels(args);
+    }
+}
+export const checkDuplicates = (labelled, unlabelled, options) => {
+    const all = new Set();
+    if (!options.allowDuplicatesLabelled)
+        addAllUnique(all, labelled, "labelled");
+    if (!options.allowDuplicatesUnlabelled)
+        addAllUnique(all, unlabelled, "unlabelled");
+};
+const addAllUnique = (all, entries, dataType) => {
+    for (const entry of entries) {
+        addUnique(all, entry, dataType);
+    }
+};
+const addUnique = (all, entry, dataType) => {
+    if (all.has(entry.data)) {
+        throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+            context: {
+                error: `Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`,
+            },
+        });
+    }
+    all.add(entry.data);
+};
+//# sourceMappingURL=generate.js.map

package/dist/commands/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EACN,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GACrB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAc7D,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAExE,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC;IACjD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/C,yBAAyB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAIH,MAAM,OAAgB,QAEpB,SAAQ,gBAAmB;IAF7B;;QAwEC,aAAQ,GAAmB,EAAE,CAAC;QAC9B,eAAU,GAAW,EAAE,CAAC;QACxB,WAAM,GAAa,EAAE,CAAC;QACtB,oBAAe,GAAG,EAAE,CAAC;QACrB,sBAAiB,GAAG,EAAE,CAAC;QACvB,kBAAa,GAAgC,EAAE,CAAC;QAChD,YAAO,GAAa,EAAE,CAAC;QACvB,eAAU,GAAG,EAAE,CAAC;IAkEjB,CAAC;IA9IgB,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,MAAM,EAAE;gBACP,WAAW,EAAE,uCAAuC;aACpD;YACD,QAAQ,EAAE;gBACT,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,4CAA4C;aACzD;YACD,UAAU,EAAE;gBACX,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,KAAK;gBACb,WAAW,EAAE,8CAA8C;aAC3D;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,kCAAkC;aAC/C;YACD,IAAI,EAAE;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,kCAAkC;aAC/C;YACD,MAAM,EAAE;gBACP,MAAM,EAAE,IAAI;gBACZ,WAAW,EACV,6IAA6I;aAC9I;YACD,eAAe,EAAE;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EACV,iEAAiE;aAClE;YACD,uBAAuB,EAAE;gBACxB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,gDAAgD;aAC7D;YACD,yBAAyB,EAAE;gBAC1B,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,kDAAkD;aAC/D;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,MAAM,CAAC,IAAU;QAEtC,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAC;QAC1D,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAC;QAC9D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE;gBACC,cAAc,EAAE,mBAAmB;aACnC,CACD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,IAAI,EAAE,CAAC;IAClD,CAAC;IAWO,QAAQ,CAAC,IAAU;QAC1B,MAAM,uBAAuB,GAC5B,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC;QAC/D,MAAM,yBAAyB,GAC9B,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC;QAGjE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,eAAe;YACnC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CACxB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CACzD,CAAC,KAAK;YACR,CAAC,CAAC,EAAE,CAAC;QACN,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,iBAAiB;YACvC,CAAC,CAAC,UAAU,CAAC,KAAK,CAChB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAC3D,CAAC,KAAK;YACR,CAAC,CAAC,EAAE,CAAC;QAGN,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,EAAE;YAC/C,uBAAuB;YACvB,yBAAyB;SACzB,CAAC,CAAC;QAGH,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;IAEO,UAAU,CAAC,IAAU;QAI5B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CACf,GAAG;gBACF,GAAG,qBAAqB,CAAC,KAAK,CAC7B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAChD,CAAC,MAAM;aACR,CACD,CAAC;QACH,CAAC;aAAM,CAAC;YAEP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACxC,CAAC;IACF,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,aAAa,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QAE9B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAEnB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEpB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;CACD;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,CAC9B,QAAwB,EACxB,UAAkB,EAClB,OAGC,EACA,EAAE;IAEH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,IAAI,CAAC,OAAO,CAAC,uBAAuB;QAAE,YAAY,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC9E,IAAI,CAAC,OAAO,CAAC,yBAAyB;QACrC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,GAAgB,EAAE,OAAe,EAAE,QAAgB,EAAE,EAAE;IAC5E,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IACjC,CAAC;AACF,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,GAAgB,EAAE,KAAW,EAAE,QAAgB,EAAE,EAAE;IACrE,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,EAAE;YACxD,OAAO,EAAE;gBACR,KAAK,EAAE,2BAA2B,QAAQ,UAAU,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE;aAC3E;SACD,CAAC,CAAC;IACJ,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACrB,CAAC,CAAC"}

package/dist/commands/generateV1.d.ts

@@ -0,0 +1,179 @@
+import * as z from "zod";
+import { Generate } from "./generate.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, {
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodNumber;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}>, {
+    solved: z.ZodOptional<z.ZodNumber>;
+    unsolved: z.ZodOptional<z.ZodNumber>;
+    minCorrect: z.ZodOptional<z.ZodNumber>;
+    maxCorrect: z.ZodOptional<z.ZodNumber>;
+}>, "strip", z.ZodTypeAny, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare class GenerateV1 extends Generate<ArgsSchemaType> {
+    getArgSchema(): z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+        output: z.ZodString;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+    }, {
+        labels: z.ZodOptional<z.ZodString>;
+        labelled: z.ZodOptional<z.ZodString>;
+        unlabelled: z.ZodOptional<z.ZodString>;
+        seed: z.ZodNumber;
+        size: z.ZodOptional<z.ZodNumber>;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+    }>, {
+        solved: z.ZodOptional<z.ZodNumber>;
+        unsolved: z.ZodOptional<z.ZodNumber>;
+        minCorrect: z.ZodOptional<z.ZodNumber>;
+        maxCorrect: z.ZodOptional<z.ZodNumber>;
+    }>, "strip", z.ZodTypeAny, {
+        output: string;
+        seed: number;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+        labels?: string | undefined;
+        labelled?: string | undefined;
+        unlabelled?: string | undefined;
+        size?: number | undefined;
+        allowDuplicatesLabelled?: boolean | undefined;
+        allowDuplicatesUnlabelled?: boolean | undefined;
+        solved?: number | undefined;
+        unsolved?: number | undefined;
+        minCorrect?: number | undefined;
+        maxCorrect?: number | undefined;
+    }, {
+        output: string;
+        seed: number;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+        labels?: string | undefined;
+        labelled?: string | undefined;
+        unlabelled?: string | undefined;
+        size?: number | undefined;
+        allowDuplicatesLabelled?: boolean | undefined;
+        allowDuplicatesUnlabelled?: boolean | undefined;
+        solved?: number | undefined;
+        unsolved?: number | undefined;
+        minCorrect?: number | undefined;
+        maxCorrect?: number | undefined;
+    }>;
+    getDescription(): string;
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        output: {
+            alias: string;
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        overwrite: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        output: {
+            description: string;
+        };
+        labelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        unlabelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        seed: {
+            number: boolean;
+            demand: boolean;
+            description: string;
+        };
+        size: {
+            number: boolean;
+            description: string;
+        };
+        labels: {
+            string: boolean;
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesLabelled: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesUnlabelled: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        solved: {
+            description: string;
+            number: boolean;
+        };
+        unsolved: {
+            description: string;
+            number: boolean;
+        };
+        minCorrect: {
+            description: string;
+            number: boolean;
+        };
+        maxCorrect: {
+            description: string;
+            number: boolean;
+        };
+    };
+    private generateSolved;
+    private generateUnsolved;
+    _run(args: Args): Promise<void>;
+}
+//# sourceMappingURL=generateV1.d.ts.map

package/dist/commands/generateV1.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateV1.d.ts","sourceRoot":"","sources":["../../src/commands/generateV1.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAoC,MAAM,eAAe,CAAC;AAE3E,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKrB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,qBAAa,UAAW,SAAQ,QAAQ,CAAC,cAAc,CAAC;IACvC,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAIZ,cAAc,IAAI,MAAM;IAIxB,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAqB1B,OAAO,CAAC,cAAc;IAkGtB,OAAO,CAAC,gBAAgB;IA0DF,IAAI,CAAC,IAAI,EAAE,IAAI;CA6CrC"}

package/dist/commands/generateV1.js

@@ -0,0 +1,169 @@
+import fs from "node:fs";
+import { ProsopoDatasetError, ProsopoEnvError } from "@prosopo/common";
+import { CaptchaTypes, CaptchasContainerSchema, } from "@prosopo/types";
+import { at, get } from "@prosopo/util";
+import { blake2AsHex } from "@prosopo/util-crypto";
+import { lodash } from "@prosopo/util/lodash";
+import bcrypt from "bcrypt";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import { Generate, ArgsSchema as GenerateArgsSchema } from "./generate.js";
+export const ArgsSchema = GenerateArgsSchema.extend({
+    solved: z.number().optional(),
+    unsolved: z.number().optional(),
+    minCorrect: z.number().optional(),
+    maxCorrect: z.number().optional(),
+});
+export class GenerateV1 extends Generate {
+    getArgSchema() {
+        return ArgsSchema;
+    }
+    getDescription() {
+        return "Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled";
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            solved: {
+                description: "Number of captchas to generate that are solved",
+                number: true,
+            },
+            unsolved: {
+                description: "Number of captchas to generate that are unsolved",
+                number: true,
+            },
+            minCorrect: {
+                description: "Minimum number of target images in each captcha",
+                number: true,
+            },
+            maxCorrect: {
+                description: "Maximum number of target images in each captcha",
+                number: true,
+            },
+        });
+    }
+    generateSolved(solved, size, minCorrect, maxCorrect, bar) {
+        const _ = lodash();
+        const solvedCaptchas = [];
+        for (let i = 0; i < solved; i++) {
+            bar.increment();
+            if (this.targets.length <= 1) {
+                throw new ProsopoDatasetError(new Error("not enough different labels in labelled data"), {
+                    translationKey: "DATASET.NOT_ENOUGH_LABELS",
+                });
+            }
+            const target = at(this.targets, i % this.targets.length);
+            const notTargets = this.targets.filter((t) => t !== target);
+            const nCorrect = _.random(minCorrect, maxCorrect);
+            const nIncorrect = size - nCorrect;
+            const targetItems = get(this.labelToImages, target);
+            const notTargetItems = notTargets.flatMap((notTarget) => get(this.labelToImages, notTarget));
+            if (targetItems.length < nCorrect) {
+                throw new ProsopoEnvError(new Error(`not enough images for target (${target})`), {
+                    translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+                });
+            }
+            if (notTargetItems.length < nIncorrect) {
+                throw new ProsopoDatasetError(new Error(`not enough non-matching images for target (${target})`), {
+                    translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+                });
+            }
+            const correctItems = _.sampleSize(targetItems, nCorrect);
+            const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
+            let items = [...correctItems, ...incorrectItems];
+            let indices = [...Array(items.length).keys()];
+            indices = _.shuffle(indices);
+            items = indices.map((i) => at(items, i));
+            items = items.map((item) => {
+                return {
+                    data: item.data,
+                    hash: item.hash,
+                    type: item.type,
+                };
+            });
+            const solution = indices
+                .map((index, i) => {
+                return {
+                    pre: index,
+                    post: i,
+                };
+            })
+                .filter((item) => item.pre < correctItems.length)
+                .map((item) => {
+                return item.post;
+            });
+            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds));
+            const captcha = {
+                salt,
+                target,
+                items,
+                solution,
+            };
+            solvedCaptchas.push(captcha);
+        }
+        return solvedCaptchas;
+    }
+    generateUnsolved(unsolved, size, bar) {
+        const _ = lodash();
+        const unsolvedCaptchas = [];
+        for (let i = 0; i < unsolved; i++) {
+            bar.increment();
+            if (this.unlabelled.length <= size) {
+                throw new ProsopoDatasetError(new Error("unlabelled map file does not contain enough data"), {
+                    translationKey: "DATASET.NOT_ENOUGH_IMAGES",
+                });
+            }
+            if (this.labels.length <= 0) {
+                throw new ProsopoDatasetError(new Error("no labels found for unlabelled data"), {
+                    translationKey: "DATASET.NOT_ENOUGH_LABELS",
+                });
+            }
+            const index = _.random(0, this.labels.length - 1);
+            const target = at(this.labels, index);
+            const itemSet = _.sampleSize(this.unlabelled, size);
+            let items = [...itemSet];
+            let indices = [...Array(items.length).keys()];
+            indices = _.shuffle(indices);
+            items = indices.map((i) => at(items, i));
+            items = items.map((item) => {
+                return {
+                    data: item.data,
+                    hash: item.hash,
+                    type: item.type,
+                };
+            });
+            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds));
+            const captcha = {
+                salt,
+                target,
+                items,
+            };
+            unsolvedCaptchas.push(captcha);
+        }
+        return unsolvedCaptchas;
+    }
+    async _run(args) {
+        await super._run(args);
+        const outFile = args.output;
+        const _ = lodash();
+        const size = args.size || 9;
+        const minCorrect = args.minCorrect || 1;
+        const maxCorrect = args.maxCorrect || size - 1;
+        const solved = args.solved || 0;
+        const unsolved = args.unsolved || 0;
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+        bar.start(solved + unsolved, 0);
+        const solvedCaptchas = this.generateSolved(solved, size, minCorrect, maxCorrect, bar);
+        const unsolvedCaptchas = this.generateUnsolved(unsolved, size, bar);
+        bar.stop();
+        const output = {
+            captchas: [...solvedCaptchas, ...unsolvedCaptchas],
+            format: CaptchaTypes.SelectAll,
+        };
+        CaptchasContainerSchema.parse(output);
+        fs.mkdirSync(args.output.split("/").slice(0, -1).join("/"), {
+            recursive: true,
+        });
+        fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+    }
+}
+//# sourceMappingURL=generateV1.js.map

package/dist/commands/generateV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateV1.js","sourceRoot":"","sources":["../../src/commands/generateV1.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvE,OAAO,EACN,YAAY,EAGZ,uBAAuB,GAGvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAE,QAAQ,EAAE,UAAU,IAAI,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE3E,MAAM,CAAC,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAIH,MAAM,OAAO,UAAW,SAAQ,QAAwB;IACvC,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,cAAc;QAC7B,OAAO,8GAA8G,CAAC;IACvH,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,MAAM,EAAE;gBACP,WAAW,EAAE,gDAAgD;gBAC7D,MAAM,EAAE,IAAI;aACZ;YACD,QAAQ,EAAE;gBACT,WAAW,EAAE,kDAAkD;gBAC/D,MAAM,EAAE,IAAI;aACZ;YACD,UAAU,EAAE;gBACX,WAAW,EAAE,iDAAiD;gBAC9D,MAAM,EAAE,IAAI;aACZ;YACD,UAAU,EAAE;gBACX,WAAW,EAAE,iDAAiD;gBAC9D,MAAM,EAAE,IAAI;aACZ;SACD,CAAC,CAAC;IACJ,CAAC;IAEO,cAAc,CACrB,MAAc,EACd,IAAY,EACZ,UAAkB,EAClB,UAAkB,EAClB,GAA0B;QAE1B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAEnB,MAAM,cAAc,GAAuB,EAAE,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAEjC,GAAG,CAAC,SAAS,EAAE,CAAC;YAEhB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,8CAA8C,CAAC,EACzD;oBACC,cAAc,EAAE,2BAA2B;iBAC3C,CACD,CAAC;YACH,CAAC;YAGD,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC;YAG5D,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YAElD,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAC;YAEnC,MAAM,WAAW,GAAW,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAC5D,MAAM,cAAc,GAAW,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE,CAC/D,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAClC,CAAC;YAEF,IAAI,WAAW,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBACnC,MAAM,IAAI,eAAe,CACxB,IAAI,KAAK,CAAC,iCAAiC,MAAM,GAAG,CAAC,EACrD;oBACC,cAAc,EAAE,2BAA2B;iBAC3C,CACD,CAAC;YACH,CAAC;YACD,IAAI,cAAc,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;gBACxC,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,EAClE;oBACC,cAAc,EAAE,2BAA2B;iBAC3C,CACD,CAAC;YACH,CAAC;YAGD,MAAM,YAAY,GAAW,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAGjE,MAAM,cAAc,GAAW,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;YAExE,IAAI,KAAK,GAAW,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,CAAC,CAAC;YACzD,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACxD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACzC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC1B,OAAO;oBACN,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,IAAI,CAAC,IAAI;iBACf,CAAC;YACH,CAAC,CAAC,CAAC;YAGH,MAAM,QAAQ,GAAkB,OAAO;iBACrC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;gBACjB,OAAO;oBACN,GAAG,EAAE,KAAK;oBACV,IAAI,EAAE,CAAC;iBACP,CAAC;YACH,CAAC,CAAC;iBACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC;iBAChD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBACb,OAAO,IAAI,CAAC,IAAI,CAAC;YAClB,CAAC,CAAC,CAAC;YAEJ,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAqB;gBACjC,IAAI;gBACJ,MAAM;gBACN,KAAK;gBACL,QAAQ;aACR,CAAC;YACF,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,cAAc,CAAC;IACvB,CAAC;IAEO,gBAAgB,CACvB,QAAgB,EAChB,IAAY,EACZ,GAA0B;QAE1B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAInB,MAAM,gBAAgB,GAAuB,EAAE,CAAC;QAChD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,GAAG,CAAC,SAAS,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,kDAAkD,CAAC,EAC7D;oBACC,cAAc,EAAE,2BAA2B;iBAC3C,CACD,CAAC;YACH,CAAC;YAGD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,mBAAmB,CAC5B,IAAI,KAAK,CAAC,qCAAqC,CAAC,EAChD;oBACC,cAAc,EAAE,2BAA2B;iBAC3C,CACD,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAClD,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAEtC,MAAM,OAAO,GAAW,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAE5D,IAAI,KAAK,GAAW,CAAC,GAAG,OAAO,CAAC,CAAC;YACjC,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACxD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC7B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;YACzC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC1B,OAAO;oBACN,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,IAAI,CAAC,IAAI;iBACf,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAE9D,MAAM,OAAO,GAAqB;gBACjC,IAAI;gBACJ,MAAM;gBACN,KAAK;aACL,CAAC;YACF,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,gBAAgB,CAAC;IACzB,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAW,IAAI,CAAC,MAAM,CAAC;QAGpC,MAAM,CAAC,GAAG,MAAM,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACpC,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAChD,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,GAAG,CAAC,CAAC;QACvD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAW,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QAG5C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QAGF,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CACzC,MAAM,EACN,IAAI,EACJ,UAAU,EACV,UAAU,EACV,GAAG,CACH,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACpE,GAAG,CAAC,IAAI,EAAE,CAAC;QAEX,MAAM,MAAM,GAAa;YACxB,QAAQ,EAAE,CAAC,GAAG,cAAc,EAAE,GAAG,gBAAgB,CAAC;YAClD,MAAM,EAAE,YAAY,CAAC,SAAS;SAC9B,CAAC;QAGF,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEtC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;YAC3D,SAAS,EAAE,IAAI;SACf,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;CACD"}