@prosopo/datasets-fs 0.1.17

package/README.md

@@ -0,0 +1,84 @@
+## How to produce captchas
+
+You need 3 files:
+
+1. a json file containing labelled data in the form:
+    ```json
+    [
+        {
+            "label": "dog",
+            "data": "http://example.com/a.png"
+        },
+        {
+            "label": "cat",
+            "data": "http://example.com/b.png"
+        },
+        ...
+    ]
+    ```
+1. a json file containing unlabelled data in the form:
+    ```json
+    [
+        {
+            "data": "http://example.com/c.png"
+        },
+        {
+            "data": "http://example.com/d.png"
+        },
+        ...
+    ]
+    ```
+1. (optional) a json file with an array of labels which unlabelled data can be categorised into. If not specified will default to the same labels as seen in the labelled data.
+
+Then build and run the cli, passing appropriate parameters:
+`npm run build && node ./dist/js/cli.js generate --labelled /path/to/my/labelled/data.json --unlabelled /path/to/my/unlabelled/data.json --seed 0 --labels /path/to/my/labels.json --output /path/to/the/output/captchas.json`
+
+Use `node ./dist/js/cli.ts --help` to inspect other parameters.
+
+Commands:
+
+1. `flatten` converts a hierarchical directory structure into a single directory with corresponding map file, e.g.
+    ```
+    data/
+        dog/
+            a.png
+            ...
+        cat/
+            b.png
+            ...
+    ```
+    into
+    ```
+    data/
+        a.png
+        b.png
+        ...
+    map.json
+    ```
+    where `map.json` looks like:
+    ```
+    [
+        {
+            "label": "dog",
+            "data": "http://example.com/a.png"
+        },
+        {
+            "label": "cat",
+            "data": "http://example.com/b.png"
+        },
+        ...
+    ]
+    ```
+    e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js flatten --in ~/bench/test/data --out ~/bench/test/flat --overwrite`
+1. `generateDistinct` takes the 3 files described above and produces captcha challenges comprising 2 rounds, one labelled and one unlabelled.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate distinct --out ~/bench/test/captchas.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --solved 1 --unsolved 1 --overwrite --labels ~/bench/test/flat/labels.json`
+1. `generateUnion` takes the 3 files described above and produces captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js generate union --out ~/bench/test/captchas-union.json --labelled ~/bench/test/flat/data.json --unlabelled ~/bench/test/flat/data.json --seed 0 --size 9 --allow-duplicates --count 2 --overwrite --labels ~/bench/test/flat/labels.json`
+1. 'labels' gets all labels from a data json.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js labels --data ~/bench/test/flat/data2.json`
+1. 'get' fetches all images using a GET request, displaying errors for any images which hit 404 or not OK.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js get --data ~/bench/test/flat/data2.json`
+1. 'relocate' rewrites the url of images in a json file.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js relocate --from example.com --to web.site --data ~/bench/test/flat/data2.json`
+1. 'scale' rescales images in a flat directory structure to a given size.
+   e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js scale --data ~/bench/test/flat/data.json --out ~/bench/test/flat/images2 --overwrite --size 128`

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,50 @@
+import { __awaiter } from "tslib";
+import { LogLevelSchema, getLogger } from '@prosopo/common';
+import { hideBin } from 'yargs/helpers';
+import esMain from 'es-main';
+import flatten from './flatten/cli.js';
+import generate from './generate/cli.js';
+import get from './get/cli.js';
+import labels from './labels/cli.js';
+import process from 'process';
+import relocate from './relocate/cli.js';
+import scale from './scale/cli.js';
+import yargs from 'yargs';
+const dirname = process.cwd();
+const logger = getLogger(LogLevelSchema.enum.Info, `${dirname}`);
+const main = () => __awaiter(void 0, void 0, void 0, function* () {
+    yield yargs(hideBin(process.argv))
+        .help()
+        .option('log-level', {
+        type: 'string',
+        choices: Object.values(LogLevelSchema.enum),
+        default: LogLevelSchema.enum.Info,
+        description: 'The log level',
+    })
+        .middleware((argv) => {
+        logger.setLogLevel(argv.logLevel);
+    })
+        .command(generate({ logger }))
+        .command(flatten({ logger }))
+        .command(scale({ logger }))
+        .command(relocate({ logger }))
+        .command(get({ logger }))
+        .command(labels({ logger }))
+        .strictCommands()
+        .showHelpOnFail(false, 'Specify --help for available options')
+        .fail(false)
+        .parse();
+});
+//if main process
+if (esMain(import.meta)) {
+    main()
+        .then(() => {
+        logger.debug('done');
+        process.exit(0);
+    })
+        .catch((err) => {
+        logger.error('error:', err);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAAA,OAAO,EAAY,cAAc,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,OAAO,MAAM,kBAAkB,CAAA;AACtC,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,GAAG,MAAM,cAAc,CAAA;AAC9B,OAAO,MAAM,MAAM,iBAAiB,CAAA;AACpC,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;AAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;AAEhE,MAAM,IAAI,GAAG,GAAS,EAAE;IACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC7B,IAAI,EAAE;SACN,MAAM,CAAC,WAAW,EAAE;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC;QAC3C,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,IAAI;QACjC,WAAW,EAAE,eAAe;KAC/B,CAAC;SACD,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE;QACjB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAoB,CAAC,CAAA;IACjD,CAAC,CAAC;SACD,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SACxB,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC3B,cAAc,EAAE;SAChB,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC;SAC7D,IAAI,CAAC,KAAK,CAAC;SACX,KAAK,EAAE,CAAA;AAChB,CAAC,CAAA,CAAA;AAED,iBAAiB;AACjB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;IACrB,IAAI,EAAE;SACD,IAAI,CAAC,GAAG,EAAE;QACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;CACT"}

package/dist/flatten/args.d.ts

@@ -0,0 +1,16 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    data: z.ZodString;
+    out: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    data: string;
+    out: string;
+    overwrite?: boolean | undefined;
+}, {
+    data: string;
+    out: string;
+    overwrite?: boolean | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/flatten/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/flatten/args.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;EAIrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/flatten/args.js

@@ -0,0 +1,8 @@
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = z.object({
+    data: z.string(),
+    out: z.string(),
+    overwrite: z.boolean().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/flatten/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/flatten/args.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAA"}

package/dist/flatten/cli.d.ts

@@ -0,0 +1,18 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        data: string;
+    } & {
+        out: string;
+    } & {
+        overwrite: boolean | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<void>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/flatten/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAKpB,IAAI;;;;;;;oBAkBC,kBAAkB;;AAvBhD,wBA2BC"}

package/dist/flatten/cli.js

@@ -0,0 +1,31 @@
+import { __awaiter } from "tslib";
+import { argsSchema } from './args.js';
+import flatten from './flatten.js';
+export default (cmdArgs) => {
+    return {
+        command: 'flatten',
+        describe: 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels',
+        builder: (yargs) => {
+            return yargs
+                .option('data', {
+                type: 'string',
+                alias: 'in',
+                demand: true,
+                description: 'Path to the data directory containing subdirectories for each image classification',
+            })
+                .option('out', {
+                type: 'string',
+                demand: true,
+                description: 'Where to put the output file containing the labels and single directory of images',
+            })
+                .option('overwrite', {
+                type: 'boolean',
+                description: 'Overwrite the output file if it already exists',
+            });
+        },
+        handler: (argv) => __awaiter(void 0, void 0, void 0, function* () {
+            yield flatten(argsSchema.parse(argv), cmdArgs === null || cmdArgs === void 0 ? void 0 : cmdArgs.logger);
+        }),
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/flatten/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,OAAO,MAAM,cAAc,CAAA;AAElC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,SAAS;QAClB,QAAQ,EACJ,wIAAwI;QAC5I,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,oFAAoF;aACpG,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,mFAAmF;aACnG,CAAC;iBACD,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,CAAO,IAAwB,EAAE,EAAE;YACxC,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,CAAC,CAAA;QAC1D,CAAC,CAAA;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/flatten/flatten.d.ts

@@ -0,0 +1,5 @@
+import { Args } from './args.js';
+import { Logger } from '@prosopo/common';
+declare const _default: (args: Args, logger?: Logger) => Promise<void>;
+export default _default;
+//# sourceMappingURL=flatten.d.ts.map

package/dist/flatten/flatten.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.d.ts","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAK/C,IAAI,WAAW,MAAM;AAAjD,wBAqEC"}

package/dist/flatten/flatten.js

@@ -0,0 +1,66 @@
+import { __awaiter } from "tslib";
+import { CaptchaItemTypes, DataSchema } from '@prosopo/types';
+import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
+import { blake2b } from '@noble/hashes/blake2b';
+import { u8aToHex } from '@polkadot/util';
+import fs from 'fs';
+export default (args, logger) => __awaiter(void 0, void 0, void 0, function* () {
+    logger = logger || getLoggerDefault();
+    logger.debug(args, 'flattening...');
+    const dataDir = args.data;
+    if (!fs.existsSync(dataDir)) {
+        throw new ProsopoEnvError(new Error(`data directory does not exist: ${dataDir}`), 'FS.DIRECTORY_NOT_FOUND');
+    }
+    const outDir = args.out;
+    const overwrite = args.overwrite || false;
+    if (!overwrite && fs.existsSync(outDir)) {
+        throw new ProsopoEnvError(new Error(`output directory already exists: ${outDir}`), 'FS.DIRECTORY_ALREADY_EXISTS');
+    }
+    // find the labels (these should be subdirectories of the data directory)
+    const labels = fs
+        .readdirSync(dataDir, { withFileTypes: true })
+        .filter((dirent) => dirent.isDirectory())
+        .map((dirent) => dirent.name);
+    // create the output directory
+    const imageDir = `${outDir}/images`;
+    fs.mkdirSync(imageDir, { recursive: true });
+    // for each label
+    const items = [];
+    for (const label of labels) {
+        // find all the images
+        const images = fs.readdirSync(`${dataDir}/${label}`);
+        // for each image
+        for (const image of images) {
+            logger.log(`flattening ${label}/${image}`);
+            // copy the image to the output directory
+            const extension = image.split('.').pop();
+            // read file to bytes
+            const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+            // hash based on the content of the image
+            const hash = blake2b(content);
+            const hex = u8aToHex(hash);
+            const name = `${hex}.${extension}`;
+            if (fs.existsSync(`${imageDir}/${name}`)) {
+                logger.log(`duplicate image: ${label}/${image} -> ${name}`);
+            }
+            fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+            const filePath = fs.realpathSync(`${imageDir}/${name}`);
+            // add the image to the map file
+            const entry = {
+                data: filePath,
+                type: CaptchaItemTypes.Image,
+                label,
+                hash: hex,
+            };
+            items.push(entry);
+        }
+    }
+    const data = {
+        items,
+    };
+    // verify data
+    DataSchema.parse(data);
+    // write map file
+    fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+});
+//# sourceMappingURL=flatten.js.map

package/dist/flatten/flatten.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,gBAAgB,EAAQ,UAAU,EAAgB,MAAM,gBAAgB,CAAA;AACjF,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,CAAO,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,IAAI,CAAA;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACzB,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC9G;IACD,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAA;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;QACrC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD,6BAA6B,CAChC,CAAA;KACJ;IAED,yEAAyE;IACzE,MAAM,MAAM,GAAa,EAAE;SACtB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;SAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAEjC,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAA;IACnC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,iBAAiB;IACjB,MAAM,KAAK,GAAmB,EAAE,CAAA;IAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QACxB,sBAAsB;QACtB,MAAM,MAAM,GAAa,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAA;QAC9D,iBAAiB;QACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,CAAC,GAAG,CAAC,cAAc,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC1C,yCAAyC;YACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;YACxC,qBAAqB;YACrB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC/D,yCAAyC;YACzC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAA;YAClC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE;gBACtC,MAAM,CAAC,GAAG,CAAC,oBAAoB,KAAK,IAAI,KAAK,OAAO,IAAI,EAAE,CAAC,CAAA;aAC9D;YACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACtE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACvD,gCAAgC;YAChC,MAAM,KAAK,GAAiB;gBACxB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;gBAC5B,KAAK;gBACL,IAAI,EAAE,GAAG;aACZ,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;SACpB;KACJ;IAED,MAAM,IAAI,GAAS;QACf,KAAK;KACR,CAAA;IAED,cAAc;IACd,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEtB,iBAAiB;IACjB,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC1E,CAAC,CAAA,CAAA"}

package/dist/flatten/index.d.ts

@@ -0,0 +1,4 @@
+export * from './flatten.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/flatten/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/flatten/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/flatten/index.js

@@ -0,0 +1,4 @@
+export * from './flatten.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.js.map

package/dist/flatten/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/flatten/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/args.d.ts

@@ -0,0 +1,37 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    labels: z.ZodOptional<z.ZodString>;
+    out: z.ZodString;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    out: string;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}, {
+    out: string;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/generate/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/generate/args.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/generate/args.js

@@ -0,0 +1,15 @@
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = z.object({
+    labels: z.string().optional(),
+    out: z.string(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number().optional(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/generate/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/generate/args.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/C,yBAAyB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAA"}

package/dist/generate/cli.d.ts

@@ -0,0 +1,32 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        overwrite: boolean | undefined;
+    } & {
+        out: string;
+    } & {
+        labelled: string;
+    } & {
+        unlabelled: string;
+    } & {
+        seed: number | undefined;
+    } & {
+        size: number | undefined;
+    } & {
+        labels: string | undefined;
+    } & {
+        "allow-duplicates": boolean | undefined;
+    } & {
+        "allow-duplicates-labelled": boolean | undefined;
+    } & {
+        "allow-duplicates-unlabelled": boolean | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<never>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/generate/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAIpB,IAAI;;;;;;;;;;;;;;;;;;;;;oBAmDC,kBAAkB;;AAvDhD,wBA2DC"}

package/dist/generate/cli.js

@@ -0,0 +1,63 @@
+import { __awaiter } from "tslib";
+import generateDistinct from './distinct/cli.js';
+import generateUnion from './union/cli.js';
+export default (cmdArgs) => {
+    return {
+        command: 'generate',
+        describe: 'Generate captchas',
+        builder: (yargs) => {
+            return yargs
+                .command(generateDistinct(cmdArgs))
+                .command(generateUnion(cmdArgs))
+                .demandCommand()
+                .option('overwrite', {
+                type: 'boolean',
+                description: 'Overwrite the output file if it already exists',
+            })
+                .option('out', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the output file',
+                default: 'captchas.json',
+            })
+                .option('labelled', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the file containing map of images urls to labels',
+            })
+                .option('unlabelled', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the file containing list of images url which are unlabelled',
+            })
+                .option('seed', {
+                type: 'number',
+                description: 'Seed for the random number generator',
+            })
+                .option('size', {
+                type: 'number',
+                description: 'Number of images in each captcha',
+            })
+                .options('labels', {
+                type: 'string',
+                description: 'Path to the labels file. This is a file containing a list of labels which unlabelled data will be assigned to.',
+            })
+                .option('allow-duplicates', {
+                type: 'boolean',
+                description: 'Allow duplicates in the data (labelled and unlabelled)',
+            })
+                .option('allow-duplicates-labelled', {
+                type: 'boolean',
+                description: 'Allow duplicates in the labelled data',
+            })
+                .option('allow-duplicates-unlabelled', {
+                type: 'boolean',
+                description: 'Allow duplicates in the unlabelled data',
+            });
+        },
+        handler: (argv) => __awaiter(void 0, void 0, void 0, function* () {
+            throw new Error('Please specify a command');
+        }),
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/generate/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":";AAEA,OAAO,gBAAgB,MAAM,mBAAmB,CAAA;AAChD,OAAO,aAAa,MAAM,gBAAgB,CAAA;AAE1C,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,mBAAmB;QAC7B,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;iBAClC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;iBAC/B,aAAa,EAAE;iBACf,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,yBAAyB;gBACtC,OAAO,EAAE,eAAe;aAC3B,CAAC;iBACD,MAAM,CAAC,UAAU,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,0DAA0D;aAC1E,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,qEAAqE;aACrF,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,sCAAsC;aACtD,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kCAAkC;aAClD,CAAC;iBACD,OAAO,CAAC,QAAQ,EAAE;gBACf,IAAI,EAAE,QAAQ;gBACd,WAAW,EACP,gHAAgH;aACvH,CAAC;iBACD,MAAM,CAAC,kBAAkB,EAAE;gBACxB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,wDAAwD;aACxE,CAAC;iBACD,MAAM,CAAC,2BAA2B,EAAE;gBACjC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,uCAAuC;aACvD,CAAC;iBACD,MAAM,CAAC,6BAA6B,EAAE;gBACnC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,yCAAyC;aACzD,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,CAAO,IAAwB,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC/C,CAAC,CAAA;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/generate/distinct/args.d.ts

@@ -0,0 +1,49 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    out: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+    solved: z.ZodOptional<z.ZodNumber>;
+    unsolved: z.ZodOptional<z.ZodNumber>;
+    minCorrect: z.ZodOptional<z.ZodNumber>;
+    maxCorrect: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/generate/distinct/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/args.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/generate/distinct/args.js

@@ -0,0 +1,10 @@
+import { argsSchema as parentSchema } from '../args.js';
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = parentSchema.extend({
+    solved: z.number().optional(),
+    unsolved: z.number().optional(),
+    minCorrect: z.number().optional(),
+    maxCorrect: z.number().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/generate/distinct/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../../src/generate/distinct/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,YAAY,EAAE,MAAM,YAAY,CAAA;AACvD,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAA"}

package/dist/generate/distinct/cli.d.ts

@@ -0,0 +1,20 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        solved: number | undefined;
+    } & {
+        unsolved: number | undefined;
+    } & {
+        "min-correct": number | undefined;
+    } & {
+        "max-correct": number | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<void>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/generate/distinct/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAKpB,IAAI;;;;;;;;;oBAmBC,kBAAkB;;AAxBhD,wBA4BC"}

package/dist/generate/distinct/cli.js

@@ -0,0 +1,32 @@
+import { __awaiter } from "tslib";
+import { argsSchema } from './args.js';
+import generate from './generate.js';
+export default (cmdArgs) => {
+    return {
+        command: 'distinct',
+        describe: 'Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled',
+        builder: (yargs) => {
+            return yargs
+                .option('solved', {
+                type: 'number',
+                description: 'Number of captchas to generate that are solved',
+            })
+                .option('unsolved', {
+                type: 'number',
+                description: 'Number of captchas to generate that are unsolved',
+            })
+                .option('min-correct', {
+                type: 'number',
+                description: 'Minimum number of target images in each captcha',
+            })
+                .option('max-correct', {
+                type: 'number',
+                description: 'Maximum number of target images in each captcha',
+            });
+        },
+        handler: (argv) => __awaiter(void 0, void 0, void 0, function* () {
+            yield generate(argsSchema.parse(argv), cmdArgs === null || cmdArgs === void 0 ? void 0 : cmdArgs.logger);
+        }),
+    };
+};
+//# sourceMappingURL=cli.js.map