@prosopo/datasets-fs 0.1.18 → 0.2.0

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,49 @@
+import { LogLevel, getLogger } from '@prosopo/common';
+import { hideBin } from 'yargs/helpers';
+import esMain from 'es-main';
+import flatten from './flatten/cli.js';
+import generate from './generate/cli.js';
+import get from './get/cli.js';
+import labels from './labels/cli.js';
+import process from 'process';
+import relocate from './relocate/cli.js';
+import scale from './scale/cli.js';
+import yargs from 'yargs';
+const dirname = process.cwd();
+const logger = getLogger(LogLevel.enum.info, `${dirname}`);
+const main = async () => {
+    await yargs(hideBin(process.argv))
+        .help()
+        .option('log-level', {
+        type: 'string',
+        choices: Object.values(LogLevel.options),
+        default: LogLevel.enum.info,
+        description: 'The log level',
+    })
+        .middleware((argv) => {
+        logger.setLogLevel(argv.logLevel);
+    })
+        .command(generate({ logger }))
+        .command(flatten({ logger }))
+        .command(scale({ logger }))
+        .command(relocate({ logger }))
+        .command(get({ logger }))
+        .command(labels({ logger }))
+        .strictCommands()
+        .showHelpOnFail(false, 'Specify --help for available options')
+        .fail(false)
+        .parse();
+};
+//if main process
+if (esMain(import.meta)) {
+    main()
+        .then(() => {
+        logger.debug('done');
+        process.exit(0);
+    })
+        .catch((err) => {
+        logger.error('error:', err);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,OAAO,MAAM,kBAAkB,CAAA;AACtC,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,GAAG,MAAM,cAAc,CAAA;AAC9B,OAAO,MAAM,MAAM,iBAAiB,CAAA;AACpC,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;AAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;AAE1D,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC7B,IAAI,EAAE;SACN,MAAM,CAAC,WAAW,EAAE;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;QAC3B,WAAW,EAAE,eAAe;KAC/B,CAAC;SACD,UAAU,CAAC,CAAC,IAAS,EAAE,EAAE;QACtB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrC,CAAC,CAAC;SACD,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SACxB,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC3B,cAAc,EAAE;SAChB,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC;SAC7D,IAAI,CAAC,KAAK,CAAC;SACX,KAAK,EAAE,CAAA;AAChB,CAAC,CAAA;AAED,iBAAiB;AACjB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;IACrB,IAAI,EAAE;SACD,IAAI,CAAC,GAAG,EAAE;QACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;CACT"}

package/dist/flatten/args.d.ts

@@ -0,0 +1,16 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    data: z.ZodString;
+    out: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    data: string;
+    out: string;
+    overwrite?: boolean | undefined;
+}, {
+    data: string;
+    out: string;
+    overwrite?: boolean | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/flatten/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/flatten/args.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;EAIrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/flatten/args.js

@@ -0,0 +1,8 @@
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = z.object({
+    data: z.string(),
+    out: z.string(),
+    overwrite: z.boolean().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/flatten/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/flatten/args.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAA"}

package/dist/flatten/cli.d.ts

@@ -0,0 +1,18 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        data: string;
+    } & {
+        out: string;
+    } & {
+        overwrite: boolean | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<void>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/flatten/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAKpB,IAAI;;;;;;;oBAkBC,kBAAkB;;AAvBhD,wBA2BC"}

package/dist/flatten/cli.js

@@ -0,0 +1,30 @@
+import { argsSchema } from './args.js';
+import flatten from './flatten.js';
+export default (cmdArgs) => {
+    return {
+        command: 'flatten',
+        describe: 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels',
+        builder: (yargs) => {
+            return yargs
+                .option('data', {
+                type: 'string',
+                alias: 'in',
+                demand: true,
+                description: 'Path to the data directory containing subdirectories for each image classification',
+            })
+                .option('out', {
+                type: 'string',
+                demand: true,
+                description: 'Where to put the output file containing the labels and single directory of images',
+            })
+                .option('overwrite', {
+                type: 'boolean',
+                description: 'Overwrite the output file if it already exists',
+            });
+        },
+        handler: async (argv) => {
+            await flatten(argsSchema.parse(argv), cmdArgs?.logger);
+        },
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/flatten/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/flatten/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,OAAO,MAAM,cAAc,CAAA;AAElC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,SAAS;QAClB,QAAQ,EACJ,wIAAwI;QAC5I,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,IAAI;gBACX,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,oFAAoF;aACpG,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,mFAAmF;aACnG,CAAC;iBACD,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC1D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/flatten/flatten.d.ts

@@ -0,0 +1,5 @@
+import { Args } from './args.js';
+import { Logger } from '@prosopo/common';
+declare const _default: (args: Args, logger?: Logger) => Promise<void>;
+export default _default;
+//# sourceMappingURL=flatten.d.ts.map

package/dist/flatten/flatten.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.d.ts","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAK/C,IAAI,WAAW,MAAM;AAAjD,wBAqEC"}

package/dist/flatten/flatten.js

@@ -0,0 +1,65 @@
+import { CaptchaItemTypes, DataSchema } from '@prosopo/types';
+import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
+import { blake2b } from '@noble/hashes/blake2b';
+import { u8aToHex } from '@polkadot/util';
+import fs from 'fs';
+export default async (args, logger) => {
+    logger = logger || getLoggerDefault();
+    logger.debug(args, 'flattening...');
+    const dataDir = args.data;
+    if (!fs.existsSync(dataDir)) {
+        throw new ProsopoEnvError(new Error(`data directory does not exist: ${dataDir}`), 'FS.DIRECTORY_NOT_FOUND');
+    }
+    const outDir = args.out;
+    const overwrite = args.overwrite || false;
+    if (!overwrite && fs.existsSync(outDir)) {
+        throw new ProsopoEnvError(new Error(`output directory already exists: ${outDir}`), 'FS.DIRECTORY_ALREADY_EXISTS');
+    }
+    // find the labels (these should be subdirectories of the data directory)
+    const labels = fs
+        .readdirSync(dataDir, { withFileTypes: true })
+        .filter((dirent) => dirent.isDirectory())
+        .map((dirent) => dirent.name);
+    // create the output directory
+    const imageDir = `${outDir}/images`;
+    fs.mkdirSync(imageDir, { recursive: true });
+    // for each label
+    const items = [];
+    for (const label of labels) {
+        // find all the images
+        const images = fs.readdirSync(`${dataDir}/${label}`);
+        // for each image
+        for (const image of images) {
+            logger.log(`flattening ${label}/${image}`);
+            // copy the image to the output directory
+            const extension = image.split('.').pop();
+            // read file to bytes
+            const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+            // hash based on the content of the image
+            const hash = blake2b(content);
+            const hex = u8aToHex(hash);
+            const name = `${hex}.${extension}`;
+            if (fs.existsSync(`${imageDir}/${name}`)) {
+                logger.log(`duplicate image: ${label}/${image} -> ${name}`);
+            }
+            fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+            const filePath = fs.realpathSync(`${imageDir}/${name}`);
+            // add the image to the map file
+            const entry = {
+                data: filePath,
+                type: CaptchaItemTypes.Image,
+                label,
+                hash: hex,
+            };
+            items.push(entry);
+        }
+    }
+    const data = {
+        items,
+    };
+    // verify data
+    DataSchema.parse(data);
+    // write map file
+    fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+};
+//# sourceMappingURL=flatten.js.map

package/dist/flatten/flatten.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/flatten/flatten.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAQ,UAAU,EAAgB,MAAM,gBAAgB,CAAA;AACjF,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,IAAI,CAAA;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACzB,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC9G;IACD,MAAM,MAAM,GAAW,IAAI,CAAC,GAAG,CAAA;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;QACrC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,EACvD,6BAA6B,CAChC,CAAA;KACJ;IAED,yEAAyE;IACzE,MAAM,MAAM,GAAa,EAAE;SACtB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;SAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAEjC,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAA;IACnC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,iBAAiB;IACjB,MAAM,KAAK,GAAmB,EAAE,CAAA;IAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QACxB,sBAAsB;QACtB,MAAM,MAAM,GAAa,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAA;QAC9D,iBAAiB;QACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,CAAC,GAAG,CAAC,cAAc,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC1C,yCAAyC;YACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;YACxC,qBAAqB;YACrB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;YAC/D,yCAAyC;YACzC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAA;YAClC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE;gBACtC,MAAM,CAAC,GAAG,CAAC,oBAAoB,KAAK,IAAI,KAAK,OAAO,IAAI,EAAE,CAAC,CAAA;aAC9D;YACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACtE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;YACvD,gCAAgC;YAChC,MAAM,KAAK,GAAiB;gBACxB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;gBAC5B,KAAK;gBACL,IAAI,EAAE,GAAG;aACZ,CAAA;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;SACpB;KACJ;IAED,MAAM,IAAI,GAAS;QACf,KAAK;KACR,CAAA;IAED,cAAc;IACd,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEtB,iBAAiB;IACjB,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC1E,CAAC,CAAA"}

package/dist/flatten/index.d.ts

@@ -0,0 +1,4 @@
+export * from './flatten.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/flatten/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/flatten/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/flatten/index.js

@@ -0,0 +1,4 @@
+export * from './flatten.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.js.map

package/dist/flatten/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/flatten/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAA;AAC5B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/args.d.ts

@@ -0,0 +1,37 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    labels: z.ZodOptional<z.ZodString>;
+    out: z.ZodString;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    out: string;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}, {
+    out: string;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/generate/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/generate/args.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAWrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/generate/args.js

@@ -0,0 +1,15 @@
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = z.object({
+    labels: z.string().optional(),
+    out: z.string(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number().optional(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/generate/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/generate/args.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,uBAAuB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/C,yBAAyB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAA"}

package/dist/generate/cli.d.ts

@@ -0,0 +1,32 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        overwrite: boolean | undefined;
+    } & {
+        out: string;
+    } & {
+        labelled: string;
+    } & {
+        unlabelled: string;
+    } & {
+        seed: number | undefined;
+    } & {
+        size: number | undefined;
+    } & {
+        labels: string | undefined;
+    } & {
+        "allow-duplicates": boolean | undefined;
+    } & {
+        "allow-duplicates-labelled": boolean | undefined;
+    } & {
+        "allow-duplicates-unlabelled": boolean | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<never>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/generate/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAIpB,IAAI;;;;;;;;;;;;;;;;;;;;;oBAmDC,kBAAkB;;AAvDhD,wBA2DC"}

package/dist/generate/cli.js

@@ -0,0 +1,62 @@
+import generateDistinct from './distinct/cli.js';
+import generateUnion from './union/cli.js';
+export default (cmdArgs) => {
+    return {
+        command: 'generate',
+        describe: 'Generate captchas',
+        builder: (yargs) => {
+            return yargs
+                .command(generateDistinct(cmdArgs))
+                .command(generateUnion(cmdArgs))
+                .demandCommand()
+                .option('overwrite', {
+                type: 'boolean',
+                description: 'Overwrite the output file if it already exists',
+            })
+                .option('out', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the output file',
+                default: 'captchas.json',
+            })
+                .option('labelled', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the file containing map of images urls to labels',
+            })
+                .option('unlabelled', {
+                type: 'string',
+                demand: true,
+                description: 'Path to the file containing list of images url which are unlabelled',
+            })
+                .option('seed', {
+                type: 'number',
+                description: 'Seed for the random number generator',
+            })
+                .option('size', {
+                type: 'number',
+                description: 'Number of images in each captcha',
+            })
+                .options('labels', {
+                type: 'string',
+                description: 'Path to the labels file. This is a file containing a list of labels which unlabelled data will be assigned to.',
+            })
+                .option('allow-duplicates', {
+                type: 'boolean',
+                description: 'Allow duplicates in the data (labelled and unlabelled)',
+            })
+                .option('allow-duplicates-labelled', {
+                type: 'boolean',
+                description: 'Allow duplicates in the labelled data',
+            })
+                .option('allow-duplicates-unlabelled', {
+                type: 'boolean',
+                description: 'Allow duplicates in the unlabelled data',
+            });
+        },
+        handler: async (argv) => {
+            throw new Error('Please specify a command');
+        },
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/generate/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/generate/cli.ts"],"names":[],"mappings":"AAEA,OAAO,gBAAgB,MAAM,mBAAmB,CAAA;AAChD,OAAO,aAAa,MAAM,gBAAgB,CAAA;AAE1C,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,UAAU;QACnB,QAAQ,EAAE,mBAAmB;QAC7B,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;iBAClC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;iBAC/B,aAAa,EAAE;iBACf,MAAM,CAAC,WAAW,EAAE;gBACjB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,gDAAgD;aAChE,CAAC;iBACD,MAAM,CAAC,KAAK,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,yBAAyB;gBACtC,OAAO,EAAE,eAAe;aAC3B,CAAC;iBACD,MAAM,CAAC,UAAU,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,0DAA0D;aAC1E,CAAC;iBACD,MAAM,CAAC,YAAY,EAAE;gBAClB,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,qEAAqE;aACrF,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,sCAAsC;aACtD,CAAC;iBACD,MAAM,CAAC,MAAM,EAAE;gBACZ,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kCAAkC;aAClD,CAAC;iBACD,OAAO,CAAC,QAAQ,EAAE;gBACf,IAAI,EAAE,QAAQ;gBACd,WAAW,EACP,gHAAgH;aACvH,CAAC;iBACD,MAAM,CAAC,kBAAkB,EAAE;gBACxB,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,wDAAwD;aACxE,CAAC;iBACD,MAAM,CAAC,2BAA2B,EAAE;gBACjC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,uCAAuC;aACvD,CAAC;iBACD,MAAM,CAAC,6BAA6B,EAAE;gBACnC,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,yCAAyC;aACzD,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAC/C,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/generate/distinct/args.d.ts

@@ -0,0 +1,49 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    out: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+    solved: z.ZodOptional<z.ZodNumber>;
+    unsolved: z.ZodOptional<z.ZodNumber>;
+    minCorrect: z.ZodOptional<z.ZodNumber>;
+    maxCorrect: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    solved?: number | undefined;
+    unsolved?: number | undefined;
+    minCorrect?: number | undefined;
+    maxCorrect?: number | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/generate/distinct/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/args.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/generate/distinct/args.js

@@ -0,0 +1,10 @@
+import { argsSchema as parentSchema } from '../args.js';
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = parentSchema.extend({
+    solved: z.number().optional(),
+    unsolved: z.number().optional(),
+    minCorrect: z.number().optional(),
+    maxCorrect: z.number().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/generate/distinct/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../../src/generate/distinct/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,YAAY,EAAE,MAAM,YAAY,CAAA;AACvD,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAA"}

package/dist/generate/distinct/cli.d.ts

@@ -0,0 +1,20 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        solved: number | undefined;
+    } & {
+        unsolved: number | undefined;
+    } & {
+        "min-correct": number | undefined;
+    } & {
+        "max-correct": number | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<void>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/generate/distinct/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAKpB,IAAI;;;;;;;;;oBAmBC,kBAAkB;;AAxBhD,wBA4BC"}

package/dist/generate/distinct/cli.js

@@ -0,0 +1,31 @@
+import { argsSchema } from './args.js';
+import generate from './generate.js';
+export default (cmdArgs) => {
+    return {
+        command: 'distinct',
+        describe: 'Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled',
+        builder: (yargs) => {
+            return yargs
+                .option('solved', {
+                type: 'number',
+                description: 'Number of captchas to generate that are solved',
+            })
+                .option('unsolved', {
+                type: 'number',
+                description: 'Number of captchas to generate that are unsolved',
+            })
+                .option('min-correct', {
+                type: 'number',
+                description: 'Minimum number of target images in each captcha',
+            })
+                .option('max-correct', {
+                type: 'number',
+                description: 'Maximum number of target images in each captcha',
+            });
+        },
+        handler: async (argv) => {
+            await generate(argsSchema.parse(argv), cmdArgs?.logger);
+        },
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/generate/distinct/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/distinct/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAA;AAEpC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,UAAU;QACnB,QAAQ,EACJ,8GAA8G;QAClH,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,QAAQ,EAAE;gBACd,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,gDAAgD;aAChE,CAAC;iBACD,MAAM,CAAC,UAAU,EAAE;gBAChB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,kDAAkD;aAClE,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC3D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/generate/distinct/dummy.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=dummy.d.ts.map

package/dist/generate/distinct/dummy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dummy.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/dummy.ts"],"names":[],"mappings":""}

package/dist/generate/distinct/dummy.js

@@ -0,0 +1,29 @@
+// generate some dummy data for testing the generate fns
+import { CaptchaItemTypes } from '@prosopo/types';
+import fs from 'fs';
+const nLabels = 4;
+const nLabelledImages = 100;
+const nUnlabelledImages = 100;
+const labelledImages = [];
+const unlabelledImages = [];
+for (let i = 0; i < nLabelledImages; i++) {
+    const label = i % nLabels;
+    labelledImages.push({
+        label: label.toString(),
+        data: `abc${i}`,
+        type: CaptchaItemTypes.Image,
+        hash: `abc${i}hash`,
+    });
+}
+for (let i = 0; i < nUnlabelledImages; i++) {
+    unlabelledImages.push({
+        data: `def${i}`,
+        type: CaptchaItemTypes.Image,
+        hash: `def${i}hash`,
+    });
+}
+fs.writeFileSync('labelled.json', JSON.stringify(labelledImages));
+fs.writeFileSync('unlabelled.json', JSON.stringify(unlabelledImages));
+fs.writeFileSync('labels.json', JSON.stringify(Array.from(Array(nLabels).keys()).map((x) => x.toString())));
+process.exit(0);
+//# sourceMappingURL=dummy.js.map

package/dist/generate/distinct/dummy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dummy.js","sourceRoot":"","sources":["../../../src/generate/distinct/dummy.ts"],"names":[],"mappings":"AAAA,wDAAwD;AAExD,OAAO,EAAE,gBAAgB,EAAsB,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,MAAM,OAAO,GAAG,CAAC,CAAA;AACjB,MAAM,eAAe,GAAG,GAAG,CAAA;AAC3B,MAAM,iBAAiB,GAAG,GAAG,CAAA;AAE7B,MAAM,cAAc,GAAmB,EAAE,CAAA;AACzC,MAAM,gBAAgB,GAAW,EAAE,CAAA;AAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE;IACtC,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAA;IACzB,cAAc,CAAC,IAAI,CAAC;QAChB,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE;QACvB,IAAI,EAAE,MAAM,CAAC,EAAE;QACf,IAAI,EAAE,gBAAgB,CAAC,KAAK;QAC5B,IAAI,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC,CAAA;CACL;AAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC,EAAE,EAAE;IACxC,gBAAgB,CAAC,IAAI,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC,EAAE;QACf,IAAI,EAAE,gBAAgB,CAAC,KAAK;QAC5B,IAAI,EAAE,MAAM,CAAC,MAAM;KACtB,CAAC,CAAA;CACL;AAED,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAA;AACjE,EAAE,CAAC,aAAa,CAAC,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAA;AACrE,EAAE,CAAC,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAA;AAE3G,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA"}

package/dist/generate/distinct/generate.d.ts

@@ -0,0 +1,5 @@
+import { Args } from './args.js';
+import { Logger } from '@prosopo/common';
+declare const _default: (args: Args, logger?: Logger) => Promise<void>;
+export default _default;
+//# sourceMappingURL=generate.d.ts.map

package/dist/generate/distinct/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAQ/C,IAAI,WAAW,MAAM;AAAjD,wBAyNC"}