@prosopo/datasets-fs 0.1.18 → 0.2.0

package/dist/generate/distinct/generate.js

@@ -0,0 +1,186 @@
+import { CaptchaTypes, CaptchasContainerSchema, DataSchema, LabelledDataSchema, LabelsContainerSchema, } from '@prosopo/types';
+import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
+import { at, get, lodash, setSeedGlobal } from '@prosopo/util';
+import { blake2AsHex } from '@polkadot/util-crypto';
+import { checkDuplicates } from '../util.js';
+import bcrypt from 'bcrypt';
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+export default async (args, logger) => {
+    logger = logger || getLoggerDefault();
+    logger.debug(args, 'generating...');
+    const outFile = args.out;
+    const overwrite = args.overwrite || false;
+    if (!overwrite && fs.existsSync(outFile)) {
+        throw new ProsopoEnvError(new Error(`output file already exists: ${outFile}`), 'FS.FILE_ALREADY_EXISTS');
+    }
+    const labelledMapFile = args.labelled;
+    if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+        throw new ProsopoEnvError(new Error(`labelled map file does not exist: ${labelledMapFile}`), 'FS.FILE_NOT_FOUND');
+    }
+    const unlabelledMapFile = args.unlabelled;
+    if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+        throw new ProsopoEnvError(new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`), 'FS.FILE_NOT_FOUND');
+    }
+    const labelsFile = args.labels;
+    const seed = args.seed || 0;
+    const size = args.size || 9;
+    const minCorrect = args.minCorrect || 1;
+    const maxCorrect = args.maxCorrect || size - 1;
+    const solved = args.solved || 0;
+    const unsolved = args.unsolved || 0;
+    const saltRounds = 10;
+    const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+    const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+    // set the seed
+    setSeedGlobal(seed);
+    // get lodash (with seeded rng)
+    const _ = lodash();
+    // load the map to get the labelled and unlabelled data
+    const labelled = labelledMapFile
+        ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, 'utf8'))).items
+        : [];
+    const unlabelled = unlabelledMapFile
+        ? DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, 'utf8'))).items
+        : [];
+    // check for duplicates
+    checkDuplicates(labelled, unlabelled, {
+        allowDuplicatesLabelled,
+        allowDuplicatesUnlabelled,
+    });
+    // split the labelled data by label
+    const labelToImages = {};
+    for (const entry of labelled) {
+        const arr = labelToImages[entry.label] || [];
+        arr.push(entry);
+        labelToImages[entry.label] = arr;
+    }
+    const targets = Object.keys(labelToImages);
+    // load the labels from file
+    // these are the labels that unlabelled data will be assigned to
+    // note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+    const labels = [];
+    if (labelsFile && fs.existsSync(labelsFile)) {
+        labels.push(...[...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, 'utf8'))).labels]);
+    }
+    else {
+        // else default to the labels in the labelled data
+        labels.push(...[...targets]);
+    }
+    // generate n solved captchas
+    const solvedCaptchas = [];
+    // create a new progress bar instance and use shades_classic theme
+    const barSolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+    logger.info(`Generating ${solved} solved captchas...`);
+    barSolved.start(solved, 0);
+    for (let i = 0; i < solved; i++) {
+        // update the current value in your application..
+        barSolved.update(i + 1);
+        if (targets.length <= 1) {
+            throw new ProsopoEnvError(new Error(`not enough different labels in labelled data: ${labelledMapFile}`), 'DATASET.NOT_ENOUGH_LABELS');
+        }
+        // uniformly sample targets
+        const target = at(targets, i % targets.length);
+        const notTargets = targets.filter((t) => t !== target);
+        // how many correct items should be in the captcha?
+        const nCorrect = _.random(minCorrect, maxCorrect);
+        // how many incorrect items should be in the captcha?
+        const nIncorrect = size - nCorrect;
+        const targetItems = get(labelToImages, target);
+        const notTargetItems = notTargets.map((notTarget) => get(labelToImages, notTarget)).flat();
+        if (targetItems.length < nCorrect) {
+            throw new ProsopoEnvError(new Error(`not enough images for target (${target})`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        if (notTargetItems.length < nIncorrect) {
+            throw new ProsopoEnvError(new Error(`not enough non-matching images for target (${target})`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        // get the correct items
+        const correctItems = _.sampleSize(targetItems, nCorrect);
+        // get the incorrect items
+        const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
+        let items = [...correctItems, ...incorrectItems];
+        let indices = [...Array(items.length).keys()];
+        indices = _.shuffle(indices);
+        items = indices.map((i) => at(items, i));
+        items = items.map((item) => {
+            return {
+                data: item.data,
+                hash: item.hash,
+                type: item.type,
+            };
+        });
+        // the first n indices are the correct items
+        const solution = indices
+            .map((index, i) => {
+            return {
+                pre: index,
+                post: i, // the index of the item in the shuffled array
+            };
+        })
+            .filter((item) => item.pre < correctItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
+            .map((item) => {
+            return item.post; // return the index in the shuffled array
+        });
+        const salt = blake2AsHex(bcrypt.genSaltSync(saltRounds));
+        // create the captcha
+        const captcha = {
+            salt,
+            target,
+            items,
+            solution,
+        };
+        solvedCaptchas.push(captcha);
+    }
+    barSolved.stop();
+    logger.info(`Generating ${unsolved} unsolved captchas...`);
+    // create a new progress bar instance and use shades_classic theme
+    const barUnsolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+    barUnsolved.start(unsolved, 0);
+    // generate n unsolved captchas
+    const unsolvedCaptchas = [];
+    for (let i = 0; i < unsolved; i++) {
+        barUnsolved.update(i + 1);
+        if (unlabelled.length <= size) {
+            throw new ProsopoEnvError(new Error(`unlabelled map file does not contain enough data: ${unlabelledMapFile}`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        // pick a random label to be the target
+        // note that these are potentially different to the labelled data labels
+        if (labels.length <= 0) {
+            throw new ProsopoEnvError(new Error(`no labels found for unlabelled data: ${labelsFile}`), 'DATASET.NOT_ENOUGH_LABELS');
+        }
+        const index = _.random(0, labels.length - 1);
+        const target = at(labels, index);
+        // randomly pick images from the unlabelled data
+        const itemSet = _.sampleSize(unlabelled, size);
+        // shuffle the items
+        let items = [...itemSet];
+        let indices = [...Array(items.length).keys()];
+        indices = _.shuffle(indices);
+        items = indices.map((i) => at(items, i));
+        items = items.map((item) => {
+            return {
+                data: item.data,
+                hash: item.hash,
+                type: item.type,
+            };
+        });
+        const salt = blake2AsHex(bcrypt.genSaltSync(saltRounds));
+        // create the captcha
+        const captcha = {
+            salt,
+            target,
+            items,
+        };
+        unsolvedCaptchas.push(captcha);
+    }
+    barUnsolved.stop();
+    // write to file
+    const output = {
+        captchas: [...solvedCaptchas, ...unsolvedCaptchas],
+        format: CaptchaTypes.SelectAll,
+    };
+    // verify the output
+    CaptchasContainerSchema.parse(output);
+    fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+};
+//# sourceMappingURL=generate.js.map

package/dist/generate/distinct/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/distinct/generate.ts"],"names":[],"mappings":"AACA,OAAO,EACH,YAAY,EAGZ,uBAAuB,EACvB,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GAExB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,WAAW,MAAM,cAAc,CAAA;AACtC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,GAAG,CAAA;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACtC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC3G;IACD,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAA;IACzD,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QACpD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAA;IAC7D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;QACxD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,UAAU,GAAuB,IAAI,CAAC,MAAM,CAAA;IAClD,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAA;IAC/C,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,IAAI,GAAG,CAAC,CAAA;IACtD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA;IACvC,MAAM,QAAQ,GAAW,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAA;IAC3C,MAAM,UAAU,GAAG,EAAE,CAAA;IACrB,MAAM,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAEjG,eAAe;IACf,aAAa,CAAC,IAAI,CAAC,CAAA;IACnB,+BAA+B;IAC/B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,uDAAuD;IACvD,MAAM,QAAQ,GAAmB,eAAe;QAC5C,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QACtF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,UAAU,GAAW,iBAAiB;QACxC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QAChF,CAAC,CAAC,EAAE,CAAA;IAER,uBAAuB;IACvB,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE;QAClC,uBAAuB;QACvB,yBAAyB;KAC5B,CAAC,CAAA;IAEF,mCAAmC;IACnC,MAAM,aAAa,GAAgC,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAA;KACnC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAE1C,4BAA4B;IAC5B,gEAAgE;IAChE,+IAA+I;IAC/I,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;KAC3G;SAAM;QACH,kDAAkD;QAClD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;KAC/B;IAED,6BAA6B;IAC7B,MAAM,cAAc,GAAuB,EAAE,CAAA;IAC7C,kEAAkE;IAClE,MAAM,SAAS,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IAEnF,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,qBAAqB,CAAC,CAAA;IACtD,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;QAC7B,iDAAiD;QACjD,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAEvB,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iDAAiD,eAAe,EAAE,CAAC,EAC7E,2BAA2B,CAC9B,CAAA;SACJ;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAA;QAEtD,mDAAmD;QACnD,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QACjD,qDAAqD;QACrD,MAAM,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAA;QAElC,MAAM,WAAW,GAAW,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QACtD,MAAM,cAAc,GAAW,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAElG,IAAI,WAAW,CAAC,MAAM,GAAG,QAAQ,EAAE;YAC/B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iCAAiC,MAAM,GAAG,CAAC,EACrD,2BAA2B,CAC9B,CAAA;SACJ;QACD,IAAI,cAAc,CAAC,MAAM,GAAG,UAAU,EAAE;YACpC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,EAClE,2BAA2B,CAC9B,CAAA;SACJ;QAED,wBAAwB;QACxB,MAAM,YAAY,GAAW,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QAEhE,0BAA0B;QAC1B,MAAM,cAAc,GAAW,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;QAEvE,IAAI,KAAK,GAAW,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,CAAC,CAAA;QACxD,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QAEF,4CAA4C;QAC5C,MAAM,QAAQ,GAAkB,OAAO;aAClC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,0GAA0G;aAC3J,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;YACL,QAAQ;SACX,CAAA;QACD,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KAC/B;IACD,SAAS,CAAC,IAAI,EAAE,CAAA;IAChB,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,uBAAuB,CAAC,CAAA;IAC1D,kEAAkE;IAClE,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;IACrF,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAC9B,+BAA+B;IAC/B,MAAM,gBAAgB,GAAuB,EAAE,CAAA;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE;QAC/B,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACzB,IAAI,UAAU,CAAC,MAAM,IAAI,IAAI,EAAE;YAC3B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qDAAqD,iBAAiB,EAAE,CAAC,EACnF,2BAA2B,CAC9B,CAAA;SACJ;QACD,uCAAuC;QACvC,wEAAwE;QACxE,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE;YACpB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,wCAAwC,UAAU,EAAE,CAAC,EAC/D,2BAA2B,CAC9B,CAAA;SACJ;QACD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAC5C,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAChC,gDAAgD;QAChD,MAAM,OAAO,GAAW,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QACtD,oBAAoB;QACpB,IAAI,KAAK,GAAW,CAAC,GAAG,OAAO,CAAC,CAAA;QAChC,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;SACR,CAAA;QACD,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KACjC;IACD,WAAW,CAAC,IAAI,EAAE,CAAA;IAClB,gBAAgB;IAChB,MAAM,MAAM,GAAa;QACrB,QAAQ,EAAE,CAAC,GAAG,cAAc,EAAE,GAAG,gBAAgB,CAAC;QAClD,MAAM,EAAE,YAAY,CAAC,SAAS;KACjC,CAAA;IAED,oBAAoB;IACpB,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAErC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC,CAAA"}

package/dist/generate/distinct/index.d.ts

@@ -0,0 +1,4 @@
+export * from './generate.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/generate/distinct/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/generate/distinct/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/distinct/index.js

@@ -0,0 +1,4 @@
+export * from './generate.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.js.map

package/dist/generate/distinct/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/generate/distinct/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/index.d.ts

@@ -0,0 +1,5 @@
+export * as distinct from './distinct/index.js';
+export * as union from './union/index.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/generate/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/generate/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAA;AACzC,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/index.js

@@ -0,0 +1,5 @@
+export * as distinct from './distinct/index.js';
+export * as union from './union/index.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.js.map

package/dist/generate/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/generate/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,qBAAqB,CAAA;AAC/C,OAAO,KAAK,KAAK,MAAM,kBAAkB,CAAA;AACzC,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/union/args.d.ts

@@ -0,0 +1,52 @@
+import { z } from 'zod';
+export declare const argsSchema: z.ZodObject<{
+    out: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+    minCorrect: z.ZodOptional<z.ZodNumber>;
+    minIncorrect: z.ZodOptional<z.ZodNumber>;
+    minLabelled: z.ZodOptional<z.ZodNumber>;
+    maxLabelled: z.ZodOptional<z.ZodNumber>;
+    count: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    minCorrect?: number | undefined;
+    minIncorrect?: number | undefined;
+    minLabelled?: number | undefined;
+    maxLabelled?: number | undefined;
+    count?: number | undefined;
+}, {
+    out: string;
+    overwrite?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    seed?: number | undefined;
+    size?: number | undefined;
+    allowDuplicates?: boolean | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+    minCorrect?: number | undefined;
+    minIncorrect?: number | undefined;
+    minLabelled?: number | undefined;
+    maxLabelled?: number | undefined;
+    count?: number | undefined;
+}>;
+export type Args = z.infer<typeof argsSchema>;
+//# sourceMappingURL=args.d.ts.map

package/dist/generate/union/args.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../../src/generate/union/args.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMrB,CAAA;AAEF,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAA"}

package/dist/generate/union/args.js

@@ -0,0 +1,11 @@
+import { argsSchema as parentSchema } from '../args.js';
+// Args for generating a dataset
+import { z } from 'zod';
+export const argsSchema = parentSchema.extend({
+    minCorrect: z.number().optional(),
+    minIncorrect: z.number().optional(),
+    minLabelled: z.number().optional(),
+    maxLabelled: z.number().optional(),
+    count: z.number().optional(),
+});
+//# sourceMappingURL=args.js.map

package/dist/generate/union/args.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"args.js","sourceRoot":"","sources":["../../../src/generate/union/args.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,IAAI,YAAY,EAAE,MAAM,YAAY,CAAA;AACvD,gCAAgC;AAEhC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IAC1C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAA"}

package/dist/generate/union/cli.d.ts

@@ -0,0 +1,22 @@
+import { ArgumentsCamelCase, Argv } from 'yargs';
+import { Logger } from '@prosopo/common';
+declare const _default: (cmdArgs?: {
+    logger?: Logger;
+}) => {
+    command: string;
+    describe: string;
+    builder: (yargs: Argv) => Argv<{
+        count: number | undefined;
+    } & {
+        "min-correct": number | undefined;
+    } & {
+        "min-incorrect": number | undefined;
+    } & {
+        "min-labelled": number | undefined;
+    } & {
+        "max-labelled": number | undefined;
+    }>;
+    handler: (argv: ArgumentsCamelCase) => Promise<void>;
+};
+export default _default;
+//# sourceMappingURL=cli.d.ts.map

package/dist/generate/union/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/generate/union/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,OAAO,CAAA;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAA;mCAId;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE;;;qBAKpB,IAAI;;;;;;;;;;;oBAuBC,kBAAkB;;AA5BhD,wBAgCC"}

package/dist/generate/union/cli.js

@@ -0,0 +1,35 @@
+import { argsSchema } from './args.js';
+import generate from './generate.js';
+export default (cmdArgs) => {
+    return {
+        command: 'union',
+        describe: 'Generate distinct captchas producing captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round',
+        builder: (yargs) => {
+            return yargs
+                .option('count', {
+                type: 'number',
+                description: 'Number of captchas to generate',
+            })
+                .option('min-correct', {
+                type: 'number',
+                description: 'Minimum number of target images in each captcha',
+            })
+                .option('min-incorrect', {
+                type: 'number',
+                description: 'Minimum number of incorrect images in each captcha',
+            })
+                .option('min-labelled', {
+                type: 'number',
+                description: 'Minimum number of labelled images in each captcha',
+            })
+                .option('max-labelled', {
+                type: 'number',
+                description: 'Maximum number of labelled images in each captcha',
+            });
+        },
+        handler: async (argv) => {
+            await generate(argsSchema.parse(argv), cmdArgs?.logger);
+        },
+    };
+};
+//# sourceMappingURL=cli.js.map

package/dist/generate/union/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/generate/union/cli.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AACtC,OAAO,QAAQ,MAAM,eAAe,CAAA;AAEpC,eAAe,CAAC,OAA6B,EAAE,EAAE;IAC7C,OAAO;QACH,OAAO,EAAE,OAAO;QAChB,QAAQ,EACJ,gJAAgJ;QACpJ,OAAO,EAAE,CAAC,KAAW,EAAE,EAAE;YACrB,OAAO,KAAK;iBACP,MAAM,CAAC,OAAO,EAAE;gBACb,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,gCAAgC;aAChD,CAAC;iBACD,MAAM,CAAC,aAAa,EAAE;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,iDAAiD;aACjE,CAAC;iBACD,MAAM,CAAC,eAAe,EAAE;gBACrB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,oDAAoD;aACpE,CAAC;iBACD,MAAM,CAAC,cAAc,EAAE;gBACpB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,mDAAmD;aACnE,CAAC;iBACD,MAAM,CAAC,cAAc,EAAE;gBACpB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,mDAAmD;aACnE,CAAC,CAAA;QACV,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,IAAwB,EAAE,EAAE;YACxC,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC3D,CAAC;KACJ,CAAA;AACL,CAAC,CAAA"}

package/dist/generate/union/generate.d.ts

@@ -0,0 +1,5 @@
+import { Args } from './args.js';
+import { Logger } from '@prosopo/common';
+declare const _default: (args: Args, logger?: Logger) => Promise<void>;
+export default _default;
+//# sourceMappingURL=generate.d.ts.map

package/dist/generate/union/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAahC,OAAO,EAAE,MAAM,EAAqC,MAAM,iBAAiB,CAAA;+BAO/C,IAAI,WAAW,MAAM;AAAjD,wBA8LC"}

package/dist/generate/union/generate.js

@@ -0,0 +1,168 @@
+import { CaptchaTypes, CaptchasContainerSchema, DataSchema, LabelledDataSchema, LabelsContainerSchema, } from '@prosopo/types';
+import { ProsopoEnvError, getLoggerDefault } from '@prosopo/common';
+import { at, get, lodash, setSeedGlobal } from '@prosopo/util';
+import { blake2AsHex } from '@polkadot/util-crypto';
+import { checkDuplicates } from '../util.js';
+import bcrypt from 'bcrypt';
+import fs from 'fs';
+export default async (args, logger) => {
+    logger = logger || getLoggerDefault();
+    logger.debug(args, 'generating...');
+    const outFile = args.out;
+    const overwrite = args.overwrite || false;
+    if (!overwrite && fs.existsSync(outFile)) {
+        throw new ProsopoEnvError(new Error(`Output file already exists: ${outFile}`), 'FS.FILE_ALREADY_EXISTS');
+    }
+    const labelledMapFile = args.labelled;
+    if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+        throw new ProsopoEnvError(new Error(`Labelled map file does not exist: ${labelledMapFile}`), 'FS.FILE_NOT_FOUND');
+    }
+    const unlabelledMapFile = args.unlabelled;
+    if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+        throw new ProsopoEnvError(new Error(`Unlabelled map file does not exist: ${unlabelledMapFile}`), 'FS.FILE_NOT_FOUND');
+    }
+    const labelsFile = args.labels;
+    const seed = args.seed || 0;
+    const size = args.size || 9;
+    const minCorrect = args.minCorrect || 1;
+    const saltRounds = 10;
+    const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+    const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+    const minIncorrect = Math.max(args.minIncorrect || 1, 1); // at least 1 incorrect image
+    const minLabelled = minCorrect + minIncorrect; // min incorrect + correct
+    const maxLabelled = Math.min(args.maxLabelled || size, size); // at least 1 labelled image
+    const count = args.count || 0;
+    // set the seed
+    setSeedGlobal(seed);
+    // get lodash (with seeded rng)
+    const _ = lodash();
+    // the captcha contains n images. Each of these images are either labelled, being correct or incorrect against the target, or unlabelled. To construct one of these captchas, we need to decide how many of the images should be labelled vs unlabelled, and then how many of the labelled images should be correct vs incorrect
+    // in the traditional captcha, two rounds are produced, one with labelled images and the other with unlabelled images. This gives 18 images overall, 9 labels produced.
+    // the parameters for generation can regulate how many labels are collected vs how much of a test the captcha posses. E.g. 18 images could have 16 unlabelled and 2 labelled, or 2 unlabelled and 16 labelled. The former is a better test of the user being human, but the latter is a better for maximising label collection.
+    // if we focus on a single captcha round of 9 images, we must have at least 1 labelled correct image in the captcha for it to work, otherwise it's just a labelling phase, which normally isn't a problem but if we're treating these as tests for humanity too then we need some kind of test in there. (e.g. we abolish the labelled then unlabelled pattern of the challenge rounds in favour of mixing labelled and unlabelled data, but we then run a small chance of serving two completely unlabelled rounds if we don't set the min number of labelled images to 1 per captcha round)
+    // load the map to get the labelled and unlabelled data
+    const labelled = labelledMapFile
+        ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, 'utf8'))).items
+        : [];
+    const unlabelled = unlabelledMapFile
+        ? DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, 'utf8'))).items
+        : [];
+    // check for duplicates
+    checkDuplicates(labelled, unlabelled, {
+        allowDuplicatesLabelled,
+        allowDuplicatesUnlabelled,
+    });
+    // split the labelled data by label
+    const labelToImages = {};
+    for (const entry of labelled) {
+        const arr = labelToImages[entry.label] || [];
+        arr.push(entry);
+        labelToImages[entry.label] = arr;
+    }
+    const targets = Object.keys(labelToImages);
+    // load the labels from file
+    // these are the labels that unlabelled data will be assigned to
+    // note that these can be differen to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+    const labels = [];
+    if (labelsFile && fs.existsSync(labelsFile)) {
+        labels.push(...[...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, 'utf8'))).labels]);
+    }
+    else {
+        // else use the labels from the labelled data
+        labels.push(...[...targets]);
+    }
+    // generate n captchas
+    const captchas = [];
+    for (let i = 0; i < count; i++) {
+        logger.info(`generating captcha ${i + 1} of ${count}`);
+        if (targets.length <= 1) {
+            throw new ProsopoEnvError(new Error(`not enough different labels in labelled data: ${labelledMapFile}`), 'DATASET.NOT_ENOUGH_LABELS');
+        }
+        // uniformly sample targets
+        const target = at(targets, i % targets.length);
+        const notTargets = targets.filter((t) => t !== target);
+        // how many labelled images should be in the captcha?
+        const nLabelled = _.random(minLabelled, maxLabelled);
+        // how many correct labelled images should be in the captcha?
+        const maxCorrect = nLabelled - minCorrect;
+        const nCorrect = _.random(minCorrect, maxCorrect);
+        const nIncorrect = nLabelled - nCorrect;
+        const nUnlabelled = size - nLabelled;
+        const targetItems = get(labelToImages, target);
+        const notTargetItems = notTargets.map((notTarget) => get(labelToImages, notTarget)).flat();
+        if (nUnlabelled > unlabelled.length) {
+            throw new ProsopoEnvError(new Error(`not enough unlabelled data`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        if (nCorrect > targetItems.length) {
+            throw new ProsopoEnvError(new Error(`not enough images for target (${target})`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        if (nIncorrect > notTargetItems.length) {
+            throw new ProsopoEnvError(new Error(`not enough non-matching images for target (${target})`), 'DATASET.NOT_ENOUGH_IMAGES');
+        }
+        // get the correct items
+        const correctItems = _.sampleSize(targetItems, nCorrect);
+        // get the incorrect items
+        const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
+        // get the unlabelled items
+        const unlabelledItems = new Set();
+        while (unlabelledItems.size < size - nLabelled) {
+            // get a random image from the unlabelled data
+            const image = at(unlabelled, _.random(0, unlabelled.length - 1));
+            unlabelledItems.add(image);
+        }
+        let items = [...correctItems, ...incorrectItems, ...unlabelledItems];
+        let indices = [...Array(items.length).keys()];
+        indices = _.shuffle(indices);
+        items = indices.map((i) => at(items, i));
+        items = items.map((item) => {
+            return {
+                data: item.data,
+                hash: item.hash,
+                type: item.type,
+            };
+        });
+        // the first n indices are the correct items
+        const solution = indices
+            .map((index, i) => {
+            return {
+                pre: index,
+                post: i, // the index of the item in the shuffled array
+            };
+        })
+            .filter((item) => item.pre < correctItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
+            .map((item) => {
+            return item.post; // return the index in the shuffled array
+        });
+        // the unlabelled indices were after the correct and incorrect
+        const unlabelledIndices = indices
+            .map((index, i) => {
+            return {
+                pre: index,
+                post: i, // the index of the item in the shuffled array
+            };
+        })
+            .filter((item) => item.pre >= correctItems.length + incorrectItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
+            .map((item) => {
+            return item.post; // return the index in the shuffled array
+        });
+        const salt = blake2AsHex(bcrypt.genSaltSync(saltRounds));
+        // create the captcha
+        const captcha = {
+            salt,
+            target,
+            items,
+            solution,
+            unlabelled: unlabelledIndices,
+        };
+        captchas.push(captcha);
+    }
+    // write to file
+    const output = {
+        captchas,
+        format: CaptchaTypes.SelectAll,
+    };
+    // verify the output
+    CaptchasContainerSchema.parse(output);
+    fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+};
+//# sourceMappingURL=generate.js.map

package/dist/generate/union/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../../src/generate/union/generate.ts"],"names":[],"mappings":"AACA,OAAO,EACH,YAAY,EAGZ,uBAAuB,EACvB,UAAU,EAEV,kBAAkB,EAElB,qBAAqB,GAExB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAU,eAAe,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAC3E,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,MAAM,MAAM,QAAQ,CAAA;AAC3B,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,eAAe,KAAK,EAAE,IAAU,EAAE,MAAe,EAAE,EAAE;IACjD,MAAM,GAAG,MAAM,IAAI,gBAAgB,EAAE,CAAA;IAErC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAA;IAEnC,MAAM,OAAO,GAAW,IAAI,CAAC,GAAG,CAAA;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAA;IACzC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QACtC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,+BAA+B,OAAO,EAAE,CAAC,EAAE,wBAAwB,CAAC,CAAA;KAC3G;IACD,MAAM,eAAe,GAAuB,IAAI,CAAC,QAAQ,CAAA;IACzD,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE;QACpD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,qCAAqC,eAAe,EAAE,CAAC,EACjE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,iBAAiB,GAAuB,IAAI,CAAC,UAAU,CAAA;IAC7D,IAAI,iBAAiB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;QACxD,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,uCAAuC,iBAAiB,EAAE,CAAC,EACrE,mBAAmB,CACtB,CAAA;KACJ;IACD,MAAM,UAAU,GAAuB,IAAI,CAAC,MAAM,CAAA;IAClD,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,IAAI,GAAW,IAAI,CAAC,IAAI,IAAI,CAAC,CAAA;IACnC,MAAM,UAAU,GAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAA;IAC/C,MAAM,UAAU,GAAG,EAAE,CAAA;IACrB,MAAM,uBAAuB,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IAC7F,MAAM,yBAAyB,GAAG,IAAI,CAAC,yBAAyB,IAAI,IAAI,CAAC,eAAe,IAAI,KAAK,CAAA;IACjG,MAAM,YAAY,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA,CAAC,6BAA6B;IAC9F,MAAM,WAAW,GAAW,UAAU,GAAG,YAAY,CAAA,CAAC,0BAA0B;IAChF,MAAM,WAAW,GAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,EAAE,IAAI,CAAC,CAAA,CAAC,4BAA4B;IACjG,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,IAAI,CAAC,CAAA;IAErC,eAAe;IACf,aAAa,CAAC,IAAI,CAAC,CAAA;IACnB,+BAA+B;IAC/B,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,gUAAgU;IAChU,uKAAuK;IACvK,+TAA+T;IAC/T,6jBAA6jB;IAC7jB,uDAAuD;IACvD,MAAM,QAAQ,GAAmB,eAAe;QAC5C,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QACtF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,UAAU,GAAW,iBAAiB;QACxC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK;QAChF,CAAC,CAAC,EAAE,CAAA;IACR,uBAAuB;IACvB,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE;QAClC,uBAAuB;QACvB,yBAAyB;KAC5B,CAAC,CAAA;IACF,mCAAmC;IACnC,MAAM,aAAa,GAAgC,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;QAC1B,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5C,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAA;KACnC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC1C,4BAA4B;IAC5B,gEAAgE;IAChE,8IAA8I;IAC9I,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,UAAU,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;KAC3G;SAAM;QACH,6CAA6C;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;KAC/B;IACD,sBAAsB;IACtB,MAAM,QAAQ,GAAuB,EAAE,CAAA;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;QAC5B,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,OAAO,KAAK,EAAE,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;YACrB,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iDAAiD,eAAe,EAAE,CAAC,EAC7E,2BAA2B,CAC9B,CAAA;SACJ;QAED,2BAA2B;QAC3B,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAA;QACtD,qDAAqD;QACrD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;QACpD,6DAA6D;QAC7D,MAAM,UAAU,GAAG,SAAS,GAAG,UAAU,CAAA;QACzC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;QACjD,MAAM,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAA;QACvC,MAAM,WAAW,GAAG,IAAI,GAAG,SAAS,CAAA;QAEpC,MAAM,WAAW,GAAG,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QAC9C,MAAM,cAAc,GAAW,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAElG,IAAI,WAAW,GAAG,UAAU,CAAC,MAAM,EAAE;YACjC,MAAM,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE,2BAA2B,CAAC,CAAA;SAClG;QACD,IAAI,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE;YAC/B,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,iCAAiC,MAAM,GAAG,CAAC,EACrD,2BAA2B,CAC9B,CAAA;SACJ;QACD,IAAI,UAAU,GAAG,cAAc,CAAC,MAAM,EAAE;YACpC,MAAM,IAAI,eAAe,CACrB,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,EAClE,2BAA2B,CAC9B,CAAA;SACJ;QAED,wBAAwB;QACxB,MAAM,YAAY,GAAW,CAAC,CAAC,UAAU,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAA;QAEhE,0BAA0B;QAC1B,MAAM,cAAc,GAAW,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;QAEvE,2BAA2B;QAC3B,MAAM,eAAe,GAAG,IAAI,GAAG,EAAQ,CAAA;QACvC,OAAO,eAAe,CAAC,IAAI,GAAG,IAAI,GAAG,SAAS,EAAE;YAC5C,8CAA8C;YAC9C,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;YAChE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;SAC7B;QAED,IAAI,KAAK,GAAW,CAAC,GAAG,YAAY,EAAE,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,CAAA;QAC5E,IAAI,OAAO,GAAa,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QACvD,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5B,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;QACxC,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO;gBACH,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,IAAI,EAAE,IAAI,CAAC,IAAI;aAClB,CAAA;QACL,CAAC,CAAC,CAAA;QAEF,4CAA4C;QAC5C,MAAM,QAAQ,GAAkB,OAAO;aAClC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,0GAA0G;aAC3J,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,8DAA8D;QAC9D,MAAM,iBAAiB,GAAkB,OAAO;aAC3C,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACd,OAAO;gBACH,GAAG,EAAE,KAAK;gBACV,IAAI,EAAE,CAAC,EAAE,8CAA8C;aAC1D,CAAA;QACL,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,IAAI,YAAY,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,0GAA0G;aACpL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,OAAO,IAAI,CAAC,IAAI,CAAA,CAAC,yCAAyC;QAC9D,CAAC,CAAC,CAAA;QAEN,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAA;QACxD,qBAAqB;QACrB,MAAM,OAAO,GAAqB;YAC9B,IAAI;YACJ,MAAM;YACN,KAAK;YACL,QAAQ;YACR,UAAU,EAAE,iBAAiB;SAChC,CAAA;QACD,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;KACzB;IACD,gBAAgB;IAChB,MAAM,MAAM,GAAa;QACrB,QAAQ;QACR,MAAM,EAAE,YAAY,CAAC,SAAS;KACjC,CAAA;IAED,oBAAoB;IACpB,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAErC,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;AAC9D,CAAC,CAAA"}

package/dist/generate/union/index.d.ts

@@ -0,0 +1,4 @@
+export * from './generate.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/generate/union/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/generate/union/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/union/index.js

@@ -0,0 +1,4 @@
+export * from './generate.js';
+export * from './cli.js';
+export * from './args.js';
+//# sourceMappingURL=index.js.map

package/dist/generate/union/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/generate/union/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA"}

package/dist/generate/util.d.ts

@@ -0,0 +1,6 @@
+import { Item, LabelledItem } from '@prosopo/types';
+export declare const checkDuplicates: (labelled: LabelledItem[], unlabelled: Item[], options: {
+    allowDuplicatesLabelled?: boolean;
+    allowDuplicatesUnlabelled?: boolean;
+}) => void;
+//# sourceMappingURL=util.d.ts.map

package/dist/generate/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAEnD,eAAO,MAAM,eAAe,aACd,YAAY,EAAE,cACZ,IAAI,EAAE,WACT;IACL,uBAAuB,CAAC,EAAE,OAAO,CAAA;IACjC,yBAAyB,CAAC,EAAE,OAAO,CAAA;CACtC,SAmBJ,CAAA"}

package/dist/generate/util.js

@@ -0,0 +1,20 @@
+export const checkDuplicates = (labelled, unlabelled, options) => {
+    // check for duplicates
+    const all = new Set();
+    if (!options.allowDuplicatesLabelled) {
+        for (const entry of labelled) {
+            if (all.has(entry.data)) {
+                throw new Error(`Duplicate data entry in labelled data: ${JSON.stringify(entry)}`);
+            }
+            all.add(entry.data);
+        }
+    }
+    if (!options.allowDuplicatesUnlabelled) {
+        for (const entry of unlabelled) {
+            if (all.has(entry.data)) {
+                throw new Error(`Duplicate data entry in unlabelled data: ${JSON.stringify(entry)}`);
+            }
+        }
+    }
+};
+//# sourceMappingURL=util.js.map

package/dist/generate/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/generate/util.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,eAAe,GAAG,CAC3B,QAAwB,EACxB,UAAkB,EAClB,OAGC,EACH,EAAE;IACA,uBAAuB;IACvB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE;QAClC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;YAC1B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACrF;YACD,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;SACtB;KACJ;IACD,IAAI,CAAC,OAAO,CAAC,yBAAyB,EAAE;QACpC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE;YAC5B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;aACvF;SACJ;KACJ;AACL,CAAC,CAAA"}