@prosopo/datasets-fs 0.2.0 → 0.2.2

package/dist/cjs/cli.cjs

@@ -0,0 +1,34 @@
+"use strict";
+const common = require("@prosopo/common");
+const helpers = require("yargs/helpers");
+const esMain = require("es-main");
+const cli$1 = require("./flatten/cli.cjs");
+const cli = require("./generate/cli.cjs");
+const cli$4 = require("./get/cli.cjs");
+const cli$5 = require("./labels/cli.cjs");
+const process = require("process");
+const cli$3 = require("./relocate/cli.cjs");
+const cli$2 = require("./scale/cli.cjs");
+const yargs = require("yargs");
+var _documentCurrentScript = typeof document !== "undefined" ? document.currentScript : null;
+const dirname = process.cwd();
+const logger = common.getLogger(common.LogLevel.enum.info, `${dirname}`);
+const main = async () => {
+  await yargs(helpers.hideBin(process.argv)).help().option("log-level", {
+    type: "string",
+    choices: Object.values(common.LogLevel.options),
+    default: common.LogLevel.enum.info,
+    description: "The log level"
+  }).middleware((argv) => {
+    logger.setLogLevel(argv.logLevel);
+  }).command(cli({ logger })).command(cli$1({ logger })).command(cli$2({ logger })).command(cli$3({ logger })).command(cli$4({ logger })).command(cli$5({ logger })).strictCommands().showHelpOnFail(false, "Specify --help for available options").fail(false).parse();
+};
+if (esMain({ url: typeof document === "undefined" ? require("url").pathToFileURL(__filename).href : _documentCurrentScript && _documentCurrentScript.src || new URL("cli.cjs", document.baseURI).href })) {
+  main().then(() => {
+    logger.debug("done");
+    process.exit(0);
+  }).catch((err) => {
+    logger.error("error:", err);
+    process.exit(1);
+  });
+}

package/dist/cjs/flatten/args.cjs

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  data: zod.z.string(),
+  out: zod.z.string(),
+  overwrite: zod.z.boolean().optional()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/flatten/cli.cjs

@@ -0,0 +1,28 @@
+"use strict";
+const args = require("./args.cjs");
+const flatten$1 = require("./flatten.cjs");
+const flatten = (cmdArgs) => {
+  return {
+    command: "flatten",
+    describe: "Restructure a directory containing directories for each image classification into a single directory with a file containing the labels",
+    builder: (yargs) => {
+      return yargs.option("data", {
+        type: "string",
+        alias: "in",
+        demand: true,
+        description: "Path to the data directory containing subdirectories for each image classification"
+      }).option("out", {
+        type: "string",
+        demand: true,
+        description: "Where to put the output file containing the labels and single directory of images"
+      }).option("overwrite", {
+        type: "boolean",
+        description: "Overwrite the output file if it already exists"
+      });
+    },
+    handler: async (argv) => {
+      await flatten$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = flatten;

package/dist/cjs/flatten/flatten.cjs

@@ -0,0 +1,55 @@
+"use strict";
+const types = require("@prosopo/types");
+const common = require("@prosopo/common");
+const blake2b = require("@noble/hashes/blake2b");
+const util = require("@polkadot/util");
+const fs = require("fs");
+const flatten = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "flattening...");
+  const dataDir = args.data;
+  if (!fs.existsSync(dataDir)) {
+    throw new common.ProsopoEnvError(new Error(`data directory does not exist: ${dataDir}`), "FS.DIRECTORY_NOT_FOUND");
+  }
+  const outDir = args.out;
+  const overwrite = args.overwrite || false;
+  if (!overwrite && fs.existsSync(outDir)) {
+    throw new common.ProsopoEnvError(
+      new Error(`output directory already exists: ${outDir}`),
+      "FS.DIRECTORY_ALREADY_EXISTS"
+    );
+  }
+  const labels = fs.readdirSync(dataDir, { withFileTypes: true }).filter((dirent) => dirent.isDirectory()).map((dirent) => dirent.name);
+  const imageDir = `${outDir}/images`;
+  fs.mkdirSync(imageDir, { recursive: true });
+  const items = [];
+  for (const label of labels) {
+    const images = fs.readdirSync(`${dataDir}/${label}`);
+    for (const image of images) {
+      logger.log(`flattening ${label}/${image}`);
+      const extension = image.split(".").pop();
+      const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+      const hash = blake2b.blake2b(content);
+      const hex = util.u8aToHex(hash);
+      const name = `${hex}.${extension}`;
+      if (fs.existsSync(`${imageDir}/${name}`)) {
+        logger.log(`duplicate image: ${label}/${image} -> ${name}`);
+      }
+      fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+      const filePath = fs.realpathSync(`${imageDir}/${name}`);
+      const entry = {
+        data: filePath,
+        type: types.CaptchaItemTypes.Image,
+        label,
+        hash: hex
+      };
+      items.push(entry);
+    }
+  }
+  const data = {
+    items
+  };
+  types.DataSchema.parse(data);
+  fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+};
+module.exports = flatten;

package/dist/cjs/generate/args.cjs

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  labels: zod.z.string().optional(),
+  out: zod.z.string(),
+  labelled: zod.z.string().optional(),
+  unlabelled: zod.z.string().optional(),
+  seed: zod.z.number().optional(),
+  size: zod.z.number().optional(),
+  overwrite: zod.z.boolean().optional(),
+  allowDuplicates: zod.z.boolean().optional(),
+  allowDuplicatesLabelled: zod.z.boolean().optional(),
+  allowDuplicatesUnlabelled: zod.z.boolean().optional()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/generate/cli.cjs

@@ -0,0 +1,50 @@
+"use strict";
+const cli = require("./distinct/cli.cjs");
+const cli$1 = require("./union/cli.cjs");
+const generate = (cmdArgs) => {
+  return {
+    command: "generate",
+    describe: "Generate captchas",
+    builder: (yargs) => {
+      return yargs.command(cli(cmdArgs)).command(cli$1(cmdArgs)).demandCommand().option("overwrite", {
+        type: "boolean",
+        description: "Overwrite the output file if it already exists"
+      }).option("out", {
+        type: "string",
+        demand: true,
+        description: "Path to the output file",
+        default: "captchas.json"
+      }).option("labelled", {
+        type: "string",
+        demand: true,
+        description: "Path to the file containing map of images urls to labels"
+      }).option("unlabelled", {
+        type: "string",
+        demand: true,
+        description: "Path to the file containing list of images url which are unlabelled"
+      }).option("seed", {
+        type: "number",
+        description: "Seed for the random number generator"
+      }).option("size", {
+        type: "number",
+        description: "Number of images in each captcha"
+      }).options("labels", {
+        type: "string",
+        description: "Path to the labels file. This is a file containing a list of labels which unlabelled data will be assigned to."
+      }).option("allow-duplicates", {
+        type: "boolean",
+        description: "Allow duplicates in the data (labelled and unlabelled)"
+      }).option("allow-duplicates-labelled", {
+        type: "boolean",
+        description: "Allow duplicates in the labelled data"
+      }).option("allow-duplicates-unlabelled", {
+        type: "boolean",
+        description: "Allow duplicates in the unlabelled data"
+      });
+    },
+    handler: async (argv) => {
+      throw new Error("Please specify a command");
+    }
+  };
+};
+module.exports = generate;

package/dist/cjs/generate/distinct/args.cjs

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const args = require("../args.cjs");
+const zod = require("zod");
+const argsSchema = args.argsSchema.extend({
+  solved: zod.z.number().optional(),
+  unsolved: zod.z.number().optional(),
+  minCorrect: zod.z.number().optional(),
+  maxCorrect: zod.z.number().optional()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/generate/distinct/cli.cjs

@@ -0,0 +1,28 @@
+"use strict";
+const args = require("./args.cjs");
+const generate = require("./generate.cjs");
+const generateDistinct = (cmdArgs) => {
+  return {
+    command: "distinct",
+    describe: "Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled",
+    builder: (yargs) => {
+      return yargs.option("solved", {
+        type: "number",
+        description: "Number of captchas to generate that are solved"
+      }).option("unsolved", {
+        type: "number",
+        description: "Number of captchas to generate that are unsolved"
+      }).option("min-correct", {
+        type: "number",
+        description: "Minimum number of target images in each captcha"
+      }).option("max-correct", {
+        type: "number",
+        description: "Maximum number of target images in each captcha"
+      });
+    },
+    handler: async (argv) => {
+      await generate(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = generateDistinct;

package/dist/cjs/generate/distinct/generate.cjs

@@ -0,0 +1,174 @@
+"use strict";
+const types = require("@prosopo/types");
+const common = require("@prosopo/common");
+const util = require("@prosopo/util");
+const utilCrypto = require("@polkadot/util-crypto");
+const util$1 = require("../util.cjs");
+const bcrypt = require("bcrypt");
+const cliProgress = require("cli-progress");
+const fs = require("fs");
+const generate = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "generating...");
+  const outFile = args.out;
+  const overwrite = args.overwrite || false;
+  if (!overwrite && fs.existsSync(outFile)) {
+    throw new common.ProsopoEnvError(new Error(`output file already exists: ${outFile}`), "FS.FILE_ALREADY_EXISTS");
+  }
+  const labelledMapFile = args.labelled;
+  if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+    throw new common.ProsopoEnvError(
+      new Error(`labelled map file does not exist: ${labelledMapFile}`),
+      "FS.FILE_NOT_FOUND"
+    );
+  }
+  const unlabelledMapFile = args.unlabelled;
+  if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+    throw new common.ProsopoEnvError(
+      new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
+      "FS.FILE_NOT_FOUND"
+    );
+  }
+  const labelsFile = args.labels;
+  const seed = args.seed || 0;
+  const size = args.size || 9;
+  const minCorrect = args.minCorrect || 1;
+  const maxCorrect = args.maxCorrect || size - 1;
+  const solved = args.solved || 0;
+  const unsolved = args.unsolved || 0;
+  const saltRounds = 10;
+  const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+  const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+  util.setSeedGlobal(seed);
+  const _ = util.lodash();
+  const labelled = labelledMapFile ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, "utf8"))).items : [];
+  const unlabelled = unlabelledMapFile ? types.DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, "utf8"))).items : [];
+  util$1.checkDuplicates(labelled, unlabelled, {
+    allowDuplicatesLabelled,
+    allowDuplicatesUnlabelled
+  });
+  const labelToImages = {};
+  for (const entry of labelled) {
+    const arr = labelToImages[entry.label] || [];
+    arr.push(entry);
+    labelToImages[entry.label] = arr;
+  }
+  const targets = Object.keys(labelToImages);
+  const labels = [];
+  if (labelsFile && fs.existsSync(labelsFile)) {
+    labels.push(...[...types.LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, "utf8"))).labels]);
+  } else {
+    labels.push(...[...targets]);
+  }
+  const solvedCaptchas = [];
+  const barSolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+  logger.info(`Generating ${solved} solved captchas...`);
+  barSolved.start(solved, 0);
+  for (let i = 0; i < solved; i++) {
+    barSolved.update(i + 1);
+    if (targets.length <= 1) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough different labels in labelled data: ${labelledMapFile}`),
+        "DATASET.NOT_ENOUGH_LABELS"
+      );
+    }
+    const target = util.at(targets, i % targets.length);
+    const notTargets = targets.filter((t) => t !== target);
+    const nCorrect = _.random(minCorrect, maxCorrect);
+    const nIncorrect = size - nCorrect;
+    const targetItems = util.get(labelToImages, target);
+    const notTargetItems = notTargets.map((notTarget) => util.get(labelToImages, notTarget)).flat();
+    if (targetItems.length < nCorrect) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough images for target (${target})`),
+        "DATASET.NOT_ENOUGH_IMAGES"
+      );
+    }
+    if (notTargetItems.length < nIncorrect) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough non-matching images for target (${target})`),
+        "DATASET.NOT_ENOUGH_IMAGES"
+      );
+    }
+    const correctItems = _.sampleSize(targetItems, nCorrect);
+    const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
+    let items = [...correctItems, ...incorrectItems];
+    let indices = [...Array(items.length).keys()];
+    indices = _.shuffle(indices);
+    items = indices.map((i2) => util.at(items, i2));
+    items = items.map((item) => {
+      return {
+        data: item.data,
+        hash: item.hash,
+        type: item.type
+      };
+    });
+    const solution = indices.map((index, i2) => {
+      return {
+        pre: index,
+        // the index of the item in the items array before shuffle
+        post: i2
+        // the index of the item in the shuffled array
+      };
+    }).filter((item) => item.pre < correctItems.length).map((item) => {
+      return item.post;
+    });
+    const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
+    const captcha = {
+      salt,
+      target,
+      items,
+      solution
+    };
+    solvedCaptchas.push(captcha);
+  }
+  barSolved.stop();
+  logger.info(`Generating ${unsolved} unsolved captchas...`);
+  const barUnsolved = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+  barUnsolved.start(unsolved, 0);
+  const unsolvedCaptchas = [];
+  for (let i = 0; i < unsolved; i++) {
+    barUnsolved.update(i + 1);
+    if (unlabelled.length <= size) {
+      throw new common.ProsopoEnvError(
+        new Error(`unlabelled map file does not contain enough data: ${unlabelledMapFile}`),
+        "DATASET.NOT_ENOUGH_IMAGES"
+      );
+    }
+    if (labels.length <= 0) {
+      throw new common.ProsopoEnvError(
+        new Error(`no labels found for unlabelled data: ${labelsFile}`),
+        "DATASET.NOT_ENOUGH_LABELS"
+      );
+    }
+    const index = _.random(0, labels.length - 1);
+    const target = util.at(labels, index);
+    const itemSet = _.sampleSize(unlabelled, size);
+    let items = [...itemSet];
+    let indices = [...Array(items.length).keys()];
+    indices = _.shuffle(indices);
+    items = indices.map((i2) => util.at(items, i2));
+    items = items.map((item) => {
+      return {
+        data: item.data,
+        hash: item.hash,
+        type: item.type
+      };
+    });
+    const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
+    const captcha = {
+      salt,
+      target,
+      items
+    };
+    unsolvedCaptchas.push(captcha);
+  }
+  barUnsolved.stop();
+  const output = {
+    captchas: [...solvedCaptchas, ...unsolvedCaptchas],
+    format: types.CaptchaTypes.SelectAll
+  };
+  types.CaptchasContainerSchema.parse(output);
+  fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+};
+module.exports = generate;

package/dist/cjs/generate/union/args.cjs

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const args = require("../args.cjs");
+const zod = require("zod");
+const argsSchema = args.argsSchema.extend({
+  minCorrect: zod.z.number().optional(),
+  minIncorrect: zod.z.number().optional(),
+  minLabelled: zod.z.number().optional(),
+  maxLabelled: zod.z.number().optional(),
+  count: zod.z.number().optional()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/generate/union/cli.cjs

@@ -0,0 +1,31 @@
+"use strict";
+const args = require("./args.cjs");
+const generate = require("./generate.cjs");
+const generateUnion = (cmdArgs) => {
+  return {
+    command: "union",
+    describe: "Generate distinct captchas producing captcha challenges comprising one or more rounds, mixing labelled and unlabelled data into a single round",
+    builder: (yargs) => {
+      return yargs.option("count", {
+        type: "number",
+        description: "Number of captchas to generate"
+      }).option("min-correct", {
+        type: "number",
+        description: "Minimum number of target images in each captcha"
+      }).option("min-incorrect", {
+        type: "number",
+        description: "Minimum number of incorrect images in each captcha"
+      }).option("min-labelled", {
+        type: "number",
+        description: "Minimum number of labelled images in each captcha"
+      }).option("max-labelled", {
+        type: "number",
+        description: "Maximum number of labelled images in each captcha"
+      });
+    },
+    handler: async (argv) => {
+      await generate(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = generateUnion;

package/dist/cjs/generate/union/generate.cjs

@@ -0,0 +1,151 @@
+"use strict";
+const types = require("@prosopo/types");
+const common = require("@prosopo/common");
+const util = require("@prosopo/util");
+const utilCrypto = require("@polkadot/util-crypto");
+const util$1 = require("../util.cjs");
+const bcrypt = require("bcrypt");
+const fs = require("fs");
+const generate = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "generating...");
+  const outFile = args.out;
+  const overwrite = args.overwrite || false;
+  if (!overwrite && fs.existsSync(outFile)) {
+    throw new common.ProsopoEnvError(new Error(`Output file already exists: ${outFile}`), "FS.FILE_ALREADY_EXISTS");
+  }
+  const labelledMapFile = args.labelled;
+  if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+    throw new common.ProsopoEnvError(
+      new Error(`Labelled map file does not exist: ${labelledMapFile}`),
+      "FS.FILE_NOT_FOUND"
+    );
+  }
+  const unlabelledMapFile = args.unlabelled;
+  if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+    throw new common.ProsopoEnvError(
+      new Error(`Unlabelled map file does not exist: ${unlabelledMapFile}`),
+      "FS.FILE_NOT_FOUND"
+    );
+  }
+  const labelsFile = args.labels;
+  const seed = args.seed || 0;
+  const size = args.size || 9;
+  const minCorrect = args.minCorrect || 1;
+  const saltRounds = 10;
+  const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false;
+  const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false;
+  const minIncorrect = Math.max(args.minIncorrect || 1, 1);
+  const minLabelled = minCorrect + minIncorrect;
+  const maxLabelled = Math.min(args.maxLabelled || size, size);
+  const count = args.count || 0;
+  util.setSeedGlobal(seed);
+  const _ = util.lodash();
+  const labelled = labelledMapFile ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(labelledMapFile, "utf8"))).items : [];
+  const unlabelled = unlabelledMapFile ? types.DataSchema.parse(JSON.parse(fs.readFileSync(unlabelledMapFile, "utf8"))).items : [];
+  util$1.checkDuplicates(labelled, unlabelled, {
+    allowDuplicatesLabelled,
+    allowDuplicatesUnlabelled
+  });
+  const labelToImages = {};
+  for (const entry of labelled) {
+    const arr = labelToImages[entry.label] || [];
+    arr.push(entry);
+    labelToImages[entry.label] = arr;
+  }
+  const targets = Object.keys(labelToImages);
+  const labels = [];
+  if (labelsFile && fs.existsSync(labelsFile)) {
+    labels.push(...[...types.LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(labelsFile, "utf8"))).labels]);
+  } else {
+    labels.push(...[...targets]);
+  }
+  const captchas = [];
+  for (let i = 0; i < count; i++) {
+    logger.info(`generating captcha ${i + 1} of ${count}`);
+    if (targets.length <= 1) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough different labels in labelled data: ${labelledMapFile}`),
+        "DATASET.NOT_ENOUGH_LABELS"
+      );
+    }
+    const target = util.at(targets, i % targets.length);
+    const notTargets = targets.filter((t) => t !== target);
+    const nLabelled = _.random(minLabelled, maxLabelled);
+    const maxCorrect = nLabelled - minCorrect;
+    const nCorrect = _.random(minCorrect, maxCorrect);
+    const nIncorrect = nLabelled - nCorrect;
+    const nUnlabelled = size - nLabelled;
+    const targetItems = util.get(labelToImages, target);
+    const notTargetItems = notTargets.map((notTarget) => util.get(labelToImages, notTarget)).flat();
+    if (nUnlabelled > unlabelled.length) {
+      throw new common.ProsopoEnvError(new Error(`not enough unlabelled data`), "DATASET.NOT_ENOUGH_IMAGES");
+    }
+    if (nCorrect > targetItems.length) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough images for target (${target})`),
+        "DATASET.NOT_ENOUGH_IMAGES"
+      );
+    }
+    if (nIncorrect > notTargetItems.length) {
+      throw new common.ProsopoEnvError(
+        new Error(`not enough non-matching images for target (${target})`),
+        "DATASET.NOT_ENOUGH_IMAGES"
+      );
+    }
+    const correctItems = _.sampleSize(targetItems, nCorrect);
+    const incorrectItems = _.sampleSize(notTargetItems, nIncorrect);
+    const unlabelledItems = /* @__PURE__ */ new Set();
+    while (unlabelledItems.size < size - nLabelled) {
+      const image = util.at(unlabelled, _.random(0, unlabelled.length - 1));
+      unlabelledItems.add(image);
+    }
+    let items = [...correctItems, ...incorrectItems, ...unlabelledItems];
+    let indices = [...Array(items.length).keys()];
+    indices = _.shuffle(indices);
+    items = indices.map((i2) => util.at(items, i2));
+    items = items.map((item) => {
+      return {
+        data: item.data,
+        hash: item.hash,
+        type: item.type
+      };
+    });
+    const solution = indices.map((index, i2) => {
+      return {
+        pre: index,
+        // the index of the item in the items array before shuffle
+        post: i2
+        // the index of the item in the shuffled array
+      };
+    }).filter((item) => item.pre < correctItems.length).map((item) => {
+      return item.post;
+    });
+    const unlabelledIndices = indices.map((index, i2) => {
+      return {
+        pre: index,
+        // the index of the item in the items array before shuffle
+        post: i2
+        // the index of the item in the shuffled array
+      };
+    }).filter((item) => item.pre >= correctItems.length + incorrectItems.length).map((item) => {
+      return item.post;
+    });
+    const salt = utilCrypto.blake2AsHex(bcrypt.genSaltSync(saltRounds));
+    const captcha = {
+      salt,
+      target,
+      items,
+      solution,
+      unlabelled: unlabelledIndices
+    };
+    captchas.push(captcha);
+  }
+  const output = {
+    captchas,
+    format: types.CaptchaTypes.SelectAll
+  };
+  types.CaptchasContainerSchema.parse(output);
+  fs.writeFileSync(outFile, JSON.stringify(output, null, 4));
+};
+module.exports = generate;

package/dist/cjs/generate/util.cjs

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const checkDuplicates = (labelled, unlabelled, options) => {
+  const all = /* @__PURE__ */ new Set();
+  if (!options.allowDuplicatesLabelled) {
+    for (const entry of labelled) {
+      if (all.has(entry.data)) {
+        throw new Error(`Duplicate data entry in labelled data: ${JSON.stringify(entry)}`);
+      }
+      all.add(entry.data);
+    }
+  }
+  if (!options.allowDuplicatesUnlabelled) {
+    for (const entry of unlabelled) {
+      if (all.has(entry.data)) {
+        throw new Error(`Duplicate data entry in unlabelled data: ${JSON.stringify(entry)}`);
+      }
+    }
+  }
+};
+exports.checkDuplicates = checkDuplicates;

package/dist/cjs/get/args.cjs

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  data: zod.z.string()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/get/cli.cjs

@@ -0,0 +1,20 @@
+"use strict";
+const args = require("./args.cjs");
+const get$1 = require("./get.cjs");
+const get = (cmdArgs) => {
+  return {
+    command: "get",
+    describe: "Test a GET request at image URLs",
+    builder: (yargs) => {
+      return yargs.option("data", {
+        type: "string",
+        demand: true,
+        description: 'JSON file containing urls under a "data" key'
+      });
+    },
+    handler: async (argv) => {
+      await get$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = get;

package/dist/cjs/get/get.cjs

@@ -0,0 +1,53 @@
+"use strict";
+const common = require("@prosopo/common");
+const util = require("@prosopo/util");
+const zod = require("zod");
+const fetch = require("node-fetch");
+const fs = require("fs");
+const get = async (args, loggerOpt) => {
+  const logger = loggerOpt || common.getLoggerDefault();
+  logger.debug(args, "getting...");
+  const traverse = async (data2) => {
+    if (data2 instanceof Array) {
+      for (let i = 0; i < data2.length; i++) {
+        data2[i] = await traverse(data2[i]);
+      }
+    } else if (data2 instanceof Object) {
+      for (const key of Object.keys(data2)) {
+        if (key == "data") {
+          const value = util.get(data2, key);
+          const url = zod.z.string().parse(value);
+          if (url.startsWith("http")) {
+            try {
+              const response = await fetch(url);
+              if (!response.ok) {
+                logger.error(`GET ${url} ${response.status} ${response.statusText}`);
+              } else {
+                logger.log(`GET ${url} OK`);
+              }
+            } catch (err) {
+              logger.error(err);
+            }
+          } else {
+            try {
+              fs.readFileSync(url);
+              logger.log(`GET ${url} OK`);
+            } catch (err) {
+              logger.error(`GET ${url} ${err}`);
+            }
+          }
+        } else {
+          await traverse(util.get(data2, key));
+        }
+      }
+    }
+    return data2;
+  };
+  const file = args.data;
+  if (!fs.existsSync(file)) {
+    throw new common.ProsopoEnvError(new Error(`file does not exist: ${file}`), "FS.FILE_NOT_FOUND");
+  }
+  const data = JSON.parse(fs.readFileSync(file, "utf8"));
+  await traverse(data);
+};
+module.exports = get;

package/dist/cjs/index.cjs

@@ -0,0 +1,2 @@
+"use strict";
+require("./cli.cjs");

package/dist/cjs/labels/args.cjs

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  data: zod.z.string()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/labels/cli.cjs

@@ -0,0 +1,20 @@
+"use strict";
+const args = require("./args.cjs");
+const labels$1 = require("./labels.cjs");
+const labels = (cmdArgs) => {
+  return {
+    command: "labels",
+    describe: "get all labels from some data",
+    builder: (yargs) => {
+      return yargs.option("data", {
+        type: "string",
+        demand: true,
+        description: "JSON file containing data"
+      });
+    },
+    handler: async (argv) => {
+      await labels$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = labels;

package/dist/cjs/labels/labels.cjs

@@ -0,0 +1,21 @@
+"use strict";
+const types = require("@prosopo/types");
+const common = require("@prosopo/common");
+const fs = require("fs");
+const labels = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "reading labels...");
+  const file = args.data;
+  if (!fs.existsSync(file)) {
+    throw new common.ProsopoEnvError(new Error(`file does not exist: ${file}`), "FS.FILE_NOT_FOUND");
+  }
+  const labelled = file ? types.LabelledDataSchema.parse(JSON.parse(fs.readFileSync(file, "utf8"))).items : [];
+  const labels2 = /* @__PURE__ */ new Set();
+  for (const item of labelled) {
+    labels2.add(item.label);
+  }
+  const labelArray = Array.from(labels2);
+  labelArray.sort();
+  logger.log(JSON.stringify({ labels: labelArray }, null, 4));
+};
+module.exports = labels;

package/dist/cjs/relocate/args.cjs

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  data: zod.z.string(),
+  from: zod.z.string(),
+  to: zod.z.string()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/relocate/cli.cjs

@@ -0,0 +1,27 @@
+"use strict";
+const args = require("./args.cjs");
+const relocate$1 = require("./relocate.cjs");
+const relocate = (cmdArgs) => {
+  return {
+    command: "relocate",
+    describe: 'Relocate a dataset by replacing the old urls with new ones. E.g. "example.com/1.jpg" to "newwebsite.com/1.jpg"',
+    builder: (yargs) => {
+      return yargs.option("from", {
+        type: "string",
+        demand: true,
+        description: "The old url to replace"
+      }).option("to", {
+        type: "string",
+        demand: true,
+        description: "The new url to replace the old one with"
+      }).option("data", {
+        type: "string",
+        description: "Path to the images JSON containing the urls of images to replace"
+      });
+    },
+    handler: async (argv) => {
+      await relocate$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = relocate;

package/dist/cjs/relocate/relocate.cjs

@@ -0,0 +1,36 @@
+"use strict";
+const common = require("@prosopo/common");
+const util = require("@prosopo/util");
+const fs = require("fs");
+const relocate = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "relocating...");
+  const replace = (data2, from, to) => {
+    if (Array.isArray(data2)) {
+      for (let i = 0; i < data2.length; i++) {
+        data2[i] = replace(data2[i], from, to);
+      }
+    } else if (typeof data2 === "object") {
+      const obj = data2;
+      for (const key of Object.keys(obj)) {
+        if (key === "data") {
+          const value = util.get(obj, key);
+          if (value instanceof String) {
+            if (value.startsWith(from)) {
+              obj[key] = to + value.slice(from.length);
+            }
+          }
+        } else {
+          obj[key] = replace(obj[key], from, to);
+        }
+      }
+    }
+    return data2;
+  };
+  const file = args.data;
+  logger.log(`relocating data in ${file} from ${args.from} to ${args.to}`);
+  let data = JSON.parse(fs.readFileSync(file, "utf8"));
+  data = replace(data, args.from, args.to);
+  fs.writeFileSync(file, JSON.stringify(data, null, 4));
+};
+module.exports = relocate;

package/dist/cjs/scale/args.cjs

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const zod = require("zod");
+const argsSchema = zod.z.object({
+  data: zod.z.string(),
+  out: zod.z.string(),
+  overwrite: zod.z.boolean().optional(),
+  size: zod.z.number(),
+  square: zod.z.boolean().optional()
+});
+exports.argsSchema = argsSchema;

package/dist/cjs/scale/cli.cjs

@@ -0,0 +1,34 @@
+"use strict";
+const args = require("./args.cjs");
+const scale$1 = require("./scale.cjs");
+const scale = (cmdArgs) => {
+  return {
+    command: "scale",
+    describe: "Scale images down to a given size",
+    builder: (yargs) => {
+      return yargs.option("data", {
+        type: "string",
+        demand: true,
+        description: "JSON file containing a list of objects with (at least) a url"
+      }).option("out", {
+        type: "string",
+        demand: true,
+        description: "Where to put the output directory containing the map file and single directory of images. The map file will contain the new urls of the scaled images, new hashes and pass through any other information, e.g. labels."
+      }).option("overwrite", {
+        type: "boolean",
+        description: "Overwrite the output if it already exists"
+      }).option("size", {
+        type: "number",
+        demand: true,
+        description: "The dimension (height/width) of the scaled image. If the image is not square, the other dimension will be scaled to maintain the aspect ratio."
+      }).option("square", {
+        type: "boolean",
+        description: "If true, the image will be cropped to a square before scaling. If false, the image will be scaled to the given size, maintaining the aspect ratio."
+      });
+    },
+    handler: async (argv) => {
+      await scale$1(args.argsSchema.parse(argv), cmdArgs == null ? void 0 : cmdArgs.logger);
+    }
+  };
+};
+module.exports = scale;

package/dist/cjs/scale/scale.cjs

@@ -0,0 +1,55 @@
+"use strict";
+const types = require("@prosopo/types");
+const common = require("@prosopo/common");
+const blake2b = require("@noble/hashes/blake2b");
+const util = require("@polkadot/util");
+const fs = require("fs");
+const sharp = require("sharp");
+const scale = async (args, logger) => {
+  logger = logger || common.getLoggerDefault();
+  logger.debug(args, "scaling...");
+  const size = args.size;
+  const square = args.square ?? false;
+  const mapFile = args.data;
+  if (!fs.existsSync(mapFile)) {
+    throw new common.ProsopoEnvError(new Error(`Map file does not exist: ${mapFile}`), "FS.FILE_NOT_FOUND");
+  }
+  const outDir = args.out;
+  const overwrite = args.overwrite || false;
+  if (!overwrite && fs.existsSync(outDir)) {
+    throw new common.ProsopoEnvError(new Error(`Output directory already exists: ${outDir}`), "FS.FILE_NOT_FOUND");
+  }
+  const imgDir = `${outDir}/images`;
+  fs.mkdirSync(imgDir, { recursive: true });
+  const inputItems = types.DataSchema.parse(JSON.parse(fs.readFileSync(mapFile, "utf8"))).items;
+  const outputItems = [];
+  for (const inputItem of inputItems) {
+    logger.log(`scaling ${inputItem.data}`);
+    const img = fs.readFileSync(inputItem.data);
+    const resized = await sharp(img).resize({
+      width: size,
+      height: size,
+      fit: square ? "fill" : "inside"
+    }).png();
+    const tmpFilePath = `${imgDir}/tmp.png`;
+    await resized.toFile(tmpFilePath);
+    const resizedImg = fs.readFileSync(tmpFilePath);
+    const hash = blake2b.blake2b(resizedImg);
+    const hex = util.u8aToHex(hash);
+    const finalFilePath = `${imgDir}/${hex}.png`;
+    fs.renameSync(tmpFilePath, finalFilePath);
+    const outputItem = {
+      ...inputItem,
+      hash: hex,
+      data: fs.realpathSync(finalFilePath)
+    };
+    outputItems.push(outputItem);
+  }
+  const outputMapFile = `${outDir}/map.json`;
+  const data = {
+    items: outputItems
+  };
+  types.DataSchema.parse(data);
+  fs.writeFileSync(outputMapFile, JSON.stringify(outputItems, null, 4));
+};
+module.exports = scale;

package/package.json

@@ -1,12 +1,13 @@
 {
     "name": "@prosopo/datasets-fs",
-    "version": "0.2.0",
+    "version": "0.2.2",
     "author": "PROSOPO LIMITED <info@prosopo.io>",
     "license": "Apache-2.0",
     "private": false,
     "scripts": {
         "clean": "tsc --build --clean",
-        "build": "tsc --build --verbose",
+        "build": "tsc --build --verbose tsconfig.json",
+        "build:cjs": "npx vite --config vite.cjs.config.ts build",
         "lint": "npx eslint .",
         "lint:fix": "npx eslint . --fix --config ../../.eslintrc.js",
         "cli": "node ./dist/js/cli.js",
@@ -14,12 +15,18 @@
     },
     "main": "./dist/index.js",
     "type": "module",
+    "exports": {
+        ".": {
+            "import": "./dist/index.js",
+            "require": "./dist/cjs/index.cjs"
+        }
+    },
     "types": "./dist/index.d.ts",
     "dependencies": {
         "@polkadot/util": "^12.3.2",
-        "@prosopo/common": "0.2.0",
-        "@prosopo/types": "0.2.0",
-        "@prosopo/util": "0.2.0",
+        "@prosopo/common": "0.2.2",
+        "@prosopo/types": "0.2.2",
+        "@prosopo/util": "0.2.2",
         "bcrypt": "^5.1.0",
         "cli-progress": "^3.12.0",
         "es-main": "^1.2.0",

package/vite.cjs.config.ts

@@ -0,0 +1,6 @@
+import { ViteCommonJSConfig } from '@prosopo/config'
+import path from 'path'
+
+export default function () {
+    return ViteCommonJSConfig('datasets-fs', path.resolve('./tsconfig.cjs.json'))
+}