@prosopo/datasets-fs 0.2.7 → 0.2.8

package/README.md

@@ -82,3 +82,24 @@ Commands:
    e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js relocate --from example.com --to web.site --data ~/bench/test/flat/data2.json`
 1. 'scale' rescales images in a flat directory structure to a given size.
    e.g. `npm run -w @prosopo/datasets-fs build && node packages/datasets-fs/dist/cli.js scale --data ~/bench/test/flat/data.json --out ~/bench/test/flat/images2 --overwrite --size 128`
+
+
+
+# Generating test data
+The test data was generated using:
+```
+cli generate-v1 --min-correct 1 --max-correct 6 --labelled $PWD/src/tests/data/flat_resized/data.json --unlabelled $PWD/src/tests/data/flat_resized/data.json --out $PWD/src/tests/data/flat_resized/captchas_v1.json --solved 50 --unsolved 50 --seed 0 --allowDuplicates --seed 0
+
+cli generate-v2 --count 100 --min-correct 1 --min-incorrect 1 --min-labelled 2 --max-labelled 7 --labelled $PWD/src/tests/data/flat_resized/data.json --unlabelled $PWD/src/tests/data/flat_resized/data.json --out $PWD/src/tests/data/flat_resized/captchas_v2.json --seed 0 --allowDuplicates --seed 0
+
+cli relocate --from '${repo}' --to newwebsite.com --input $PWD/src/tests/data/flat_resized/data.json --output $PWD/src/tests/data/flat_resized/relocated_data.json --overwrite
+
+cli labels --input $PWD/src/tests/data/flat_resized/data.json --output $PWD/src/tests/data/flat_resized/labels.json --overwrite
+
+cli flatten --in $PWD/src/tests/data/hierarchical --out $PWD/src/tests/data/flat --overwrite
+
+cli resize --square --size 128 --in $PWD/src/tests/data/flat/data.json --out $PWD/src/tests/data/flat_resized --overwrite
+```
+
+
+`cli` is an alias to `npm run build && npm run cli --`

package/dist/cjs/cli/cli.cjs

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const common = require("@prosopo/common");
+const helpers = require("yargs/helpers");
+const yargs = require("yargs");
+const dirname = process.cwd();
+const logger = common.getLogger(common.LogLevel.enum.info, `${dirname}`);
+class Cli extends common.Loggable {
+  #commands;
+  constructor(commands) {
+    super();
+    this.#commands = commands;
+    this.logger = logger;
+  }
+  config() {
+    let y = yargs(helpers.hideBin(process.argv)).option("log-level", {
+      type: "string",
+      choices: Object.values(common.LogLevel.options),
+      default: common.LogLevel.enum.info,
+      description: "The log level"
+    }).middleware((argv) => {
+      this.logger.setLogLevel(argv.logLevel);
+    }, true);
+    this.#commands.forEach((command) => {
+      y = y.command({
+        command: command.getCommandName(),
+        describe: command.getDescription(),
+        builder: command.getOptions(),
+        handler: async (argv) => {
+          this.logger.debug(`running ${command.getCommandName()}}`);
+          const args = await command.parse(argv);
+          await command.exec(args);
+        }
+      });
+    });
+    if (!this.#commands.find((c) => c.getCommandName() === "$0")) {
+      y = y.command(
+        "$0",
+        "default command",
+        (y2) => y2,
+        (argv) => {
+          throw new Error(`no command specified`);
+        }
+      );
+    }
+    y = y.demandCommand().strict().showHelpOnFail(false, "Specify --help for available options");
+    return y;
+  }
+  async exec(args = process.argv.slice(2)) {
+    const config = this.config();
+    this.logger.debug("parsing", args);
+    await config.parse(args);
+  }
+}
+exports.Cli = Cli;

package/dist/cjs/index.cjs

@@ -1,2 +1,4 @@
 "use strict";
-require("./cli.cjs");
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const cli = require("./cli/cli.cjs");
+exports.Cli = cli.Cli;

package/dist/cli/cli.d.ts

@@ -0,0 +1,9 @@
+import { CliCommandAny } from './cliCommand.js';
+import { Loggable } from '@prosopo/common';
+export declare class Cli extends Loggable {
+    #private;
+    constructor(commands: CliCommandAny[]);
+    private config;
+    exec(args?: string[]): Promise<void>;
+}
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAY,QAAQ,EAAa,MAAM,iBAAiB,CAAA;AAO/D,qBAAa,GAAI,SAAQ,QAAQ;;gBAGjB,QAAQ,EAAE,aAAa,EAAE;IAMrC,OAAO,CAAC,MAAM;IAsCD,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B;CAK3D"}

package/dist/cli/cli.js

@@ -0,0 +1,54 @@
+var _Cli_commands;
+import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
+import { LogLevel, Loggable, getLogger } from '@prosopo/common';
+import { hideBin } from 'yargs/helpers';
+import yargs from 'yargs';
+const dirname = process.cwd();
+const logger = getLogger(LogLevel.enum.info, `${dirname}`);
+export class Cli extends Loggable {
+    constructor(commands) {
+        super();
+        _Cli_commands.set(this, void 0);
+        __classPrivateFieldSet(this, _Cli_commands, commands, "f");
+        this.logger = logger;
+    }
+    config() {
+        let y = yargs(hideBin(process.argv))
+            .option('log-level', {
+            type: 'string',
+            choices: Object.values(LogLevel.options),
+            default: LogLevel.enum.info,
+            description: 'The log level',
+        })
+            .middleware((argv) => {
+            this.logger.setLogLevel(argv.logLevel);
+        }, true);
+        __classPrivateFieldGet(this, _Cli_commands, "f").forEach((command) => {
+            y = y.command({
+                command: command.getCommandName(),
+                describe: command.getDescription(),
+                builder: command.getOptions(),
+                handler: async (argv) => {
+                    this.logger.debug(`running ${command.getCommandName()}}`);
+                    const args = await command.parse(argv);
+                    await command.exec(args);
+                },
+            });
+        });
+        if (!__classPrivateFieldGet(this, _Cli_commands, "f").find((c) => c.getCommandName() === '$0')) {
+            // no default command
+            y = y.command('$0', 'default command', (y) => y, (argv) => {
+                throw new Error(`no command specified`);
+            });
+        }
+        y = y.demandCommand().strict().showHelpOnFail(false, 'Specify --help for available options');
+        return y;
+    }
+    async exec(args = process.argv.slice(2)) {
+        const config = this.config();
+        this.logger.debug('parsing', args);
+        await config.parse(args);
+    }
+}
+_Cli_commands = new WeakMap();
+//# sourceMappingURL=cli.js.map

package/dist/cli/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";;AACA,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,KAAe,MAAM,OAAO,CAAA;AAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;AAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;AAE1D,MAAM,OAAO,GAAI,SAAQ,QAAQ;IAG7B,YAAY,QAAyB;QACjC,KAAK,EAAE,CAAA;QAHX,gCAA0B;QAItB,uBAAA,IAAI,iBAAa,QAAQ,MAAA,CAAA;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACxB,CAAC;IAEO,MAAM;QACV,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAC/B,MAAM,CAAC,WAAW,EAAE;YACjB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;YAC3B,WAAW,EAAE,eAAe;SAC/B,CAAC;aACD,UAAU,CAAC,CAAC,IAAS,EAAE,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAC1C,CAAC,EAAE,IAAI,CAAC,CAAA;QACZ,uBAAA,IAAI,qBAAU,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC/B,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;gBACV,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE;gBACjC,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE;gBAClC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE;gBAC7B,OAAO,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;oBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAA;oBACzD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;oBACtC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAC5B,CAAC;aACJ,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QACF,IAAI,CAAC,uBAAA,IAAI,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,EAAE,KAAK,IAAI,CAAC,EAAE;YAC1D,qBAAqB;YACrB,CAAC,GAAG,CAAC,CAAC,OAAO,CACT,IAAI,EACJ,iBAAiB,EACjB,CAAC,CAAO,EAAE,EAAE,CAAC,CAAC,EACd,CAAC,IAAS,EAAE,EAAE;gBACV,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;YAC3C,CAAC,CACJ,CAAA;SACJ;QACD,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,MAAM,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC,CAAA;QAC5F,OAAO,CAAC,CAAA;IACZ,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;QAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;QAClC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;CACJ"}

package/dist/cli/cliCommand.d.ts

@@ -0,0 +1,17 @@
+import { Loggable } from '@prosopo/common';
+import { Options } from 'yargs';
+import { z } from 'zod';
+export declare abstract class CliCommand<T extends z.ZodTypeAny> extends Loggable {
+    getOptions(): {
+        [key: string]: Options;
+    };
+    abstract getArgSchema(): T;
+    parse(args: unknown): Promise<z.infer<T>>;
+    _check(args: z.infer<T>): Promise<void>;
+    _run(args: z.infer<T>): Promise<void>;
+    exec(args: z.infer<T>): Promise<void>;
+    getCommandName(): string;
+    abstract getDescription(): string;
+}
+export type CliCommandAny = CliCommand<z.ZodTypeAny>;
+//# sourceMappingURL=cliCommand.d.ts.map

package/dist/cli/cliCommand.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommand.d.ts","sourceRoot":"","sources":["../../src/cli/cliCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAA;AAE/B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,8BAAsB,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,CAAE,SAAQ,QAAQ;IAE9D,UAAU,IAAI;QACjB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KACzB;aAKe,YAAY,IAAI,CAAC;IAIpB,KAAK,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAQzC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAKvB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAKrB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAQ3B,cAAc;aAIL,cAAc,IAAI,MAAM;CAC3C;AAED,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAA"}

package/dist/cli/cliCommand.js

@@ -0,0 +1,36 @@
+import { Loggable } from '@prosopo/common';
+import { kebabCase } from '@prosopo/util';
+export class CliCommand extends Loggable {
+    // get the options for the CLI
+    getOptions() {
+        return {};
+    }
+    // parse args using the schema
+    // use this when you have an args object but not sure if it's in the right format / contains the correct fields, e.g. args from cmdline
+    async parse(args) {
+        const argsSchema = this.getArgSchema();
+        const parsed = await argsSchema.parse(args);
+        this.logger.debug('parsed args:', parsed);
+        return parsed;
+    }
+    // run any checks before the main run function. This function should have no side effects, i.e. not write to disk. It's purely for conducting checks / setting up
+    async _check(args) {
+        this.logger.debug('runChecks');
+    }
+    // the main run function. This should be the function that does the work
+    async _run(args) {
+        this.logger.debug('run');
+    }
+    // exec is a public facing function that should be called by the CLI. It will run the preRun, run, and postRun functions in order
+    async exec(args) {
+        this.logger.debug('exec');
+        // run any checks
+        await this._check(args);
+        // then run the command
+        await this._run(args);
+    }
+    getCommandName() {
+        return kebabCase(this.constructor.name);
+    }
+}
+//# sourceMappingURL=cliCommand.js.map

package/dist/cli/cliCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommand.js","sourceRoot":"","sources":["../../src/cli/cliCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAA;AAE1C,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAGzC,MAAM,OAAgB,UAAmC,SAAQ,QAAQ;IACrE,8BAA8B;IACvB,UAAU;QAGb,OAAO,EAAE,CAAA;IACb,CAAC;IAKD,8BAA8B;IAC9B,uIAAuI;IAChI,KAAK,CAAC,KAAK,CAAC,IAAa;QAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAA;QACzC,OAAO,MAAM,CAAA;IACjB,CAAC;IAED,iKAAiK;IAC1J,KAAK,CAAC,MAAM,CAAC,IAAgB;QAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IAClC,CAAC;IAED,wEAAwE;IACjE,KAAK,CAAC,IAAI,CAAC,IAAgB;QAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAC5B,CAAC;IAED,iIAAiI;IAC1H,KAAK,CAAC,IAAI,CAAC,IAAgB;QAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACzB,iBAAiB;QACjB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACvB,uBAAuB;QACvB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACzB,CAAC;IAEM,cAAc;QACjB,OAAO,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;IAC3C,CAAC;CAGJ"}

package/dist/cli/cliCommandComposite.d.ts

@@ -0,0 +1,14 @@
+import { CliCommand, CliCommandAny } from './cliCommand.js';
+import { Options } from 'yargs';
+import { z } from 'zod';
+export declare abstract class CliCommandComposite<T extends z.ZodTypeAny> extends CliCommand<T> {
+    #private;
+    getCommands(): CliCommandAny[];
+    constructor(commands: CliCommandAny[]);
+    getOptions(): {
+        [key: string]: Options;
+    };
+    _run(args: z.TypeOf<T>): Promise<void>;
+    _check(args: z.TypeOf<T>): Promise<void>;
+}
+//# sourceMappingURL=cliCommandComposite.d.ts.map

package/dist/cli/cliCommandComposite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommandComposite.d.ts","sourceRoot":"","sources":["../../src/cli/cliCommandComposite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,OAAO,CAAA;AAC/B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,8BAAsB,mBAAmB,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,CAAE,SAAQ,UAAU,CAAC,CAAC,CAAC;;IAG5E,WAAW;gBAIN,QAAQ,EAAE,aAAa,EAAE;IAMrB,UAAU,IAAI;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE;IAOxC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAM3D"}

package/dist/cli/cliCommandComposite.js

@@ -0,0 +1,34 @@
+var _CliCommandComposite_commands;
+import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
+import { CliCommand } from './cliCommand.js';
+export class CliCommandComposite extends CliCommand {
+    getCommands() {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f");
+    }
+    constructor(commands) {
+        super();
+        _CliCommandComposite_commands.set(this, []);
+        // take a copy of the array to avoid leakage
+        __classPrivateFieldSet(this, _CliCommandComposite_commands, [...commands], "f");
+    }
+    getOptions() {
+        // merge options in turn from each command. Command order matters, latter commands will overwrite earlier ones
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce((prev, command) => {
+            return { ...prev, ...command.getOptions() };
+        }, {});
+    }
+    _run(args) {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce(async (prev, command) => {
+            await prev;
+            await command._run(args);
+        }, Promise.resolve());
+    }
+    _check(args) {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce(async (prev, command) => {
+            await prev;
+            await command._check(args);
+        }, Promise.resolve());
+    }
+}
+_CliCommandComposite_commands = new WeakMap();
+//# sourceMappingURL=cliCommandComposite.js.map

package/dist/cli/cliCommandComposite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommandComposite.js","sourceRoot":"","sources":["../../src/cli/cliCommandComposite.ts"],"names":[],"mappings":";;AAAA,OAAO,EAAE,UAAU,EAAiB,MAAM,iBAAiB,CAAA;AAI3D,MAAM,OAAgB,mBAA4C,SAAQ,UAAa;IAG5E,WAAW;QACd,OAAO,uBAAA,IAAI,qCAAU,CAAA;IACzB,CAAC;IAED,YAAY,QAAyB;QACjC,KAAK,EAAE,CAAA;QAPX,wCAA6B,EAAE,EAAA;QAQ3B,4CAA4C;QAC5C,uBAAA,IAAI,iCAAa,CAAC,GAAG,QAAQ,CAAC,MAAA,CAAA;IAClC,CAAC;IAEe,UAAU;QACtB,8GAA8G;QAC9G,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC3C,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAA;QAC/C,CAAC,EAAE,EAAE,CAAC,CAAA;IACV,CAAC;IAEe,IAAI,CAAC,IAAiB;QAClC,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,IAAI,CAAA;YACV,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC5B,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IACzB,CAAC;IAEe,MAAM,CAAC,IAAiB;QACpC,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YACjD,MAAM,IAAI,CAAA;YACV,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9B,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IACzB,CAAC;CACJ"}

package/dist/cli.js

@@ -1,48 +1,34 @@
-import { LogLevel, getLogger } from '@prosopo/common';
-import { hideBin } from 'yargs/helpers';
+import { Cli } from './index.js';
+import { Flatten } from './commands/flatten.js';
+import { GenerateV1 } from './commands/generateV1.js';
+import { GenerateV2 } from './commands/generateV2.js';
+import { Get } from './commands/get.js';
+import { Labels } from './commands/labels.js';
+import { Relocate } from './commands/relocate.js';
+import { Resize } from './commands/resize.js';
 import esMain from 'es-main';
-import flatten from './flatten/cli.js';
-import generate from './generate/cli.js';
-import get from './get/cli.js';
-import labels from './labels/cli.js';
-import process from 'process';
-import relocate from './relocate/cli.js';
-import scale from './scale/cli.js';
-import yargs from 'yargs';
-const dirname = process.cwd();
-const logger = getLogger(LogLevel.enum.info, `${dirname}`);
 const main = async () => {
-    await yargs(hideBin(process.argv))
-        .help()
-        .option('log-level', {
-        type: 'string',
-        choices: Object.values(LogLevel.options),
-        default: LogLevel.enum.info,
-        description: 'The log level',
-    })
-        .middleware((argv) => {
-        logger.setLogLevel(argv.logLevel);
-    })
-        .command(generate({ logger }))
-        .command(flatten({ logger }))
-        .command(scale({ logger }))
-        .command(relocate({ logger }))
-        .command(get({ logger }))
-        .command(labels({ logger }))
-        .strictCommands()
-        .showHelpOnFail(false, 'Specify --help for available options')
-        .fail(false)
-        .parse();
+    const commands = [
+        new Flatten(),
+        new GenerateV1(),
+        new GenerateV2(),
+        new Get(),
+        new Labels(),
+        new Relocate(),
+        new Resize(),
+    ];
+    const cli = new Cli(commands);
+    cli.logger.setLogLevel('debug');
+    await cli.exec();
 };
 //if main process
 if (esMain(import.meta)) {
     main()
         .then(() => {
-        logger.debug('done');
         process.exit(0);
     })
         .catch((err) => {
-        logger.error('error:', err);
+        console.log('error', err);
         process.exit(1);
     });
 }

package/dist/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AACvC,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,OAAO,MAAM,kBAAkB,CAAA;AACtC,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,GAAG,MAAM,cAAc,CAAA;AAC9B,OAAO,MAAM,MAAM,iBAAiB,CAAA;AACpC,OAAO,OAAO,MAAM,SAAS,CAAA;AAC7B,OAAO,QAAQ,MAAM,mBAAmB,CAAA;AACxC,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;AAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAA;AAE1D,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;SAC7B,IAAI,EAAE;SACN,MAAM,CAAC,WAAW,EAAE;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;QACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;QAC3B,WAAW,EAAE,eAAe;KAC/B,CAAC;SACD,UAAU,CAAC,CAAC,IAAS,EAAE,EAAE;QACtB,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrC,CAAC,CAAC;SACD,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC5B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC1B,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC7B,OAAO,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SACxB,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;SAC3B,cAAc,EAAE;SAChB,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC;SAC7D,IAAI,CAAC,KAAK,CAAC;SACX,KAAK,EAAE,CAAA;AAChB,CAAC,CAAA;AAED,iBAAiB;AACjB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;IACrB,IAAI,EAAE;SACD,IAAI,CAAC,GAAG,EAAE;QACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;CACT"}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAA;AAEhC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAA;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAA;AACjD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAC7C,OAAO,MAAM,MAAM,SAAS,CAAA;AAE5B,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACpB,MAAM,QAAQ,GAAoB;QAC9B,IAAI,OAAO,EAAE;QACb,IAAI,UAAU,EAAE;QAChB,IAAI,UAAU,EAAE;QAChB,IAAI,GAAG,EAAE;QACT,IAAI,MAAM,EAAE;QACZ,IAAI,QAAQ,EAAE;QACd,IAAI,MAAM,EAAE;KACf,CAAA;IACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAA;IAC7B,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAC/B,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;AACpB,CAAC,CAAA;AAED,iBAAiB;AACjB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;IACrB,IAAI,EAAE;SACD,IAAI,CAAC,GAAG,EAAE;QACP,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACX,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;QACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;CACT"}

package/dist/commands/flatten.d.ts

@@ -0,0 +1,56 @@
+/// <reference types="yargs" />
+import { InputOutputCliCommand } from '../utils/inputOutput.js';
+import { z } from 'zod';
+export declare const ArgsSchema: z.ZodObject<{
+    input: z.ZodString;
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+    getArgSchema(): z.ZodObject<{
+        input: z.ZodString;
+        output: z.ZodString;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+        allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    }, "strip", z.ZodTypeAny, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }>;
+    getDescription(): string;
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        input: {
+            description: string;
+        };
+        output: {
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+    };
+    _run(args: Args): Promise<void>;
+}
+//# sourceMappingURL=flatten.d.ts.map

package/dist/commands/flatten.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.d.ts","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":";AACA,OAAO,EAAkD,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAI/G,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAIvB,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;EAErB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAA;AAC9C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;AAE1C,qBAAa,OAAQ,SAAQ,qBAAqB,CAAC,cAAc,CAAC;IAC9C,YAAY;;;;;;;;;;;;;;;;IAIZ,cAAc,IAAI,MAAM;IAIxB,UAAU;;;;;;;;;;;;;;IAeJ,IAAI,CAAC,IAAI,EAAE,IAAI;CAgFxC"}

package/dist/commands/flatten.js

@@ -0,0 +1,103 @@
+import { CaptchaItemTypes, DataSchema } from '@prosopo/types';
+import { InputOutputArgsSchema as InputOutputArgsSchema, InputOutputCliCommand } from '../utils/inputOutput.js';
+import { at, lodash } from '@prosopo/util';
+import { blake2b } from '@noble/hashes/blake2b';
+import { u8aToHex } from '@polkadot/util';
+import { z } from 'zod';
+import cliProgress from 'cli-progress';
+import fs from 'fs';
+export const ArgsSchema = InputOutputArgsSchema.extend({
+    allowDuplicates: z.boolean().optional(),
+});
+export class Flatten extends InputOutputCliCommand {
+    getArgSchema() {
+        return ArgsSchema;
+    }
+    getDescription() {
+        return 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels';
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            input: {
+                description: 'Path to the data directory containing subdirectories for each image classification',
+            },
+            output: {
+                description: 'Where to put the output file containing the labels and single directory of images',
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: 'If true, allow duplicates in the data',
+            },
+        });
+    }
+    async _run(args) {
+        this.logger.debug('flatten run');
+        await super._run(args);
+        const dataDir = args.input;
+        const outDir = args.output;
+        // find the labels (these should be subdirectories of the data directory)
+        this.logger.info('reading data');
+        const labels = fs
+            .readdirSync(dataDir, { withFileTypes: true })
+            .filter((dirent) => dirent.isDirectory())
+            .map((dirent) => dirent.name)
+            .sort();
+        const imagesByLabel = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`));
+        // create the output directory
+        const imageDir = `${outDir}/images`;
+        fs.mkdirSync(imageDir, { recursive: true });
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+        // for each label
+        const items = [];
+        bar.start(imagesByLabel.reduce((acc, images) => acc + images.length, 0), 0);
+        labels.forEach((label, i) => {
+            const images = at(imagesByLabel, i);
+            // for each image
+            for (const image of images) {
+                bar.increment();
+                // this.logger.log(`flattening ${label}/${image}`)
+                // copy the image to the output directory
+                const extension = image.split('.').pop();
+                // read file to bytes
+                const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+                // hash based on the content of the image
+                const hash = blake2b(content);
+                const hex = u8aToHex(hash);
+                const name = `${hex}.${extension}`;
+                if (fs.existsSync(`${imageDir}/${name}`)) {
+                    for (const item of items) {
+                        if (item.hash === hex) {
+                            this.logger.log(`\ndupe: ${label}/${image}`);
+                            this.logger.log('item hash', item.hash);
+                            this.logger.log('item label', item.label);
+                        }
+                    }
+                    if (!args.allowDuplicates) {
+                        throw new Error(`duplicate image detected`);
+                    }
+                }
+                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+                const filePath = fs.realpathSync(`${imageDir}/${name}`);
+                // add the image to the map file
+                const entry = {
+                    data: filePath,
+                    type: CaptchaItemTypes.Image,
+                    label,
+                    hash: hex,
+                };
+                items.push(entry);
+            }
+        });
+        bar.stop();
+        const data = {
+            items,
+        };
+        // verify data
+        this.logger.info('verifying data');
+        DataSchema.parse(data);
+        // write map file
+        this.logger.info('writing data');
+        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+    }
+}
+//# sourceMappingURL=flatten.js.map

package/dist/commands/flatten.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAQ,UAAU,EAAgB,MAAM,gBAAgB,CAAA;AACjF,OAAO,EAAE,qBAAqB,IAAI,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAC/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,WAAW,MAAM,cAAc,CAAA;AACtC,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACnD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAA;AAIF,MAAM,OAAO,OAAQ,SAAQ,qBAAqC;IAC9C,YAAY;QACxB,OAAO,UAAU,CAAA;IACrB,CAAC;IAEe,cAAc;QAC1B,OAAO,wIAAwI,CAAA;IACnJ,CAAC;IAEe,UAAU;QACtB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACtC,KAAK,EAAE;gBACH,WAAW,EAAE,oFAAoF;aACpG;YACD,MAAM,EAAE;gBACJ,WAAW,EAAE,mFAAmF;aACnG;YACD,eAAe,EAAE;gBACb,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,uCAAuC;aACvD;SACJ,CAAC,CAAA;IACN,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAA;QAChC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEtB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAA;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAE1B,yEAAyE;QACzE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAChC,MAAM,MAAM,GAAa,EAAE;aACtB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;aACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;aAC5B,IAAI,EAAE,CAAA;QACX,MAAM,aAAa,GAAe,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAA;QAE9F,8BAA8B;QAC9B,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAA;QACnC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAE3C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;QAE7E,iBAAiB;QACjB,MAAM,KAAK,GAAmB,EAAE,CAAA;QAChC,GAAG,CAAC,KAAK,CACL,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAC7D,CAAC,CACJ,CAAA;QACD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YACxB,MAAM,MAAM,GAAa,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC,CAAA;YAC7C,iBAAiB;YACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBACxB,GAAG,CAAC,SAAS,EAAE,CAAA;gBACf,kDAAkD;gBAClD,yCAAyC;gBACzC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxC,qBAAqB;gBACrB,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;gBAC/D,yCAAyC;gBACzC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;gBAC7B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;gBAC1B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAA;gBAClC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE;oBACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;wBACtB,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,EAAE;4BACnB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,KAAK,EAAE,CAAC,CAAA;4BAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;4BACvC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAA;yBAC5C;qBACJ;oBACD,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;wBACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;qBAC9C;iBACJ;gBACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;gBACtE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAA;gBACvD,gCAAgC;gBAChC,MAAM,KAAK,GAAiB;oBACxB,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;oBAC5B,KAAK;oBACL,IAAI,EAAE,GAAG;iBACZ,CAAA;gBACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;aACpB;QACL,CAAC,CAAC,CAAA;QACF,GAAG,CAAC,IAAI,EAAE,CAAA;QAEV,MAAM,IAAI,GAAS;YACf,KAAK;SACR,CAAA;QAED,cAAc;QACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAClC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAEtB,iBAAiB;QACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAChC,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IAC1E,CAAC;CACJ"}