@prosopo/datasets-fs 2.1.8 → 2.1.9

package/dist/cli/cli.d.ts

@@ -0,0 +1,9 @@
+import { Loggable } from "@prosopo/common";
+import type { CliCommandAny } from "./cliCommand.js";
+export declare class Cli extends Loggable {
+    #private;
+    constructor(commands: CliCommandAny[]);
+    private config;
+    exec(args?: string[]): Promise<void>;
+}
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,QAAQ,EAGR,MAAM,iBAAiB,CAAC;AAgBzB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAKrD,qBAAa,GAAI,SAAQ,QAAQ;;gBAGpB,QAAQ,EAAE,aAAa,EAAE;IAMrC,OAAO,CAAC,MAAM;IA+CD,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B;CAKxD"}

package/dist/cli/cli.js

@@ -0,0 +1,58 @@
+var _Cli_commands;
+import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
+import { LogLevel, Loggable, ProsopoCliError, getLogger, } from "@prosopo/common";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+const dirname = process.cwd();
+const logger = getLogger(LogLevel.enum.info, `${dirname}`);
+export class Cli extends Loggable {
+    constructor(commands) {
+        super();
+        _Cli_commands.set(this, void 0);
+        __classPrivateFieldSet(this, _Cli_commands, commands, "f");
+        this.logger = logger;
+    }
+    config() {
+        let y = yargs(hideBin(process.argv))
+            .option("log-level", {
+            type: "string",
+            choices: Object.values(LogLevel.options),
+            default: LogLevel.enum.info,
+            description: "The log level",
+        })
+            .middleware((argv) => {
+            this.logger.setLogLevel(argv.logLevel);
+        }, true);
+        for (const command of __classPrivateFieldGet(this, _Cli_commands, "f")) {
+            y = y.command({
+                command: command.getCommandName(),
+                describe: command.getDescription(),
+                builder: command.getOptions(),
+                handler: async (argv) => {
+                    this.logger.debug(`running ${command.getCommandName()}}`);
+                    const args = await command.parse(argv);
+                    await command.exec(args);
+                },
+            });
+        }
+        if (!__classPrivateFieldGet(this, _Cli_commands, "f").find((c) => c.getCommandName() === "$0")) {
+            y = y.command("$0", "default command", (y) => y, () => {
+                throw new ProsopoCliError("CLI.PARAMETER_ERROR", {
+                    context: { error: "no command specified" },
+                });
+            });
+        }
+        y = y
+            .demandCommand()
+            .strict()
+            .showHelpOnFail(false, "Specify --help for available options");
+        return y;
+    }
+    async exec(args = process.argv.slice(2)) {
+        const config = this.config();
+        this.logger.debug("parsing", args);
+        await config.parse(args);
+    }
+}
+_Cli_commands = new WeakMap();
+//# sourceMappingURL=cli.js.map

package/dist/cli/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";;AAAA,OAAO,EACN,QAAQ,EACR,QAAQ,EACR,eAAe,EACf,SAAS,GACT,MAAM,iBAAiB,CAAC;AACzB,OAAO,KAAoB,MAAM,OAAO,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAgBxC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAC9B,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;AAE3D,MAAM,OAAO,GAAI,SAAQ,QAAQ;IAGhC,YAAY,QAAyB;QACpC,KAAK,EAAE,CAAC;QAHT,gCAA2B;QAI1B,uBAAA,IAAI,iBAAa,QAAQ,MAAA,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACtB,CAAC;IAEO,MAAM;QACb,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;aAClC,MAAM,CAAC,WAAW,EAAE;YACpB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI;YAC3B,WAAW,EAAE,eAAe;SAC5B,CAAC;aAED,UAAU,CAAC,CAAC,IAAS,EAAE,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEV,KAAK,MAAM,OAAO,IAAI,uBAAA,IAAI,qBAAU,EAAE,CAAC;YACtC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;gBACb,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE;gBACjC,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE;gBAClC,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE;gBAE7B,OAAO,EAAE,KAAK,EAAE,IAAS,EAAE,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;oBAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACvC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,CAAC;aACD,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,uBAAA,IAAI,qBAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAE9D,CAAC,GAAG,CAAC,CAAC,OAAO,CACZ,IAAI,EACJ,iBAAiB,EACjB,CAAC,CAAO,EAAE,EAAE,CAAC,CAAC,EACd,GAAG,EAAE;gBACJ,MAAM,IAAI,eAAe,CAAC,qBAAqB,EAAE;oBAChD,OAAO,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE;iBAC1C,CAAC,CAAC;YACJ,CAAC,CACD,CAAC;QACH,CAAC;QACD,CAAC,GAAG,CAAC;aACH,aAAa,EAAE;aACf,MAAM,EAAE;aACR,cAAc,CAAC,KAAK,EAAE,sCAAsC,CAAC,CAAC;QAChE,OAAO,CAAC,CAAC;IACV,CAAC;IAEM,KAAK,CAAC,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACnC,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACD"}

package/dist/cli/cliCommand.d.ts

@@ -0,0 +1,17 @@
+import { Loggable } from "@prosopo/common";
+import type { Options } from "yargs";
+import type * as z from "zod";
+export declare abstract class CliCommand<T extends z.ZodTypeAny> extends Loggable {
+    getOptions(): {
+        [key: string]: Options;
+    };
+    abstract getArgSchema(): T;
+    parse(args: unknown): Promise<z.infer<T>>;
+    _check(args: z.infer<T>): Promise<void>;
+    _run(args: z.infer<T>): Promise<void>;
+    exec(args: z.infer<T>): Promise<void>;
+    getCommandName(): string;
+    abstract getDescription(): string;
+}
+export type CliCommandAny = CliCommand<z.ZodTypeAny>;
+//# sourceMappingURL=cliCommand.d.ts.map

package/dist/cli/cliCommand.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommand.d.ts","sourceRoot":"","sources":["../../src/cli/cliCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAcrC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,CAAC;AAE9B,8BAAsB,UAAU,CAAC,CAAC,SAAS,CAAC,CAAC,UAAU,CAAE,SAAQ,QAAQ;IAEjE,UAAU,IAAI;QACpB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB;aAKe,YAAY,IAAI,CAAC;IAIpB,KAAK,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAQzC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAKvB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAKrB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAQ3B,cAAc;aAIL,cAAc,IAAI,MAAM;CACxC;AAED,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC"}

package/dist/cli/cliCommand.js

@@ -0,0 +1,28 @@
+import { Loggable } from "@prosopo/common";
+import { kebabCase } from "@prosopo/util";
+export class CliCommand extends Loggable {
+    getOptions() {
+        return {};
+    }
+    async parse(args) {
+        const argsSchema = this.getArgSchema();
+        const parsed = await argsSchema.parse(args);
+        this.logger.debug("parsed args:", parsed);
+        return parsed;
+    }
+    async _check(args) {
+        this.logger.debug("runChecks");
+    }
+    async _run(args) {
+        this.logger.debug("run");
+    }
+    async exec(args) {
+        this.logger.debug("exec");
+        await this._check(args);
+        await this._run(args);
+    }
+    getCommandName() {
+        return kebabCase(this.constructor.name);
+    }
+}
+//# sourceMappingURL=cliCommand.js.map

package/dist/cli/cliCommand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommand.js","sourceRoot":"","sources":["../../src/cli/cliCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAiB1C,MAAM,OAAgB,UAAmC,SAAQ,QAAQ;IAEjE,UAAU;QAGhB,OAAO,EAAE,CAAC;IACX,CAAC;IAOM,KAAK,CAAC,KAAK,CAAC,IAAa;QAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,MAAM,CAAC;IACf,CAAC;IAGM,KAAK,CAAC,MAAM,CAAC,IAAgB;QACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAChC,CAAC;IAGM,KAAK,CAAC,IAAI,CAAC,IAAgB;QACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAGM,KAAK,CAAC,IAAI,CAAC,IAAgB;QACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE1B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAExB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAEM,cAAc;QACpB,OAAO,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;CAGD"}

package/dist/cli/cliCommandComposite.d.ts

@@ -0,0 +1,14 @@
+import type { Options } from "yargs";
+import type * as z from "zod";
+import { CliCommand, type CliCommandAny } from "./cliCommand.js";
+export declare abstract class CliCommandComposite<T extends z.ZodTypeAny> extends CliCommand<T> {
+    #private;
+    getCommands(): CliCommandAny[];
+    constructor(commands: CliCommandAny[]);
+    getOptions(): {
+        [key: string]: Options;
+    };
+    _run(args: z.TypeOf<T>): Promise<void>;
+    _check(args: z.TypeOf<T>): Promise<void>;
+}
+//# sourceMappingURL=cliCommandComposite.d.ts.map

package/dist/cli/cliCommandComposite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommandComposite.d.ts","sourceRoot":"","sources":["../../src/cli/cliCommandComposite.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAcrC,OAAO,KAAK,KAAK,CAAC,MAAM,KAAK,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE,8BAAsB,mBAAmB,CACxC,CAAC,SAAS,CAAC,CAAC,UAAU,CACrB,SAAQ,UAAU,CAAC,CAAC,CAAC;;IAGf,WAAW;gBAIN,QAAQ,EAAE,aAAa,EAAE;IAMrB,UAAU,IAAI;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE;IAQxC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAMxD"}

package/dist/cli/cliCommandComposite.js

@@ -0,0 +1,32 @@
+var _CliCommandComposite_commands;
+import { __classPrivateFieldGet, __classPrivateFieldSet } from "tslib";
+import { CliCommand } from "./cliCommand.js";
+export class CliCommandComposite extends CliCommand {
+    getCommands() {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f");
+    }
+    constructor(commands) {
+        super();
+        _CliCommandComposite_commands.set(this, []);
+        __classPrivateFieldSet(this, _CliCommandComposite_commands, [...commands], "f");
+    }
+    getOptions() {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce((prev, command) => {
+            return { ...prev, ...command.getOptions() };
+        }, {});
+    }
+    _run(args) {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce(async (prev, command) => {
+            await prev;
+            await command._run(args);
+        }, Promise.resolve());
+    }
+    _check(args) {
+        return __classPrivateFieldGet(this, _CliCommandComposite_commands, "f").reduce(async (prev, command) => {
+            await prev;
+            await command._check(args);
+        }, Promise.resolve());
+    }
+}
+_CliCommandComposite_commands = new WeakMap();
+//# sourceMappingURL=cliCommandComposite.js.map

package/dist/cli/cliCommandComposite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cliCommandComposite.js","sourceRoot":"","sources":["../../src/cli/cliCommandComposite.ts"],"names":[],"mappings":";;AAeA,OAAO,EAAE,UAAU,EAAsB,MAAM,iBAAiB,CAAC;AAEjE,MAAM,OAAgB,mBAEpB,SAAQ,UAAa;IAGf,WAAW;QACjB,OAAO,uBAAA,IAAI,qCAAU,CAAC;IACvB,CAAC;IAED,YAAY,QAAyB;QACpC,KAAK,EAAE,CAAC;QAPT,wCAA6B,EAAE,EAAC;QAS/B,uBAAA,IAAI,iCAAa,CAAC,GAAG,QAAQ,CAAC,MAAA,CAAC;IAChC,CAAC;IAEe,UAAU;QAEzB,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAE9C,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QAC7C,CAAC,EAAE,EAAE,CAAC,CAAC;IACR,CAAC;IAEe,IAAI,CAAC,IAAiB;QACrC,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YACpD,MAAM,IAAI,CAAC;YACX,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACvB,CAAC;IAEe,MAAM,CAAC,IAAiB;QACvC,OAAO,uBAAA,IAAI,qCAAU,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;YACpD,MAAM,IAAI,CAAC;YACX,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACvB,CAAC;CACD"}

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,34 @@
+import { isMain } from "@prosopo/util";
+import { Flatten } from "./commands/flatten.js";
+import { GenerateV1 } from "./commands/generateV1.js";
+import { GenerateV2 } from "./commands/generateV2.js";
+import { Get } from "./commands/get.js";
+import { Labels } from "./commands/labels.js";
+import { Relocate } from "./commands/relocate.js";
+import { Resize } from "./commands/resize.js";
+import { Cli } from "./index.js";
+const main = async () => {
+    const commands = [
+        new Flatten(),
+        new GenerateV1(),
+        new GenerateV2(),
+        new Get(),
+        new Labels(),
+        new Relocate(),
+        new Resize(),
+    ];
+    const cli = new Cli(commands);
+    cli.logger.setLogLevel("debug");
+    await cli.exec();
+};
+if (isMain(import.meta.url)) {
+    main()
+        .then(() => {
+        process.exit(0);
+    })
+        .catch((err) => {
+        console.log("error", err);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAEvC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAc9C,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEjC,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACvB,MAAM,QAAQ,GAAoB;QACjC,IAAI,OAAO,EAAE;QACb,IAAI,UAAU,EAAE;QAChB,IAAI,UAAU,EAAE;QAChB,IAAI,GAAG,EAAE;QACT,IAAI,MAAM,EAAE;QACZ,IAAI,QAAQ,EAAE;QACd,IAAI,MAAM,EAAE;KACZ,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9B,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC,CAAC;AAGF,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC7B,IAAI,EAAE;SACJ,IAAI,CAAC,GAAG,EAAE;QACV,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC,CAAC;SACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACd,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/commands/flatten.d.ts

@@ -0,0 +1,59 @@
+import * as z from "zod";
+import { InputOutputCliCommand } from "../utils/inputOutput.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+    input: z.ZodString;
+}, {
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}>, {
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+}>, "strip", z.ZodTypeAny, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}, {
+    input: string;
+    output: string;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+    getArgSchema(): z.ZodObject<z.objectUtil.extendShape<z.objectUtil.extendShape<{
+        input: z.ZodString;
+    }, {
+        output: z.ZodString;
+        overwrite: z.ZodOptional<z.ZodBoolean>;
+    }>, {
+        allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    }>, "strip", z.ZodTypeAny, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }, {
+        input: string;
+        output: string;
+        overwrite?: boolean | undefined;
+        allowDuplicates?: boolean | undefined;
+    }>;
+    getDescription(): string;
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        input: {
+            description: string;
+        };
+        output: {
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+    };
+    _run(args: Args): Promise<void>;
+}
+//# sourceMappingURL=flatten.d.ts.map

package/dist/commands/flatten.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.d.ts","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAEN,qBAAqB,EACrB,MAAM,yBAAyB,CAAC;AAEjC,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;EAErB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,qBAAa,OAAQ,SAAQ,qBAAqB,CAAC,cAAc,CAAC;IACjD,YAAY;;;;;;;;;;;;;;;;;;IAIZ,cAAc,IAAI,MAAM;IAIxB,UAAU;;;;;;;;;;;;;;IAiBJ,IAAI,CAAC,IAAI,EAAE,IAAI;CAuFrC"}

package/dist/commands/flatten.js

@@ -0,0 +1,96 @@
+import fs from "node:fs";
+import { blake2b } from "@noble/hashes/blake2b";
+import { u8aToHex } from "@polkadot/util/u8a";
+import { ProsopoDatasetError } from "@prosopo/common";
+import { CaptchaItemTypes, DataSchema, } from "@prosopo/types";
+import { at } from "@prosopo/util";
+import { lodash } from "@prosopo/util/lodash";
+import cliProgress from "cli-progress";
+import * as z from "zod";
+import { InputOutputArgsSchema, InputOutputCliCommand, } from "../utils/inputOutput.js";
+export const ArgsSchema = InputOutputArgsSchema.extend({
+    allowDuplicates: z.boolean().optional(),
+});
+export class Flatten extends InputOutputCliCommand {
+    getArgSchema() {
+        return ArgsSchema;
+    }
+    getDescription() {
+        return "Restructure a directory containing directories for each image classification into a single directory with a file containing the labels";
+    }
+    getOptions() {
+        return lodash().merge(super.getOptions(), {
+            input: {
+                description: "Path to the data directory containing subdirectories for each image classification",
+            },
+            output: {
+                description: "Where to put the output file containing the labels and single directory of images",
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: "If true, allow duplicates in the data",
+            },
+        });
+    }
+    async _run(args) {
+        this.logger.debug("flatten run");
+        await super._run(args);
+        const dataDir = args.input;
+        const outDir = args.output;
+        this.logger.info("reading data");
+        const labels = fs
+            .readdirSync(dataDir, { withFileTypes: true })
+            .filter((dirent) => dirent.isDirectory())
+            .map((dirent) => dirent.name)
+            .sort();
+        const imagesByLabel = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`));
+        const imageDir = `${outDir}/images`;
+        fs.mkdirSync(imageDir, { recursive: true });
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
+        const items = [];
+        bar.start(imagesByLabel.reduce((acc, images) => acc + images.length, 0), 0);
+        labels.forEach((label, i) => {
+            const images = at(imagesByLabel, i);
+            for (const image of images) {
+                bar.increment();
+                const extension = image.split(".").pop();
+                const content = fs.readFileSync(`${dataDir}/${label}/${image}`);
+                const hash = blake2b(content);
+                const hex = u8aToHex(hash);
+                const name = `${hex}.${extension}`;
+                if (fs.existsSync(`${imageDir}/${name}`)) {
+                    for (const item of items) {
+                        if (item.hash === hex) {
+                            this.logger.log(`\ndupe: ${label}/${image}`);
+                            this.logger.log("item hash", item.hash);
+                            this.logger.log("item label", item.label);
+                        }
+                    }
+                    if (!args.allowDuplicates) {
+                        throw new ProsopoDatasetError("DATASET.DUPLICATE_IMAGE", {
+                            context: { image: `${label}/${image}` },
+                        });
+                    }
+                }
+                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`);
+                const filePath = fs.realpathSync(`${imageDir}/${name}`);
+                const entry = {
+                    data: filePath,
+                    type: CaptchaItemTypes.Image,
+                    label,
+                    hash: hex,
+                };
+                items.push(entry);
+            }
+        });
+        bar.stop();
+        const data = {
+            items,
+        };
+        this.logger.info("verifying data");
+        DataSchema.parse(data);
+        this.logger.info("writing data");
+        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4));
+    }
+}
+//# sourceMappingURL=flatten.js.map

package/dist/commands/flatten.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flatten.js","sourceRoot":"","sources":["../../src/commands/flatten.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EACN,gBAAgB,EAEhB,UAAU,GAEV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,MAAM,eAAe,CAAC;AACnC,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,WAAW,MAAM,cAAc,CAAC;AAcvC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EACN,qBAAqB,EACrB,qBAAqB,GACrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,UAAU,GAAG,qBAAqB,CAAC,MAAM,CAAC;IACtD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACvC,CAAC,CAAC;AAIH,MAAM,OAAO,OAAQ,SAAQ,qBAAqC;IACjD,YAAY;QAC3B,OAAO,UAAU,CAAC;IACnB,CAAC;IAEe,cAAc;QAC7B,OAAO,wIAAwI,CAAC;IACjJ,CAAC;IAEe,UAAU;QACzB,OAAO,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE;YACzC,KAAK,EAAE;gBACN,WAAW,EACV,oFAAoF;aACrF;YACD,MAAM,EAAE;gBACP,WAAW,EACV,mFAAmF;aACpF;YACD,eAAe,EAAE;gBAChB,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,uCAAuC;aACpD;SACD,CAAC,CAAC;IACJ,CAAC;IAEe,KAAK,CAAC,IAAI,CAAC,IAAU;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACjC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAG3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjC,MAAM,MAAM,GAAa,EAAE;aACzB,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAC7C,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;aACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;aAC5B,IAAI,EAAE,CAAC;QACT,MAAM,aAAa,GAAe,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACtD,EAAE,CAAC,WAAW,CAAC,GAAG,OAAO,IAAI,KAAK,EAAE,CAAC,CACrC,CAAC;QAGF,MAAM,QAAQ,GAAG,GAAG,MAAM,SAAS,CAAC;QACpC,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE5C,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACpC,EAAE,EACF,WAAW,CAAC,OAAO,CAAC,cAAc,CAClC,CAAC;QAGF,MAAM,KAAK,GAAmB,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,CACR,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,EAC7D,CAAC,CACD,CAAC;QACF,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAC3B,MAAM,MAAM,GAAa,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;YAE9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC5B,GAAG,CAAC,SAAS,EAAE,CAAC;gBAGhB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;gBAEzC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;gBAEhE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM,IAAI,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;gBACnC,IAAI,EAAE,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;oBAC1C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;wBAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC;4BACvB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC;4BAC7C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;4BACxC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC3C,CAAC;oBACF,CAAC;oBACD,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;wBAC3B,MAAM,IAAI,mBAAmB,CAAC,yBAAyB,EAAE;4BACxD,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,KAAK,IAAI,KAAK,EAAE,EAAE;yBACvC,CAAC,CAAC;oBACJ,CAAC;gBACF,CAAC;gBACD,EAAE,CAAC,YAAY,CAAC,GAAG,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,EAAE,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;gBACvE,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;gBAExD,MAAM,KAAK,GAAiB;oBAC3B,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,gBAAgB,CAAC,KAAK;oBAC5B,KAAK;oBACL,IAAI,EAAE,GAAG;iBACT,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,CAAC;QACF,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,IAAI,EAAE,CAAC;QAEX,MAAM,IAAI,GAAS;YAClB,KAAK;SACL,CAAC;QAGF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACnC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAGvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACjC,EAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;CACD"}

package/dist/commands/generate.d.ts

@@ -0,0 +1,115 @@
+import { type Item, type LabelledItem } from "@prosopo/types";
+import * as z from "zod";
+import { OutputCliCommand } from "../utils/output.js";
+export declare const ArgsSchema: z.ZodObject<z.objectUtil.extendShape<{
+    output: z.ZodString;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+}, {
+    labels: z.ZodOptional<z.ZodString>;
+    labelled: z.ZodOptional<z.ZodString>;
+    unlabelled: z.ZodOptional<z.ZodString>;
+    seed: z.ZodNumber;
+    size: z.ZodOptional<z.ZodNumber>;
+    overwrite: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicates: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesLabelled: z.ZodOptional<z.ZodBoolean>;
+    allowDuplicatesUnlabelled: z.ZodOptional<z.ZodBoolean>;
+}>, "strip", z.ZodTypeAny, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}, {
+    output: string;
+    seed: number;
+    overwrite?: boolean | undefined;
+    allowDuplicates?: boolean | undefined;
+    labels?: string | undefined;
+    labelled?: string | undefined;
+    unlabelled?: string | undefined;
+    size?: number | undefined;
+    allowDuplicatesLabelled?: boolean | undefined;
+    allowDuplicatesUnlabelled?: boolean | undefined;
+}>;
+export type ArgsSchemaType = typeof ArgsSchema;
+export type Args = z.infer<ArgsSchemaType>;
+export declare abstract class Generate<T extends ArgsSchemaType> extends OutputCliCommand<T> {
+    getOptions(): {
+        [key: string]: import("yargs").Options;
+    } & {
+        output: {
+            alias: string;
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        overwrite: {
+            boolean: boolean;
+            description: string;
+        };
+    } & {
+        output: {
+            description: string;
+        };
+        labelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        unlabelled: {
+            string: boolean;
+            demand: boolean;
+            description: string;
+        };
+        seed: {
+            number: boolean;
+            demand: boolean;
+            description: string;
+        };
+        size: {
+            number: boolean;
+            description: string;
+        };
+        labels: {
+            string: boolean;
+            description: string;
+        };
+        allowDuplicates: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesLabelled: {
+            boolean: boolean;
+            description: string;
+        };
+        allowDuplicatesUnlabelled: {
+            boolean: boolean;
+            description: string;
+        };
+    };
+    _check(args: Args): Promise<void>;
+    labelled: LabelledItem[];
+    unlabelled: Item[];
+    labels: string[];
+    labelledMapFile: string;
+    unlabelledMapFile: string;
+    labelToImages: {
+        [label: string]: Item[];
+    };
+    targets: string[];
+    saltRounds: number;
+    private loadData;
+    private loadLabels;
+    _run(args: Args): Promise<void>;
+}
+export declare const checkDuplicates: (labelled: LabelledItem[], unlabelled: Item[], options: {
+    allowDuplicatesLabelled?: boolean;
+    allowDuplicatesUnlabelled?: boolean;
+}) => void;
+//# sourceMappingURL=generate.d.ts.map

package/dist/commands/generate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.d.ts","sourceRoot":"","sources":["../../src/commands/generate.ts"],"names":[],"mappings":"AAEA,OAAO,EAEN,KAAK,IAAI,EAET,KAAK,YAAY,EAEjB,MAAM,gBAAgB,CAAC;AAexB,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AACzB,OAAO,EAAoB,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAExE,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAUrB,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,UAAU,CAAC;AAC/C,MAAM,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;AAE3C,8BAAsB,QAAQ,CAC7B,CAAC,SAAS,cAAc,CACvB,SAAQ,gBAAgB,CAAC,CAAC,CAAC;IACZ,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA6CJ,MAAM,CAAC,IAAI,EAAE,IAAI;IAwBvC,QAAQ,EAAE,YAAY,EAAE,CAAM;IAC9B,UAAU,EAAE,IAAI,EAAE,CAAM;IACxB,MAAM,EAAE,MAAM,EAAE,CAAM;IACtB,eAAe,SAAM;IACrB,iBAAiB,SAAM;IACvB,aAAa,EAAE;QAAE,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,EAAE,CAAA;KAAE,CAAM;IAChD,OAAO,EAAE,MAAM,EAAE,CAAM;IACvB,UAAU,SAAM;IAEhB,OAAO,CAAC,QAAQ;IAkChB,OAAO,CAAC,UAAU;IAmBI,IAAI,CAAC,IAAI,EAAE,IAAI;CAWrC;AAED,eAAO,MAAM,eAAe,aACjB,YAAY,EAAE,cACZ,IAAI,EAAE,WACT;IACR,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,yBAAyB,CAAC,EAAE,OAAO,CAAC;CACpC,SAOD,CAAC"}