@prosopo/datasets-fs 0.2.11 → 0.2.14

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@prosopo/datasets-fs",
-    "version": "0.2.11",
+    "version": "0.2.14",
     "author": "PROSOPO LIMITED <info@prosopo.io>",
     "license": "Apache-2.0",
     "private": false,
@@ -28,9 +28,9 @@
     "types": "./dist/index.d.ts",
     "dependencies": {
         "@polkadot/util": "^12.3.2",
-        "@prosopo/common": "0.2.11",
-        "@prosopo/types": "0.2.11",
-        "@prosopo/util": "0.2.11",
+        "@prosopo/common": "0.2.14",
+        "@prosopo/types": "0.2.14",
+        "@prosopo/util": "0.2.14",
         "bcrypt": "^5.1.0",
         "cli-progress": "^3.12.0",
         "es-main": "^1.2.0",

package/src/cli/cli.ts

@@ -0,0 +1,61 @@
+import { CliCommandAny } from './cliCommand.js'
+import { LogLevel, Loggable, getLogger } from '@prosopo/common'
+import { hideBin } from 'yargs/helpers'
+import yargs, { Argv } from 'yargs'
+
+const dirname = process.cwd()
+const logger = getLogger(LogLevel.enum.info, `${dirname}`)
+
+export class Cli extends Loggable {
+    #commands: CliCommandAny[]
+
+    constructor(commands: CliCommandAny[]) {
+        super()
+        this.#commands = commands
+        this.logger = logger
+    }
+
+    private config() {
+        let y = yargs(hideBin(process.argv))
+            .option('log-level', {
+                type: 'string',
+                choices: Object.values(LogLevel.options),
+                default: LogLevel.enum.info,
+                description: 'The log level',
+            })
+            .middleware((argv: any) => {
+                this.logger.setLogLevel(argv.logLevel)
+            }, true)
+        this.#commands.forEach((command) => {
+            y = y.command({
+                command: command.getCommandName(),
+                describe: command.getDescription(),
+                builder: command.getOptions(),
+                handler: async (argv: any) => {
+                    this.logger.debug(`running ${command.getCommandName()}}`)
+                    const args = await command.parse(argv)
+                    await command.exec(args)
+                },
+            })
+        })
+        if (!this.#commands.find((c) => c.getCommandName() === '$0')) {
+            // no default command
+            y = y.command(
+                '$0',
+                'default command',
+                (y: Argv) => y,
+                (argv: any) => {
+                    throw new Error(`no command specified`)
+                }
+            )
+        }
+        y = y.demandCommand().strict().showHelpOnFail(false, 'Specify --help for available options')
+        return y
+    }
+
+    public async exec(args: string[] = process.argv.slice(2)) {
+        const config = this.config()
+        this.logger.debug('parsing', args)
+        await config.parse(args)
+    }
+}

package/src/cli/cliCommand.ts

@@ -0,0 +1,52 @@
+import * as z from 'zod'
+import { Loggable } from '@prosopo/common'
+import { Options } from 'yargs'
+import { kebabCase } from '@prosopo/util'
+
+export abstract class CliCommand<T extends z.ZodTypeAny> extends Loggable {
+    // get the options for the CLI
+    public getOptions(): {
+        [key: string]: Options
+    } {
+        return {}
+    }
+
+    // get the schema for the args to this command
+    public abstract getArgSchema(): T
+
+    // parse args using the schema
+    // use this when you have an args object but not sure if it's in the right format / contains the correct fields, e.g. args from cmdline
+    public async parse(args: unknown): Promise<z.infer<T>> {
+        const argsSchema = this.getArgSchema()
+        const parsed = await argsSchema.parse(args)
+        this.logger.debug('parsed args:', parsed)
+        return parsed
+    }
+
+    // run any checks before the main run function. This function should have no side effects, i.e. not write to disk. It's purely for conducting checks / setting up
+    public async _check(args: z.infer<T>) {
+        this.logger.debug('runChecks')
+    }
+
+    // the main run function. This should be the function that does the work
+    public async _run(args: z.infer<T>) {
+        this.logger.debug('run')
+    }
+
+    // exec is a public facing function that should be called by the CLI. It will run the preRun, run, and postRun functions in order
+    public async exec(args: z.infer<T>) {
+        this.logger.debug('exec')
+        // run any checks
+        await this._check(args)
+        // then run the command
+        await this._run(args)
+    }
+
+    public getCommandName() {
+        return kebabCase(this.constructor.name)
+    }
+
+    public abstract getDescription(): string
+}
+
+export type CliCommandAny = CliCommand<z.ZodTypeAny>

package/src/cli/cliCommandComposite.ts

@@ -0,0 +1,38 @@
+import * as z from 'zod'
+import { CliCommand, CliCommandAny } from './cliCommand.js'
+import { Options } from 'yargs'
+
+export abstract class CliCommandComposite<T extends z.ZodTypeAny> extends CliCommand<T> {
+    #commands: CliCommandAny[] = []
+
+    public getCommands() {
+        return this.#commands
+    }
+
+    constructor(commands: CliCommandAny[]) {
+        super()
+        // take a copy of the array to avoid leakage
+        this.#commands = [...commands]
+    }
+
+    public override getOptions(): { [key: string]: Options } {
+        // merge options in turn from each command. Command order matters, latter commands will overwrite earlier ones
+        return this.#commands.reduce((prev, command) => {
+            return { ...prev, ...command.getOptions() }
+        }, {})
+    }
+
+    public override _run(args: z.TypeOf<T>): Promise<void> {
+        return this.#commands.reduce(async (prev, command) => {
+            await prev
+            await command._run(args)
+        }, Promise.resolve())
+    }
+
+    public override _check(args: z.TypeOf<T>): Promise<void> {
+        return this.#commands.reduce(async (prev, command) => {
+            await prev
+            await command._check(args)
+        }, Promise.resolve())
+    }
+}

package/src/cli.ts

@@ -0,0 +1,37 @@
+import { Cli } from './index.js'
+import { CliCommandAny } from './cli/cliCommand.js'
+import { Flatten } from './commands/flatten.js'
+import { GenerateV1 } from './commands/generateV1.js'
+import { GenerateV2 } from './commands/generateV2.js'
+import { Get } from './commands/get.js'
+import { Labels } from './commands/labels.js'
+import { Relocate } from './commands/relocate.js'
+import { Resize } from './commands/resize.js'
+import esMain from 'es-main'
+
+const main = async () => {
+    const commands: CliCommandAny[] = [
+        new Flatten(),
+        new GenerateV1(),
+        new GenerateV2(),
+        new Get(),
+        new Labels(),
+        new Relocate(),
+        new Resize(),
+    ]
+    const cli = new Cli(commands)
+    cli.logger.setLogLevel('debug')
+    await cli.exec()
+}
+
+//if main process
+if (esMain(import.meta)) {
+    main()
+        .then(() => {
+            process.exit(0)
+        })
+        .catch((err) => {
+            console.log('error', err)
+            process.exit(1)
+        })
+}

package/src/commands/flatten.ts

@@ -0,0 +1,121 @@
+import * as z from 'zod'
+import { CaptchaItemTypes, Data, DataSchema, LabelledItem } from '@prosopo/types'
+import { InputOutputArgsSchema as InputOutputArgsSchema, InputOutputCliCommand } from '../utils/inputOutput.js'
+import { at } from '@prosopo/util'
+import { blake2b } from '@noble/hashes/blake2b'
+import { lodash } from '@prosopo/util/lodash'
+import { u8aToHex } from '@polkadot/util/u8a'
+import cliProgress from 'cli-progress'
+import fs from 'fs'
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+    allowDuplicates: z.boolean().optional(),
+})
+export type ArgsSchemaType = typeof ArgsSchema
+export type Args = z.infer<ArgsSchemaType>
+
+export class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+    public override getArgSchema() {
+        return ArgsSchema
+    }
+
+    public override getDescription(): string {
+        return 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels'
+    }
+
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            input: {
+                description: 'Path to the data directory containing subdirectories for each image classification',
+            },
+            output: {
+                description: 'Where to put the output file containing the labels and single directory of images',
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: 'If true, allow duplicates in the data',
+            },
+        })
+    }
+
+    public override async _run(args: Args) {
+        this.logger.debug('flatten run')
+        await super._run(args)
+
+        const dataDir = args.input
+        const outDir = args.output
+
+        // find the labels (these should be subdirectories of the data directory)
+        this.logger.info('reading data')
+        const labels: string[] = fs
+            .readdirSync(dataDir, { withFileTypes: true })
+            .filter((dirent) => dirent.isDirectory())
+            .map((dirent) => dirent.name)
+            .sort()
+        const imagesByLabel: string[][] = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`))
+
+        // create the output directory
+        const imageDir = `${outDir}/images`
+        fs.mkdirSync(imageDir, { recursive: true })
+
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
+
+        // for each label
+        const items: LabelledItem[] = []
+        bar.start(
+            imagesByLabel.reduce((acc, images) => acc + images.length, 0),
+            0
+        )
+        labels.forEach((label, i) => {
+            const images: string[] = at(imagesByLabel, i)
+            // for each image
+            for (const image of images) {
+                bar.increment()
+                // this.logger.log(`flattening ${label}/${image}`)
+                // copy the image to the output directory
+                const extension = image.split('.').pop()
+                // read file to bytes
+                const content = fs.readFileSync(`${dataDir}/${label}/${image}`)
+                // hash based on the content of the image
+                const hash = blake2b(content)
+                const hex = u8aToHex(hash)
+                const name = `${hex}.${extension}`
+                if (fs.existsSync(`${imageDir}/${name}`)) {
+                    for (const item of items) {
+                        if (item.hash === hex) {
+                            this.logger.log(`\ndupe: ${label}/${image}`)
+                            this.logger.log('item hash', item.hash)
+                            this.logger.log('item label', item.label)
+                        }
+                    }
+                    if (!args.allowDuplicates) {
+                        throw new Error(`duplicate image detected`)
+                    }
+                }
+                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`)
+                const filePath = fs.realpathSync(`${imageDir}/${name}`)
+                // add the image to the map file
+                const entry: LabelledItem = {
+                    data: filePath,
+                    type: CaptchaItemTypes.Image,
+                    label,
+                    hash: hex,
+                }
+                items.push(entry)
+            }
+        })
+        bar.stop()
+
+        const data: Data = {
+            items,
+        }
+
+        // verify data
+        this.logger.info('verifying data')
+        DataSchema.parse(data)
+
+        // write map file
+        this.logger.info('writing data')
+        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4))
+    }
+}

package/src/commands/generate.ts

@@ -0,0 +1,177 @@
+import * as z from 'zod'
+import { DataSchema, Item, LabelledDataSchema, LabelledItem, LabelsContainerSchema } from '@prosopo/types'
+import { OutputArgsSchema, OutputCliCommand } from '../utils/output.js'
+import { ProsopoEnvError } from '@prosopo/common'
+import { lodash, setSeedGlobal } from '@prosopo/util/lodash'
+import fs from 'fs'
+
+export const ArgsSchema = OutputArgsSchema.extend({
+    labels: z.string().optional(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+})
+export type ArgsSchemaType = typeof ArgsSchema
+export type Args = z.infer<ArgsSchemaType>
+
+export abstract class Generate<T extends ArgsSchemaType> extends OutputCliCommand<T> {
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            output: {
+                description: 'Where to write the captchas JSON file',
+            },
+            labelled: {
+                string: true,
+                demand: true,
+                description: 'Path to JSON file containing labelled data',
+            },
+            unlabelled: {
+                string: true,
+                demand: true,
+                description: 'Path to JSON file containing unlabelled data',
+            },
+            seed: {
+                number: true,
+                demand: true,
+                description: 'Seed for random number generator',
+            },
+            size: {
+                number: true,
+                description: 'Number of images in each captcha',
+            },
+            labels: {
+                string: true,
+                description:
+                    'Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.',
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: 'If true, allow duplicates in the data (labelled and unlabelled)',
+            },
+            allowDuplicatesLabelled: {
+                boolean: true,
+                description: 'If true, allow duplicates in the labelled data',
+            },
+            allowDuplicatesUnlabelled: {
+                boolean: true,
+                description: 'If true, allow duplicates in the unlabelled data',
+            },
+        })
+    }
+
+    public override async _check(args: Args) {
+        // if specified, check files exist
+        const labelledMapFile: string | undefined = args.labelled
+        if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+            throw new ProsopoEnvError(
+                new Error(`labelled map file does not exist: ${labelledMapFile}`),
+                'FS.FILE_NOT_FOUND'
+            )
+        }
+        const unlabelledMapFile: string | undefined = args.unlabelled
+        if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+            throw new ProsopoEnvError(
+                new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
+                'FS.FILE_NOT_FOUND'
+            )
+        }
+        this.labelledMapFile = labelledMapFile || ''
+        this.unlabelledMapFile = unlabelledMapFile || ''
+    }
+
+    labelled: LabelledItem[] = []
+    unlabelled: Item[] = []
+    labels: string[] = []
+    labelledMapFile = ''
+    unlabelledMapFile = ''
+    labelToImages: { [label: string]: Item[] } = {}
+    targets: string[] = []
+    saltRounds = 10
+
+    private loadData(args: Args) {
+        const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false
+        const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false
+
+        // load the map to get the labelled and unlabelled data
+        this.labelled = this.labelledMapFile
+            ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(this.labelledMapFile, 'utf8'))).items
+            : []
+        this.unlabelled = this.unlabelledMapFile
+            ? DataSchema.parse(JSON.parse(fs.readFileSync(this.unlabelledMapFile, 'utf8'))).items
+            : []
+
+        // check for duplicates
+        checkDuplicates(this.labelled, this.unlabelled, {
+            allowDuplicatesLabelled,
+            allowDuplicatesUnlabelled,
+        })
+
+        // split the labelled data by label
+        this.labelToImages = {}
+        for (const entry of this.labelled) {
+            const arr = this.labelToImages[entry.label] || []
+            arr.push(entry)
+            this.labelToImages[entry.label] = arr
+        }
+        this.targets = Object.keys(this.labelToImages)
+    }
+
+    private loadLabels(args: Args) {
+        // load the labels from file
+        // these are the labels that unlabelled data will be assigned to
+        // note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+        this.labels = []
+        if (args.labels && fs.existsSync(args.labels)) {
+            this.labels.push(
+                ...[...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(args.labels, 'utf8'))).labels]
+            )
+        } else {
+            // else default to the labels in the labelled data
+            this.labels.push(...[...this.targets])
+        }
+    }
+
+    public override async _run(args: Args) {
+        await super._run(args)
+        // set the seed
+        setSeedGlobal(args.seed || 0)
+        // get lodash (with seeded rng)
+        const _ = lodash()
+
+        this.loadData(args)
+
+        this.loadLabels(args)
+    }
+}
+
+export const checkDuplicates = (
+    labelled: LabelledItem[],
+    unlabelled: Item[],
+    options: {
+        allowDuplicatesLabelled?: boolean
+        allowDuplicatesUnlabelled?: boolean
+    }
+) => {
+    // check for duplicates
+    const all = new Set<string>()
+    if (!options.allowDuplicatesLabelled) addAllUnique(all, labelled, 'labelled')
+    if (!options.allowDuplicatesUnlabelled) addAllUnique(all, unlabelled, 'unlabelled')
+}
+
+const addAllUnique = (all: Set<string>, entries: Item[], dataType: string) => {
+    for (const entry of entries) {
+        addUnique(all, entry, dataType)
+    }
+}
+
+const addUnique = (all: Set<string>, entry: Item, dataType: string) => {
+    if (all.has(entry.data)) {
+        throw new Error(`Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`)
+    }
+    all.add(entry.data)
+}