@prosopo/datasets-fs 0.2.10 → 0.2.13

package/src/commands/flatten.ts

@@ -0,0 +1,120 @@
+import { CaptchaItemTypes, Data, DataSchema, LabelledItem } from '@prosopo/types'
+import { InputOutputArgsSchema as InputOutputArgsSchema, InputOutputCliCommand } from '../utils/inputOutput.js'
+import { at, lodash } from '@prosopo/util'
+import { blake2b } from '@noble/hashes/blake2b'
+import { u8aToHex } from '@polkadot/util'
+import { z } from 'zod'
+import cliProgress from 'cli-progress'
+import fs from 'fs'
+
+export const ArgsSchema = InputOutputArgsSchema.extend({
+    allowDuplicates: z.boolean().optional(),
+})
+export type ArgsSchemaType = typeof ArgsSchema
+export type Args = z.infer<ArgsSchemaType>
+
+export class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
+    public override getArgSchema() {
+        return ArgsSchema
+    }
+
+    public override getDescription(): string {
+        return 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels'
+    }
+
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            input: {
+                description: 'Path to the data directory containing subdirectories for each image classification',
+            },
+            output: {
+                description: 'Where to put the output file containing the labels and single directory of images',
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: 'If true, allow duplicates in the data',
+            },
+        })
+    }
+
+    public override async _run(args: Args) {
+        this.logger.debug('flatten run')
+        await super._run(args)
+
+        const dataDir = args.input
+        const outDir = args.output
+
+        // find the labels (these should be subdirectories of the data directory)
+        this.logger.info('reading data')
+        const labels: string[] = fs
+            .readdirSync(dataDir, { withFileTypes: true })
+            .filter((dirent) => dirent.isDirectory())
+            .map((dirent) => dirent.name)
+            .sort()
+        const imagesByLabel: string[][] = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`))
+
+        // create the output directory
+        const imageDir = `${outDir}/images`
+        fs.mkdirSync(imageDir, { recursive: true })
+
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
+
+        // for each label
+        const items: LabelledItem[] = []
+        bar.start(
+            imagesByLabel.reduce((acc, images) => acc + images.length, 0),
+            0
+        )
+        labels.forEach((label, i) => {
+            const images: string[] = at(imagesByLabel, i)
+            // for each image
+            for (const image of images) {
+                bar.increment()
+                // this.logger.log(`flattening ${label}/${image}`)
+                // copy the image to the output directory
+                const extension = image.split('.').pop()
+                // read file to bytes
+                const content = fs.readFileSync(`${dataDir}/${label}/${image}`)
+                // hash based on the content of the image
+                const hash = blake2b(content)
+                const hex = u8aToHex(hash)
+                const name = `${hex}.${extension}`
+                if (fs.existsSync(`${imageDir}/${name}`)) {
+                    for (const item of items) {
+                        if (item.hash === hex) {
+                            this.logger.log(`\ndupe: ${label}/${image}`)
+                            this.logger.log('item hash', item.hash)
+                            this.logger.log('item label', item.label)
+                        }
+                    }
+                    if (!args.allowDuplicates) {
+                        throw new Error(`duplicate image detected`)
+                    }
+                }
+                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`)
+                const filePath = fs.realpathSync(`${imageDir}/${name}`)
+                // add the image to the map file
+                const entry: LabelledItem = {
+                    data: filePath,
+                    type: CaptchaItemTypes.Image,
+                    label,
+                    hash: hex,
+                }
+                items.push(entry)
+            }
+        })
+        bar.stop()
+
+        const data: Data = {
+            items,
+        }
+
+        // verify data
+        this.logger.info('verifying data')
+        DataSchema.parse(data)
+
+        // write map file
+        this.logger.info('writing data')
+        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4))
+    }
+}

package/src/commands/generate.ts

@@ -0,0 +1,177 @@
+import { DataSchema, Item, LabelledDataSchema, LabelledItem, LabelsContainerSchema } from '@prosopo/types'
+import { OutputArgsSchema, OutputCliCommand } from '../utils/output.js'
+import { ProsopoEnvError } from '@prosopo/common'
+import { lodash, setSeedGlobal } from '@prosopo/util'
+import { z } from 'zod'
+import fs from 'fs'
+
+export const ArgsSchema = OutputArgsSchema.extend({
+    labels: z.string().optional(),
+    labelled: z.string().optional(),
+    unlabelled: z.string().optional(),
+    seed: z.number(),
+    size: z.number().optional(),
+    overwrite: z.boolean().optional(),
+    allowDuplicates: z.boolean().optional(),
+    allowDuplicatesLabelled: z.boolean().optional(),
+    allowDuplicatesUnlabelled: z.boolean().optional(),
+})
+export type ArgsSchemaType = typeof ArgsSchema
+export type Args = z.infer<ArgsSchemaType>
+
+export abstract class Generate<T extends ArgsSchemaType> extends OutputCliCommand<T> {
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            output: {
+                description: 'Where to write the captchas JSON file',
+            },
+            labelled: {
+                string: true,
+                demand: true,
+                description: 'Path to JSON file containing labelled data',
+            },
+            unlabelled: {
+                string: true,
+                demand: true,
+                description: 'Path to JSON file containing unlabelled data',
+            },
+            seed: {
+                number: true,
+                demand: true,
+                description: 'Seed for random number generator',
+            },
+            size: {
+                number: true,
+                description: 'Number of images in each captcha',
+            },
+            labels: {
+                string: true,
+                description:
+                    'Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.',
+            },
+            allowDuplicates: {
+                boolean: true,
+                description: 'If true, allow duplicates in the data (labelled and unlabelled)',
+            },
+            allowDuplicatesLabelled: {
+                boolean: true,
+                description: 'If true, allow duplicates in the labelled data',
+            },
+            allowDuplicatesUnlabelled: {
+                boolean: true,
+                description: 'If true, allow duplicates in the unlabelled data',
+            },
+        })
+    }
+
+    public override async _check(args: Args) {
+        // if specified, check files exist
+        const labelledMapFile: string | undefined = args.labelled
+        if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
+            throw new ProsopoEnvError(
+                new Error(`labelled map file does not exist: ${labelledMapFile}`),
+                'FS.FILE_NOT_FOUND'
+            )
+        }
+        const unlabelledMapFile: string | undefined = args.unlabelled
+        if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
+            throw new ProsopoEnvError(
+                new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
+                'FS.FILE_NOT_FOUND'
+            )
+        }
+        this.labelledMapFile = labelledMapFile || ''
+        this.unlabelledMapFile = unlabelledMapFile || ''
+    }
+
+    labelled: LabelledItem[] = []
+    unlabelled: Item[] = []
+    labels: string[] = []
+    labelledMapFile = ''
+    unlabelledMapFile = ''
+    labelToImages: { [label: string]: Item[] } = {}
+    targets: string[] = []
+    saltRounds = 10
+
+    private loadData(args: Args) {
+        const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false
+        const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false
+
+        // load the map to get the labelled and unlabelled data
+        this.labelled = this.labelledMapFile
+            ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(this.labelledMapFile, 'utf8'))).items
+            : []
+        this.unlabelled = this.unlabelledMapFile
+            ? DataSchema.parse(JSON.parse(fs.readFileSync(this.unlabelledMapFile, 'utf8'))).items
+            : []
+
+        // check for duplicates
+        checkDuplicates(this.labelled, this.unlabelled, {
+            allowDuplicatesLabelled,
+            allowDuplicatesUnlabelled,
+        })
+
+        // split the labelled data by label
+        this.labelToImages = {}
+        for (const entry of this.labelled) {
+            const arr = this.labelToImages[entry.label] || []
+            arr.push(entry)
+            this.labelToImages[entry.label] = arr
+        }
+        this.targets = Object.keys(this.labelToImages)
+    }
+
+    private loadLabels(args: Args) {
+        // load the labels from file
+        // these are the labels that unlabelled data will be assigned to
+        // note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
+        this.labels = []
+        if (args.labels && fs.existsSync(args.labels)) {
+            this.labels.push(
+                ...[...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(args.labels, 'utf8'))).labels]
+            )
+        } else {
+            // else default to the labels in the labelled data
+            this.labels.push(...[...this.targets])
+        }
+    }
+
+    public override async _run(args: Args) {
+        await super._run(args)
+        // set the seed
+        setSeedGlobal(args.seed || 0)
+        // get lodash (with seeded rng)
+        const _ = lodash()
+
+        this.loadData(args)
+
+        this.loadLabels(args)
+    }
+}
+
+export const checkDuplicates = (
+    labelled: LabelledItem[],
+    unlabelled: Item[],
+    options: {
+        allowDuplicatesLabelled?: boolean
+        allowDuplicatesUnlabelled?: boolean
+    }
+) => {
+    // check for duplicates
+    const all = new Set<string>()
+    if (!options.allowDuplicatesLabelled) addAllUnique(all, labelled, 'labelled')
+    if (!options.allowDuplicatesUnlabelled) addAllUnique(all, unlabelled, 'unlabelled')
+}
+
+const addAllUnique = (all: Set<string>, entries: Item[], dataType: string) => {
+    for (const entry of entries) {
+        addUnique(all, entry, dataType)
+    }
+}
+
+const addUnique = (all: Set<string>, entry: Item, dataType: string) => {
+    if (all.has(entry.data)) {
+        throw new Error(`Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`)
+    }
+    all.add(entry.data)
+}

package/src/commands/generateV1.ts

@@ -0,0 +1,221 @@
+import { CaptchaTypes, CaptchaWithoutId, Captchas, CaptchasContainerSchema, Item, RawSolution } from '@prosopo/types'
+import { Generate, ArgsSchema as GenerateArgsSchema } from './generate.js'
+import { ProsopoEnvError } from '@prosopo/common'
+import { at, get, lodash } from '@prosopo/util'
+import { blake2AsHex } from '@polkadot/util-crypto'
+import { z } from 'zod'
+import bcrypt from 'bcrypt'
+import cliProgress from 'cli-progress'
+import fs from 'fs'
+
+export const ArgsSchema = GenerateArgsSchema.extend({
+    solved: z.number().optional(),
+    unsolved: z.number().optional(),
+    minCorrect: z.number().optional(),
+    maxCorrect: z.number().optional(),
+})
+export type ArgsSchemaType = typeof ArgsSchema
+export type Args = z.infer<ArgsSchemaType>
+
+export class GenerateV1 extends Generate<ArgsSchemaType> {
+    public override getArgSchema() {
+        return ArgsSchema
+    }
+
+    public override getDescription(): string {
+        return 'Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled'
+    }
+
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            solved: {
+                description: 'Number of captchas to generate that are solved',
+                number: true,
+            },
+            unsolved: {
+                description: 'Number of captchas to generate that are unsolved',
+                number: true,
+            },
+            minCorrect: {
+                description: 'Minimum number of target images in each captcha',
+                number: true,
+            },
+            maxCorrect: {
+                description: 'Maximum number of target images in each captcha',
+                number: true,
+            },
+        })
+    }
+
+    private generateSolved(
+        solved: number,
+        size: number,
+        minCorrect: number,
+        maxCorrect: number,
+        bar: cliProgress.SingleBar
+    ) {
+        const _ = lodash()
+        // generate n solved captchas
+        const solvedCaptchas: CaptchaWithoutId[] = []
+        for (let i = 0; i < solved; i++) {
+            // update the current value in your application..
+            bar.increment()
+
+            if (this.targets.length <= 1) {
+                throw new ProsopoEnvError(
+                    new Error(`not enough different labels in labelled data`),
+                    'DATASET.NOT_ENOUGH_LABELS'
+                )
+            }
+
+            // uniformly sample targets
+            const target = at(this.targets, i % this.targets.length)
+            const notTargets = this.targets.filter((t) => t !== target)
+
+            // how many correct items should be in the captcha?
+            const nCorrect = _.random(minCorrect, maxCorrect)
+            // how many incorrect items should be in the captcha?
+            const nIncorrect = size - nCorrect
+
+            const targetItems: Item[] = get(this.labelToImages, target)
+            const notTargetItems: Item[] = notTargets.map((notTarget) => get(this.labelToImages, notTarget)).flat()
+
+            if (targetItems.length < nCorrect) {
+                throw new ProsopoEnvError(
+                    new Error(`not enough images for target (${target})`),
+                    'DATASET.NOT_ENOUGH_IMAGES'
+                )
+            }
+            if (notTargetItems.length < nIncorrect) {
+                throw new ProsopoEnvError(
+                    new Error(`not enough non-matching images for target (${target})`),
+                    'DATASET.NOT_ENOUGH_IMAGES'
+                )
+            }
+
+            // get the correct items
+            const correctItems: Item[] = _.sampleSize(targetItems, nCorrect)
+
+            // get the incorrect items
+            const incorrectItems: Item[] = _.sampleSize(notTargetItems, nIncorrect)
+
+            let items: Item[] = [...correctItems, ...incorrectItems]
+            let indices: number[] = [...Array(items.length).keys()]
+            indices = _.shuffle(indices)
+            items = indices.map((i) => at(items, i))
+            items = items.map((item) => {
+                return {
+                    data: item.data,
+                    hash: item.hash,
+                    type: item.type,
+                }
+            })
+
+            // the first n indices are the correct items
+            const solution: RawSolution[] = indices
+                .map((index, i) => {
+                    return {
+                        pre: index, // the index of the item in the items array before shuffle
+                        post: i, // the index of the item in the shuffled array
+                    }
+                })
+                .filter((item) => item.pre < correctItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
+                .map((item) => {
+                    return item.post // return the index in the shuffled array
+                })
+
+            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds))
+            // create the captcha
+            const captcha: CaptchaWithoutId = {
+                salt,
+                target,
+                items,
+                solution,
+            }
+            solvedCaptchas.push(captcha)
+        }
+        return solvedCaptchas
+    }
+
+    private generateUnsolved(unsolved: number, size: number, bar: cliProgress.SingleBar) {
+        const _ = lodash()
+        // this.logger.info(`Generating ${unsolved} unsolved captchas...`)
+        // create a new progress bar instance and use shades_classic theme
+        // generate n unsolved captchas
+        const unsolvedCaptchas: CaptchaWithoutId[] = []
+        for (let i = 0; i < unsolved; i++) {
+            bar.increment()
+            if (this.unlabelled.length <= size) {
+                throw new ProsopoEnvError(
+                    new Error(`unlabelled map file does not contain enough data`),
+                    'DATASET.NOT_ENOUGH_IMAGES'
+                )
+            }
+            // pick a random label to be the target
+            // note that these are potentially different to the labelled data labels
+            if (this.labels.length <= 0) {
+                throw new ProsopoEnvError(new Error(`no labels found for unlabelled data`), 'DATASET.NOT_ENOUGH_LABELS')
+            }
+            const index = _.random(0, this.labels.length - 1)
+            const target = at(this.labels, index)
+            // randomly pick images from the unlabelled data
+            const itemSet: Item[] = _.sampleSize(this.unlabelled, size)
+            // shuffle the items
+            let items: Item[] = [...itemSet]
+            let indices: number[] = [...Array(items.length).keys()]
+            indices = _.shuffle(indices)
+            items = indices.map((i) => at(items, i))
+            items = items.map((item) => {
+                return {
+                    data: item.data,
+                    hash: item.hash,
+                    type: item.type,
+                }
+            })
+            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds))
+            // create the captcha
+            const captcha: CaptchaWithoutId = {
+                salt,
+                target,
+                items,
+            }
+            unsolvedCaptchas.push(captcha)
+        }
+        return unsolvedCaptchas
+    }
+
+    public override async _run(args: Args) {
+        await super._run(args)
+
+        const outFile: string = args.output
+
+        // get lodash (with seeded rng)
+        const _ = lodash()
+
+        const size: number = args.size || 9
+        const minCorrect: number = args.minCorrect || 1
+        const maxCorrect: number = args.maxCorrect || size - 1
+        const solved: number = args.solved || 0
+        const unsolved: number = args.unsolved || 0
+
+        // create a new progress bar instance and use shades_classic theme
+        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
+
+        // this.logger.info(`Generating ${solved} solved captchas...`)
+        bar.start(solved + unsolved, 0)
+        const solvedCaptchas = this.generateSolved(solved, size, minCorrect, maxCorrect, bar)
+        const unsolvedCaptchas = this.generateUnsolved(unsolved, size, bar)
+        bar.stop()
+        // write to file
+        const output: Captchas = {
+            captchas: [...solvedCaptchas, ...unsolvedCaptchas],
+            format: CaptchaTypes.SelectAll,
+        }
+
+        // verify the output
+        CaptchasContainerSchema.parse(output)
+
+        fs.mkdirSync(args.output.split('/').slice(0, -1).join('/'), { recursive: true })
+        fs.writeFileSync(outFile, JSON.stringify(output, null, 4))
+    }
+}