@prosopo/datasets-fs 0.2.15 → 0.2.17

package/src/commands/flatten.ts

@@ -1,121 +0,0 @@
-import * as z from 'zod'
-import { CaptchaItemTypes, Data, DataSchema, LabelledItem } from '@prosopo/types'
-import { InputOutputArgsSchema as InputOutputArgsSchema, InputOutputCliCommand } from '../utils/inputOutput.js'
-import { at } from '@prosopo/util'
-import { blake2b } from '@noble/hashes/blake2b'
-import { lodash } from '@prosopo/util/lodash'
-import { u8aToHex } from '@polkadot/util/u8a'
-import cliProgress from 'cli-progress'
-import fs from 'fs'
-
-export const ArgsSchema = InputOutputArgsSchema.extend({
-    allowDuplicates: z.boolean().optional(),
-})
-export type ArgsSchemaType = typeof ArgsSchema
-export type Args = z.infer<ArgsSchemaType>
-
-export class Flatten extends InputOutputCliCommand<ArgsSchemaType> {
-    public override getArgSchema() {
-        return ArgsSchema
-    }
-
-    public override getDescription(): string {
-        return 'Restructure a directory containing directories for each image classification into a single directory with a file containing the labels'
-    }
-
-    public override getOptions() {
-        return lodash().merge(super.getOptions(), {
-            input: {
-                description: 'Path to the data directory containing subdirectories for each image classification',
-            },
-            output: {
-                description: 'Where to put the output file containing the labels and single directory of images',
-            },
-            allowDuplicates: {
-                boolean: true,
-                description: 'If true, allow duplicates in the data',
-            },
-        })
-    }
-
-    public override async _run(args: Args) {
-        this.logger.debug('flatten run')
-        await super._run(args)
-
-        const dataDir = args.input
-        const outDir = args.output
-
-        // find the labels (these should be subdirectories of the data directory)
-        this.logger.info('reading data')
-        const labels: string[] = fs
-            .readdirSync(dataDir, { withFileTypes: true })
-            .filter((dirent) => dirent.isDirectory())
-            .map((dirent) => dirent.name)
-            .sort()
-        const imagesByLabel: string[][] = labels.map((label) => fs.readdirSync(`${dataDir}/${label}`))
-
-        // create the output directory
-        const imageDir = `${outDir}/images`
-        fs.mkdirSync(imageDir, { recursive: true })
-
-        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
-
-        // for each label
-        const items: LabelledItem[] = []
-        bar.start(
-            imagesByLabel.reduce((acc, images) => acc + images.length, 0),
-            0
-        )
-        labels.forEach((label, i) => {
-            const images: string[] = at(imagesByLabel, i)
-            // for each image
-            for (const image of images) {
-                bar.increment()
-                // this.logger.log(`flattening ${label}/${image}`)
-                // copy the image to the output directory
-                const extension = image.split('.').pop()
-                // read file to bytes
-                const content = fs.readFileSync(`${dataDir}/${label}/${image}`)
-                // hash based on the content of the image
-                const hash = blake2b(content)
-                const hex = u8aToHex(hash)
-                const name = `${hex}.${extension}`
-                if (fs.existsSync(`${imageDir}/${name}`)) {
-                    for (const item of items) {
-                        if (item.hash === hex) {
-                            this.logger.log(`\ndupe: ${label}/${image}`)
-                            this.logger.log('item hash', item.hash)
-                            this.logger.log('item label', item.label)
-                        }
-                    }
-                    if (!args.allowDuplicates) {
-                        throw new Error(`duplicate image detected`)
-                    }
-                }
-                fs.copyFileSync(`${dataDir}/${label}/${image}`, `${imageDir}/${name}`)
-                const filePath = fs.realpathSync(`${imageDir}/${name}`)
-                // add the image to the map file
-                const entry: LabelledItem = {
-                    data: filePath,
-                    type: CaptchaItemTypes.Image,
-                    label,
-                    hash: hex,
-                }
-                items.push(entry)
-            }
-        })
-        bar.stop()
-
-        const data: Data = {
-            items,
-        }
-
-        // verify data
-        this.logger.info('verifying data')
-        DataSchema.parse(data)
-
-        // write map file
-        this.logger.info('writing data')
-        fs.writeFileSync(`${outDir}/data.json`, JSON.stringify(data, null, 4))
-    }
-}

package/src/commands/generate.ts

@@ -1,177 +0,0 @@
-import * as z from 'zod'
-import { DataSchema, Item, LabelledDataSchema, LabelledItem, LabelsContainerSchema } from '@prosopo/types'
-import { OutputArgsSchema, OutputCliCommand } from '../utils/output.js'
-import { ProsopoEnvError } from '@prosopo/common'
-import { lodash, setSeedGlobal } from '@prosopo/util/lodash'
-import fs from 'fs'
-
-export const ArgsSchema = OutputArgsSchema.extend({
-    labels: z.string().optional(),
-    labelled: z.string().optional(),
-    unlabelled: z.string().optional(),
-    seed: z.number(),
-    size: z.number().optional(),
-    overwrite: z.boolean().optional(),
-    allowDuplicates: z.boolean().optional(),
-    allowDuplicatesLabelled: z.boolean().optional(),
-    allowDuplicatesUnlabelled: z.boolean().optional(),
-})
-export type ArgsSchemaType = typeof ArgsSchema
-export type Args = z.infer<ArgsSchemaType>
-
-export abstract class Generate<T extends ArgsSchemaType> extends OutputCliCommand<T> {
-    public override getOptions() {
-        return lodash().merge(super.getOptions(), {
-            output: {
-                description: 'Where to write the captchas JSON file',
-            },
-            labelled: {
-                string: true,
-                demand: true,
-                description: 'Path to JSON file containing labelled data',
-            },
-            unlabelled: {
-                string: true,
-                demand: true,
-                description: 'Path to JSON file containing unlabelled data',
-            },
-            seed: {
-                number: true,
-                demand: true,
-                description: 'Seed for random number generator',
-            },
-            size: {
-                number: true,
-                description: 'Number of images in each captcha',
-            },
-            labels: {
-                string: true,
-                description:
-                    'Path to JSON file containing labels which unlabelled data will be assigned to. If not given, labels will be deduced from the labelled data.',
-            },
-            allowDuplicates: {
-                boolean: true,
-                description: 'If true, allow duplicates in the data (labelled and unlabelled)',
-            },
-            allowDuplicatesLabelled: {
-                boolean: true,
-                description: 'If true, allow duplicates in the labelled data',
-            },
-            allowDuplicatesUnlabelled: {
-                boolean: true,
-                description: 'If true, allow duplicates in the unlabelled data',
-            },
-        })
-    }
-
-    public override async _check(args: Args) {
-        // if specified, check files exist
-        const labelledMapFile: string | undefined = args.labelled
-        if (labelledMapFile && !fs.existsSync(labelledMapFile)) {
-            throw new ProsopoEnvError(
-                new Error(`labelled map file does not exist: ${labelledMapFile}`),
-                'FS.FILE_NOT_FOUND'
-            )
-        }
-        const unlabelledMapFile: string | undefined = args.unlabelled
-        if (unlabelledMapFile && !fs.existsSync(unlabelledMapFile)) {
-            throw new ProsopoEnvError(
-                new Error(`unlabelled map file does not exist: ${unlabelledMapFile}`),
-                'FS.FILE_NOT_FOUND'
-            )
-        }
-        this.labelledMapFile = labelledMapFile || ''
-        this.unlabelledMapFile = unlabelledMapFile || ''
-    }
-
-    labelled: LabelledItem[] = []
-    unlabelled: Item[] = []
-    labels: string[] = []
-    labelledMapFile = ''
-    unlabelledMapFile = ''
-    labelToImages: { [label: string]: Item[] } = {}
-    targets: string[] = []
-    saltRounds = 10
-
-    private loadData(args: Args) {
-        const allowDuplicatesLabelled = args.allowDuplicatesLabelled || args.allowDuplicates || false
-        const allowDuplicatesUnlabelled = args.allowDuplicatesUnlabelled || args.allowDuplicates || false
-
-        // load the map to get the labelled and unlabelled data
-        this.labelled = this.labelledMapFile
-            ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(this.labelledMapFile, 'utf8'))).items
-            : []
-        this.unlabelled = this.unlabelledMapFile
-            ? DataSchema.parse(JSON.parse(fs.readFileSync(this.unlabelledMapFile, 'utf8'))).items
-            : []
-
-        // check for duplicates
-        checkDuplicates(this.labelled, this.unlabelled, {
-            allowDuplicatesLabelled,
-            allowDuplicatesUnlabelled,
-        })
-
-        // split the labelled data by label
-        this.labelToImages = {}
-        for (const entry of this.labelled) {
-            const arr = this.labelToImages[entry.label] || []
-            arr.push(entry)
-            this.labelToImages[entry.label] = arr
-        }
-        this.targets = Object.keys(this.labelToImages)
-    }
-
-    private loadLabels(args: Args) {
-        // load the labels from file
-        // these are the labels that unlabelled data will be assigned to
-        // note that these can be different to the labels in the map file as the labelled data is independent of the unlabelled data in terms of labels
-        this.labels = []
-        if (args.labels && fs.existsSync(args.labels)) {
-            this.labels.push(
-                ...[...LabelsContainerSchema.parse(JSON.parse(fs.readFileSync(args.labels, 'utf8'))).labels]
-            )
-        } else {
-            // else default to the labels in the labelled data
-            this.labels.push(...[...this.targets])
-        }
-    }
-
-    public override async _run(args: Args) {
-        await super._run(args)
-        // set the seed
-        setSeedGlobal(args.seed || 0)
-        // get lodash (with seeded rng)
-        const _ = lodash()
-
-        this.loadData(args)
-
-        this.loadLabels(args)
-    }
-}
-
-export const checkDuplicates = (
-    labelled: LabelledItem[],
-    unlabelled: Item[],
-    options: {
-        allowDuplicatesLabelled?: boolean
-        allowDuplicatesUnlabelled?: boolean
-    }
-) => {
-    // check for duplicates
-    const all = new Set<string>()
-    if (!options.allowDuplicatesLabelled) addAllUnique(all, labelled, 'labelled')
-    if (!options.allowDuplicatesUnlabelled) addAllUnique(all, unlabelled, 'unlabelled')
-}
-
-const addAllUnique = (all: Set<string>, entries: Item[], dataType: string) => {
-    for (const entry of entries) {
-        addUnique(all, entry, dataType)
-    }
-}
-
-const addUnique = (all: Set<string>, entry: Item, dataType: string) => {
-    if (all.has(entry.data)) {
-        throw new Error(`Duplicate data entry in ${dataType} data: ${JSON.stringify(entry)}`)
-    }
-    all.add(entry.data)
-}

package/src/commands/generateV1.ts

@@ -1,222 +0,0 @@
-import * as z from 'zod'
-import { CaptchaTypes, CaptchaWithoutId, Captchas, CaptchasContainerSchema, Item, RawSolution } from '@prosopo/types'
-import { Generate, ArgsSchema as GenerateArgsSchema } from './generate.js'
-import { ProsopoEnvError } from '@prosopo/common'
-import { at, get } from '@prosopo/util'
-import { blake2AsHex } from '@polkadot/util-crypto/blake2'
-import { lodash } from '@prosopo/util/lodash'
-import bcrypt from 'bcrypt'
-import cliProgress from 'cli-progress'
-import fs from 'fs'
-
-export const ArgsSchema = GenerateArgsSchema.extend({
-    solved: z.number().optional(),
-    unsolved: z.number().optional(),
-    minCorrect: z.number().optional(),
-    maxCorrect: z.number().optional(),
-})
-export type ArgsSchemaType = typeof ArgsSchema
-export type Args = z.infer<ArgsSchemaType>
-
-export class GenerateV1 extends Generate<ArgsSchemaType> {
-    public override getArgSchema() {
-        return ArgsSchema
-    }
-
-    public override getDescription(): string {
-        return 'Generate distinct captchas producing captcha challenges comprising 2 rounds, one labelled and one unlabelled'
-    }
-
-    public override getOptions() {
-        return lodash().merge(super.getOptions(), {
-            solved: {
-                description: 'Number of captchas to generate that are solved',
-                number: true,
-            },
-            unsolved: {
-                description: 'Number of captchas to generate that are unsolved',
-                number: true,
-            },
-            minCorrect: {
-                description: 'Minimum number of target images in each captcha',
-                number: true,
-            },
-            maxCorrect: {
-                description: 'Maximum number of target images in each captcha',
-                number: true,
-            },
-        })
-    }
-
-    private generateSolved(
-        solved: number,
-        size: number,
-        minCorrect: number,
-        maxCorrect: number,
-        bar: cliProgress.SingleBar
-    ) {
-        const _ = lodash()
-        // generate n solved captchas
-        const solvedCaptchas: CaptchaWithoutId[] = []
-        for (let i = 0; i < solved; i++) {
-            // update the current value in your application..
-            bar.increment()
-
-            if (this.targets.length <= 1) {
-                throw new ProsopoEnvError(
-                    new Error(`not enough different labels in labelled data`),
-                    'DATASET.NOT_ENOUGH_LABELS'
-                )
-            }
-
-            // uniformly sample targets
-            const target = at(this.targets, i % this.targets.length)
-            const notTargets = this.targets.filter((t) => t !== target)
-
-            // how many correct items should be in the captcha?
-            const nCorrect = _.random(minCorrect, maxCorrect)
-            // how many incorrect items should be in the captcha?
-            const nIncorrect = size - nCorrect
-
-            const targetItems: Item[] = get(this.labelToImages, target)
-            const notTargetItems: Item[] = notTargets.map((notTarget) => get(this.labelToImages, notTarget)).flat()
-
-            if (targetItems.length < nCorrect) {
-                throw new ProsopoEnvError(
-                    new Error(`not enough images for target (${target})`),
-                    'DATASET.NOT_ENOUGH_IMAGES'
-                )
-            }
-            if (notTargetItems.length < nIncorrect) {
-                throw new ProsopoEnvError(
-                    new Error(`not enough non-matching images for target (${target})`),
-                    'DATASET.NOT_ENOUGH_IMAGES'
-                )
-            }
-
-            // get the correct items
-            const correctItems: Item[] = _.sampleSize(targetItems, nCorrect)
-
-            // get the incorrect items
-            const incorrectItems: Item[] = _.sampleSize(notTargetItems, nIncorrect)
-
-            let items: Item[] = [...correctItems, ...incorrectItems]
-            let indices: number[] = [...Array(items.length).keys()]
-            indices = _.shuffle(indices)
-            items = indices.map((i) => at(items, i))
-            items = items.map((item) => {
-                return {
-                    data: item.data,
-                    hash: item.hash,
-                    type: item.type,
-                }
-            })
-
-            // the first n indices are the correct items
-            const solution: RawSolution[] = indices
-                .map((index, i) => {
-                    return {
-                        pre: index, // the index of the item in the items array before shuffle
-                        post: i, // the index of the item in the shuffled array
-                    }
-                })
-                .filter((item) => item.pre < correctItems.length) // keep all items that were in the first n slots of the original item array - these were the correct items
-                .map((item) => {
-                    return item.post // return the index in the shuffled array
-                })
-
-            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds))
-            // create the captcha
-            const captcha: CaptchaWithoutId = {
-                salt,
-                target,
-                items,
-                solution,
-            }
-            solvedCaptchas.push(captcha)
-        }
-        return solvedCaptchas
-    }
-
-    private generateUnsolved(unsolved: number, size: number, bar: cliProgress.SingleBar) {
-        const _ = lodash()
-        // this.logger.info(`Generating ${unsolved} unsolved captchas...`)
-        // create a new progress bar instance and use shades_classic theme
-        // generate n unsolved captchas
-        const unsolvedCaptchas: CaptchaWithoutId[] = []
-        for (let i = 0; i < unsolved; i++) {
-            bar.increment()
-            if (this.unlabelled.length <= size) {
-                throw new ProsopoEnvError(
-                    new Error(`unlabelled map file does not contain enough data`),
-                    'DATASET.NOT_ENOUGH_IMAGES'
-                )
-            }
-            // pick a random label to be the target
-            // note that these are potentially different to the labelled data labels
-            if (this.labels.length <= 0) {
-                throw new ProsopoEnvError(new Error(`no labels found for unlabelled data`), 'DATASET.NOT_ENOUGH_LABELS')
-            }
-            const index = _.random(0, this.labels.length - 1)
-            const target = at(this.labels, index)
-            // randomly pick images from the unlabelled data
-            const itemSet: Item[] = _.sampleSize(this.unlabelled, size)
-            // shuffle the items
-            let items: Item[] = [...itemSet]
-            let indices: number[] = [...Array(items.length).keys()]
-            indices = _.shuffle(indices)
-            items = indices.map((i) => at(items, i))
-            items = items.map((item) => {
-                return {
-                    data: item.data,
-                    hash: item.hash,
-                    type: item.type,
-                }
-            })
-            const salt = blake2AsHex(bcrypt.genSaltSync(this.saltRounds))
-            // create the captcha
-            const captcha: CaptchaWithoutId = {
-                salt,
-                target,
-                items,
-            }
-            unsolvedCaptchas.push(captcha)
-        }
-        return unsolvedCaptchas
-    }
-
-    public override async _run(args: Args) {
-        await super._run(args)
-
-        const outFile: string = args.output
-
-        // get lodash (with seeded rng)
-        const _ = lodash()
-
-        const size: number = args.size || 9
-        const minCorrect: number = args.minCorrect || 1
-        const maxCorrect: number = args.maxCorrect || size - 1
-        const solved: number = args.solved || 0
-        const unsolved: number = args.unsolved || 0
-
-        // create a new progress bar instance and use shades_classic theme
-        const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic)
-
-        // this.logger.info(`Generating ${solved} solved captchas...`)
-        bar.start(solved + unsolved, 0)
-        const solvedCaptchas = this.generateSolved(solved, size, minCorrect, maxCorrect, bar)
-        const unsolvedCaptchas = this.generateUnsolved(unsolved, size, bar)
-        bar.stop()
-        // write to file
-        const output: Captchas = {
-            captchas: [...solvedCaptchas, ...unsolvedCaptchas],
-            format: CaptchaTypes.SelectAll,
-        }
-
-        // verify the output
-        CaptchasContainerSchema.parse(output)
-
-        fs.mkdirSync(args.output.split('/').slice(0, -1).join('/'), { recursive: true })
-        fs.writeFileSync(outFile, JSON.stringify(output, null, 4))
-    }
-}