@prosopo/datasets-fs 0.2.11 → 0.2.14

package/src/tests/mocked.test.ts

@@ -0,0 +1,289 @@
+import { CaptchasContainerSchema, DataSchema } from '@prosopo/types'
+import { Flatten } from '../commands/flatten.js'
+import { GenerateV1 } from '../commands/generateV1.js'
+import { GenerateV2 } from '../commands/generateV2.js'
+import { Labels } from '../commands/labels.js'
+import { Relocate } from '../commands/relocate.js'
+import { Resize } from '../commands/resize.js'
+import { afterAll, beforeAll, describe, test } from 'vitest'
+import { blake2b } from '@noble/hashes/blake2b'
+import { captchasEqFs, fsEq, fsWalk, restoreRepoDir, substituteRepoDir } from './utils.js'
+import { u8aToHex } from '@polkadot/util/u8a'
+import fs from 'fs'
+import sharp from 'sharp'
+
+describe('dataset commands', () => {
+    beforeAll(() => {
+        // substitute the repo path in the data for tests
+        substituteRepoDir()
+        // remove previous test results
+        if (fs.existsSync(`${__dirname}/test_results`)) {
+            fs.rmdirSync(`${__dirname}/test_results`, { recursive: true })
+        }
+        fs.mkdirSync(`${__dirname}/test_results`, { recursive: true })
+    })
+
+    afterAll(() => {
+        // restore repo path back to placeholder
+        restoreRepoDir()
+    })
+
+    test('labels', async () => {
+        const input = `${__dirname}/data/flat_resized/data.json`
+        const output = `${__dirname}/test_results/labels.json`
+        const labels = new Labels()
+        labels.logger.setLogLevel('error')
+        await labels.exec({
+            input,
+            output,
+        })
+        // make sure the results are the same as the expected results
+        fsEq(output, `${__dirname}/data/flat_resized/labels.json`)
+
+        // make sure there's one class per hierarchical directory
+        const content = fs.readFileSync(output).toString()
+        const labelsJson = JSON.parse(content.toString())
+        const foundLabels = labelsJson.labels as string[]
+        const categories = fs.readdirSync(`${__dirname}/data/hierarchical`)
+        if (JSON.stringify(foundLabels) !== JSON.stringify(categories)) {
+            throw new Error(`expected ${categories} but found ${foundLabels}`)
+        }
+    })
+
+    test('generate v2', async () => {
+        const input = `${__dirname}/data/flat_resized/data.json`
+        const output = `${__dirname}/test_results/captchas_v2.json`
+        const minIncorrect = 1
+        const minCorrect = 1
+        const minLabelled = 2
+        const maxLabelled = 7
+        const generate = new GenerateV2()
+        generate.logger.setLogLevel('error')
+        await generate.exec({
+            labelled: input,
+            unlabelled: input,
+            overwrite: true,
+            output,
+            count: 100,
+            seed: 0,
+            minIncorrect,
+            minCorrect,
+            minLabelled,
+            maxLabelled,
+            allowDuplicates: true,
+        })
+        // make sure the results are the same as the expected results
+        if (!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v2.json`)) {
+            throw new Error(`captchas not equal`)
+        }
+
+        // test that the solutions array and unlabelled array per captcha never conflict
+        const content = fs.readFileSync(output).toString()
+        const captchasJson = JSON.parse(content.toString())
+        const captchas = CaptchasContainerSchema.parse(captchasJson)
+        for (const captcha of captchas.captchas) {
+            const solutions = captcha.solution
+            const unlabelled = captcha.unlabelled
+            // both should not be undefined
+            if (solutions === undefined || unlabelled === undefined) {
+                console.log(captcha)
+                throw new Error(`solutions or unlabelled array is undefined`)
+            }
+            for (const solution of solutions) {
+                // solution should not be in unlabelled
+                if (unlabelled.includes(solution)) {
+                    throw new Error(`solution ${solution} is also in unlabelled array`)
+                }
+            }
+            // check the correct, incorrect, labelled and unlabelled distribution
+            const nCorrect = solutions.length
+            const nUnlabelled = unlabelled.length
+            const nIncorrect = 9 - nCorrect - nUnlabelled
+            const nLabelled = nCorrect + nIncorrect
+            if (nCorrect < minCorrect) {
+                throw new Error(`expected at least ${minCorrect} correct but found ${nCorrect}`)
+            }
+            if (nIncorrect < minIncorrect) {
+                throw new Error(`expected at least ${minIncorrect} incorrect but found ${nIncorrect}`)
+            }
+            if (nLabelled < minLabelled) {
+                throw new Error(`expected at least ${minLabelled} labelled but found ${nLabelled}`)
+            }
+            if (nLabelled > maxLabelled) {
+                throw new Error(`expected at most ${maxLabelled} labelled but found ${nLabelled}`)
+            }
+            if (nUnlabelled !== 9 - nLabelled) {
+                throw new Error(`expected ${9 - nLabelled} unlabelled but found ${nUnlabelled}`)
+            }
+        }
+    })
+
+    test('generate v1', async () => {
+        const input = `${__dirname}/data/flat_resized/data.json`
+        const output = `${__dirname}/test_results/captchas_v1.json`
+        const generate = new GenerateV1()
+        generate.logger.setLogLevel('error')
+        await generate.exec({
+            labelled: input,
+            unlabelled: input,
+            overwrite: true,
+            output,
+            solved: 50,
+            unsolved: 50,
+            seed: 0,
+            minCorrect: 1,
+            maxCorrect: 6,
+            allowDuplicates: true,
+        })
+        // make sure the results are the same as the expected results
+        if (!captchasEqFs(output, `${__dirname}/data/flat_resized/captchas_v1.json`)) {
+            throw new Error(`captchas not equal`)
+        }
+    })
+
+    test('resizes data', async () => {
+        const input = `${__dirname}/data/flat/data.json`
+        const output = `${__dirname}/test_results/flat_resized`
+        const resize = new Resize()
+        resize.logger.setLogLevel('error')
+        await resize.exec({
+            input,
+            output,
+            overwrite: true,
+            square: true,
+            size: 128,
+        })
+
+        // make sure the results are the same as the expected results
+        const expected = `${__dirname}/data/flat_resized`
+        fsEq(output, expected)
+
+        // check data in resized dir is all 128x128
+        for (const pth of fsWalk(output)) {
+            // if pth is not img, ignore
+            if (!pth.endsWith('.jpg') && !pth.endsWith('.jpeg') && !pth.endsWith('.png')) {
+                continue
+            }
+            const image = sharp(pth)
+            const metadata = await image.metadata()
+            if (metadata.width !== 128 || metadata.height !== 128) {
+                throw new Error(`image ${pth} is not 128x128`)
+            }
+        }
+    })
+
+    test('relocates data', async () => {
+        const input = `${__dirname}/data/flat_resized/data.json`
+        const output = `${__dirname}/test_results/relocated_data.json`
+        const relocate = new Relocate()
+        relocate.logger.setLogLevel('error')
+        await relocate.exec({
+            input,
+            output,
+            overwrite: true,
+            to: 'newwebsite.com',
+            from: '${repo}',
+        })
+
+        // make sure the results are the same as the expected results
+        const expected = `${__dirname}/data/flat_resized/relocated_data.json`
+        fsEq(output, expected)
+    })
+
+    test('flattens hierarchical data', async () => {
+        const input = `${__dirname}/data/hierarchical`
+        const output = `${__dirname}/test_results/flat`
+        const flatten = new Flatten()
+        flatten.logger.setLogLevel('error')
+        await flatten.exec({
+            input,
+            output,
+            overwrite: true,
+        })
+
+        // make sure the results are the same as the expected results
+        const expected = `${__dirname}/data/flat`
+        fsEq(output, expected)
+
+        // read data json
+        const content = fs.readFileSync(`${output}/data.json`).toString()
+        const dataJson = JSON.parse(content.toString())
+        const data = DataSchema.parse(dataJson)
+
+        // check each image from the hierarchical data is in the flat data
+        let hierCount = 0
+        for (const pth of fsWalk(input)) {
+            // if pth is not img, ignore
+            if (!pth.endsWith('.jpg') && !pth.endsWith('.jpeg') && !pth.endsWith('.png')) {
+                continue
+            }
+            hierCount++
+            // get the category
+            const category = pth.split('/').slice(-2)[0]
+            const content = fs.readFileSync(pth)
+            // check the image is in the flat data
+            let name = ''
+            for (const pth2 of fsWalk(output)) {
+                // if pth2 is not file, ignore
+                if (!fs.statSync(pth2).isFile()) {
+                    continue
+                }
+                const content2 = fs.readFileSync(pth2)
+                if (content.equals(content2)) {
+                    const str = pth2.split('/').slice(-1)[0]
+                    if (str === undefined) {
+                        throw new Error(`unable to parse ${pth2}`)
+                    }
+                    name = str
+                    break
+                }
+            }
+
+            if (!name) {
+                throw new Error(`unable to find image ${pth} in output`)
+            }
+            // check the image is in the data json
+            const item = data.items.find((item) => {
+                const name2 = item.data.split('/').slice(-1)[0]
+                if (name2 !== name) {
+                    return false
+                }
+                return true
+            })
+            if (item === undefined) {
+                throw new Error(`unable to find image ${pth} in data.json`)
+            }
+            // correct name, so check category
+            if (item.label != category) {
+                throw new Error(`expected ${category} but found ${item.label}`)
+            }
+            // type should be image
+            if (item.type !== 'image') {
+                throw new Error(`expected image type but found ${item.type}`)
+            }
+            // file name should be the hash
+            if (item.hash !== name.split('.')[0]) {
+                throw new Error(`expected ${name.split('.')[0]} hash but found ${item.hash}`)
+            }
+            // hash should be the hash of the image content
+            const hash = u8aToHex(blake2b(content))
+            if (item.hash !== hash) {
+                throw new Error(`expected ${hash} hash but found ${item.hash}`)
+            }
+        }
+
+        // count the number of images in the flat data
+        let flatCount = 0
+        for (const pth of fsWalk(output)) {
+            if (!pth.endsWith('.jpg') && !pth.endsWith('.jpeg') && !pth.endsWith('.png')) {
+                continue
+            }
+            flatCount++
+        }
+
+        // check same number of images in flat data
+        if (hierCount !== flatCount) {
+            throw new Error(`expected ${hierCount} images but found ${flatCount}`)
+        }
+    })
+})

package/src/tests/utils.ts

@@ -0,0 +1,132 @@
+import { CaptchasContainer, CaptchasContainerSchema, DataSchema } from '@prosopo/types'
+import { at } from '@prosopo/util'
+import fs from 'fs'
+
+// recursively list files in a directory
+export function* fsWalk(
+    pth: string,
+    options?: {
+        filesFirst?: boolean // if true, depth first (i.e. yield files inside a dir before the dir itself, recursively), else yield in encounter order (i.e. dir first, then files inside, recursively)
+    }
+): Generator<string> {
+    // if path is a directory, recurse
+    const isDir = fs.existsSync(pth) && fs.statSync(pth).isDirectory()
+    let subpaths = isDir ? fs.readdirSync(pth) : []
+    subpaths = subpaths.map((f) => `${pth}/${f}`)
+    if (!options?.filesFirst) {
+        yield pth
+    }
+    for (const subpath of subpaths) {
+        yield* fsWalk(subpath, options)
+    }
+    if (options?.filesFirst) {
+        yield pth
+    }
+}
+
+// test whether two files/dirs are equal, recursively comparing contents as required
+export const fsEq = (pth1: string, pth2: string) => {
+    const it1 = fsWalk(pth1)
+    const it2 = fsWalk(pth2)
+    // drop the root path
+    const files1 = [...it1].map((f) => f.slice(pth1.length))
+    const files2 = [...it2].map((f) => f.slice(pth2.length))
+    if (files1.length !== files2.length) {
+        return false
+    }
+    // sort files so in same order
+    files1.sort()
+    files2.sort()
+    if (!files1.every((f, i) => f === files2[i])) {
+        // the two lists of files are not the same
+        return false
+    }
+    // compare contents of each file
+    for (let i = 0; i < files1.length; i++) {
+        const file1 = pth1 + at(files1, i)
+        const file2 = pth2 + at(files2, i)
+        const stat1 = fs.statSync(file1)
+        if (stat1.isDirectory()) {
+            // already checked above
+            continue
+        }
+        // files, so compare contents
+        const content1 = fs.readFileSync(file1)
+        const content2 = fs.readFileSync(file2)
+        if (!content1.equals(content2)) {
+            return false
+        }
+    }
+    return true
+}
+
+export const readDataJson = (pth: string) => {
+    let content = fs.readFileSync(pth).toString()
+    // TODO use getPaths() here to find the repo dir
+    content = content.replaceAll('${repo}', __dirname + '/../../../..')
+    const dataJson = JSON.parse(content.toString())
+    const data = DataSchema.parse(dataJson)
+    return data
+}
+
+export const readCaptchasJson = (pth: string) => {
+    const content = fs.readFileSync(pth).toString()
+    const dataJson = JSON.parse(content)
+    const captchas = CaptchasContainerSchema.parse(dataJson)
+    return captchas
+}
+
+export const captchasEqFs = (pth1: string, pth2: string) => {
+    const data1 = readCaptchasJson(pth1)
+    const data2 = readCaptchasJson(pth2)
+    return captchasEq(data1, data2)
+}
+
+export const captchasEq = (first: CaptchasContainer, second: CaptchasContainer) => {
+    if (first.captchas.length !== second.captchas.length) {
+        return false
+    }
+    for (let i = 0; i < first.captchas.length; i++) {
+        const captcha1 = at(first.captchas, i)
+        const captcha2 = at(second.captchas, i)
+        // salts should not be the same as generated on the fly and not stored!
+        if (captcha1.salt === captcha2.salt) {
+            return false
+        }
+        // everything else should be the same
+        const { salt: salt1, ...rest1 } = captcha1
+        const { salt: salt2, ...rest2 } = captcha2
+        if (JSON.stringify(rest1) !== JSON.stringify(rest2)) {
+            return false
+        }
+    }
+    return true
+}
+
+export const substituteRepoDir = () => {
+    // read all json files in the test data dir
+    for (const pth of fsWalk(__dirname + '/data')) {
+        if (!pth.endsWith('.json')) {
+            continue
+        }
+        // make a backup of each file
+        fs.copyFileSync(pth, pth + '.bak')
+        // replace ${repo} with the path to the repo
+        let content = fs.readFileSync(pth).toString()
+        // TODO use getPaths() here to find the repo dir
+        content = content.replaceAll('${repo}', __dirname + '/../../../..')
+        // rewrite the file
+        fs.writeFileSync(pth, content)
+    }
+}
+
+export const restoreRepoDir = () => {
+    // read all json files in the test data dir
+    for (const pth of fsWalk(__dirname + '/data')) {
+        if (!pth.endsWith('.json')) {
+            continue
+        }
+        // restore the backup of each file
+        fs.renameSync(pth + '.bak', pth)
+    }
+}

package/src/utils/input.ts

@@ -0,0 +1,38 @@
+import * as z from 'zod'
+import { CliCommand } from '../cli/cliCommand.js'
+import { ProsopoEnvError } from '@prosopo/common'
+import fs from 'fs'
+
+export const InputArgsSchema = z.object({
+    input: z.string(),
+})
+
+export type InputArgs = z.infer<typeof InputArgsSchema>
+
+export class InputCliCommand<T extends typeof InputArgsSchema> extends CliCommand<T> {
+    public override getArgSchema(): T {
+        throw new Error('Method not implemented.')
+    }
+    public override getDescription(): string {
+        throw new Error('Method not implemented.')
+    }
+    public override getOptions() {
+        return {
+            input: {
+                string: true,
+                alias: 'in',
+                demand: true,
+                description: 'The input path',
+            },
+        }
+    }
+
+    public override async _check(args: InputArgs) {
+        this.logger.debug('input _check')
+        await super._check(args)
+        // input must exist
+        if (!fs.existsSync(args.input)) {
+            throw new ProsopoEnvError(new Error(`input path does not exist: ${args.input}`), 'FS.FILE_NOT_FOUND')
+        }
+    }
+}

package/src/utils/inputOutput.ts

@@ -0,0 +1,14 @@
+import * as z from 'zod'
+import { CliCommandComposite } from '../cli/cliCommandComposite.js'
+import { InputArgsSchema, InputCliCommand } from './input.js'
+import { OutputArgsSchema, OutputCliCommand } from './output.js'
+
+export const InputOutputArgsSchema = InputArgsSchema.merge(OutputArgsSchema)
+
+export type InputOutputArgs = z.infer<typeof InputOutputArgsSchema>
+
+export abstract class InputOutputCliCommand<T extends typeof InputOutputArgsSchema> extends CliCommandComposite<T> {
+    constructor() {
+        super([new InputCliCommand<T>(), new OutputCliCommand<T>()])
+    }
+}

package/src/utils/output.ts

@@ -0,0 +1,69 @@
+import { CliCommand } from '../cli/cliCommand.js'
+import { ProsopoEnvError } from '@prosopo/common'
+import { boolean, object, string, infer as zInfer } from 'zod'
+import { lodash } from '@prosopo/util/lodash'
+import fs from 'fs'
+
+export const OutputArgsSchema = object({
+    output: string(),
+    overwrite: boolean().optional(),
+})
+
+export type OutputArgs = zInfer<typeof OutputArgsSchema>
+
+export class OutputCliCommand<T extends typeof OutputArgsSchema> extends CliCommand<T> {
+    #outputExists = false
+
+    public outputExists() {
+        return this.#outputExists
+    }
+
+    public override getArgSchema(): T {
+        throw new Error('Method not implemented.')
+    }
+    public override getDescription(): string {
+        throw new Error('Method not implemented.')
+    }
+    public override getOptions() {
+        return lodash().merge(super.getOptions(), {
+            output: {
+                alias: 'out',
+                string: true,
+                demand: true,
+                description: 'The output path',
+            },
+            overwrite: {
+                boolean: true,
+                description: 'Overwrite files in the output path if they already exist',
+            },
+        })
+    }
+
+    public override async _check(args: OutputArgs) {
+        this.logger.debug('Output _check')
+        await super._check(args)
+
+        // record whether output already exists
+        this.#outputExists = fs.existsSync(args.output)
+
+        // output must not exist, unless overwrite is true
+        if (this.outputExists()) {
+            if (!args.overwrite) {
+                throw new ProsopoEnvError(
+                    new Error(`output path already exists: ${args.output}`),
+                    'FS.FILE_ALREADY_EXISTS'
+                )
+            }
+        }
+    }
+
+    public override async _run(args: OutputArgs) {
+        this.logger.debug('Output run')
+        await super._run(args)
+        if (args.overwrite && this.outputExists()) {
+            // if overwrite is true, delete the output directory
+            this.logger.info('cleaning output directory...')
+            fs.rmSync(args.output, { recursive: true })
+        }
+    }
+}

package/tsconfig.cjs.json

@@ -0,0 +1,16 @@
+{
+    "extends": "../../tsconfig.cjs.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./dist/cjs"
+    },
+    "include": ["./src/**/*.ts", "./src/**/*.json", "./src/**/*.d.ts", "./src/**/*.tsx"],
+    "references": [
+        {
+            "path": "../util/tsconfig.cjs.json"
+        },
+        {
+            "path": "../types/tsconfig.cjs.json"
+        }
+    ]
+}

package/tsconfig.json

@@ -0,0 +1,16 @@
+{
+    "extends": "../../tsconfig.esm.json",
+    "compilerOptions": {
+        "rootDir": "./src",
+        "outDir": "./dist"
+    },
+    "include": ["src", "src/**/*.json"],
+    "references": [
+        {
+            "path": "../util"
+        },
+        {
+            "path": "../types"
+        }
+    ]
+}