@prosopo/datasets-fs 0.1.17 → 0.1.18

package/src/index.ts

@@ -1,14 +0,0 @@
-// Copyright 2021-2023 Prosopo (UK) Ltd.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-export * from './cli.js'

package/src/labels/args.ts

@@ -1,9 +0,0 @@
-// Args for generating a dataset
-
-import { z } from 'zod'
-
-export const argsSchema = z.object({
-    data: z.string(),
-})
-
-export type Args = z.infer<typeof argsSchema>

package/src/labels/cli.ts

@@ -1,21 +0,0 @@
-import { ArgumentsCamelCase, Argv } from 'yargs'
-import { Logger } from '@prosopo/common'
-import { argsSchema } from './args.js'
-import labels from './labels.js'
-
-export default (cmdArgs?: { logger?: Logger }) => {
-    return {
-        command: 'labels',
-        describe: 'get all labels from some data',
-        builder: (yargs: Argv) => {
-            return yargs.option('data', {
-                type: 'string',
-                demand: true,
-                description: 'JSON file containing data',
-            })
-        },
-        handler: async (argv: ArgumentsCamelCase) => {
-            await labels(argsSchema.parse(argv), cmdArgs?.logger)
-        },
-    }
-}

package/src/labels/index.ts

@@ -1,3 +0,0 @@
-export * from './labels.js'
-export * from './cli.js'
-export * from './args.js'

package/src/labels/labels.ts

@@ -1,27 +0,0 @@
-import { Args } from './args.js'
-import { LabelledDataSchema, LabelledItem } from '@prosopo/types'
-import { Logger, ProsopoEnvError, getLoggerDefault } from '@prosopo/common'
-import fs from 'fs'
-
-export default async (args: Args, logger?: Logger) => {
-    logger = logger || getLoggerDefault()
-
-    logger.debug(args, 'reading labels...')
-
-    const file = args.data
-    if (!fs.existsSync(file)) {
-        throw new ProsopoEnvError(new Error(`file does not exist: ${file}`), 'FS.FILE_NOT_FOUND')
-    }
-
-    const labelled: LabelledItem[] = file
-        ? LabelledDataSchema.parse(JSON.parse(fs.readFileSync(file, 'utf8'))).items
-        : []
-
-    const labels = new Set<string>()
-    for (const item of labelled) {
-        labels.add(item.label)
-    }
-    const labelArray = Array.from(labels)
-    labelArray.sort()
-    logger.log(JSON.stringify({ labels: labelArray }, null, 4))
-}

package/src/relocate/args.ts

@@ -1,11 +0,0 @@
-// Args for generating a dataset
-
-import { z } from 'zod'
-
-export const argsSchema = z.object({
-    data: z.string(),
-    from: z.string(),
-    to: z.string(),
-})
-
-export type Args = z.infer<typeof argsSchema>

package/src/relocate/cli.ts

@@ -1,32 +0,0 @@
-import { ArgumentsCamelCase, Argv } from 'yargs'
-import { Logger } from '@prosopo/common'
-import { argsSchema } from './args.js'
-import relocate from './relocate.js'
-
-export default (cmdArgs?: { logger?: Logger }) => {
-    return {
-        command: 'relocate',
-        describe:
-            'Relocate a dataset by replacing the old urls with new ones. E.g. "example.com/1.jpg" to "newwebsite.com/1.jpg"',
-        builder: (yargs: Argv) => {
-            return yargs
-                .option('from', {
-                    type: 'string',
-                    demand: true,
-                    description: 'The old url to replace',
-                })
-                .option('to', {
-                    type: 'string',
-                    demand: true,
-                    description: 'The new url to replace the old one with',
-                })
-                .option('data', {
-                    type: 'string',
-                    description: 'Path to the images JSON containing the urls of images to replace',
-                })
-        },
-        handler: async (argv: ArgumentsCamelCase) => {
-            await relocate(argsSchema.parse(argv), cmdArgs?.logger)
-        },
-    }
-}

package/src/relocate/relocate.ts

@@ -1,38 +0,0 @@
-import { Args } from './args.js'
-import { Logger, getLoggerDefault } from '@prosopo/common'
-import fs from 'fs'
-
-export default async (args: Args, logger?: Logger) => {
-    logger = logger || getLoggerDefault()
-    logger.debug(args, 'relocating...')
-
-    const replace = (data: unknown, from: string, to: string) => {
-        if (Array.isArray(data)) {
-            for (let i = 0; i < data.length; i++) {
-                data[i] = replace(data[i], from, to)
-            }
-        } else if (typeof data === 'object') {
-            const obj = data as object
-            for (const key of Object.keys(obj)) {
-                if (key === 'data') {
-                    const value = obj[key]
-                    if (value.startsWith(from)) {
-                        obj[key] = to + value.slice(from.length)
-                    }
-                } else {
-                    obj[key] = replace(obj[key], from, to)
-                }
-            }
-        }
-        return data
-    }
-
-    const file: string = args.data
-    logger.log(`relocating data in ${file} from ${args.from} to ${args.to}`)
-    // read the file
-    let data = JSON.parse(fs.readFileSync(file, 'utf8'))
-    // replace the urls by recursively traversing the data
-    data = replace(data, args.from, args.to)
-    // write the file
-    fs.writeFileSync(file, JSON.stringify(data, null, 4))
-}

package/src/scale/args.ts

@@ -1,13 +0,0 @@
-// Args for generating a dataset
-
-import { z } from 'zod'
-
-export const argsSchema = z.object({
-    data: z.string(),
-    out: z.string(),
-    overwrite: z.boolean().optional(),
-    size: z.number(),
-    square: z.boolean().optional(),
-})
-
-export type Args = z.infer<typeof argsSchema>

package/src/scale/cli.ts

@@ -1,43 +0,0 @@
-import { ArgumentsCamelCase, Argv } from 'yargs'
-import { Logger } from '@prosopo/common'
-import { argsSchema } from './args.js'
-import scale from './scale.js'
-
-export default (cmdArgs?: { logger?: Logger }) => {
-    return {
-        command: 'scale',
-        describe: 'Scale images down to a given size',
-        builder: (yargs: Argv) => {
-            return yargs
-                .option('data', {
-                    type: 'string',
-                    demand: true,
-                    description: 'JSON file containing a list of objects with (at least) a url',
-                })
-                .option('out', {
-                    type: 'string',
-                    demand: true,
-                    description:
-                        'Where to put the output directory containing the map file and single directory of images. The map file will contain the new urls of the scaled images, new hashes and pass through any other information, e.g. labels.',
-                })
-                .option('overwrite', {
-                    type: 'boolean',
-                    description: 'Overwrite the output if it already exists',
-                })
-                .option('size', {
-                    type: 'number',
-                    demand: true,
-                    description:
-                        'The dimension (height/width) of the scaled image. If the image is not square, the other dimension will be scaled to maintain the aspect ratio.',
-                })
-                .option('square', {
-                    type: 'boolean',
-                    description:
-                        'If true, the image will be cropped to a square before scaling. If false, the image will be scaled to the given size, maintaining the aspect ratio.',
-                })
-        },
-        handler: async (argv: ArgumentsCamelCase) => {
-            await scale(argsSchema.parse(argv), cmdArgs?.logger)
-        },
-    }
-}

package/src/scale/index.ts

@@ -1,3 +0,0 @@
-export * from './scale.js'
-export * from './cli.js'
-export * from './args.js'

package/src/scale/scale.ts

@@ -1,79 +0,0 @@
-import { Args } from './args.js'
-import { Data, DataSchema, Item } from '@prosopo/types'
-import { Logger, ProsopoEnvError, getLoggerDefault } from '@prosopo/common'
-import { blake2b } from '@noble/hashes/blake2b'
-import { u8aToHex } from '@polkadot/util'
-import fs from 'fs'
-import sharp from 'sharp'
-
-export default async (args: Args, logger?: Logger) => {
-    logger = logger || getLoggerDefault()
-
-    logger.debug(args, 'scaling...')
-
-    const size = args.size
-    const square = args.square ?? false
-
-    const mapFile: string = args.data
-    if (!fs.existsSync(mapFile)) {
-        throw new ProsopoEnvError(new Error(`Map file does not exist: ${mapFile}`), 'FS.FILE_NOT_FOUND')
-    }
-    const outDir: string = args.out
-    const overwrite = args.overwrite || false
-    if (!overwrite && fs.existsSync(outDir)) {
-        throw new ProsopoEnvError(new Error(`Output directory already exists: ${outDir}`), 'FS.FILE_NOT_FOUND')
-    }
-
-    // create the output directory
-    const imgDir = `${outDir}/images`
-    fs.mkdirSync(imgDir, { recursive: true })
-
-    // read the map file
-    const inputItems: Item[] = DataSchema.parse(JSON.parse(fs.readFileSync(mapFile, 'utf8'))).items
-
-    // for each item
-    const outputItems: Item[] = []
-    for (const inputItem of inputItems) {
-        logger.log(`scaling ${inputItem.data}`)
-        // read the file
-        const img = fs.readFileSync(inputItem.data)
-        // resize the image
-        const resized = await sharp(img)
-            .resize({
-                width: size,
-                height: size,
-                fit: square ? 'fill' : 'inside',
-            })
-            .png()
-        const tmpFilePath = `${imgDir}/tmp.png`
-        await resized.toFile(tmpFilePath)
-        // read the resized image
-        const resizedImg = fs.readFileSync(tmpFilePath)
-        // hash the image
-        const hash = blake2b(resizedImg)
-        const hex = u8aToHex(hash)
-        // move the image
-        const finalFilePath = `${imgDir}/${hex}.png`
-        fs.renameSync(tmpFilePath, finalFilePath)
-
-        // add the item to the output
-        const outputItem: Item = {
-            ...inputItem,
-            hash: hex,
-            data: fs.realpathSync(finalFilePath),
-        }
-        outputItems.push(outputItem)
-    }
-
-    // write the map file
-    const outputMapFile = `${outDir}/map.json`
-
-    const data: Data = {
-        items: outputItems,
-    }
-
-    // verify the output
-    DataSchema.parse(data)
-
-    fs.writeFileSync(outputMapFile, JSON.stringify(outputItems, null, 4))
-}

package/tsconfig.json

@@ -1,17 +0,0 @@
-{
-    "extends": "../../tsconfig.build.json",
-    "compilerOptions": {
-        "rootDir": "./src",
-        "outDir": "./dist"
-    },
-    "include": ["./src/**/*.ts"],
-    "exclude": ["**/*.test.ts"],
-    "references": [
-        {
-            "path": "../util"
-        },
-        {
-            "path": "../types"
-        }
-    ]
-}