@helloao/cli 0.0.5 → 0.0.7

package/files.d.ts

@@ -1,113 +0,0 @@
-import { InputFile, OutputFile } from "@helloao/tools/generation/common-types";
-import { Readable } from "stream";
-/**
- * Defines an interface that contains information about a serialized file.
- */
-export interface Uploader {
-    /**
-     * Gets the ideal batch size for the uploader.
-     * Null if the uploader does not need batching.
-     */
-    idealBatchSize: number | null;
-    /**
-     * Uploads the given file.
-     * @param file The file to upload.
-     * @param overwrite Whether the file should be overwritten if it already exists.
-     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
-     */
-    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
-    /**
-     * Disposes resources that the uploader uses.
-     */
-    dispose?(): Promise<void>;
-}
-/**
- * Defines an interface for a file that has been serialized.
- */
-export interface SerializedFile {
-    path: string;
-    content: string | Readable;
-    /**
-     * Gets the base64-encoded SHA256 hash of the content of the file.
-     */
-    sha256?(): string;
-}
-/**
- * The options for serializing API files.
- */
-export interface SerializeApiOptions {
-    /**
-     * Whether the output should be pretty-printed.
-     */
-    pretty?: boolean;
-}
-/**
- * Serializes the given output files into serialized files using the given options.
- *
- * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
- *
- * @param files The files that should be serialized.
- * @param options The options for serialization.
- */
-export declare function serializeOutputFiles(files: AsyncIterable<OutputFile[]>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
-/**
- * Serializes the given output file content into a serialized file.
- * @param path The path that the file should be saved to.
- * @param content The content of the file.
- * @param options The options for serialization.
- */
-export declare function serializeFile(path: string, content: OutputFile['content'], options: SerializeApiOptions): Promise<SerializedFile | null>;
-/**
- * Loads the files for the given translations.
- * @param dir The directory that the translations exist in.
- */
-export declare function loadTranslationsFiles(dirs: string[]): Promise<InputFile[]>;
-/**
- * Loads the files for the given translation.
- * @param translation The directory that the translation exists in.
- * @returns
- */
-export declare function loadTranslationFiles(translation: string): Promise<InputFile[]>;
-export interface CollectionTranslationMetadata {
-    name: {
-        local: string;
-        abbrev: string;
-        english: string;
-    };
-    language: string;
-    year: number;
-    direction: 'ltr' | 'rtl';
-    copyright: {
-        licenses: any[];
-        attribution: string;
-        attribution_url: string;
-    };
-    id: string | null;
-    source: {
-        id: string;
-    };
-}
-/**
- * Defines an uploader that is able to upload files to a directory.
- */
-export declare class FilesUploader implements Uploader {
-    private _dir;
-    constructor(dir: string);
-    get idealBatchSize(): number | null;
-    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
-}
-/**
- * Defines an uploader that is able to upload files into a zip file.
- */
-export declare class ZipUploader implements Uploader {
-    private _path;
-    private _initPromise;
-    private _fileHandle;
-    private _zip;
-    constructor(filePath: string);
-    private _init;
-    get idealBatchSize(): number | null;
-    upload(file: SerializedFile, _overwrite: boolean): Promise<boolean>;
-    dispose(): Promise<void>;
-}
-//# sourceMappingURL=files.d.ts.map

package/files.js

@@ -1,341 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ZipUploader = exports.FilesUploader = void 0;
-exports.serializeOutputFiles = serializeOutputFiles;
-exports.serializeFile = serializeFile;
-exports.loadTranslationsFiles = loadTranslationsFiles;
-exports.loadTranslationFiles = loadTranslationFiles;
-const promises_1 = require("fs/promises");
-const path_1 = require("path");
-const path = __importStar(require("path"));
-const fs_extra_1 = require("fs-extra");
-const zip_js_1 = require("@zip.js/zip.js");
-const stream_1 = require("stream");
-const hash_js_1 = require("hash.js");
-const usx_parser_1 = require("@helloao/tools/parser/usx-parser");
-const lodash_1 = require("lodash");
-const base64_js_1 = require("base64-js");
-/**
- * Serializes the given output files into serialized files using the given options.
- *
- * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
- *
- * @param files The files that should be serialized.
- * @param options The options for serialization.
- */
-async function* serializeOutputFiles(files, options) {
-    const mergableFiles = new Map();
-    for await (let batch of files) {
-        let serializedFiles = [];
-        for (let file of batch) {
-            if (file.mergable) {
-                let arr = mergableFiles.get(file.path);
-                if (!arr) {
-                    arr = [];
-                    mergableFiles.set(file.path, arr);
-                }
-                arr.push(file);
-                continue;
-            }
-            const serialized = await serializeFile(file.path, file.content, options);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-        yield serializedFiles;
-    }
-    let serializedFiles = [];
-    for (let [path, files] of mergableFiles) {
-        let content = {};
-        for (let file of files) {
-            if (!content) {
-                content = file.content;
-            }
-            else {
-                content = (0, lodash_1.mergeWith)(content, file.content, (objValue, srcValue) => {
-                    if (Array.isArray(objValue)) {
-                        return objValue.concat(srcValue);
-                    }
-                    return undefined;
-                });
-            }
-        }
-        if (content) {
-            const serialized = await serializeFile(path, content, options);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-    }
-    yield serializedFiles;
-}
-/**
- * Serializes the given output file content into a serialized file.
- * @param path The path that the file should be saved to.
- * @param content The content of the file.
- * @param options The options for serialization.
- */
-async function serializeFile(path, content, options) {
-    let fileContent;
-    if (typeof content === 'function') {
-        fileContent = await content();
-    }
-    else {
-        fileContent = content;
-    }
-    const ext = (0, path_1.extname)(path);
-    if (ext === '.json') {
-        let json;
-        if (fileContent instanceof ReadableStream) {
-            json = '';
-            for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
-                encoding: 'utf-8'
-            })) {
-                json += chunk;
-            }
-        }
-        else {
-            json = JSON.stringify(content, undefined, options.pretty ? 2 : undefined);
-        }
-        return {
-            path,
-            content: json,
-            sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest()))
-        };
-    }
-    else if (ext === '.mp3') {
-        if (fileContent instanceof ReadableStream) {
-            return {
-                path,
-                content: stream_1.Readable.fromWeb(fileContent),
-            };
-        }
-        else {
-            console.warn('Expected content to be a readable stream for', path);
-            console.warn('Skipping file');
-            return null;
-        }
-    }
-    console.warn('Unknown file type', path);
-    console.warn('Skipping file');
-    return null;
-}
-/**
- * Loads the files for the given translations.
- * @param dir The directory that the translations exist in.
- */
-async function loadTranslationsFiles(dirs) {
-    const promises = [];
-    for (let dir of dirs) {
-        const fullPath = path.resolve(dir);
-        promises.push(loadTranslationFiles(fullPath));
-    }
-    const allFiles = await Promise.all(promises);
-    const files = allFiles.flat();
-    return files;
-}
-/**
- * Loads the files for the given translation.
- * @param translation The directory that the translation exists in.
- * @returns
- */
-async function loadTranslationFiles(translation) {
-    const metadata = await loadTranslationMetadata(translation);
-    if (!metadata) {
-        console.error('Could not load metadata for translation!', translation);
-        return [];
-    }
-    let files = await (0, promises_1.readdir)(translation);
-    let usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm' || (0, path_1.extname)(f) === '.usx' || (0, path_1.extname)(f) === '.json');
-    if (usfmFiles.length <= 0) {
-        translation = path.resolve(translation, 'usfm');
-        if ((0, fs_extra_1.existsSync)(translation)) {
-            files = await (0, promises_1.readdir)(translation);
-            usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm');
-        }
-    }
-    if (usfmFiles.length <= 0) {
-        console.error('Could not find USFM files for translation!', translation);
-        return [];
-    }
-    let promises = [];
-    for (let file of usfmFiles) {
-        if (path.parse(file).name === 'metadata') {
-            continue;
-        }
-        const filePath = path.resolve(translation, file);
-        promises.push(loadFile(filePath, {
-            translation: metadata
-        }));
-    }
-    return await Promise.all(promises);
-}
-/**
- * Loads the metadata for the given translation.
- * @param translation The translation that the metadata should be loaded for.
- * @returns
- */
-async function loadTranslationMetadata(translation) {
-    const metadataTs = path.resolve(translation, 'metadata.ts');
-    if ((0, fs_extra_1.existsSync)(metadataTs)) {
-        return (await Promise.resolve(`${metadataTs}`).then(s => __importStar(require(s)))).default;
-    }
-    else {
-        const metadataJson = path.resolve(translation, 'meta.json');
-        if ((0, fs_extra_1.existsSync)(metadataJson)) {
-            const data = await (0, promises_1.readFile)(metadataJson, { encoding: 'utf-8' });
-            const metadata = JSON.parse(data);
-            return {
-                id: metadata.id ?? metadata.source.id,
-                language: metadata.language,
-                name: metadata.name.local,
-                englishName: metadata.name.english,
-                licenseUrl: metadata.copyright.attribution_url,
-                website: metadata.copyright.attribution_url,
-                shortName: metadata.name.abbrev,
-                direction: metadata.direction
-            };
-        }
-        else {
-            const metadataJson = path.resolve(translation, 'metadata.json');
-            if ((0, fs_extra_1.existsSync)(metadataJson)) {
-                const data = await (0, promises_1.readFile)(metadataJson, { encoding: 'utf-8' });
-                return JSON.parse(data);
-            }
-        }
-    }
-    console.error('Could not find metadata for translation!', translation);
-    return null;
-}
-/**
- * Loads the file from the given path using the given metadata.
- * @param file The file that should be loaded.
- * @param metadata The metadata.
- */
-async function loadFile(file, metadata) {
-    const extension = path.extname(file);
-    const content = await (0, promises_1.readFile)(file, {
-        encoding: 'utf-8'
-    });
-    const hash = (0, hash_js_1.sha256)()
-        .update(content)
-        // Hack to ensure that file hashes are different for different versions of the parser.
-        .update(usx_parser_1.PARSER_VERSION)
-        .digest('hex');
-    return {
-        content,
-        metadata: metadata,
-        name: file,
-        sha256: hash,
-        fileType: extension.slice(1),
-    };
-}
-/**
- * Defines an uploader that is able to upload files to a directory.
- */
-class FilesUploader {
-    _dir;
-    constructor(dir) {
-        this._dir = dir;
-    }
-    get idealBatchSize() {
-        return null;
-    }
-    async upload(file, overwrite) {
-        const filePath = path.resolve(this._dir, makeRelativePath(file.path));
-        await (0, promises_1.mkdir)(path.dirname(filePath), { recursive: true });
-        if (overwrite || !(0, fs_extra_1.existsSync)(filePath)) {
-            await (0, promises_1.writeFile)(filePath, file.content, 'utf-8');
-            return true;
-        }
-        return false;
-    }
-}
-exports.FilesUploader = FilesUploader;
-/**
- * Defines an uploader that is able to upload files into a zip file.
- */
-class ZipUploader {
-    _path;
-    _initPromise;
-    _fileHandle = null;
-    _zip = null;
-    constructor(filePath) {
-        this._path = filePath;
-        this._initPromise = this._init();
-    }
-    async _init() {
-        this._fileHandle = await (0, promises_1.open)(path.resolve(this._path), 'w');
-        const writableStream = this._fileHandle.createWriteStream();
-        this._zip = new zip_js_1.ZipWriter(stream_1.Writable.toWeb(writableStream));
-        return this._zip;
-    }
-    get idealBatchSize() {
-        return 50;
-    }
-    async upload(file, _overwrite) {
-        const zip = await this._initPromise;
-        let reader;
-        if (file.content instanceof stream_1.Readable) {
-            reader = stream_1.Readable.toWeb(file.content);
-        }
-        else if (typeof file.content === 'string') {
-            reader = new zip_js_1.TextReader(file.content);
-        }
-        else {
-            throw new Error('Unknown file content type');
-        }
-        await zip.add(trimRelativePath(file.path), reader);
-        return true;
-    }
-    async dispose() {
-        if (this._zip) {
-            await this._zip.close();
-        }
-        if (this._fileHandle) {
-            await this._fileHandle.close();
-        }
-    }
-}
-exports.ZipUploader = ZipUploader;
-function trimRelativePath(path) {
-    if (path.startsWith('./')) {
-        return path.substring(2);
-    }
-    else if (path.startsWith('../')) {
-        return path.substring(3);
-    }
-    else if (path.startsWith('/')) {
-        return path.substring(1);
-    }
-    return path;
-}
-function makeRelativePath(path) {
-    if (path.startsWith('/')) {
-        return '.' + path;
-    }
-    return path;
-}

package/index.d.ts

@@ -1,8 +0,0 @@
-import * as db from './db';
-import * as downloads from './downloads';
-import * as uploads from './uploads';
-import * as actions from './actions';
-import * as files from './files';
-import * as s3 from './s3';
-export { db, downloads, uploads, actions, files, s3, };
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,38 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.s3 = exports.files = exports.actions = exports.uploads = exports.downloads = exports.db = void 0;
-const db = __importStar(require("./db"));
-exports.db = db;
-const downloads = __importStar(require("./downloads"));
-exports.downloads = downloads;
-const uploads = __importStar(require("./uploads"));
-exports.uploads = uploads;
-const actions = __importStar(require("./actions"));
-exports.actions = actions;
-const files = __importStar(require("./files"));
-exports.files = files;
-const s3 = __importStar(require("./s3"));
-exports.s3 = s3;

package/s3.d.ts

@@ -1,14 +0,0 @@
-import { SerializedFile, Uploader } from "./files";
-export declare class S3Uploader implements Uploader {
-    private _client;
-    private _bucketName;
-    private _keyPrefix;
-    get idealBatchSize(): number;
-    constructor(bucketName: string, keyPrefix: string, profile: string | null);
-    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
-}
-export declare function parseS3Url(url: string): {
-    bucketName: string;
-    objectKey: string;
-} | undefined;
-//# sourceMappingURL=s3.d.ts.map

package/s3.js

@@ -1,76 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.S3Uploader = void 0;
-exports.parseS3Url = parseS3Url;
-const client_s3_1 = require("@aws-sdk/client-s3");
-const credential_providers_1 = require("@aws-sdk/credential-providers"); // ES6 import
-class S3Uploader {
-    _client;
-    _bucketName;
-    _keyPrefix;
-    get idealBatchSize() {
-        return 50;
-    }
-    constructor(bucketName, keyPrefix, profile) {
-        this._bucketName = bucketName;
-        this._keyPrefix = keyPrefix;
-        this._client = new client_s3_1.S3Client({
-            credentials: (0, credential_providers_1.fromNodeProviderChain)({
-                profile: profile ?? undefined,
-            })
-        });
-    }
-    async upload(file, overwrite) {
-        const path = file.path.startsWith('/') ? file.path.substring(1) : file.path;
-        const key = this._keyPrefix ? `${this._keyPrefix}/${path}` : path;
-        const hash = file.sha256?.();
-        const head = new client_s3_1.HeadObjectCommand({
-            Bucket: this._bucketName,
-            Key: key,
-        });
-        if (hash || !overwrite) {
-            try {
-                const existingFile = await this._client.send(head);
-                if (hash && hash.localeCompare(existingFile?.ChecksumSHA256 ?? "", undefined, { sensitivity: 'base' }) === 0) {
-                    // File is already uploaded and matches the checksum.
-                    console.log(`[s3] Matches checksum: ${key}`);
-                    return false;
-                }
-                if (!overwrite) {
-                    return false;
-                }
-            }
-            catch (err) {
-                if (err instanceof client_s3_1.NotFound) {
-                    // not found, so we can try to write the file.
-                }
-                else {
-                    throw err;
-                }
-            }
-        }
-        const command = new client_s3_1.PutObjectCommand({
-            Bucket: this._bucketName,
-            Key: key,
-            Body: file.content,
-            ContentType: 'application/json',
-            ChecksumSHA256: hash,
-        });
-        await this._client.send(command);
-        return true;
-    }
-}
-exports.S3Uploader = S3Uploader;
-function parseS3Url(url) {
-    const regex = /^s3:\/\/([a-z0-9.\-]+)(\/[^${}]*)?$/;
-    const matched = url.match(regex);
-    if (matched) {
-        const arr = [...matched];
-        return {
-            bucketName: arr[1],
-            objectKey: arr[2] ?? "",
-        };
-    }
-    return undefined;
-}
-;

package/uploads.d.ts

@@ -1,73 +0,0 @@
-import { SerializedFile, Uploader } from "./files";
-import { DatasetOutput } from "@helloao/tools/generation/dataset";
-import { PrismaClient } from "@prisma/client";
-export interface UploadApiFromDatabaseOptions extends UploadApiOptions {
-    /**
-     * The number of files to upload in each batch.
-     */
-    batchSize: string | number;
-}
-export interface UploadApiOptions {
-    /**
-     * Whether to overwrite existing files.
-     */
-    overwrite?: boolean;
-    /**
-     * Whether to only overwrite common files.
-     * "Common files" are files that are similar between translations, like the books.json endpoint, or individual chapter endpoints.
-     */
-    overwriteCommonFiles?: boolean;
-    /**
-     * The file pattern regex that should be used to filter the files that are uploaded.
-     */
-    filePattern?: string;
-    /**
-     * The translations to generate API files for.
-     */
-    translations?: string[];
-    /**
-     * The AWS profile to use for uploading to S3.
-     */
-    profile?: string;
-    /**
-     * Whether to generate API files that use the common name instead of book IDs.
-     */
-    useCommonName?: boolean;
-    /**
-     * Whether to generate audio files for the API.
-     */
-    generateAudioFiles?: boolean;
-    /**
-     * Whether to generate pretty-printed JSON files.
-     */
-    pretty?: boolean;
-}
-/**
- * Loads and generates the API files from the database and uploads them to the specified destination.
- * @param db The database that the datasets should be loaded from.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- */
-export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
-/**
- * Generates the API files from the given datasets and uploads them to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions): Promise<void>;
-/**
- * Uploads the given serialized files to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function uploadFiles(dest: string, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
-/**
- * Uploads the given serialized files using the given uploader.
- * @param uploader The uploader to use.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function uploadFilesUsingUploader(uploader: Uploader, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
-//# sourceMappingURL=uploads.d.ts.map