@helloao/cli 0.0.6 → 0.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helloao/cli",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "A CLI and related tools for managing HelloAO's Free Bible API",
   "module": "index.js",
   "types": "index.d.ts",

package/actions.d.ts

@@ -1,121 +0,0 @@
-import { UploadApiFromDatabaseOptions, UploadApiOptions } from './uploads';
-export interface InitDbOptions {
-    /**
-     * The path to the source database to copy the schema from.
-     */
-    source?: string;
-    /**
-     * The languages to copy from the source database. If not specified, then all languages will be copied.
-     */
-    language?: string[];
-}
-/**
- * Initializes a new Bible API DB.
- * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
- * @param options The options for the initialization.
- */
-export declare function initDb(dbPath: string | null, options: InitDbOptions): Promise<void>;
-export interface ImportTranslationOptions {
-    /**
-     * Whether to forcibly import the translations, even if they have already been imported.
-     */
-    overwrite?: boolean;
-}
-/**
- * Imports a translation from the given directory into the database in the current working directory.
- * @param dir The directory that the translation is located in.
- * @param dirs Any extra directories that should be imported.
- * @param options The options for the import.
- */
-export declare function importTranslation(dir: string, dirs: string[], options: ImportTranslationOptions): Promise<void>;
-/**
- * Imports all the translations from the given directory into the database in the current working directory.
- * @param dir The directory that the translations are located in.
- * @param options The options.
- */
-export declare function importTranslations(dir: string, options: ImportTranslationOptions): Promise<void>;
-export interface FetchTranslationsOptions {
-    /**
-     * Fetch all translations. If omitted, only undownloaded translations will be fetched.
-     */
-    all?: boolean;
-}
-/**
- * Fetches the specified translations from fetch.bible and places them in the given directory.
- * @param dir The directory that the translations should be placed in.
- * @param translations The translations that should be downloaded. If not specified, then all translations will be downloaded.
- * @param options The options.
- */
-export declare function fetchTranslations(dir: string, translations?: string[], options?: FetchTranslationsOptions): Promise<void>;
-/**
- * Fetches the specified audio translations and places them in the given directory.
- * Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
- * @param dir The directory that the translations should be placed in.
- * @param translations The translations that should be downloaded.
- * @param options The options.
- */
-export declare function fetchAudio(dir: string, translations: string[], options?: FetchTranslationsOptions): Promise<void>;
-/**
- * Generates the translation files directly from the translations stored in the given input directory.
- * @param input The input directory that the translations are stored in.
- * @param dest The destination to upload the API files to.
- * @param options The options for the generation.
- */
-export declare function generateTranslationsFiles(input: string, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
-/**
- * Generates the translation files directly from the translation stored in the given input directory.
- * @param input The input directory that the translation is stored in.
- * @param dest The destination to upload the API files to.
- * @param options The options for the generation.
- */
-export declare function generateTranslationFiles(input: string, dest: string, options: UploadApiOptions): Promise<void>;
-/**
- * The options for uploading the test translations.
- */
-export interface UploadTestTranslationOptions extends UploadApiOptions {
-    /**
-     * The s3 URL to upload the translations to.
-     * Defaults to "s3://ao-bible-api-public-uploads"
-     */
-    s3Url?: string;
-}
-export interface UploadTestTranslationResult {
-    /**
-     * The S3 URL where the translations were uploaded to.
-     */
-    uploadS3Url: string;
-    /**
-     * The HTTP URL that the version can be accessed at.
-     */
-    url: string;
-    /**
-     * The URL that the available translations can be accessed at.
-     */
-    availableTranslationsUrl: string;
-    /**
-     * The version that was uploaded.
-     * This is a SHA-256 hash of the input files.
-     */
-    version: string;
-}
-/**
- * Generates the API files directly from the translations stored in the given input directory and
- * uploads them to the HelloAO test s3 bucket.
- *
- * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
- *
- * @param input The input directory that the translations are stored in.
- * @param options The options to use for the upload.
- */
-export declare function uploadTestTranslations(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult>;
-/**
- * Generates the API files directly from the given translation and
- * uploads them to the HelloAO test s3 bucket.
- *
- * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
- *
- * @param input The input directory that the translations are stored in.
- * @param options The options to use for the upload.
- */
-export declare function uploadTestTranslation(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult>;
-//# sourceMappingURL=actions.d.ts.map

package/actions.js

@@ -1,378 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.initDb = initDb;
-exports.importTranslation = importTranslation;
-exports.importTranslations = importTranslations;
-exports.fetchTranslations = fetchTranslations;
-exports.fetchAudio = fetchAudio;
-exports.generateTranslationsFiles = generateTranslationsFiles;
-exports.generateTranslationFiles = generateTranslationFiles;
-exports.uploadTestTranslations = uploadTestTranslations;
-exports.uploadTestTranslation = uploadTestTranslation;
-const node_path_1 = __importStar(require("node:path"));
-const database = __importStar(require("./db"));
-const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
-const linkedom_1 = require("linkedom");
-const promises_1 = require("node:fs/promises");
-const fetch_client_1 = require("@gracious.tech/fetch-client");
-const utils_1 = require("@helloao/tools/utils");
-const fs_extra_1 = require("fs-extra");
-const audio_1 = require("@helloao/tools/generation/audio");
-const book_order_1 = require("@helloao/tools/generation/book-order");
-const downloads_1 = require("./downloads");
-const iterators_1 = require("@helloao/tools/parser/iterators");
-const files_1 = require("./files");
-const dataset_1 = require("@helloao/tools/generation/dataset");
-const uploads_1 = require("./uploads");
-const s3_1 = require("./s3");
-/**
- * Initializes a new Bible API DB.
- * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
- * @param options The options for the initialization.
- */
-async function initDb(dbPath, options) {
-    console.log('Initializing new Bible API DB...');
-    if (options.source) {
-        const db = new better_sqlite3_1.default(database.getDbPath(dbPath), {});
-        const sourcePath = node_path_1.default.resolve(options.source);
-        try {
-            console.log('Copying schema from source DB...');
-            if (options.language) {
-                console.log('Copying only the following languages:', options.language);
-                const languages = `(${options.language
-                    .map((l) => `'${l}'`)
-                    .join(', ')})`;
-                db.exec(`
-                    ATTACH DATABASE "${sourcePath}" AS source;
-
-                    CREATE TABLE "_prisma_migrations" AS SELECT * FROM source._prisma_migrations;
-                    
-                    CREATE TABLE "Translation" AS SELECT * FROM source.Translation
-                    WHERE language IN ${languages};
-
-                    CREATE TABLE "Book" AS SELECT * FROM source.Book
-                    INNER JOIN source.Translation ON source.Translation.id = source.Book.translationId
-                    WHERE source.Translation.language IN ${languages};
-
-                    CREATE TABLE "Chapter" AS SELECT * FROM source.Chapter
-                    INNER JOIN source.Translation ON source.Translation.id = source.Chapter.translationId
-                    WHERE source.Translation.language IN ${languages};
-
-                    CREATE TABLE "ChapterVerse" AS SELECT * FROM source.ChapterVerse
-                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterVerse.translationId
-                    WHERE source.Translation.language IN ${languages};
-
-                    CREATE TABLE "ChapterFootnote" AS SELECT * FROM source.ChapterFootnote
-                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterFootnote.translationId
-                    WHERE source.Translation.language IN ${languages};
-
-                    CREATE TABLE "ChapterAudioUrl" AS SELECT * FROM source.ChapterAudioUrl
-                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterAudioUrl.translationId
-                    WHERE source.Translation.language IN ${languages};
-                `);
-            }
-            else {
-                db.exec(`
-                    ATTACH DATABASE "${sourcePath}" AS source;
-
-                    CREATE TABLE "_prisma_migrations" AS SELECT * FROM source._prisma_migrations;
-                    CREATE TABLE "Translation" AS SELECT * FROM source.Translation;
-                    CREATE TABLE "Book" AS SELECT * FROM source.Book;
-                    CREATE TABLE "Chapter" AS SELECT * FROM source.Chapter;
-                    CREATE TABLE "ChapterVerse" AS SELECT * FROM source.ChapterVerse;
-                    CREATE TABLE "ChapterFootnote" AS SELECT * FROM source.ChapterFootnote;
-                    CREATE TABLE "ChapterAudioUrl" AS SELECT * FROM source.ChapterAudioUrl;
-                `);
-            }
-            console.log('Done.');
-        }
-        finally {
-            db.close();
-        }
-    }
-    else {
-        const db = await database.getDb(database.getDbPath(dbPath));
-        db.close();
-    }
-}
-/**
- * Imports a translation from the given directory into the database in the current working directory.
- * @param dir The directory that the translation is located in.
- * @param dirs Any extra directories that should be imported.
- * @param options The options for the import.
- */
-async function importTranslation(dir, dirs, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const db = await database.getDbFromDir(process.cwd());
-    try {
-        await database.importTranslations(db, [dir, ...dirs], parser, !!options.overwrite);
-    }
-    finally {
-        db.close();
-    }
-}
-/**
- * Imports all the translations from the given directory into the database in the current working directory.
- * @param dir The directory that the translations are located in.
- * @param options The options.
- */
-async function importTranslations(dir, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const db = await database.getDbFromDir(process.cwd());
-    try {
-        const files = await (0, promises_1.readdir)(dir);
-        const translationDirs = files.map((f) => node_path_1.default.resolve(dir, f));
-        console.log(`Importing ${translationDirs.length} translations`);
-        await database.importTranslations(db, translationDirs, parser, !!options.overwrite);
-    }
-    finally {
-        db.close();
-    }
-}
-/**
- * Fetches the specified translations from fetch.bible and places them in the given directory.
- * @param dir The directory that the translations should be placed in.
- * @param translations The translations that should be downloaded. If not specified, then all translations will be downloaded.
- * @param options The options.
- */
-async function fetchTranslations(dir, translations, options = {}) {
-    const translationsSet = new Set(translations);
-    const client = new fetch_client_1.BibleClient({
-        remember_fetches: false,
-    });
-    const collection = await client.fetch_collection();
-    const collectionTranslations = collection.get_translations();
-    console.log(`Discovered ${collectionTranslations.length} translations`);
-    const filtered = translations && translations.length <= 0
-        ? collectionTranslations
-        : collectionTranslations.filter((t) => translationsSet.has(t.id));
-    let batches = [];
-    while (filtered.length > 0) {
-        batches.push(filtered.splice(0, 10));
-    }
-    console.log(`Downloading ${filtered.length} translations in ${batches.length} batches`);
-    for (let i = 0; i < batches.length; i++) {
-        const batch = batches[i];
-        console.log(`Downloading batch ${i + 1} of ${batches.length}`);
-        const translations = await Promise.all(batch.map(async (t) => {
-            const id = (0, utils_1.getTranslationId)(t.id);
-            const translation = {
-                id,
-                name: (0, utils_1.getFirstNonEmpty)(t.name_local, t.name_english, t.name_abbrev),
-                direction: (0, utils_1.getFirstNonEmpty)(t.direction, 'ltr'),
-                englishName: (0, utils_1.getFirstNonEmpty)(t.name_english, t.name_abbrev, t.name_local),
-                language: (0, utils_1.normalizeLanguage)(t.language),
-                licenseUrl: t.attribution_url,
-                shortName: (0, utils_1.getFirstNonEmpty)(t.name_abbrev, id),
-                website: t.attribution_url,
-            };
-            const books = await Promise.all(collection.get_books(t.id).map(async (b) => {
-                const name = `${b.id}.usx`;
-                if (!options.all &&
-                    (await (0, fs_extra_1.exists)(node_path_1.default.resolve(dir, translation.id, name)))) {
-                    return null;
-                }
-                const content = await collection.fetch_book(t.id, b.id, 'usx');
-                const contentString = content.get_whole();
-                const file = {
-                    fileType: 'usx',
-                    content: contentString,
-                    metadata: {
-                        translation,
-                    },
-                    name,
-                };
-                return file;
-            }));
-            return {
-                translation,
-                books,
-            };
-        }));
-        console.log(`Writing batch ${i + 1} of ${batches.length}`);
-        let promises = [];
-        for (let { translation, books } of translations) {
-            for (let book of books) {
-                if (!book) {
-                    continue;
-                }
-                if (!book.name) {
-                    throw new Error('Book name is required');
-                }
-                const fullPath = node_path_1.default.resolve(dir, translation.id, book.name);
-                await (0, promises_1.mkdir)(node_path_1.default.dirname(fullPath), { recursive: true });
-                const promise = (0, promises_1.writeFile)(fullPath, book.content);
-                promises.push(promise);
-            }
-            const translationPath = node_path_1.default.resolve(dir, translation.id, 'metadata.json');
-            await (0, promises_1.mkdir)(node_path_1.default.dirname(translationPath), { recursive: true });
-            const translationData = JSON.stringify(translation, null, 2);
-            promises.push((0, promises_1.writeFile)(translationPath, translationData));
-        }
-        await Promise.all(promises);
-    }
-}
-/**
- * Fetches the specified audio translations and places them in the given directory.
- * Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
- * @param dir The directory that the translations should be placed in.
- * @param translations The translations that should be downloaded.
- * @param options The options.
- */
-async function fetchAudio(dir, translations, options = {}) {
-    for (let translation of translations) {
-        const [translationId, reader] = translation.split('/');
-        const generator = audio_1.KNOWN_AUDIO_TRANSLATIONS.get(translationId)?.get(reader);
-        if (!generator) {
-            console.warn('Unknown translation:', translation);
-            continue;
-        }
-        for (let [bookId, chapters] of book_order_1.bookChapterCountMap) {
-            for (let chapter = 1; chapter <= chapters; chapter++) {
-                const url = generator(bookId, chapter);
-                const ext = (0, node_path_1.extname)(url);
-                const [translationId, reader] = translation.split('/');
-                const name = `${chapter}.${reader}${ext}`;
-                const fullPath = node_path_1.default.resolve(dir, 'audio', translationId, bookId, name);
-                if (!options.all && (await (0, fs_extra_1.exists)(fullPath))) {
-                    continue;
-                }
-                await (0, downloads_1.downloadFile)(url, fullPath);
-            }
-        }
-    }
-}
-/**
- * Generates the translation files directly from the translations stored in the given input directory.
- * @param input The input directory that the translations are stored in.
- * @param dest The destination to upload the API files to.
- * @param options The options for the generation.
- */
-async function generateTranslationsFiles(input, dest, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const dirs = await (0, promises_1.readdir)(node_path_1.default.resolve(input));
-    const batchSize = typeof options.batchSize === 'number'
-        ? options.batchSize
-        : parseInt(options.batchSize);
-    for (let b of (0, iterators_1.batch)(dirs, batchSize)) {
-        const files = await (0, files_1.loadTranslationsFiles)(b);
-        const dataset = (0, dataset_1.generateDataset)(files, parser);
-        await (0, uploads_1.serializeAndUploadDatasets)(dest, (0, iterators_1.toAsyncIterable)([dataset]), options);
-    }
-}
-/**
- * Generates the translation files directly from the translation stored in the given input directory.
- * @param input The input directory that the translation is stored in.
- * @param dest The destination to upload the API files to.
- * @param options The options for the generation.
- */
-async function generateTranslationFiles(input, dest, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const files = await (0, files_1.loadTranslationFiles)(node_path_1.default.resolve(input));
-    const dataset = (0, dataset_1.generateDataset)(files, parser);
-    await (0, uploads_1.serializeAndUploadDatasets)(dest, (0, iterators_1.toAsyncIterable)([dataset]), options);
-}
-/**
- * Generates the API files directly from the translations stored in the given input directory and
- * uploads them to the HelloAO test s3 bucket.
- *
- * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
- *
- * @param input The input directory that the translations are stored in.
- * @param options The options to use for the upload.
- */
-async function uploadTestTranslations(input, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const dirs = await (0, promises_1.readdir)(node_path_1.default.resolve(input));
-    const files = await (0, files_1.loadTranslationsFiles)(dirs);
-    const hash = (0, files_1.hashInputFiles)(files);
-    const dataset = (0, dataset_1.generateDataset)(files, parser);
-    const url = options.s3Url || 's3://ao-bible-api-public-uploads';
-    await (0, uploads_1.serializeAndUploadDatasets)(url, (0, iterators_1.toAsyncIterable)([dataset]), {
-        ...options,
-        pathPrefix: `/${hash}`,
-    });
-    const urls = getUrls(url);
-    return {
-        ...urls,
-        version: hash,
-        availableTranslationsUrl: `${urls.url}/${hash}/api/available_translations.json`,
-    };
-}
-/**
- * Generates the API files directly from the given translation and
- * uploads them to the HelloAO test s3 bucket.
- *
- * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
- *
- * @param input The input directory that the translations are stored in.
- * @param options The options to use for the upload.
- */
-async function uploadTestTranslation(input, options) {
-    const parser = new linkedom_1.DOMParser();
-    globalThis.DOMParser = linkedom_1.DOMParser;
-    globalThis.Element = linkedom_1.Element;
-    globalThis.Node = linkedom_1.Node;
-    const files = await (0, files_1.loadTranslationFiles)(node_path_1.default.resolve(input));
-    const hash = (0, files_1.hashInputFiles)(files);
-    const dataset = (0, dataset_1.generateDataset)(files, parser);
-    const url = options.s3Url || 's3://ao-bible-api-public-uploads';
-    await (0, uploads_1.serializeAndUploadDatasets)(url, (0, iterators_1.toAsyncIterable)([dataset]), {
-        ...options,
-        pathPrefix: `/${hash}`,
-    });
-    const urls = getUrls(url);
-    return {
-        ...urls,
-        version: hash,
-        availableTranslationsUrl: `${urls.url}/${hash}/api/available_translations.json`,
-    };
-}
-function getUrls(dest) {
-    const url = (0, s3_1.getHttpUrl)(dest);
-    return {
-        uploadS3Url: dest,
-        url: url,
-    };
-}

package/cli.d.ts

@@ -1,3 +0,0 @@
-#!/usr/bin/env node
-export {};
-//# sourceMappingURL=cli.d.ts.map