@helloao/cli 0.0.7 → 0.0.8-alpha

package/dist/types/actions.d.ts

@@ -0,0 +1,153 @@
+import { InputTranslationMetadata } from '@helloao/tools/generation/index.js';
+import { UploadApiFromDatabaseOptions, UploadApiOptions } from './uploads.js';
+export interface GetTranslationsItem {
+    id: string;
+    language: string;
+    direction: 'ltr' | 'rtl';
+    year: number;
+    name_local: string;
+    name_english: string;
+    name_abbrev: string;
+    attribution: string;
+    attribution_url: string;
+    licenses: RuntimeLicense[];
+}
+export interface RuntimeLicense {
+    id: string | null;
+    name: string;
+    restrictions: MetaRestrictions;
+    url: string;
+}
+export interface MetaRestrictions {
+    limit_verses: number | null;
+    limit_book_ratio: number | null;
+    limit_content_ratio: number | null;
+    forbid_commercial: boolean;
+    forbid_derivatives: boolean | 'same-license';
+    forbid_attributionless: boolean;
+    forbid_other: boolean;
+}
+export interface InitDbOptions {
+    /**
+     * The path to the source database to copy the schema from.
+     */
+    source?: string;
+    /**
+     * The languages to copy from the source database. If not specified, then all languages will be copied.
+     */
+    language?: string[];
+}
+/**
+ * Initializes a new Bible API DB.
+ * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
+ * @param options The options for the initialization.
+ */
+export declare function initDb(dbPath: string | null, options: InitDbOptions): Promise<void>;
+export interface ImportTranslationOptions {
+    /**
+     * Whether to forcibly import the translations, even if they have already been imported.
+     */
+    overwrite?: boolean;
+}
+/**
+ * Imports a translation from the given directory into the database in the current working directory.
+ * @param dir The directory that the translation is located in.
+ * @param dirs Any extra directories that should be imported.
+ * @param options The options for the import.
+ */
+export declare function importTranslation(dir: string, dirs: string[], options: ImportTranslationOptions): Promise<void>;
+/**
+ * Imports all the translations from the given directory into the database in the current working directory.
+ * @param dir The directory that the translations are located in.
+ * @param options The options.
+ */
+export declare function importTranslations(dir: string, options: ImportTranslationOptions): Promise<void>;
+export interface FetchTranslationsOptions {
+    /**
+     * Fetch all translations. If omitted, only undownloaded translations will be fetched.
+     */
+    all?: boolean;
+}
+/**
+ * Fetches the specified translations from fetch.bible and places them in the given directory.
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded. If not specified, then all translations will be downloaded.
+ * @param options The options.
+ */
+export declare function fetchTranslations(dir: string, translations?: string[], options?: FetchTranslationsOptions): Promise<void>;
+/**
+ * Fetches the specified audio translations and places them in the given directory.
+ * Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded.
+ * @param options The options.
+ */
+export declare function fetchAudio(dir: string, translations: string[], options?: FetchTranslationsOptions): Promise<void>;
+/**
+ * Generates the translation files directly from the translations stored in the given input directory.
+ * @param input The input directory that the translations are stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+export declare function generateTranslationsFiles(input: string, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
+/**
+ * Generates the translation files directly from the translation stored in the given input directory.
+ * @param input The input directory that the translation is stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+export declare function generateTranslationFiles(input: string, dest: string, options: UploadApiOptions): Promise<void>;
+/**
+ * The options for uploading the test translations.
+ */
+export interface UploadTestTranslationOptions extends UploadApiOptions {
+    /**
+     * The s3 URL to upload the translations to.
+     * Defaults to "s3://ao-bible-api-public-uploads"
+     */
+    s3Url?: string;
+}
+export interface UploadTestTranslationResult {
+    /**
+     * The S3 URL where the translations were uploaded to.
+     */
+    uploadS3Url: string;
+    /**
+     * The HTTP URL that the version can be accessed at.
+     */
+    url: string;
+    /**
+     * The URL that the available translations can be accessed at.
+     */
+    availableTranslationsUrl: string;
+    /**
+     * The version that was uploaded.
+     * This is a SHA-256 hash of the input files.
+     */
+    version: string;
+}
+/**
+ * Generates the API files directly from the translations stored in the given input directory and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadTestTranslations(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult>;
+/**
+ * Generates the API files directly from the given translation and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadTestTranslation(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult | undefined>;
+/**
+ * Asks the user for the metadata for the translation.
+ */
+export declare function askForMetadata(defaultId?: string): Promise<InputTranslationMetadata>;
+//# sourceMappingURL=actions.d.ts.map

package/dist/types/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/types/db.d.ts

@@ -0,0 +1,93 @@
+import { PrismaClient } from './prisma-gen/index.js';
+import { Database } from 'better-sqlite3';
+import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from '@helloao/tools/generation/dataset.js';
+import { InputFile, TranslationBookChapter } from '@helloao/tools/generation/index.js';
+import { GenerateApiOptions } from '@helloao/tools/generation/api.js';
+import type { DOMParser } from 'linkedom';
+import { Readable } from 'stream';
+export declare function getMigrationsPath(): Promise<string | null>;
+/**
+ * Imports the translations from the given directories into the database.
+ * @param db The database to import the translations into.
+ * @param dirs The directories to import the translations from.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslations(db: Database, dirs: string[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Imports a batch of translations from the given directories into the database.
+ * @param db The database to import the translations into.
+ * @param dirs The directories that contain the translations.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslationBatch(db: Database, dirs: string[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Parses and imports the given files into the database.
+ * @param db The database to import the files into.
+ * @param files The files that should be parsed.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslationFileBatch(db: Database, files: InputFile[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Filters the given input files to only include those that have changed.
+ * @param db The database to check for changes.
+ * @param files The files to filter.
+ */
+export declare function getChangedOrNewInputFiles(db: Database, files: InputFile[]): InputFile[];
+export declare function insertFileMetadata(db: Database, files: InputFile[]): void;
+export declare function insertTranslations(db: Database, translations: DatasetTranslation[]): void;
+export declare function insertTranslationBooks(db: Database, translation: DatasetTranslation, translationBooks: DatasetTranslationBook[]): void;
+export declare function insertTranslationContent(db: Database, translation: DatasetTranslation, book: DatasetTranslationBook, chapters: TranslationBookChapter[]): void;
+export declare function getDbPathFromDir(dir: string): string;
+export declare function getDbPath(p: string | null): string;
+export declare function getPrismaDbFromDir(dir: string): PrismaClient<{
+    datasources: {
+        db: {
+            url: string;
+        };
+    };
+}, never, import("prisma-gen/runtime/library.js").DefaultArgs>;
+export declare function getDbFromDir(dir: string): Promise<Database>;
+export declare function getDb(dbPath: string): Promise<Database>;
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
+/**
+ * Loads the datasets from the database in a series of batches.
+ * @param db The database.
+ * @param translationsPerBatch The number of translations to load per batch.
+ * @param translationsToLoad The list of translations to load. If not provided, all translations will be loaded.
+ */
+export declare function loadDatasets(db: PrismaClient, translationsPerBatch?: number, translationsToLoad?: string[]): AsyncGenerator<DatasetOutput>;
+export interface SerializeApiOptions extends GenerateApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
+/**
+ * Generates and serializes the API files for the datasets that are stored in the database.
+ * Yields each batch of serialized files.
+ * @param db The database that the dataset should be loaded from.
+ * @param options The options to use for serializing the files.
+ * @param apiOptions The options to use for generating the API files.
+ * @param translationsPerBatch The number of translations that should be loaded and written per batch.
+ * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
+ */
+export declare function serializeFilesFromDatabase(db: PrismaClient, options?: SerializeApiOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
+/**
+ * Generates and serializes the API files for the given datasets.
+ * Yields each batch of serialized files.
+ *
+ * @param datasets The datasets to serialize.
+ * @param options The options to use for generating and serializing the files.
+ */
+export declare function serializeDatasets(datasets: AsyncIterable<DatasetOutput>, options?: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+//# sourceMappingURL=db.d.ts.map

package/dist/types/downloads.d.ts

@@ -0,0 +1,2 @@
+export declare function downloadFile(url: string, path: string): Promise<void>;
+//# sourceMappingURL=downloads.d.ts.map

package/dist/types/files.d.ts

@@ -0,0 +1,118 @@
+import { InputFile, OutputFile } from '@helloao/tools/generation/common-types.js';
+import { Readable } from 'stream';
+/**
+ * Defines an interface that contains information about a serialized file.
+ */
+export interface Uploader {
+    /**
+     * Gets the ideal batch size for the uploader.
+     * Null if the uploader does not need batching.
+     */
+    idealBatchSize: number | null;
+    /**
+     * Uploads the given file.
+     * @param file The file to upload.
+     * @param overwrite Whether the file should be overwritten if it already exists.
+     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
+     */
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+    /**
+     * Disposes resources that the uploader uses.
+     */
+    dispose?(): Promise<void>;
+}
+/**
+ * Defines an interface for a file that has been serialized.
+ */
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
+/**
+ * The options for serializing API files.
+ */
+export interface SerializeApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
+/**
+ * Serializes the given output files into serialized files using the given options.
+ *
+ * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
+ *
+ * @param files The files that should be serialized.
+ * @param options The options for serialization.
+ */
+export declare function serializeOutputFiles(files: AsyncIterable<OutputFile[]>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+/**
+ * Serializes the given output file content into a serialized file.
+ * @param path The path that the file should be saved to.
+ * @param content The content of the file.
+ * @param options The options for serialization.
+ */
+export declare function serializeFile(path: string, content: OutputFile['content'], options: SerializeApiOptions): Promise<SerializedFile | null>;
+/**
+ * Loads the files for the given translations.
+ * @param dir The directory that the translations exist in.
+ */
+export declare function loadTranslationsFiles(dirs: string[]): Promise<InputFile[]>;
+/**
+ * Loads the files for the given translation.
+ * @param translation The directory that the translation exists in.
+ * @returns The list of files that were loaded, or null if the translation has no metadata.
+ */
+export declare function loadTranslationFiles(translation: string): Promise<InputFile[] | null>;
+export interface CollectionTranslationMetadata {
+    name: {
+        local: string;
+        abbrev: string;
+        english: string;
+    };
+    language: string;
+    year: number;
+    direction: 'ltr' | 'rtl';
+    copyright: {
+        licenses: any[];
+        attribution: string;
+        attribution_url: string;
+    };
+    id: string | null;
+    source: {
+        id: string;
+    };
+}
+/**
+ * Defines an uploader that is able to upload files to a directory.
+ */
+export declare class FilesUploader implements Uploader {
+    private _dir;
+    constructor(dir: string);
+    get idealBatchSize(): number | null;
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+}
+/**
+ * Defines an uploader that is able to upload files into a zip file.
+ */
+export declare class ZipUploader implements Uploader {
+    private _path;
+    private _initPromise;
+    private _fileHandle;
+    private _zip;
+    constructor(filePath: string);
+    private _init;
+    get idealBatchSize(): number | null;
+    upload(file: SerializedFile, _overwrite: boolean): Promise<boolean>;
+    dispose(): Promise<void>;
+}
+/**
+ * Calculates the SHa256 hash of the given input files.
+ * @param files The files to hash.
+ */
+export declare function hashInputFiles(files: InputFile[]): string;
+//# sourceMappingURL=files.d.ts.map

package/dist/types/index.d.ts

@@ -0,0 +1,8 @@
+import * as db from './db.js';
+import * as downloads from './downloads.js';
+import * as uploads from './uploads.js';
+import * as actions from './actions.js';
+import * as files from './files.js';
+import * as s3 from './s3.js';
+export { db, downloads, uploads, actions, files, s3 };
+//# sourceMappingURL=index.d.ts.map

package/dist/types/s3.d.ts

@@ -0,0 +1,44 @@
+import { SerializedFile, Uploader } from './files.js';
+import { AwsCredentialIdentity, Provider } from '@smithy/types';
+export declare class S3Uploader implements Uploader {
+    private _client;
+    private _bucketName;
+    private _keyPrefix;
+    get idealBatchSize(): number;
+    constructor(bucketName: string, keyPrefix: string, profile: string | null | AwsCredentialIdentity | Provider<AwsCredentialIdentity>);
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+}
+/**
+ * Parses the given S3 URL into its bucket name and object key.
+ * @param url The URL to parse.
+ */
+export declare function parseS3Url(url: string): {
+    bucketName: string;
+    objectKey: string;
+} | undefined;
+/**
+ * Gets the HTTP URL for the given S3 URL.
+ * @param s3Url The S3 URL to convert.
+ */
+export declare function getHttpUrl(s3Url: string): string | undefined;
+/**
+ * A provider that gets the credentials directly from the user input.
+ */
+export declare const askForAccessKeyProvider: Provider<AwsCredentialIdentity>;
+/**
+ * Defines a provider that tries to get the credentials from the given list of providers.
+ * @param providers The providers to try.
+ */
+export declare function providerChain(...providers: Provider<AwsCredentialIdentity>[]): Provider<AwsCredentialIdentity>;
+/**
+ * Gets the default provider for the given options.
+ *
+ * Defaults first to using the provided access key and secret access key, then to using the given profile, then finally to asking the user for the access key.
+ * @param options
+ */
+export declare function defaultProviderForOptions(options: {
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    profile?: string;
+}): Provider<AwsCredentialIdentity> | AwsCredentialIdentity;
+//# sourceMappingURL=s3.d.ts.map

package/dist/types/uploads.d.ts

@@ -0,0 +1,83 @@
+import { SerializedFile } from './db.js';
+import { Uploader } from './files.js';
+import { DatasetOutput } from '@helloao/tools/generation/dataset.js';
+import { PrismaClient } from './prisma-gen/index.js';
+import { GenerateApiOptions } from '@helloao/tools/generation/api.js';
+export interface UploadApiFromDatabaseOptions extends UploadApiOptions, GenerateApiOptions {
+    /**
+     * The number of files to upload in each batch.
+     */
+    batchSize: string | number;
+}
+export interface UploadApiOptions {
+    /**
+     * Whether to overwrite existing files.
+     */
+    overwrite?: boolean;
+    /**
+     * Whether to only overwrite common files.
+     * "Common files" are files that are similar between translations, like the books.json endpoint, or individual chapter endpoints.
+     */
+    overwriteCommonFiles?: boolean;
+    /**
+     * The file pattern regex that should be used to filter the files that are uploaded.
+     */
+    filePattern?: string;
+    /**
+     * The translations to generate API files for.
+     */
+    translations?: string[];
+    /**
+     * The AWS profile to use for uploading to S3.
+     */
+    profile?: string;
+    /**
+     * The AWS access key ID to use for uploading to S3.
+     */
+    accessKeyId?: string;
+    /**
+     * The AWS secret access key to use for uploading to S3.
+     */
+    secretAccessKey?: string;
+    /**
+     * Whether to generate API files that use the common name instead of book IDs.
+     */
+    useCommonName?: boolean;
+    /**
+     * Whether to generate audio files for the API.
+     */
+    generateAudioFiles?: boolean;
+    /**
+     * Whether to generate pretty-printed JSON files.
+     */
+    pretty?: boolean;
+}
+/**
+ * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param db The database that the datasets should be loaded from.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
+/**
+ * Generates the API files from the given datasets and uploads them to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions & GenerateApiOptions): Promise<void>;
+/**
+ * Uploads the given serialized files to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFiles(dest: string, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
+/**
+ * Uploads the given serialized files using the given uploader.
+ * @param uploader The uploader to use.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFilesUsingUploader(uploader: Uploader, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
+//# sourceMappingURL=uploads.d.ts.map

package/package.json

@@ -1,16 +1,21 @@
 {
   "name": "@helloao/cli",
-  "version": "0.0.7",
+  "version": "0.0.8-alpha",
   "description": "A CLI and related tools for managing HelloAO's Free Bible API",
-  "module": "index.js",
-  "types": "index.d.ts",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "require": "./dist/cjs/index.cjs"
+    }
+  },
   "bin": {
-    "helloao": "./cli.js"
+    "helloao": "./dist/cjs/cli.cjs"
   },
   "author": "Kallyn Gowdy <kal@helloao.org>",
   "license": "MIT",
   "dependencies": {
-    "@helloao/tools": "^0.0.5",
+    "@helloao/tools": "^0.0.6-alpha",
     "commander": "12.1.0",
     "@gracious.tech/fetch-client": "^0.7.0",
     "prisma": "^5.12.1",

package/prisma-gen/edge.js

@@ -209,7 +209,7 @@ const config = {
     "db"
   ],
   "activeProvider": "sqlite",
-  "postinstall": false,
+  "postinstall": true,
   "inlineDatasources": {
     "db": {
       "url": {

package/prisma-gen/index.js

@@ -210,7 +210,7 @@ const config = {
     "db"
   ],
   "activeProvider": "sqlite",
-  "postinstall": false,
+  "postinstall": true,
   "inlineDatasources": {
     "db": {
       "url": {
@@ -229,8 +229,8 @@ const fs = require('fs')
 config.dirname = __dirname
 if (!fs.existsSync(path.join(__dirname, 'schema.prisma'))) {
   const alternativePaths = [
-    "packages/helloao-cli/prisma-gen",
-    "helloao-cli/prisma-gen",
+    "prisma-gen",
+    "",
   ]
   
   const alternativePath = alternativePaths.find((altPath) => {
@@ -259,7 +259,7 @@ Object.assign(exports, Prisma)
 
 // file annotations for bundling tools to include these files
 path.join(__dirname, "query_engine-windows.dll.node");
-path.join(process.cwd(), "packages/helloao-cli/prisma-gen/query_engine-windows.dll.node")
+path.join(process.cwd(), "prisma-gen/query_engine-windows.dll.node")
 // file annotations for bundling tools to include these files
 path.join(__dirname, "schema.prisma");
-path.join(process.cwd(), "packages/helloao-cli/prisma-gen/schema.prisma")
+path.join(process.cwd(), "prisma-gen/schema.prisma")