@helloao/cli 0.0.13 → 0.0.15

package/dist/types/actions.d.ts

@@ -66,8 +66,14 @@ export interface SourceTranslationsOptions {
     bibleMultiConverterPath?: string;
     /**
      * Whether to overwrite existing files in output directory.
+     *
+     * If database tracking is enabled, this will prevent overwriting files that were not tracked in the database.
      */
     overwrite?: boolean;
+    /**
+     * The database to use for tracking downloads.
+     */
+    db?: string | null;
 }
 export declare function getMetaPath(): Promise<string | null>;
 /**
@@ -81,6 +87,10 @@ export interface ImportTranslationOptions {
      * Whether to forcibly import the translations, even if they have already been imported.
      */
     overwrite?: boolean;
+    /**
+     * The database to import the translations into.
+     */
+    db?: string | null;
 }
 /**
  * Imports a translation from the given directory into the database in the current working directory.
@@ -108,6 +118,12 @@ export declare function importCommentary(dir: string, dirs: string[], options: I
  * @param options The options.
  */
 export declare function importCommentaries(dir: string, options: ImportTranslationOptions): Promise<void>;
+/**
+ * Imports the API from the given directory into the database in the current working directory.
+ * @param dir The directory that the API is located in.
+ * @param options The options.
+ */
+export declare function importApi(dir: string, options: ImportTranslationOptions): Promise<void>;
 export interface FetchTranslationsOptions {
     /**
      * Fetch all translations. If omitted, only undownloaded translations will be fetched.

package/dist/types/db.d.ts

@@ -1,7 +1,7 @@
 import { PrismaClient } from './prisma-gen/index.js';
 import { Database } from 'better-sqlite3';
-import { DatasetCommentary, DatasetCommentaryBook, DatasetCommentaryProfile, DatasetOutput, DatasetTranslation, DatasetTranslationBook } from '@helloao/tools/generation/dataset.js';
-import { InputFile, TranslationBookChapter, CommentaryBookChapter } from '@helloao/tools/generation/index.js';
+import { DatasetCommentary, DatasetCommentaryBook, DatasetCommentaryProfile, DatasetDataset, DatasetDatasetBook, DatasetOutput, DatasetTranslation, DatasetTranslationBook } from '@helloao/tools/generation/dataset.js';
+import { InputFile, TranslationBookChapter, CommentaryBookChapter, DatasetBookChapter } from '@helloao/tools/generation/index.js';
 import { GenerateApiOptions } from '@helloao/tools/generation/api.js';
 import type { DOMParser } from 'linkedom';
 import { Readable } from 'stream';
@@ -56,6 +56,12 @@ export declare function loadAndImportBatch(loadFilesFromDir: (path: string) => P
  * @param overwrite Whether to force a reload of the translations.
  */
 export declare function importFileBatch(db: Database, files: InputFile[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Imports the given dataset output into the database.
+ * @param db The database to import the dataset into.
+ * @param output The dataset output to import.
+ */
+export declare function importDatasetOutput(db: Database, output: DatasetOutput): void;
 /**
  * Filters the given input files to only include those that have changed.
  * @param db The database to check for changes.
@@ -70,17 +76,24 @@ export declare function insertCommentaries(db: Database, commentaries: DatasetCo
 export declare function insertCommentaryBooks(db: Database, commentary: DatasetCommentary, commentaryBooks: DatasetCommentaryBook[]): void;
 export declare function insertCommentaryProfiles(db: Database, commentary: DatasetCommentary, commentaryProfiles: DatasetCommentaryProfile[]): void;
 export declare function insertCommentaryContent(db: Database, commentary: DatasetCommentary, book: DatasetCommentaryBook, chapters: CommentaryBookChapter[]): void;
+export declare function insertDatasets(db: Database, datasets: DatasetDataset[]): void;
+export declare function insertDatasetBooks(db: Database, dataset: DatasetDataset, datasetBooks: DatasetDatasetBook[]): void;
+export declare function insertDatasetContent(db: Database, dataset: DatasetDataset, book: DatasetDatasetBook, chapters: DatasetBookChapter[]): void;
 export declare function getDbPathFromDir(dir: string): string;
-export declare function getDbPath(p: string | null): string;
-export declare function getPrismaDbFromDir(dir: string): PrismaClient<{
+export declare function getDbPath(p?: string | null): string;
+export declare function getPrismaDb(path?: string | null): PrismaClient<{
     datasources: {
         db: {
             url: string;
         };
     };
 }, never, import("prisma-gen/runtime/library.js").DefaultArgs>;
-export declare function getDbFromDir(dir: string): Promise<Database>;
-export declare function getDb(dbPath: string): Promise<Database>;
+/**
+ * Gets the database from the given path. If no path is provided, the current working directory is used.
+ * @param path The path to the database. If not provided, the current working directory is used.
+ * @returns
+ */
+export declare function getDb(path?: string | null): Promise<Database>;
 export interface SerializedFile {
     path: string;
     content: string | Readable;
@@ -110,6 +123,13 @@ export declare function loadTranslationDatasets(db: PrismaClient, translationsPe
  * @param commentariesToLoad The list of commentaries to load. If not provided, all commentaries will be loaded.
  */
 export declare function loadCommentaryDatasets(db: PrismaClient, perBatch?: number, commentariesToLoad?: string[]): AsyncGenerator<DatasetOutput, void, unknown>;
+/**
+ * Loads the datasets from the database as a dataset.
+ * @param db The database.
+ * @param perBatch The number of translations to load per batch.
+ * @param datasetsToLoad The list of commentaries to load. If not provided, all commentaries will be loaded.
+ */
+export declare function loadDatasetDatasets(db: PrismaClient, perBatch?: number, datasetsToLoad?: string[]): AsyncGenerator<DatasetOutput, void, unknown>;
 export interface SerializeApiOptions extends GenerateApiOptions {
     /**
      * Whether the output should be pretty-printed.

package/dist/types/files.d.ts

@@ -1,5 +1,6 @@
 import { InputFile, InputTranslationMetadata, OutputFile } from '@helloao/tools/generation/common-types.js';
 import { Readable } from 'stream';
+import { DatasetDataset } from '@helloao/tools/generation/dataset.js';
 /**
  * Defines an interface that contains information about a serialized file.
  */
@@ -79,6 +80,12 @@ export declare function loadCommentariesFiles(dirs: string[]): Promise<InputFile
  * @returns The list of files that were loaded, or null if the commentary has no metadata.
  */
 export declare function loadCommentaryFiles(commentary: string): Promise<InputFile[] | null>;
+/**
+ * Imports all the datasets from the given directory into the database in the current working directory.
+ * @param dir The directory that the datasets are located in.
+ * @param options The options.
+ */
+export declare function loadDatasetsFromDirectory(dir: string): Promise<DatasetDataset[]>;
 export interface CollectionTranslationMetadata {
     name: {
         local: string;

package/migrations/20251028135135_add_datasets/migration.sql

@@ -0,0 +1,72 @@
+-- CreateTable
+CREATE TABLE "Dataset" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "website" TEXT NOT NULL,
+    "licenseUrl" TEXT NOT NULL,
+    "licenseNotes" TEXT,
+    "englishName" TEXT NOT NULL,
+    "language" TEXT NOT NULL,
+    "textDirection" TEXT NOT NULL,
+    "sha256" TEXT
+);
+
+-- CreateTable
+CREATE TABLE "DatasetBook" (
+    "id" TEXT NOT NULL,
+    "datasetId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "commonName" TEXT NOT NULL,
+    "introduction" TEXT,
+    "introductionSummary" TEXT,
+    "order" INTEGER NOT NULL,
+    "numberOfChapters" INTEGER NOT NULL,
+    "sha256" TEXT,
+
+    PRIMARY KEY ("datasetId", "id"),
+    CONSTRAINT "DatasetBook_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "DatasetChapter" (
+    "number" INTEGER NOT NULL,
+    "bookId" TEXT NOT NULL,
+    "datasetId" TEXT NOT NULL,
+    "json" TEXT NOT NULL,
+    "sha256" TEXT,
+
+    PRIMARY KEY ("datasetId", "bookId", "number"),
+    CONSTRAINT "DatasetChapter_datasetId_bookId_fkey" FOREIGN KEY ("datasetId", "bookId") REFERENCES "DatasetBook" ("datasetId", "id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetChapter_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "DatasetChapterVerse" (
+    "number" INTEGER NOT NULL,
+    "chapterNumber" INTEGER NOT NULL,
+    "bookId" TEXT NOT NULL,
+    "datasetId" TEXT NOT NULL,
+    "contentJson" TEXT NOT NULL,
+    "sha256" TEXT,
+
+    PRIMARY KEY ("datasetId", "bookId", "chapterNumber", "number"),
+    CONSTRAINT "DatasetChapterVerse_datasetId_bookId_chapterNumber_fkey" FOREIGN KEY ("datasetId", "bookId", "chapterNumber") REFERENCES "DatasetChapter" ("datasetId", "bookId", "number") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetChapterVerse_datasetId_bookId_fkey" FOREIGN KEY ("datasetId", "bookId") REFERENCES "DatasetBook" ("datasetId", "id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetChapterVerse_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "DatasetReference" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "datasetId" TEXT NOT NULL,
+    "bookId" TEXT NOT NULL,
+    "chapterNumber" INTEGER NOT NULL,
+    "verseNumber" INTEGER NOT NULL,
+    "endVerseNumber" INTEGER,
+    "score" INTEGER,
+    "sha256" TEXT,
+    CONSTRAINT "DatasetReference_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_fkey" FOREIGN KEY ("datasetId", "bookId") REFERENCES "DatasetBook" ("datasetId", "id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_chapterNumber_fkey" FOREIGN KEY ("datasetId", "bookId", "chapterNumber") REFERENCES "DatasetChapter" ("datasetId", "bookId", "number") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_chapterNumber_verseNumber_fkey" FOREIGN KEY ("datasetId", "bookId", "chapterNumber", "verseNumber") REFERENCES "DatasetChapterVerse" ("datasetId", "bookId", "chapterNumber", "number") ON DELETE RESTRICT ON UPDATE CASCADE
+);

package/migrations/20251028144932_fix_dataset_types/migration.sql

@@ -0,0 +1,50 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `commonName` on the `DatasetBook` table. All the data in the column will be lost.
+  - You are about to drop the column `introduction` on the `DatasetBook` table. All the data in the column will be lost.
+  - You are about to drop the column `introductionSummary` on the `DatasetBook` table. All the data in the column will be lost.
+  - You are about to drop the column `name` on the `DatasetBook` table. All the data in the column will be lost.
+  - You are about to drop the column `sha256` on the `DatasetReference` table. All the data in the column will be lost.
+  - Added the required column `referenceBookId` to the `DatasetReference` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `referenceChapter` to the `DatasetReference` table without a default value. This is not possible if the table is not empty.
+  - Added the required column `referenceVerse` to the `DatasetReference` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- RedefineTables
+PRAGMA defer_foreign_keys=ON;
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_DatasetBook" (
+    "id" TEXT NOT NULL,
+    "datasetId" TEXT NOT NULL,
+    "order" INTEGER NOT NULL,
+    "numberOfChapters" INTEGER NOT NULL,
+    "sha256" TEXT,
+
+    PRIMARY KEY ("datasetId", "id"),
+    CONSTRAINT "DatasetBook_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_DatasetBook" ("datasetId", "id", "numberOfChapters", "order", "sha256") SELECT "datasetId", "id", "numberOfChapters", "order", "sha256" FROM "DatasetBook";
+DROP TABLE "DatasetBook";
+ALTER TABLE "new_DatasetBook" RENAME TO "DatasetBook";
+CREATE TABLE "new_DatasetReference" (
+    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    "datasetId" TEXT NOT NULL,
+    "bookId" TEXT NOT NULL,
+    "chapterNumber" INTEGER NOT NULL,
+    "verseNumber" INTEGER NOT NULL,
+    "referenceBookId" TEXT NOT NULL,
+    "referenceChapter" INTEGER NOT NULL,
+    "referenceVerse" INTEGER NOT NULL,
+    "endVerseNumber" INTEGER,
+    "score" INTEGER,
+    CONSTRAINT "DatasetReference_datasetId_fkey" FOREIGN KEY ("datasetId") REFERENCES "Dataset" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_fkey" FOREIGN KEY ("datasetId", "bookId") REFERENCES "DatasetBook" ("datasetId", "id") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_chapterNumber_fkey" FOREIGN KEY ("datasetId", "bookId", "chapterNumber") REFERENCES "DatasetChapter" ("datasetId", "bookId", "number") ON DELETE RESTRICT ON UPDATE CASCADE,
+    CONSTRAINT "DatasetReference_datasetId_bookId_chapterNumber_verseNumber_fkey" FOREIGN KEY ("datasetId", "bookId", "chapterNumber", "verseNumber") REFERENCES "DatasetChapterVerse" ("datasetId", "bookId", "chapterNumber", "number") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_DatasetReference" ("bookId", "chapterNumber", "datasetId", "endVerseNumber", "id", "score", "verseNumber") SELECT "bookId", "chapterNumber", "datasetId", "endVerseNumber", "id", "score", "verseNumber" FROM "DatasetReference";
+DROP TABLE "DatasetReference";
+ALTER TABLE "new_DatasetReference" RENAME TO "DatasetReference";
+PRAGMA foreign_keys=ON;
+PRAGMA defer_foreign_keys=OFF;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helloao/cli",
-  "version": "0.0.13",
+  "version": "0.0.15",
   "description": "A CLI and related tools for managing HelloAO's Free Bible API",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/esm/index.js",
@@ -43,7 +43,7 @@
     "all-iso-language-codes": "1.0.17",
     "papaparse": "5.4.1",
     "luxon": "3.5.0",
-    "@helloao/tools": "0.0.13"
+    "@helloao/tools": "0.0.15"
   },
   "files": [
     "/README.md",