@helloao/tools 0.0.1

package/generation/common-types.d.ts

@@ -0,0 +1,327 @@
+/**
+ * Defines an interface that contains information about a input file.
+ */
+export interface InputFile {
+    name?: string;
+    metadata: ParseTreeMetadata;
+    content: string;
+    sha256?: string;
+    fileType: 'usfm' | 'usx' | 'json';
+}
+export type OutputFileContent = object | ReadableStream;
+/**
+ * Defines an interface that contains information about a output file.
+ */
+export interface OutputFile {
+    /**
+     * The path that the file should be stored at.
+     */
+    path: string;
+    /**
+     * The content of the file.
+     */
+    content: OutputFileContent | (() => Promise<OutputFileContent>);
+    /**
+     * Whether the file can be merged with files of the same name but from other datasets.
+     */
+    mergable?: boolean;
+}
+/**
+ * Defines an interface that contains metadata for a parse tree.
+ */
+export interface ParseTreeMetadata {
+    /**
+     * Information about the translation.
+     */
+    translation: InputTranslationMetadata;
+}
+/**
+ * The metadata for a translation that is input into the generator.
+ */
+export interface InputTranslationMetadata {
+    /**
+     * The ID of the translation.
+     */
+    id: string;
+    /**
+     * The name of the translation.
+     */
+    name: string;
+    /**
+     * The english name of the translation.
+     */
+    englishName: string;
+    /**
+     * The website for the translation.
+     */
+    website: string;
+    /**
+     * The URL that the license for the translation can be found.
+     */
+    licenseUrl: string;
+    /**
+     * The short name for the translation.
+     */
+    shortName?: string;
+    /**
+     * The ISO 639 letter language tag that the translation is primarily in.
+     */
+    language: string;
+    /**
+     * The direction that the text is written in.
+     */
+    direction: 'ltr' | 'rtl';
+}
+/**
+ * Defines an interface that contains information about a translation.
+ */
+export interface Translation {
+    /**
+     * The ID of the translation.
+     */
+    id: string;
+    /**
+     * The name of the translation.
+     */
+    name: string;
+    /**
+     * The website for the translation.
+     */
+    website: string;
+    /**
+     * The URL that the license for the translation can be found.
+     */
+    licenseUrl: string;
+    /**
+     * The short name for the translation.
+     */
+    shortName?: string;
+    /**
+     * The English name for the translation.
+     */
+    englishName: string;
+    /**
+     * The ISO 639 3-letter language tag that the translation is primarily in.
+     */
+    language: string;
+    /**
+     * The direction that the language is written in.
+     * "ltr" indicates that the text is written from the left side of the page to the right.
+     * "rtl" indicates that the text is written from the right side of the page to the left.
+     */
+    textDirection: 'ltr' | 'rtl';
+}
+/**
+ * Defines an interface that contains information about a book.
+ */
+export interface TranslationBook {
+    /**
+     * The ID of the book.
+     */
+    id: string;
+    /**
+     * The name that the translation provided for the book.
+     */
+    name: string;
+    /**
+     * The common name for the book.
+     */
+    commonName: string;
+    /**
+     * The title of the book.
+     * This is usually a more descriptive version of the book name.
+     * If not available, then one was not provided by the translation.
+     */
+    title: string | null;
+    /**
+     * The numerical order of the book in the translation.
+     */
+    order: number;
+}
+/**
+ * Defines an interface that contains information about a book chapter.
+ */
+export interface TranslationBookChapter {
+    /**
+     * The information for the chapter.
+     */
+    chapter: ChapterData;
+    /**
+     * The links to different audio versions for the chapter.
+     */
+    thisChapterAudioLinks: TranslationBookChapterAudioLinks;
+}
+/**
+ * Defines an interface that contains the audio links for a book chapter.
+ */
+export interface TranslationBookChapterAudioLinks {
+    /**
+     * The reader for the chapter and the URL link to the audio file.
+     */
+    [reader: string]: string;
+}
+/**
+ * Defines an interface that represents data in a chapter.
+ */
+export interface ChapterData {
+    /**
+     * The number of the chapter.
+     */
+    number: number;
+    /**
+     * The content of the chapter.
+     */
+    content: ChapterContent[];
+    /**
+     * The list of footnotes for the chapter.
+     */
+    footnotes: ChapterFootnote[];
+}
+/**
+ * A union type that represents a single piece of chapter content.
+ * A piece of chapter content can be one of the following things:
+ * - A heading.
+ * - A line break.
+ * - A verse.
+ * - A Hebrew Subtitle.
+ */
+export type ChapterContent = ChapterHeading | ChapterLineBreak | ChapterVerse | ChapterHebrewSubtitle;
+/**
+ * A heading in a chapter.
+ */
+export interface ChapterHeading {
+    /**
+     * Indicates that the content represents a heading.
+     */
+    type: 'heading';
+    /**
+     * The content for the heading.
+     * If multiple strings are included in the array, they should be concatenated with a space.
+     */
+    content: string[];
+}
+/**
+ * A line break in a chapter.
+ */
+export interface ChapterLineBreak {
+    /**
+     * Indicates that the content represents a line break.
+     */
+    type: 'line_break';
+}
+/**
+ * A Hebrew Subtitle in a chapter.
+ * These are often used included as informational content that appeared in the original manuscripts.
+ * For example, Psalms 49 has the Hebrew Subtitle "To the choirmaster. A Psalm of the Sons of Korah."
+ */
+export interface ChapterHebrewSubtitle {
+    /**
+     * Indicates that the content represents a Hebrew Subtitle.
+     */
+    type: 'hebrew_subtitle';
+    /**
+     * The list of content that is contained in the subtitle.
+     * Each element in the list could be a string, formatted text, or a footnote reference.
+     */
+    content: (string | FormattedText | VerseFootnoteReference)[];
+}
+/**
+ * A verse in a chapter.
+ */
+export interface ChapterVerse {
+    /**
+     * Indicates that the content is a verse.
+     */
+    type: 'verse';
+    /**
+     * The number of the verse.
+     */
+    number: number;
+    /**
+     * The list of content for the verse.
+     * Each element in the list could be a string, formatted text, or a footnote reference.
+     */
+    content: (string | FormattedText | InlineHeading | InlineLineBreak | VerseFootnoteReference)[];
+}
+/**
+ * Formatted text. That is, text that is formated in a particular manner.
+ */
+export interface FormattedText {
+    /**
+     * The text that is formatted.
+     */
+    text: string;
+    /**
+     * Whether the text represents a poem.
+     * The number indicates the level of indent.
+     *
+     * Common in Psalms.
+     */
+    poem?: number;
+    /**
+     * Whether the text represents the Words of Jesus.
+     */
+    wordsOfJesus?: boolean;
+}
+/**
+ * Defines an interface that represents a heading that is embedded in a verse.
+ */
+export interface InlineHeading {
+    /**
+     * The text of the heading.
+     */
+    heading: string;
+}
+/**
+ * Defines an interface that represents a line break that is embedded in a verse.
+ */
+export interface InlineLineBreak {
+    lineBreak: true;
+}
+/**
+ * A footnote reference in a verse or a Hebrew Subtitle.
+ */
+export interface VerseFootnoteReference {
+    /**
+     * The ID of the note.
+     */
+    noteId: number;
+}
+/**
+ * Information about a footnote.
+ */
+export interface ChapterFootnote {
+    /**
+     * The ID of the note that is referenced.
+     */
+    noteId: number;
+    /**
+     * The text of the footnote.
+     */
+    text: string;
+    /**
+     * The verse reference for the footnote.
+     */
+    reference?: {
+        chapter: number;
+        verse: number;
+    };
+    /**
+     * The caller that should be used for the footnote.
+     * For footnotes, a "caller" is the character that is used in the text to reference to footnote.
+     *
+     * For example, in the text:
+     * Hello (a) World
+     *
+     * ----
+     * (a) This is a footnote.
+     *
+     * The "(a)" is the caller.
+     *
+     * If "+", then the caller should be autogenerated.
+     * If null, then the caller should be empty.
+     * If a string, then the caller should be that string.
+     */
+    caller: '+' | string | null;
+}
+//# sourceMappingURL=common-types.d.ts.map

package/generation/common-types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/generation/dataset.d.ts

@@ -0,0 +1,34 @@
+import { InputFile, Translation, TranslationBook, TranslationBookChapter } from "./common-types";
+/**
+ * Defines an interface that contains generated dataset info.
+ */
+export interface DatasetOutput {
+    /**
+     * The list of translations that are available in the dataset.
+     */
+    translations: DatasetTranslation[];
+}
+/**
+ * Defines a translation that is used in the dataset.
+ */
+export interface DatasetTranslation extends Translation {
+    /**
+     * The list of books that are available for the translation.
+     */
+    books: DatasetTranslationBook[];
+}
+/**
+ * Defines a translation book that is used in the dataset.
+ */
+export interface DatasetTranslationBook extends TranslationBook {
+    /**
+     * The list of chapters that are available for the book.
+     */
+    chapters: TranslationBookChapter[];
+}
+/**
+ * Generates a list of output files from the given list of input files.
+ * @param file The list of files.
+ */
+export declare function generateDataset(files: InputFile[], parser?: DOMParser): DatasetOutput;
+//# sourceMappingURL=dataset.d.ts.map

package/generation/dataset.js

@@ -0,0 +1,106 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateDataset = generateDataset;
+const parser_1 = require("../parser");
+const usx_parser_1 = require("../parser/usx-parser");
+const book_order_1 = require("./book-order");
+const lodash_1 = require("lodash");
+const audio_1 = require("./audio");
+const codex_parser_1 = require("../parser/codex-parser");
+/**
+ * Generates a list of output files from the given list of input files.
+ * @param file The list of files.
+ */
+function generateDataset(files, parser = new globalThis.DOMParser()) {
+    let output = {
+        translations: [],
+    };
+    let usfmParser = new parser_1.UsfmParser();
+    let usxParser = new usx_parser_1.USXParser(parser);
+    let codexParser = new codex_parser_1.CodexParser();
+    let parsedTranslations = new Map();
+    const unknownLanguages = new Set();
+    for (let file of files) {
+        try {
+            let parser;
+            if (file.fileType === 'usfm') {
+                parser = usfmParser;
+            }
+            else if (file.fileType === 'usx') {
+                parser = usxParser;
+            }
+            else if (file.fileType === 'json') {
+                parser = codexParser;
+            }
+            else {
+                console.warn('[generate] File does not have a valid type!', file.name);
+                continue;
+            }
+            const parsed = parser.parse(file.content);
+            const id = parsed.id;
+            if (!id) {
+                console.warn('[generate] File does not have a valid book ID!', file.name, id);
+                continue;
+            }
+            const order = book_order_1.bookOrderMap.get(id);
+            if (typeof order !== 'number') {
+                console.warn('[generate] Book does not have an order!', id);
+                continue;
+            }
+            const bookMap = book_order_1.bookIdMap.get(file.metadata.translation.language);
+            if (!bookMap) {
+                if (!unknownLanguages.has(file.metadata.translation.language)) {
+                    console.warn('[generate] File does not have a known language!', file.name, file.metadata.translation.language);
+                    unknownLanguages.add(file.metadata.translation.language);
+                }
+            }
+            const bookName = bookMap?.get(id);
+            if (!!bookMap && !bookName) {
+                console.warn('[generate] Book name not found for ID!', file.name, id);
+            }
+            let translation = parsedTranslations.get(file.metadata.translation.id);
+            if (!translation) {
+                translation = {
+                    ...(0, lodash_1.omit)(file.metadata.translation, 'direction'),
+                    textDirection: file.metadata.translation.direction,
+                    books: []
+                };
+                output.translations.push(translation);
+                parsedTranslations.set(file.metadata.translation.id, translation);
+            }
+            const name = parsed.header ?? bookName?.commonName ?? parsed.title;
+            if (!name) {
+                console.warn('[generate] Book does not have a name!', file.name, id);
+                continue;
+            }
+            const commonName = bookName?.commonName ?? parsed.header ?? parsed.title ?? id;
+            const book = {
+                id: id,
+                name,
+                commonName,
+                title: parsed.title ?? null,
+                order,
+                chapters: [],
+            };
+            for (let content of parsed.content) {
+                if (content.type === 'chapter') {
+                    book.chapters.push({
+                        chapter: {
+                            number: content.number,
+                            content: content.content,
+                            footnotes: content.footnotes
+                        },
+                        thisChapterAudioLinks: (0, audio_1.getAudioUrlsForChapter)(translation.id, id, content.number)
+                    });
+                }
+            }
+            book.chapters = (0, lodash_1.sortBy)(book.chapters, c => c.chapter.number);
+            const index = (0, lodash_1.sortedIndexBy)(translation.books, book, b => b.order);
+            translation.books.splice(index, 0, book);
+        }
+        catch (err) {
+            console.error(`[generate] Error occurred while parsing ${file.name}`, err);
+        }
+    }
+    return output;
+}

package/generation/index.d.ts

@@ -0,0 +1,7 @@
+import * as api from './api';
+import * as audio from './audio';
+import * as dataset from './dataset';
+import * as bookOrder from './book-order';
+export * from './common-types';
+export { api, audio, dataset, bookOrder };
+//# sourceMappingURL=index.d.ts.map

package/generation/index.js

@@ -0,0 +1,38 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.bookOrder = exports.dataset = exports.audio = exports.api = void 0;
+const api = __importStar(require("./api"));
+exports.api = api;
+const audio = __importStar(require("./audio"));
+exports.audio = audio;
+const dataset = __importStar(require("./dataset"));
+exports.dataset = dataset;
+const bookOrder = __importStar(require("./book-order"));
+exports.bookOrder = bookOrder;
+__exportStar(require("./common-types"), exports);

package/index.d.ts

@@ -0,0 +1,5 @@
+import * as utils from './utils';
+import * as generation from './generation';
+import * as parser from './parser';
+export { utils, generation, parser, };
+//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -0,0 +1,32 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parser = exports.generation = exports.utils = void 0;
+const utils = __importStar(require("./utils"));
+exports.utils = utils;
+const generation = __importStar(require("./generation"));
+exports.generation = generation;
+const parser = __importStar(require("./parser"));
+exports.parser = parser;

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@helloao/tools",
+  "version": "0.0.1",
+  "description": "A set of tools for managing HelloAO's Free Bible API",
+  "module": "index.js",
+  "types": "index.d.ts",
+  "author": "Kallyn Gowdy <kal@helloao.org>",
+  "license": "MIT",
+  "dependencies": {
+    "lodash": "4.17.21",
+    "all-iso-language-codes": "1.0.17",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "linkedom": "0.18.4",
+    "hash.js": "1.1.7"
+  },
+  "files": [
+    "/README.md",
+    "/LICENSE.txt",
+    "/migrations/**/*.sql",
+    "**/*.js",
+    "**/*.js.map",
+    "**/*.d.ts"
+  ],
+  "scripts": {}
+}

package/parser/codex-parser.d.ts

@@ -0,0 +1,97 @@
+import { z } from 'zod';
+import { ParseTree } from './types';
+export declare const chapterHeadingSchema: z.ZodObject<{
+    type: z.ZodLiteral<"chapter-heading">;
+    data: z.ZodObject<{
+        chapter: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        chapter: number;
+    }, {
+        chapter: number;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    type: "chapter-heading";
+    data: {
+        chapter: number;
+    };
+}, {
+    type: "chapter-heading";
+    data: {
+        chapter: number;
+    };
+}>;
+export declare const metadataSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    type: z.ZodLiteral<"chapter-heading">;
+    data: z.ZodObject<{
+        chapter: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        chapter: number;
+    }, {
+        chapter: number;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    type: "chapter-heading";
+    data: {
+        chapter: number;
+    };
+}, {
+    type: "chapter-heading";
+    data: {
+        chapter: number;
+    };
+}>]>;
+export declare const codexSchema: z.ZodObject<{
+    cells: z.ZodArray<z.ZodObject<{
+        language: z.ZodString;
+        value: z.ZodString;
+        metadata: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+            type: z.ZodString;
+        }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough">>>>;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        language: string;
+        metadata?: z.objectOutputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough"> | null | undefined;
+    }, {
+        value: string;
+        language: string;
+        metadata?: z.objectInputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough"> | null | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    cells: {
+        value: string;
+        language: string;
+        metadata?: z.objectOutputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough"> | null | undefined;
+    }[];
+}, {
+    cells: {
+        value: string;
+        language: string;
+        metadata?: z.objectInputType<{
+            type: z.ZodString;
+        }, z.ZodTypeAny, "passthrough"> | null | undefined;
+    }[];
+}>;
+/**
+ * Defines a parser for codex files.
+ */
+export declare class CodexParser {
+    private _noteCounter;
+    /**
+     * Parses the specified USX content.
+     *
+     * @param usx The USX content to parse.
+     * @returns The parse tree that was generated.
+     */
+    parse(codex: string): ParseTree;
+}
+//# sourceMappingURL=codex-parser.d.ts.map