@helloao/tools 0.2.0 → 0.4.0

package/dist/types/generation/dataset.d.ts

@@ -1,4 +1,4 @@
-import { Commentary, CommentaryBook, CommentaryBookChapter, CommentaryProfile, InputFile, Translation, TranslationBook, TranslationBookChapter, Dataset as CommonDataset, DatasetBook, DatasetBookChapter } from './common-types.js';
+import { Commentary, CommentaryBook, CommentaryBookChapter, CommentaryProfile, InputFile, Translation, TranslationBook, TranslationBookChapter, Dataset as CommonDataset, DatasetBook, DatasetBookChapter, DatasetPerson, DatasetPlace, DatasetEvent, DatasetPeopleGroup } from './common-types.js';
 import { ParseMessage } from '../parser/types.js';
 export declare const DATASET_VERSION = 1;
 /**
@@ -69,6 +69,22 @@ export interface DatasetCommentaryProfile extends CommentaryProfile {
 }
 export interface DatasetDataset extends CommonDataset {
     books: DatasetDatasetBook[];
+    /**
+     * The list of people that are contained in the dataset.
+     */
+    people?: DatasetPerson[];
+    /**
+     * The list of places that are contained in the dataset.
+     */
+    places?: DatasetPlace[];
+    /**
+     * The list of events that are contained in the dataset.
+     */
+    events?: DatasetEvent[];
+    /**
+     * The list of people groups that are contained in the dataset.
+     */
+    peopleGroups?: DatasetPeopleGroup[];
 }
 export interface DatasetDatasetBook extends DatasetBook {
     chapters: DatasetBookChapter[];

package/dist/types/generation/index.d.ts

@@ -2,6 +2,7 @@ import * as api from './api.js';
 import * as audio from './audio.js';
 import * as dataset from './dataset.js';
 import * as bookOrder from './book-order.js';
+import * as simple from './simple.js';
 export * from './common-types.js';
-export { api, audio, dataset, bookOrder };
+export { api, audio, dataset, bookOrder, simple };
 //# sourceMappingURL=index.d.ts.map

package/dist/types/generation/simple.d.ts

@@ -0,0 +1,110 @@
+import { ChapterData, ChapterFootnote, ChapterHebrewSubtitle, ChapterVerse, ChapterWord, CommentaryChapterData, SimpleChapterData, SimpleChapterHebrewSubtitle, SimpleChapterVerse, SimpleChapterWord, SimpleCommentaryChapterData, SimpleInlineHeading, SimplePoemRange, SimpleTextRange, SimpleVerseFootnote, TranslationBookChapterWords, SimpleTranslationBookChapterWords } from './common-types.js';
+/**
+ * The types of content that can be contained in a verse or Hebrew Subtitle.
+ */
+type VerseContent = ChapterVerse['content'][number] | ChapterHebrewSubtitle['content'][number];
+/**
+ * The result of flattening the content of a verse into a single string.
+ */
+export interface SimplifiedVerseContent {
+    /**
+     * The text of the verse.
+     */
+    text: string;
+    /**
+     * The footnotes that occur in the verse.
+     */
+    footnotes: SimpleVerseFootnote[];
+    /**
+     * The headings that occur in the middle of the verse.
+     */
+    headings: SimpleInlineHeading[];
+    /**
+     * The ranges of text that represent the Words of Jesus.
+     */
+    wordsOfJesus: SimpleTextRange[];
+    /**
+     * The ranges of text that represent lines of poetry.
+     */
+    poem: SimplePoemRange[];
+    /**
+     * The word-level annotations for the verse, with their offsets remapped onto the text.
+     */
+    words: SimpleChapterWord[];
+}
+/**
+ * Flattens the content of a verse into a single string, recording the positions of the
+ * footnotes, headings, and formatting that occur in it.
+ *
+ * The rules used to build the text are:
+ * - Strings and formatted text are appended as-is. The source data already contains the
+ *   spacing that is needed between them.
+ * - Lines of poetry and line breaks start a new line.
+ * - Footnote and heading markers are replaced with a single space when one is needed to
+ *   keep the surrounding words apart.
+ *
+ * All of the recorded offsets are measured in UTF-16 code units, which is what
+ * `String.prototype.slice()` and `String.prototype.length` use.
+ *
+ * Word-level annotations are anchored to a range of characters in a single item of the
+ * content array. Since the simplified format replaces that array with a single string,
+ * their offsets are remapped onto the text that is built here.
+ *
+ * @param content The content of the verse.
+ * @param footnotes The footnotes for the chapter, keyed by note ID.
+ * @param usedFootnotes A set that the IDs of the referenced footnotes are added to.
+ * @param words The word-level annotations for the verse.
+ */
+export declare function simplifyVerseContent(content: readonly VerseContent[], footnotes?: Map<number, ChapterFootnote>, usedFootnotes?: Set<number>, words?: readonly ChapterWord[]): SimplifiedVerseContent;
+/**
+ * Converts the given verse into its simplified representation.
+ * @param verse The verse to convert.
+ * @param footnotes The footnotes for the chapter, keyed by note ID.
+ * @param usedFootnotes A set that the IDs of the referenced footnotes are added to.
+ * @param words The word-level annotations for the verse.
+ * @param remappedWords An array that the remapped word-level annotations are added to.
+ */
+export declare function simplifyVerse(verse: ChapterVerse, footnotes?: Map<number, ChapterFootnote>, usedFootnotes?: Set<number>, words?: readonly ChapterWord[], remappedWords?: SimpleChapterWord[]): SimpleChapterVerse;
+/**
+ * Converts the given Hebrew Subtitle into its simplified representation.
+ * @param subtitle The subtitle to convert.
+ * @param footnotes The footnotes for the chapter, keyed by note ID.
+ * @param usedFootnotes A set that the IDs of the referenced footnotes are added to.
+ */
+export declare function simplifyHebrewSubtitle(subtitle: ChapterHebrewSubtitle, footnotes?: Map<number, ChapterFootnote>, usedFootnotes?: Set<number>): SimpleChapterHebrewSubtitle;
+/**
+ * The result of converting a chapter into the simplified chapter format.
+ */
+export interface SimplifiedChapter {
+    /**
+     * The simplified chapter.
+     */
+    chapter: SimpleChapterData;
+    /**
+     * The word-level annotations for the chapter, with their offsets remapped onto the
+     * text of each simplified verse. Empty if the chapter has no annotations.
+     */
+    words: SimpleTranslationBookChapterWords;
+}
+/**
+ * Converts the given chapter into the simplified chapter format.
+ *
+ * In the simplified format, the content of each verse is a single string instead of a list
+ * of formatted content, and the footnotes are available on the verse that they occur in,
+ * along with the offset that they occur at.
+ *
+ * The word-level annotations are returned separately, since they are published in their
+ * own file. They are converted here because their offsets have to be remapped onto the
+ * text that is built for each verse.
+ *
+ * @param chapter The chapter to convert.
+ * @param words The word-level annotations for the chapter, keyed by verse number.
+ */
+export declare function simplifyChapter(chapter: ChapterData, words?: TranslationBookChapterWords): SimplifiedChapter;
+/**
+ * Converts the given commentary chapter into the simplified chapter format.
+ * @param chapter The chapter to convert.
+ */
+export declare function simplifyCommentaryChapter(chapter: CommentaryChapterData): SimpleCommentaryChapterData;
+export {};
+//# sourceMappingURL=simple.d.ts.map

package/dist/types/generation/theographic.d.ts

@@ -0,0 +1,64 @@
+import { Dataset } from './common-types.js';
+import { DatasetDataset } from './dataset.js';
+/**
+ * The dataset information for the Theographic Bible Metadata dataset.
+ * See https://github.com/robertrouse/theographic-bible-metadata
+ */
+export declare const THEOGRAPHIC_DATASET: Dataset;
+/**
+ * Defines an interface for a raw Theographic record.
+ * The Theographic JSON files are Airtable exports, where each record has an Airtable record ID and a map of fields.
+ */
+export interface TheographicRecord {
+    /**
+     * The Airtable record ID of the record. e.g. "recO5ToygNrV47m4M"
+     */
+    id: string;
+    /**
+     * The fields of the record.
+     */
+    fields: {
+        [key: string]: any;
+    };
+}
+/**
+ * The parsed contents of the Theographic JSON files that are needed
+ * to generate the dataset.
+ */
+export interface TheographicFiles {
+    books: TheographicRecord[];
+    verses: TheographicRecord[];
+    people: TheographicRecord[];
+    places: TheographicRecord[];
+    events: TheographicRecord[];
+    peopleGroups: TheographicRecord[];
+}
+/**
+ * Maps Theographic OSIS book names to USFM book IDs.
+ */
+export declare const osisBookMap: Map<string, "GEN" | "EXO" | "LEV" | "NUM" | "DEU" | "JOS" | "JDG" | "RUT" | "1SA" | "2SA" | "1KI" | "2KI" | "1CH" | "2CH" | "EZR" | "NEH" | "EST" | "JOB" | "PSA" | "PRO" | "ECC" | "SNG" | "ISA" | "JER" | "LAM" | "EZK" | "DAN" | "HOS" | "JOL" | "AMO" | "OBA" | "JON" | "MIC" | "NAM" | "HAB" | "ZEP" | "HAG" | "ZEC" | "MAL" | "MAT" | "MRK" | "LUK" | "JHN" | "ACT" | "ROM" | "1CO" | "2CO" | "GAL" | "EPH" | "PHP" | "COL" | "1TH" | "2TH" | "1TI" | "2TI" | "TIT" | "PHM" | "HEB" | "JAS" | "1PE" | "2PE" | "1JN" | "2JN" | "3JN" | "JUD" | "REV" | "TOB" | "JDT" | "ESG" | "WIS" | "SIR" | "BAR" | "LJE" | "S3Y" | "SUS" | "BEL" | "1MA" | "2MA" | "3MA" | "4MA" | "1ES" | "2ES" | "MAN" | "PS2" | "ODA" | "PSS" | "EZA" | "5EZ" | "6EZ" | "DAG" | "PS3" | "2BA" | "LBA" | "JUB" | "ENO" | "1MQ" | "2MQ" | "3MQ" | "REP" | "4BA" | "LAO">;
+/**
+ * Slugifies the given string so that it can be used as an entity ID.
+ * @param str The string to slugify.
+ */
+export declare function slugify(str: string): string;
+/**
+ * Removes markdown links from the given text, keeping only the link text.
+ * For example, "[Esther 1:10](/esth#Esth.1.10)" becomes "Esther 1:10".
+ * @param text The text to clean.
+ */
+export declare function stripMarkdownLinks(text: string): string;
+/**
+ * Generates a dataset from the given Theographic Bible Metadata files.
+ *
+ * The returned dataset contains the people, places, events, and people groups
+ * from the Theographic data, with all cross-entity relationships resolved to entity IDs
+ * and all verse links resolved to Bible verse references.
+ *
+ * The transformation is deterministic: the same input always produces the same output.
+ *
+ * @param files The parsed Theographic JSON files.
+ * @param dataset The dataset information to use. Defaults to {@link THEOGRAPHIC_DATASET}.
+ */
+export declare function generateDatasetFromTheographic(files: TheographicFiles, dataset?: Dataset): DatasetDataset;
+//# sourceMappingURL=theographic.d.ts.map

package/dist/types/parser/commentary-csv-parser.d.ts

@@ -1,4 +1,4 @@
-import { CommentaryParseTree } from './types';
+import { CommentaryParseTree } from './types.js';
 export interface CsvLine {
     book: string;
     chapter: string;

package/dist/types/parser/lockman-parser.d.ts

@@ -1,4 +1,4 @@
-import { ParseTree } from './types';
+import { ParseTree } from './types.js';
 export declare const LOCKMAN_PARSER_VERSION = 3;
 export declare class LockmanParser {
     parse(text: string): ParseTree[];

package/dist/types/parser/tyndale-xml-parser.d.ts

@@ -1,4 +1,4 @@
-import { CommentaryParseTree } from './types';
+import { CommentaryParseTree } from './types.js';
 export declare class TyndaleXmlParser {
     private _domParser;
     constructor(domParser: DOMParser);

package/dist/types/parser/types.d.ts

@@ -49,6 +49,69 @@ export interface Chapter {
      * The list of footnotes for the chapter.
      */
     footnotes: Footnote[];
+    /**
+     * The word-level annotations for the chapter's verses.
+     * Undefined if the source didn't contain any word-level annotations.
+     */
+    words?: ChapterWords;
+}
+/**
+ * Defines the word-level annotations for a chapter, keyed by verse number.
+ */
+export type ChapterWords = {
+    [verseNumber: string]: ChapterWord[];
+};
+/**
+ * Defines an interface that represents the annotations that a source associated
+ * with a specific range of characters in a verse.
+ *
+ * The range is anchored to a single item of the verse's content array, so that
+ * consumers can highlight the exact characters that an annotation applies to.
+ */
+export interface ChapterWord {
+    /**
+     * The index of the item in the verse's content array that the annotation applies to.
+     */
+    contentIndex: number;
+    /**
+     * The index of the first character of the annotated word in the content item's text.
+     */
+    start: number;
+    /**
+     * The index after the last character of the annotated word in the content item's text.
+     * That is, `text.slice(start, end)` is the annotated word.
+     */
+    end: number;
+    /**
+     * The Strong's number(s) for the word.
+     * Undefined if the source only provided other annotations for the word.
+     */
+    strongs?: string[];
+    /**
+     * The dictionary (citation) form of the word.
+     * Taken from the `lemma` attribute.
+     */
+    lemma?: string;
+    /**
+     * The morphology parse code for the word.
+     * Taken from the `x-morph` attribute.
+     */
+    morph?: string;
+    /**
+     * The pointer to the word in the source text, in the `<sourceName>:<location>` format.
+     * Taken from the `srcloc` attribute.
+     */
+    srcloc?: string;
+    /**
+     * Which occurrence of the source word this word is. 1-based.
+     * Taken from the `x-occurrence` attribute.
+     */
+    occurrence?: number;
+    /**
+     * The total number of times that the source word occurs.
+     * Taken from the `x-occurrences` attribute.
+     */
+    occurrences?: number;
 }
 /**
  * Defines an interface that represents a hebrew subtitle.

package/dist/types/parser/usx-parser.d.ts

@@ -1,10 +1,11 @@
 import { Chapter, ChapterContent, FootnoteReference, ParseTree, Verse, Text, HebrewSubtitle, InlineLineBreak } from './types.js';
 import { RewindableIterator } from './iterators.js';
+import { AnnotatedContent } from './words.js';
 /**
  * The version of the parser.
  * Used to determine whether input files need to be re-parsed.
  */
-export declare const PARSER_VERSION = 3;
+export declare const PARSER_VERSION = 4;
 /**
  * Defines a class that is able to parse USX content.
  */
@@ -25,13 +26,13 @@ export declare class USXParser {
     parse(usx: string): ParseTree;
     iterateRootContent(usxElement: Element): Generator<ParseTree['content'][0]>;
     iterateChapterContent(chapter: Chapter, nodes: RewindableIterator<Node>): IterableIterator<ChapterContent>;
-    iterateVerseContent(chapter: Chapter, verse: Verse, nodes: RewindableIterator<Node>, previousVerse?: Verse | null): IterableIterator<string | FootnoteReference | Text | InlineLineBreak>;
+    iterateVerseContent(chapter: Chapter, verse: Verse, nodes: RewindableIterator<Node>, previousVerse?: Verse | null): IterableIterator<string | FootnoteReference | Text | InlineLineBreak | AnnotatedContent<string | Text>>;
     parseVerse(element: Element, chapter: Chapter, nodes: RewindableIterator<Node>): Verse;
     parseHebrewSubtitle(para: Element, chapter: Chapter, nodes: RewindableIterator<Node>): IterableIterator<HebrewSubtitle | Verse>;
-    iterateHebrewSubtitleContent(element: Element, chapter: Chapter, nodes: RewindableIterator<Node>): IterableIterator<Verse | string | Text | FootnoteReference | InlineLineBreak>;
-    iterateNodeTextContent(nodes: RewindableIterator<Node>, node: Node, chapter: Chapter, verse?: Verse): IterableIterator<string | Text | FootnoteReference | InlineLineBreak>;
+    iterateHebrewSubtitleContent(element: Element, chapter: Chapter, nodes: RewindableIterator<Node>): IterableIterator<Verse | string | Text | FootnoteReference | InlineLineBreak | AnnotatedContent<string | Text>>;
+    iterateNodeTextContent(nodes: RewindableIterator<Node>, node: Node, chapter: Chapter, verse?: Verse): IterableIterator<string | Text | FootnoteReference | InlineLineBreak | AnnotatedContent<string | Text>>;
     iterateCharContent(char: Element): IterableIterator<string | Text>;
     iterateNote(nodes: RewindableIterator<Node>, node: Element, chapter: Chapter, verse?: Verse): IterableIterator<FootnoteReference>;
-    iterateChar(nodes: RewindableIterator<Node>, node: Element): IterableIterator<string | Text>;
+    iterateChar(nodes: RewindableIterator<Node>, node: Element): IterableIterator<string | Text | AnnotatedContent<string | Text>>;
 }
 //# sourceMappingURL=usx-parser.d.ts.map

package/dist/types/parser/words.d.ts

@@ -0,0 +1,104 @@
+import { Chapter, ChapterWord } from './types.js';
+/**
+ * The annotations that a source associated with a word, without any information
+ * about where the word is located.
+ */
+export type WordAnnotations = Omit<ChapterWord, 'contentIndex' | 'start' | 'end'>;
+/**
+ * Defines an interface that represents a range of characters within a single
+ * piece of text, along with the annotations that apply to it.
+ */
+export interface WordRange {
+    /**
+     * The index of the first character of the word.
+     */
+    start: number;
+    /**
+     * The index after the last character of the word.
+     */
+    end: number;
+    /**
+     * The annotations that apply to the word.
+     */
+    annotations: WordAnnotations;
+}
+/**
+ * Defines an interface that pairs a piece of verse content with the word
+ * annotations that apply to it, so that the annotations can be passed along
+ * with the content while it is being parsed.
+ *
+ * The ranges are relative to the text of the content, and are never included in
+ * the final parse tree.
+ */
+export interface AnnotatedContent<T> {
+    /**
+     * The content that the annotations apply to.
+     */
+    annotatedContent: T;
+    /**
+     * The ranges of the content that are annotated.
+     */
+    words: WordRange[];
+}
+/**
+ * Determines whether the given value is content that has word annotations
+ * attached to it.
+ * @param value The value to test.
+ */
+export declare function isAnnotatedContent(value: unknown): value is AnnotatedContent<unknown>;
+/**
+ * Reads the word annotations that are attached to the given element.
+ * Returns null if the element doesn't have any recognized annotations.
+ * @param element The element to read the annotations from.
+ */
+export declare function readWordAnnotations(element: Element): WordAnnotations | null;
+/**
+ * Shrinks the given range so that it doesn't include any leading or trailing
+ * whitespace. Returns null if the range only contains whitespace.
+ * @param text The text that the range is in.
+ * @param start The index of the first character of the range.
+ * @param end The index after the last character of the range.
+ */
+export declare function trimWordRange(text: string, start: number, end: number): {
+    start: number;
+    end: number;
+} | null;
+/**
+ * Collapses the whitespace in the given text in the same manner that the
+ * parsers normalize text (each run of whitespace becomes a single space, and the
+ * result is trimmed), and returns a map from the indexes in the given text to
+ * the indexes in the returned text.
+ *
+ * The map contains an entry for every index in the given text, plus one for the
+ * index just past the end of it, so that both ends of a range can be mapped.
+ * @param text The text to collapse.
+ */
+export declare function collapseWhitespaceMap(text: string): {
+    text: string;
+    map: number[];
+};
+/**
+ * Moves the words that apply to the given content item to the indexes that they
+ * have after the item's text was collapsed with collapseWhitespaceMap().
+ * Words that no longer cover any characters are removed.
+ * @param words The words to remap.
+ * @param contentIndex The index of the content item that was collapsed.
+ * @param map The index map that collapseWhitespaceMap() returned.
+ */
+export declare function remapChapterWords(words: ChapterWord[], contentIndex: number, map: number[]): void;
+/**
+ * Removes the words that apply to the given content item, and moves the words
+ * that apply to later items back by one to account for the item being removed.
+ * @param words The words to update.
+ * @param contentIndex The index of the content item that was removed.
+ */
+export declare function removeChapterWordsForContent(words: ChapterWord[], contentIndex: number): void;
+/**
+ * Adds the given words to the chapter for the given verse number.
+ * Does nothing if there are no words.
+ * @param chapter The chapter to add the words to.
+ * @param verseNumber The number of the verse that the words are in.
+ * @param words The words to add.
+ */
+export declare function addChapterWords(chapter: Chapter, verseNumber: number, words: ChapterWord[]): void;
+//# sourceMappingURL=words.d.ts.map

package/package.json

@@ -1,7 +1,8 @@
 {
   "name": "@helloao/tools",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "A set of tools for managing HelloAO's Free Bible API",
+  "type": "module",
   "main": "./dist/cjs/index.cjs",
   "module": "./dist/esm/index.js",
   "exports": {
@@ -49,7 +50,7 @@
   "author": "Kallyn Gowdy <kal@helloao.org>",
   "license": "MIT",
   "dependencies": {
-    "lodash": "4.17.21",
+    "es-toolkit": "1.46.1",
     "zod": "^4.4.3",
     "papaparse": "5.4.1"
   },