@helloao/tools 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/generation/api.cjs +69 -1
- package/dist/cjs/generation/api.cjs.map +2 -2
- package/dist/cjs/generation/common-types.cjs.map +1 -1
- package/dist/cjs/generation/dataset.cjs +59 -42
- package/dist/cjs/generation/dataset.cjs.map +3 -3
- package/dist/cjs/index.cjs +3 -0
- package/dist/cjs/index.cjs.map +2 -2
- package/dist/cjs/log.cjs +50 -0
- package/dist/cjs/log.cjs.map +7 -0
- package/dist/cjs/parser/codex-parser.cjs +15 -12
- package/dist/cjs/parser/codex-parser.cjs.map +2 -2
- package/dist/cjs/parser/commentary-csv-parser.cjs.map +2 -2
- package/dist/cjs/parser/iterators.cjs +17 -6
- package/dist/cjs/parser/iterators.cjs.map +2 -2
- package/dist/cjs/parser/tyndale-xml-parser.cjs +212 -0
- package/dist/cjs/parser/tyndale-xml-parser.cjs.map +7 -0
- package/dist/cjs/parser/types.cjs.map +1 -1
- package/dist/cjs/parser/usfm-parser.cjs +4 -2
- package/dist/cjs/parser/usfm-parser.cjs.map +2 -2
- package/dist/cjs/utils.cjs +26 -4
- package/dist/cjs/utils.cjs.map +2 -2
- package/dist/esm/generation/api.js +65 -1
- package/dist/esm/generation/api.js.map +2 -2
- package/dist/esm/generation/dataset.js +60 -43
- package/dist/esm/generation/dataset.js.map +2 -2
- package/dist/esm/index.js +2 -1
- package/dist/esm/index.js.map +2 -2
- package/dist/esm/log.js +20 -0
- package/dist/esm/log.js.map +7 -0
- package/dist/esm/parser/codex-parser.js +15 -12
- package/dist/esm/parser/codex-parser.js.map +2 -2
- package/dist/esm/parser/commentary-csv-parser.js.map +2 -2
- package/dist/esm/parser/iterators.js +17 -6
- package/dist/esm/parser/iterators.js.map +2 -2
- package/dist/esm/parser/tyndale-xml-parser.js +184 -0
- package/dist/esm/parser/tyndale-xml-parser.js.map +7 -0
- package/dist/esm/parser/usfm-parser.js +4 -2
- package/dist/esm/parser/usfm-parser.js.map +2 -2
- package/dist/esm/utils.js +26 -4
- package/dist/esm/utils.js.map +2 -2
- package/dist/types/generation/api.d.ts +76 -1
- package/dist/types/generation/common-types.d.ts +31 -1
- package/dist/types/generation/dataset.d.ts +17 -3
- package/dist/types/index.d.ts +2 -1
- package/dist/types/log.d.ts +23 -0
- package/dist/types/parser/tyndale-xml-parser.d.ts +8 -0
- package/dist/types/parser/types.d.ts +25 -0
- package/dist/types/utils.d.ts +8 -0
- package/package.json +35 -13
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
import { UsfmParser } from "../parser/usfm-parser.js";
|
|
3
3
|
import { USXParser } from "../parser/usx-parser.js";
|
|
4
|
-
import { bookIdMap, bookOrderMap } from "./book-order.js";
|
|
4
|
+
import { bookIdMap as defaultBookIdMap, bookOrderMap } from "./book-order.js";
|
|
5
5
|
import { omit, sortBy, sortedIndexBy } from "lodash";
|
|
6
6
|
import { getAudioUrlsForChapter } from "./audio.js";
|
|
7
7
|
import { CodexParser } from "../parser/codex-parser.js";
|
|
8
8
|
import { CommentaryCsvParser } from "../parser/commentary-csv-parser.js";
|
|
9
|
-
|
|
9
|
+
import { TyndaleXmlParser } from "../parser/tyndale-xml-parser.js";
|
|
10
|
+
import { getLogger } from "../log.js";
|
|
11
|
+
export function generateDataset(files, parser = new globalThis.DOMParser(), bookMap) {
|
|
12
|
+
const logger = getLogger();
|
|
10
13
|
let output = {
|
|
11
14
|
translations: [],
|
|
12
15
|
commentaries: []
|
|
@@ -15,6 +18,7 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
15
18
|
let usxParser = new USXParser(parser);
|
|
16
19
|
let codexParser = new CodexParser();
|
|
17
20
|
let csvCommentaryParser = new CommentaryCsvParser();
|
|
21
|
+
let tyndaleXmlParser = new TyndaleXmlParser(parser);
|
|
18
22
|
let parsedTranslations = /* @__PURE__ */ new Map();
|
|
19
23
|
let parsedCommentaries = /* @__PURE__ */ new Map();
|
|
20
24
|
const unknownLanguages = /* @__PURE__ */ new Set();
|
|
@@ -29,8 +33,10 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
29
33
|
parser2 = codexParser;
|
|
30
34
|
} else if (file.fileType === "commentary/csv") {
|
|
31
35
|
parser2 = csvCommentaryParser;
|
|
36
|
+
} else if (file.fileType === "commentary/tyndale-xml") {
|
|
37
|
+
parser2 = tyndaleXmlParser;
|
|
32
38
|
} else {
|
|
33
|
-
|
|
39
|
+
logger.warn(
|
|
34
40
|
"[generate] File does not have a valid type!",
|
|
35
41
|
file.name
|
|
36
42
|
);
|
|
@@ -43,7 +49,7 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
43
49
|
addCommentaryTree(file, parsed);
|
|
44
50
|
}
|
|
45
51
|
} catch (err) {
|
|
46
|
-
|
|
52
|
+
logger.error(
|
|
47
53
|
`[generate] Error occurred while parsing ${file.name}`,
|
|
48
54
|
err
|
|
49
55
|
);
|
|
@@ -52,7 +58,7 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
52
58
|
function addTranslationTree(file, parsed) {
|
|
53
59
|
const id = parsed.id;
|
|
54
60
|
if (!id) {
|
|
55
|
-
|
|
61
|
+
logger.warn(
|
|
56
62
|
"[generate] File does not have a valid book ID!",
|
|
57
63
|
file.name,
|
|
58
64
|
id
|
|
@@ -61,28 +67,10 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
61
67
|
}
|
|
62
68
|
const order = bookOrderMap.get(id);
|
|
63
69
|
if (typeof order !== "number") {
|
|
64
|
-
|
|
70
|
+
logger.warn("[generate] Book does not have an order!", id);
|
|
65
71
|
return;
|
|
66
72
|
}
|
|
67
|
-
const
|
|
68
|
-
if (!bookMap) {
|
|
69
|
-
if (!unknownLanguages.has(file.metadata.language)) {
|
|
70
|
-
console.warn(
|
|
71
|
-
"[generate] File does not have a known language!",
|
|
72
|
-
file.name,
|
|
73
|
-
file.metadata.language
|
|
74
|
-
);
|
|
75
|
-
unknownLanguages.add(file.metadata.language);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
const bookName = bookMap?.get(id);
|
|
79
|
-
if (!!bookMap && !bookName) {
|
|
80
|
-
console.warn(
|
|
81
|
-
"[generate] Book name not found for ID!",
|
|
82
|
-
file.name,
|
|
83
|
-
id
|
|
84
|
-
);
|
|
85
|
-
}
|
|
73
|
+
const bookName = getBookNames(file, file.metadata, id);
|
|
86
74
|
let translation = parsedTranslations.get(file.metadata.id);
|
|
87
75
|
if (!translation) {
|
|
88
76
|
translation = {
|
|
@@ -93,16 +81,12 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
93
81
|
output.translations.push(translation);
|
|
94
82
|
parsedTranslations.set(file.metadata.id, translation);
|
|
95
83
|
}
|
|
96
|
-
const name = parsed.header ?? bookName?.commonName ?? parsed.title;
|
|
84
|
+
const name = parsed.header ?? bookName?.bookName?.commonName ?? parsed.title;
|
|
97
85
|
if (!name) {
|
|
98
|
-
|
|
99
|
-
"[generate] Book does not have a name!",
|
|
100
|
-
file.name,
|
|
101
|
-
id
|
|
102
|
-
);
|
|
86
|
+
logger.warn("[generate] Book does not have a name!", file.name, id);
|
|
103
87
|
return;
|
|
104
88
|
}
|
|
105
|
-
const commonName = bookName?.commonName ?? parsed.header ?? parsed.title ?? id;
|
|
89
|
+
const commonName = bookName?.bookName?.commonName ?? parsed.header ?? parsed.title ?? id;
|
|
106
90
|
const book = {
|
|
107
91
|
id,
|
|
108
92
|
name,
|
|
@@ -137,15 +121,16 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
137
121
|
commentary = {
|
|
138
122
|
...omit(file.metadata, "direction"),
|
|
139
123
|
textDirection: file.metadata.direction,
|
|
140
|
-
books: []
|
|
124
|
+
books: [],
|
|
125
|
+
profiles: []
|
|
141
126
|
};
|
|
142
127
|
output.commentaries.push(commentary);
|
|
143
128
|
parsedCommentaries.set(file.metadata.id, commentary);
|
|
144
129
|
}
|
|
145
130
|
for (let parsedBook of parsed.books) {
|
|
146
131
|
let book = commentary.books.find((b) => b.id === parsedBook.book);
|
|
147
|
-
if (book && book.introduction) {
|
|
148
|
-
|
|
132
|
+
if (book && book.introduction && parsedBook.introduction) {
|
|
133
|
+
logger.warn(
|
|
149
134
|
"[generate] Book already exists in commentary!",
|
|
150
135
|
file.name,
|
|
151
136
|
parsedBook.book
|
|
@@ -165,6 +150,9 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
165
150
|
if (parsedBook.introduction) {
|
|
166
151
|
book.introduction = parsedBook.introduction;
|
|
167
152
|
}
|
|
153
|
+
if (parsedBook.introductionSummary && !book.introductionSummary) {
|
|
154
|
+
book.introductionSummary = parsedBook.introductionSummary;
|
|
155
|
+
}
|
|
168
156
|
for (let chapter of parsedBook.chapters) {
|
|
169
157
|
let data = {
|
|
170
158
|
number: chapter.number,
|
|
@@ -180,23 +168,52 @@ export function generateDataset(files, parser = new globalThis.DOMParser()) {
|
|
|
180
168
|
book.chapters = sortBy(book.chapters, (c) => c.chapter.number);
|
|
181
169
|
commentary.books.push(book);
|
|
182
170
|
}
|
|
171
|
+
if (parsed.profiles) {
|
|
172
|
+
for (let profile of parsed.profiles) {
|
|
173
|
+
const existing = commentary.profiles.find(
|
|
174
|
+
(p) => p.id === profile.id
|
|
175
|
+
);
|
|
176
|
+
if (existing) {
|
|
177
|
+
logger.warn(
|
|
178
|
+
"[generate] Profile already exists in commentary!",
|
|
179
|
+
file.name,
|
|
180
|
+
profile.id
|
|
181
|
+
);
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
const datasetProfile = {
|
|
185
|
+
id: profile.id,
|
|
186
|
+
subject: profile.subject,
|
|
187
|
+
reference: profile.reference,
|
|
188
|
+
content: profile.content
|
|
189
|
+
};
|
|
190
|
+
commentary.profiles.push(datasetProfile);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
183
193
|
commentary.books = sortBy(commentary.books, (b) => b.order);
|
|
194
|
+
commentary.profiles = sortBy(commentary.profiles, (p) => p.id);
|
|
184
195
|
}
|
|
185
196
|
function getBookNames(file, metadata, id) {
|
|
186
|
-
const bookMap = bookIdMap.get(metadata.language);
|
|
187
197
|
if (!bookMap) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
198
|
+
bookMap = defaultBookIdMap.get(metadata.language);
|
|
199
|
+
if (!bookMap) {
|
|
200
|
+
if (!unknownLanguages.has(metadata.language)) {
|
|
201
|
+
logger.warn(
|
|
202
|
+
"[generate] File does not have a known language!",
|
|
203
|
+
file.name,
|
|
204
|
+
metadata.language
|
|
205
|
+
);
|
|
206
|
+
unknownLanguages.add(metadata.language);
|
|
207
|
+
}
|
|
208
|
+
logger.warn(
|
|
209
|
+
"[generate] Using English book map for unknown language! This might result in outputting english book names."
|
|
193
210
|
);
|
|
194
|
-
|
|
211
|
+
bookMap = defaultBookIdMap.get("en");
|
|
195
212
|
}
|
|
196
213
|
}
|
|
197
214
|
const bookName = bookMap?.get(id);
|
|
198
215
|
if (!!bookMap && !bookName) {
|
|
199
|
-
|
|
216
|
+
logger.warn(
|
|
200
217
|
"[generate] Book name not found for ID!",
|
|
201
218
|
file.name,
|
|
202
219
|
id
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../generation/dataset.ts"],
|
|
4
|
-
"sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\r\nimport { USXParser } from '../parser/usx-parser.js';\r\nimport {\r\n Commentary,\r\n CommentaryBook,\r\n CommentaryBookChapter,\r\n CommentaryChapterData,\r\n InputCommentaryFile,\r\n InputFile,\r\n InputFileBase,\r\n InputTranslationFile,\r\n MetadataBase,\r\n Translation,\r\n TranslationBook,\r\n TranslationBookChapter,\r\n} from './common-types.js';\r\nimport { bookIdMap, bookOrderMap } from './book-order.js';\r\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\r\nimport { getAudioUrlsForChapter } from './audio.js';\r\nimport { CodexParser } from '../parser/codex-parser.js';\r\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\r\nimport { CommentaryParseTree, ParseTree } from '../parser/types.js';\r\n\r\n/**\r\n * Defines an interface that contains generated dataset info.\r\n */\r\nexport interface DatasetOutput {\r\n /**\r\n * The list of translations that are available in the dataset.\r\n */\r\n translations: DatasetTranslation[];\r\n\r\n /**\r\n * The list of commentaries that are available in the dataset.\r\n */\r\n commentaries: DatasetCommentary[];\r\n}\r\n\r\n/**\r\n * Defines a translation that is used in the dataset.\r\n */\r\nexport interface DatasetTranslation extends Translation {\r\n /**\r\n * The list of books that are available for the translation.\r\n */\r\n books: DatasetTranslationBook[];\r\n}\r\n\r\n/**\r\n * Defines a commentary that is used in the dataset.\r\n */\r\nexport interface DatasetCommentary extends Commentary {\r\n /**\r\n * The list of books that are available for the translation.\r\n */\r\n books: DatasetCommentaryBook[];\r\n}\r\n\r\n/**\r\n * Defines a translation book that is used in the dataset.\r\n */\r\nexport interface DatasetTranslationBook extends TranslationBook {\r\n /**\r\n * The list of chapters that are available for the book.\r\n */\r\n chapters: TranslationBookChapter[];\r\n}\r\n\r\n/**\r\n * Defines a commentary book that is used in the dataset.\r\n */\r\nexport interface DatasetCommentaryBook extends CommentaryBook {\r\n /**\r\n * The list of chapters that are available for the book.\r\n */\r\n chapters: CommentaryBookChapter[];\r\n}\r\n\r\n/**\r\n * Generates a list of output files from the given list of input files.\r\n * @param file The list of files.\r\n */\r\nexport function generateDataset(\r\n files: InputFile[],\r\n parser: DOMParser = new globalThis.DOMParser()\r\n): DatasetOutput {\r\n let output: DatasetOutput = {\r\n translations: [],\r\n commentaries: [],\r\n };\r\n\r\n let usfmParser = new UsfmParser();\r\n let usxParser = new USXParser(parser);\r\n let codexParser = new CodexParser();\r\n let csvCommentaryParser = new CommentaryCsvParser();\r\n\r\n let parsedTranslations = new Map<string, DatasetTranslation>();\r\n let parsedCommentaries = new Map<string, DatasetCommentary>();\r\n\r\n const unknownLanguages = new Set<string>();\r\n for (let file of files) {\r\n try {\r\n let parser:\r\n | UsfmParser\r\n | USXParser\r\n | CodexParser\r\n | CommentaryCsvParser;\r\n if (file.fileType === 'usfm') {\r\n parser = usfmParser;\r\n } else if (file.fileType === 'usx') {\r\n parser = usxParser;\r\n } else if (file.fileType === 'json') {\r\n parser = codexParser;\r\n } else if (file.fileType === 'commentary/csv') {\r\n parser = csvCommentaryParser;\r\n } else {\r\n console.warn(\r\n '[generate] File does not have a valid type!',\r\n file.name\r\n );\r\n continue;\r\n }\r\n\r\n const parsed = parser.parse(file.content);\r\n\r\n if (parsed.type === 'root') {\r\n addTranslationTree(file as InputTranslationFile, parsed);\r\n } else {\r\n addCommentaryTree(file as InputCommentaryFile, parsed);\r\n }\r\n } catch (err) {\r\n console.error(\r\n `[generate] Error occurred while parsing ${file.name}`,\r\n err\r\n );\r\n }\r\n }\r\n\r\n function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\r\n const id = parsed.id;\r\n\r\n if (!id) {\r\n console.warn(\r\n '[generate] File does not have a valid book ID!',\r\n file.name,\r\n id\r\n );\r\n return;\r\n }\r\n\r\n const order = bookOrderMap.get(id);\r\n\r\n if (typeof order !== 'number') {\r\n console.warn('[generate] Book does not have an order!', id);\r\n return;\r\n }\r\n\r\n const bookMap = bookIdMap.get(file.metadata.language);\r\n\r\n if (!bookMap) {\r\n if (!unknownLanguages.has(file.metadata.language)) {\r\n console.warn(\r\n '[generate] File does not have a known language!',\r\n file.name,\r\n file.metadata.language\r\n );\r\n unknownLanguages.add(file.metadata.language);\r\n }\r\n }\r\n\r\n const bookName = bookMap?.get(id);\r\n\r\n if (!!bookMap && !bookName) {\r\n console.warn(\r\n '[generate] Book name not found for ID!',\r\n file.name,\r\n id\r\n );\r\n }\r\n\r\n let translation = parsedTranslations.get(file.metadata.id);\r\n\r\n if (!translation) {\r\n translation = {\r\n ...omit(file.metadata, 'direction'),\r\n textDirection: file.metadata.direction,\r\n books: [],\r\n };\r\n output.translations.push(translation);\r\n parsedTranslations.set(file.metadata.id, translation);\r\n }\r\n\r\n const name = parsed.header ?? bookName?.commonName ?? parsed.title;\r\n\r\n if (!name) {\r\n console.warn(\r\n '[generate] Book does not have a name!',\r\n file.name,\r\n id\r\n );\r\n return;\r\n }\r\n\r\n const commonName =\r\n bookName?.commonName ?? parsed.header ?? parsed.title ?? id;\r\n\r\n const book: DatasetTranslationBook = {\r\n id: id,\r\n name,\r\n commonName,\r\n title: parsed.title ?? null,\r\n order,\r\n chapters: [],\r\n };\r\n\r\n for (let content of parsed.content) {\r\n if (content.type === 'chapter') {\r\n book.chapters.push({\r\n chapter: {\r\n number: content.number,\r\n content: content.content,\r\n footnotes: content.footnotes,\r\n },\r\n thisChapterAudioLinks: getAudioUrlsForChapter(\r\n translation.id,\r\n id,\r\n content.number\r\n ),\r\n });\r\n }\r\n }\r\n\r\n book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\r\n\r\n const index = sortedIndexBy(translation.books, book, (b) => b.order);\r\n translation.books.splice(index, 0, book);\r\n }\r\n\r\n function addCommentaryTree(\r\n file: InputCommentaryFile,\r\n parsed: CommentaryParseTree\r\n ) {\r\n let commentary = parsedCommentaries.get(file.metadata.id);\r\n\r\n if (!commentary) {\r\n commentary = {\r\n ...omit(file.metadata, 'direction'),\r\n textDirection: file.metadata.direction,\r\n books: [],\r\n };\r\n output.commentaries.push(commentary);\r\n parsedCommentaries.set(file.metadata.id, commentary);\r\n }\r\n\r\n for (let parsedBook of parsed.books) {\r\n let book = commentary.books.find((b) => b.id === parsedBook.book);\r\n if (book && book.introduction) {\r\n console.warn(\r\n '[generate] Book already exists in commentary!',\r\n file.name,\r\n parsedBook.book\r\n );\r\n continue;\r\n }\r\n\r\n if (!book) {\r\n const name = getBookNames(file, file.metadata, parsedBook.book);\r\n book = {\r\n id: parsedBook.book,\r\n order: bookOrderMap.get(parsedBook.book) ?? -1,\r\n commonName: name?.bookName?.commonName ?? parsedBook.book,\r\n name: name?.bookName?.commonName ?? parsedBook.book,\r\n chapters: [],\r\n };\r\n }\r\n\r\n if (parsedBook.introduction) {\r\n book.introduction = parsedBook.introduction;\r\n }\r\n\r\n for (let chapter of parsedBook.chapters) {\r\n let data: CommentaryChapterData = {\r\n number: chapter.number,\r\n content: chapter.verses,\r\n };\r\n\r\n if (chapter.introduction) {\r\n data.introduction = chapter.introduction;\r\n }\r\n\r\n book.chapters.push({\r\n chapter: data,\r\n });\r\n }\r\n\r\n book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\r\n commentary.books.push(book);\r\n }\r\n\r\n commentary.books = sortBy(commentary.books, (b) => b.order);\r\n }\r\n\r\n function getBookNames(\r\n file: InputFileBase,\r\n metadata: MetadataBase,\r\n id: string\r\n ) {\r\n const bookMap = bookIdMap.get(metadata.language);\r\n\r\n if (!bookMap) {\r\n if (!unknownLanguages.has(metadata.language)) {\r\n console.warn(\r\n '[generate] File does not have a known language!',\r\n file.name,\r\n metadata.language\r\n );\r\n unknownLanguages.add(metadata.language);\r\n }\r\n }\r\n\r\n const bookName = bookMap?.get(id);\r\n\r\n if (!!bookMap && !bookName) {\r\n console.warn(\r\n '[generate] Book name not found for ID!',\r\n file.name,\r\n id\r\n );\r\n return null;\r\n }\r\n\r\n return {\r\n bookName,\r\n };\r\n }\r\n\r\n return output;\r\n}\r\n"],
|
|
5
|
-
"mappings": ";AAAA,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;
|
|
4
|
+
"sourcesContent": ["import { UsfmParser } from '../parser/usfm-parser.js';\nimport { USXParser } from '../parser/usx-parser.js';\nimport {\n Commentary,\n CommentaryBook,\n CommentaryBookChapter,\n CommentaryChapterData,\n CommentaryProfile,\n InputCommentaryFile,\n InputFile,\n InputFileBase,\n InputTranslationFile,\n MetadataBase,\n Translation,\n TranslationBook,\n TranslationBookChapter,\n} from './common-types.js';\nimport { bookIdMap as defaultBookIdMap, bookOrderMap } from './book-order.js';\nimport { omit, sortBy, sortedIndexBy } from 'lodash';\nimport { getAudioUrlsForChapter } from './audio.js';\nimport { CodexParser } from '../parser/codex-parser.js';\nimport { CommentaryCsvParser } from '../parser/commentary-csv-parser.js';\nimport { CommentaryParseTree, ParseTree } from '../parser/types.js';\nimport { TyndaleXmlParser } from '../parser/tyndale-xml-parser.js';\nimport { getLogger } from '../log.js';\n\n/**\n * Defines an interface that contains generated dataset info.\n */\nexport interface DatasetOutput {\n /**\n * The list of translations that are available in the dataset.\n */\n translations: DatasetTranslation[];\n\n /**\n * The list of commentaries that are available in the dataset.\n */\n commentaries: DatasetCommentary[];\n}\n\n/**\n * Defines a translation that is used in the dataset.\n */\nexport interface DatasetTranslation extends Translation {\n /**\n * The list of books that are available for the translation.\n */\n books: DatasetTranslationBook[];\n}\n\n/**\n * Defines a commentary that is used in the dataset.\n */\nexport interface DatasetCommentary extends Commentary {\n /**\n * The list of books that are available for the commentary.\n */\n books: DatasetCommentaryBook[];\n\n /**\n * The list of profiles that are available for the commentary.\n */\n profiles: DatasetCommentaryProfile[];\n}\n\n/**\n * Defines a translation book that is used in the dataset.\n */\nexport interface DatasetTranslationBook extends TranslationBook {\n /**\n * The list of chapters that are available for the book.\n */\n chapters: TranslationBookChapter[];\n}\n\n/**\n * Defines a commentary book that is used in the dataset.\n */\nexport interface DatasetCommentaryBook extends CommentaryBook {\n /**\n * The list of chapters that are available for the book.\n */\n chapters: CommentaryBookChapter[];\n}\n\nexport interface DatasetCommentaryProfile extends CommentaryProfile {\n /**\n * The contents of the profile.\n */\n content: string[];\n}\n\n/**\n * Generates a list of output files from the given list of input files.\n * @param file The list of files.\n * @param parser The parser to use for parsing the files.\n * @param bookMap The map of book IDs to names. If undefined, then a default map will be used.\n */\nexport function generateDataset(\n files: InputFile[],\n parser: DOMParser = new globalThis.DOMParser(),\n bookMap?: Map<string, { commonName: string }> | undefined\n): DatasetOutput {\n const logger = getLogger();\n let output: DatasetOutput = {\n translations: [],\n commentaries: [],\n };\n\n let usfmParser = new UsfmParser();\n let usxParser = new USXParser(parser);\n let codexParser = new CodexParser();\n let csvCommentaryParser = new CommentaryCsvParser();\n let tyndaleXmlParser = new TyndaleXmlParser(parser);\n\n let parsedTranslations = new Map<string, DatasetTranslation>();\n let parsedCommentaries = new Map<string, DatasetCommentary>();\n\n const unknownLanguages = new Set<string>();\n for (let file of files) {\n try {\n let parser:\n | UsfmParser\n | USXParser\n | CodexParser\n | CommentaryCsvParser\n | TyndaleXmlParser;\n if (file.fileType === 'usfm') {\n parser = usfmParser;\n } else if (file.fileType === 'usx') {\n parser = usxParser;\n } else if (file.fileType === 'json') {\n parser = codexParser;\n } else if (file.fileType === 'commentary/csv') {\n parser = csvCommentaryParser;\n } else if (file.fileType === 'commentary/tyndale-xml') {\n parser = tyndaleXmlParser;\n } else {\n logger.warn(\n '[generate] File does not have a valid type!',\n file.name\n );\n continue;\n }\n\n const parsed = parser.parse(file.content);\n\n if (parsed.type === 'root') {\n addTranslationTree(file as InputTranslationFile, parsed);\n } else {\n addCommentaryTree(file as InputCommentaryFile, parsed);\n }\n } catch (err) {\n logger.error(\n `[generate] Error occurred while parsing ${file.name}`,\n err\n );\n }\n }\n\n function addTranslationTree(file: InputTranslationFile, parsed: ParseTree) {\n const id = parsed.id;\n\n if (!id) {\n logger.warn(\n '[generate] File does not have a valid book ID!',\n file.name,\n id\n );\n return;\n }\n\n const order = bookOrderMap.get(id);\n\n if (typeof order !== 'number') {\n logger.warn('[generate] Book does not have an order!', id);\n return;\n }\n\n const bookName = getBookNames(file, file.metadata, id);\n\n let translation = parsedTranslations.get(file.metadata.id);\n\n if (!translation) {\n translation = {\n ...omit(file.metadata, 'direction'),\n textDirection: file.metadata.direction,\n books: [],\n };\n output.translations.push(translation);\n parsedTranslations.set(file.metadata.id, translation);\n }\n\n const name =\n parsed.header ?? bookName?.bookName?.commonName ?? parsed.title;\n\n if (!name) {\n logger.warn('[generate] Book does not have a name!', file.name, id);\n return;\n }\n\n const commonName =\n bookName?.bookName?.commonName ??\n parsed.header ??\n parsed.title ??\n id;\n\n const book: DatasetTranslationBook = {\n id: id,\n name,\n commonName,\n title: parsed.title ?? null,\n order,\n chapters: [],\n };\n\n for (let content of parsed.content) {\n if (content.type === 'chapter') {\n book.chapters.push({\n chapter: {\n number: content.number,\n content: content.content,\n footnotes: content.footnotes,\n },\n thisChapterAudioLinks: getAudioUrlsForChapter(\n translation.id,\n id,\n content.number\n ),\n });\n }\n }\n\n book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n\n const index = sortedIndexBy(translation.books, book, (b) => b.order);\n translation.books.splice(index, 0, book);\n }\n\n function addCommentaryTree(\n file: InputCommentaryFile,\n parsed: CommentaryParseTree\n ) {\n let commentary = parsedCommentaries.get(file.metadata.id);\n\n if (!commentary) {\n commentary = {\n ...omit(file.metadata, 'direction'),\n textDirection: file.metadata.direction,\n books: [],\n profiles: [],\n };\n output.commentaries.push(commentary);\n parsedCommentaries.set(file.metadata.id, commentary);\n }\n\n for (let parsedBook of parsed.books) {\n let book = commentary.books.find((b) => b.id === parsedBook.book);\n if (book && book.introduction && parsedBook.introduction) {\n logger.warn(\n '[generate] Book already exists in commentary!',\n file.name,\n parsedBook.book\n );\n continue;\n }\n\n if (!book) {\n const name = getBookNames(file, file.metadata, parsedBook.book);\n book = {\n id: parsedBook.book,\n order: bookOrderMap.get(parsedBook.book) ?? -1,\n commonName: name?.bookName?.commonName ?? parsedBook.book,\n name: name?.bookName?.commonName ?? parsedBook.book,\n chapters: [],\n };\n }\n\n if (parsedBook.introduction) {\n book.introduction = parsedBook.introduction;\n }\n\n if (parsedBook.introductionSummary && !book.introductionSummary) {\n book.introductionSummary = parsedBook.introductionSummary;\n }\n\n for (let chapter of parsedBook.chapters) {\n let data: CommentaryChapterData = {\n number: chapter.number,\n content: chapter.verses,\n };\n\n if (chapter.introduction) {\n data.introduction = chapter.introduction;\n }\n\n book.chapters.push({\n chapter: data,\n });\n }\n\n book.chapters = sortBy(book.chapters, (c) => c.chapter.number);\n commentary.books.push(book);\n }\n\n if (parsed.profiles) {\n for (let profile of parsed.profiles) {\n const existing = commentary.profiles.find(\n (p) => p.id === profile.id\n );\n if (existing) {\n logger.warn(\n '[generate] Profile already exists in commentary!',\n file.name,\n profile.id\n );\n continue;\n }\n\n const datasetProfile: DatasetCommentaryProfile = {\n id: profile.id,\n subject: profile.subject,\n reference: profile.reference,\n content: profile.content,\n };\n\n commentary.profiles.push(datasetProfile);\n }\n }\n\n commentary.books = sortBy(commentary.books, (b) => b.order);\n commentary.profiles = sortBy(commentary.profiles, (p) => p.id);\n }\n\n function getBookNames(\n file: InputFileBase,\n metadata: MetadataBase,\n id: string\n ) {\n if (!bookMap) {\n bookMap = defaultBookIdMap.get(metadata.language);\n\n if (!bookMap) {\n if (!unknownLanguages.has(metadata.language)) {\n logger.warn(\n '[generate] File does not have a known language!',\n file.name,\n metadata.language\n );\n unknownLanguages.add(metadata.language);\n }\n\n logger.warn(\n '[generate] Using English book map for unknown language! This might result in outputting english book names.'\n );\n bookMap = defaultBookIdMap.get('en');\n }\n }\n\n const bookName = bookMap?.get(id);\n\n if (!!bookMap && !bookName) {\n logger.warn(\n '[generate] Book name not found for ID!',\n file.name,\n id\n );\n return null;\n }\n\n return {\n bookName,\n };\n }\n\n return output;\n}\n"],
|
|
5
|
+
"mappings": ";AAAA,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;AAgB1B,SAAS,aAAa,kBAAkB,oBAAoB;AAC5D,SAAS,MAAM,QAAQ,qBAAqB;AAC5C,SAAS,8BAA8B;AACvC,SAAS,mBAAmB;AAC5B,SAAS,2BAA2B;AAEpC,SAAS,wBAAwB;AACjC,SAAS,iBAAiB;AA2EnB,gBAAS,gBACZ,OACA,SAAoB,IAAI,WAAW,UAAU,GAC7C,SACa;AACb,QAAM,SAAS,UAAU;AACzB,MAAI,SAAwB;AAAA,IACxB,cAAc,CAAC;AAAA,IACf,cAAc,CAAC;AAAA,EACnB;AAEA,MAAI,aAAa,IAAI,WAAW;AAChC,MAAI,YAAY,IAAI,UAAU,MAAM;AACpC,MAAI,cAAc,IAAI,YAAY;AAClC,MAAI,sBAAsB,IAAI,oBAAoB;AAClD,MAAI,mBAAmB,IAAI,iBAAiB,MAAM;AAElD,MAAI,qBAAqB,oBAAI,IAAgC;AAC7D,MAAI,qBAAqB,oBAAI,IAA+B;AAE5D,QAAM,mBAAmB,oBAAI,IAAY;AACzC,WAAS,QAAQ,OAAO;AACpB,QAAI;AACA,UAAIA;AAMJ,UAAI,KAAK,aAAa,QAAQ;AAC1B,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,OAAO;AAChC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,QAAQ;AACjC,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,kBAAkB;AAC3C,QAAAA,UAAS;AAAA,MACb,WAAW,KAAK,aAAa,0BAA0B;AACnD,QAAAA,UAAS;AAAA,MACb,OAAO;AACH,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,QACT;AACA;AAAA,MACJ;AAEA,YAAM,SAASA,QAAO,MAAM,KAAK,OAAO;AAExC,UAAI,OAAO,SAAS,QAAQ;AACxB,2BAAmB,MAA8B,MAAM;AAAA,MAC3D,OAAO;AACH,0BAAkB,MAA6B,MAAM;AAAA,MACzD;AAAA,IACJ,SAAS,KAAK;AACV,aAAO;AAAA,QACH,2CAA2C,KAAK,IAAI;AAAA,QACpD;AAAA,MACJ;AAAA,IACJ;AAAA,EACJ;AAEA,WAAS,mBAAmB,MAA4B,QAAmB;AACvE,UAAM,KAAK,OAAO;AAElB,QAAI,CAAC,IAAI;AACL,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA;AAAA,IACJ;AAEA,UAAM,QAAQ,aAAa,IAAI,EAAE;AAEjC,QAAI,OAAO,UAAU,UAAU;AAC3B,aAAO,KAAK,2CAA2C,EAAE;AACzD;AAAA,IACJ;AAEA,UAAM,WAAW,aAAa,MAAM,KAAK,UAAU,EAAE;AAErD,QAAI,cAAc,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAEzD,QAAI,CAAC,aAAa;AACd,oBAAc;AAAA,QACV,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,MACZ;AACA,aAAO,aAAa,KAAK,WAAW;AACpC,yBAAmB,IAAI,KAAK,SAAS,IAAI,WAAW;AAAA,IACxD;AAEA,UAAM,OACF,OAAO,UAAU,UAAU,UAAU,cAAc,OAAO;AAE9D,QAAI,CAAC,MAAM;AACP,aAAO,KAAK,yCAAyC,KAAK,MAAM,EAAE;AAClE;AAAA,IACJ;AAEA,UAAM,aACF,UAAU,UAAU,cACpB,OAAO,UACP,OAAO,SACP;AAEJ,UAAM,OAA+B;AAAA,MACjC;AAAA,MACA;AAAA,MACA;AAAA,MACA,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU,CAAC;AAAA,IACf;AAEA,aAAS,WAAW,OAAO,SAAS;AAChC,UAAI,QAAQ,SAAS,WAAW;AAC5B,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,YACL,QAAQ,QAAQ;AAAA,YAChB,SAAS,QAAQ;AAAA,YACjB,WAAW,QAAQ;AAAA,UACvB;AAAA,UACA,uBAAuB;AAAA,YACnB,YAAY;AAAA,YACZ;AAAA,YACA,QAAQ;AAAA,UACZ;AAAA,QACJ,CAAC;AAAA,MACL;AAAA,IACJ;AAEA,SAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAE7D,UAAM,QAAQ,cAAc,YAAY,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK;AACnE,gBAAY,MAAM,OAAO,OAAO,GAAG,IAAI;AAAA,EAC3C;AAEA,WAAS,kBACL,MACA,QACF;AACE,QAAI,aAAa,mBAAmB,IAAI,KAAK,SAAS,EAAE;AAExD,QAAI,CAAC,YAAY;AACb,mBAAa;AAAA,QACT,GAAG,KAAK,KAAK,UAAU,WAAW;AAAA,QAClC,eAAe,KAAK,SAAS;AAAA,QAC7B,OAAO,CAAC;AAAA,QACR,UAAU,CAAC;AAAA,MACf;AACA,aAAO,aAAa,KAAK,UAAU;AACnC,yBAAmB,IAAI,KAAK,SAAS,IAAI,UAAU;AAAA,IACvD;AAEA,aAAS,cAAc,OAAO,OAAO;AACjC,UAAI,OAAO,WAAW,MAAM,KAAK,CAAC,MAAM,EAAE,OAAO,WAAW,IAAI;AAChE,UAAI,QAAQ,KAAK,gBAAgB,WAAW,cAAc;AACtD,eAAO;AAAA,UACH;AAAA,UACA,KAAK;AAAA,UACL,WAAW;AAAA,QACf;AACA;AAAA,MACJ;AAEA,UAAI,CAAC,MAAM;AACP,cAAM,OAAO,aAAa,MAAM,KAAK,UAAU,WAAW,IAAI;AAC9D,eAAO;AAAA,UACH,IAAI,WAAW;AAAA,UACf,OAAO,aAAa,IAAI,WAAW,IAAI,KAAK;AAAA,UAC5C,YAAY,MAAM,UAAU,cAAc,WAAW;AAAA,UACrD,MAAM,MAAM,UAAU,cAAc,WAAW;AAAA,UAC/C,UAAU,CAAC;AAAA,QACf;AAAA,MACJ;AAEA,UAAI,WAAW,cAAc;AACzB,aAAK,eAAe,WAAW;AAAA,MACnC;AAEA,UAAI,WAAW,uBAAuB,CAAC,KAAK,qBAAqB;AAC7D,aAAK,sBAAsB,WAAW;AAAA,MAC1C;AAEA,eAAS,WAAW,WAAW,UAAU;AACrC,YAAI,OAA8B;AAAA,UAC9B,QAAQ,QAAQ;AAAA,UAChB,SAAS,QAAQ;AAAA,QACrB;AAEA,YAAI,QAAQ,cAAc;AACtB,eAAK,eAAe,QAAQ;AAAA,QAChC;AAEA,aAAK,SAAS,KAAK;AAAA,UACf,SAAS;AAAA,QACb,CAAC;AAAA,MACL;AAEA,WAAK,WAAW,OAAO,KAAK,UAAU,CAAC,MAAM,EAAE,QAAQ,MAAM;AAC7D,iBAAW,MAAM,KAAK,IAAI;AAAA,IAC9B;AAEA,QAAI,OAAO,UAAU;AACjB,eAAS,WAAW,OAAO,UAAU;AACjC,cAAM,WAAW,WAAW,SAAS;AAAA,UACjC,CAAC,MAAM,EAAE,OAAO,QAAQ;AAAA,QAC5B;AACA,YAAI,UAAU;AACV,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,QAAQ;AAAA,UACZ;AACA;AAAA,QACJ;AAEA,cAAM,iBAA2C;AAAA,UAC7C,IAAI,QAAQ;AAAA,UACZ,SAAS,QAAQ;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,SAAS,QAAQ;AAAA,QACrB;AAEA,mBAAW,SAAS,KAAK,cAAc;AAAA,MAC3C;AAAA,IACJ;AAEA,eAAW,QAAQ,OAAO,WAAW,OAAO,CAAC,MAAM,EAAE,KAAK;AAC1D,eAAW,WAAW,OAAO,WAAW,UAAU,CAAC,MAAM,EAAE,EAAE;AAAA,EACjE;AAEA,WAAS,aACL,MACA,UACA,IACF;AACE,QAAI,CAAC,SAAS;AACV,gBAAU,iBAAiB,IAAI,SAAS,QAAQ;AAEhD,UAAI,CAAC,SAAS;AACV,YAAI,CAAC,iBAAiB,IAAI,SAAS,QAAQ,GAAG;AAC1C,iBAAO;AAAA,YACH;AAAA,YACA,KAAK;AAAA,YACL,SAAS;AAAA,UACb;AACA,2BAAiB,IAAI,SAAS,QAAQ;AAAA,QAC1C;AAEA,eAAO;AAAA,UACH;AAAA,QACJ;AACA,kBAAU,iBAAiB,IAAI,IAAI;AAAA,MACvC;AAAA,IACJ;AAEA,UAAM,WAAW,SAAS,IAAI,EAAE;AAEhC,QAAI,CAAC,CAAC,WAAW,CAAC,UAAU;AACxB,aAAO;AAAA,QACH;AAAA,QACA,KAAK;AAAA,QACL;AAAA,MACJ;AACA,aAAO;AAAA,IACX;AAEA,WAAO;AAAA,MACH;AAAA,IACJ;AAAA,EACJ;AAEA,SAAO;AACX;",
|
|
6
6
|
"names": ["parser"]
|
|
7
7
|
}
|
package/dist/esm/index.js
CHANGED
|
@@ -2,5 +2,6 @@
|
|
|
2
2
|
import * as utils from "./utils.js";
|
|
3
3
|
import * as generation from "./generation/index.js";
|
|
4
4
|
import * as parser from "./parser/index.js";
|
|
5
|
-
|
|
5
|
+
import * as log from "./log.js";
|
|
6
|
+
export { utils, generation, parser, log };
|
|
6
7
|
//# sourceMappingURL=index.js.map
|
package/dist/esm/index.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../index.ts"],
|
|
4
|
-
"sourcesContent": ["import * as utils from './utils.js';\
|
|
5
|
-
"mappings": ";AAAA,YAAY,WAAW;AACvB,YAAY,gBAAgB;AAC5B,YAAY,YAAY;
|
|
4
|
+
"sourcesContent": ["import * as utils from './utils.js';\nimport * as generation from './generation/index.js';\nimport * as parser from './parser/index.js';\nimport * as log from './log.js';\n\nexport { utils, generation, parser, log };\n"],
|
|
5
|
+
"mappings": ";AAAA,YAAY,WAAW;AACvB,YAAY,gBAAgB;AAC5B,YAAY,YAAY;AACxB,YAAY,SAAS;AAErB,SAAS,OAAO,YAAY,QAAQ;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/esm/log.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
export const consoleLogger = {
|
|
3
|
+
log: (message, ...args) => {
|
|
4
|
+
console.log(message, ...args);
|
|
5
|
+
},
|
|
6
|
+
warn: (message, ...args) => {
|
|
7
|
+
console.warn(message, ...args);
|
|
8
|
+
},
|
|
9
|
+
error: (message, ...args) => {
|
|
10
|
+
console.error(message, ...args);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
let logger = consoleLogger;
|
|
14
|
+
export function getLogger() {
|
|
15
|
+
return logger;
|
|
16
|
+
}
|
|
17
|
+
export function setLogger(newLogger) {
|
|
18
|
+
logger = newLogger;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=log.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../log.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * A simple logging interface that can be used to log messages.\n */\nexport interface Logger {\n log: (message: any, ...args: any[]) => void;\n warn: (message: string, ...args: any[]) => void;\n error: (message: string, ...args: any[]) => void;\n}\n\n/**\n * A logger that logs messages to the console.\n */\nexport const consoleLogger: Logger = {\n log: (message: string, ...args) => {\n console.log(message, ...args);\n },\n warn: (message: string, ...args) => {\n console.warn(message, ...args);\n },\n error: (message: string, ...args) => {\n console.error(message, ...args);\n },\n};\n\n/**\n * The logger that should be used for logging messages.\n */\nlet logger: Logger = consoleLogger;\n\n/**\n * Gets the logger that is currently being used for logging messages.\n * @returns The logger that is currently being used.\n */\nexport function getLogger(): Logger {\n return logger;\n}\n\n/**\n * Sets the logger to use for logging messages.\n * @param newLogger The new logger to use.\n */\nexport function setLogger(newLogger: Logger) {\n logger = newLogger;\n}\n"],
|
|
5
|
+
"mappings": ";AAYO,aAAM,gBAAwB;AAAA,EACjC,KAAK,CAAC,YAAoB,SAAS;AAC/B,YAAQ,IAAI,SAAS,GAAG,IAAI;AAAA,EAChC;AAAA,EACA,MAAM,CAAC,YAAoB,SAAS;AAChC,YAAQ,KAAK,SAAS,GAAG,IAAI;AAAA,EACjC;AAAA,EACA,OAAO,CAAC,YAAoB,SAAS;AACjC,YAAQ,MAAM,SAAS,GAAG,IAAI;AAAA,EAClC;AACJ;AAKA,IAAI,SAAiB;AAMd,gBAAS,YAAoB;AAChC,SAAO;AACX;AAMO,gBAAS,UAAU,WAAmB;AACzC,WAAS;AACb;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -87,7 +87,7 @@ export class CodexParser {
|
|
|
87
87
|
};
|
|
88
88
|
}
|
|
89
89
|
for (let cell of data.cells) {
|
|
90
|
-
if (cell.languageId === "html") {
|
|
90
|
+
if (cell.languageId === "html" && cell.value) {
|
|
91
91
|
if (!cell.metadata) continue;
|
|
92
92
|
const metadata = metadataSchema.parse(cell.metadata);
|
|
93
93
|
if (metadata.type === "text") {
|
|
@@ -96,17 +96,20 @@ export class CodexParser {
|
|
|
96
96
|
throw new Error("Could not find verse reference.");
|
|
97
97
|
if (!root.id) root.id = reference.book;
|
|
98
98
|
const content = stripHTML(cell.value);
|
|
99
|
-
const lines = content.split("\n").reduce(
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
99
|
+
const lines = content.split("\n").reduce(
|
|
100
|
+
(prev, current) => {
|
|
101
|
+
if (prev.length === 0) return [current];
|
|
102
|
+
else
|
|
103
|
+
return [
|
|
104
|
+
...prev,
|
|
105
|
+
{
|
|
106
|
+
lineBreak: true
|
|
107
|
+
},
|
|
108
|
+
current
|
|
109
|
+
];
|
|
110
|
+
},
|
|
111
|
+
[]
|
|
112
|
+
);
|
|
110
113
|
addReference(reference, lines);
|
|
111
114
|
} else if (metadata.type === "paratext") {
|
|
112
115
|
const reference = parseVerseReference(`${metadata.id}:1`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../parser/codex-parser.ts"],
|
|
4
|
-
"sourcesContent": ["import { z } from 'zod';\
|
|
5
|
-
"mappings": ";AAAA,SAAS,SAAS;AAQlB,SAAS,2BAAqC;AAEvC,aAAM,uBAAuB,EAAE,OAAO;AAAA,EACzC,MAAM,EAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,EAAE,OAAO;AAAA,IACX,SAAS,EAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,MAAM,EAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,OAAO;AAAA,EACnC,MAAM,EAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,OAAO,EAAE;AAAA,IACL,EAAE,OAAO;AAAA,MACL,MAAM,EAAE,OAAO;AAAA,MACf,YAAY,EAAE,OAAO;AAAA,MACrB,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EACL,OAAO;AAAA,QACJ,MAAM,EAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,aAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQxB,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,kBACL,eACA,SACF;AACE,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,aAAa,KAAe,SAA2B;AAC5D,wBAAkB,IAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQ,IAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,
|
|
4
|
+
"sourcesContent": ["import { z } from 'zod';\nimport {\n Chapter,\n ChapterContent,\n ParseTree,\n Verse,\n VerseContent,\n} from './types.js';\nimport { parseVerseReference, VerseRef } from '../utils.js';\n\nexport const chapterHeadingSchema = z.object({\n type: z.literal('chapter-heading'),\n data: z.object({\n chapter: z.coerce.number().int(),\n }),\n});\n\nexport const verseSchema = z.object({\n type: z.literal('text'),\n id: z.string(),\n});\n\nexport const paratextSchema = z.object({\n type: z.literal('paratext'),\n id: z.string(),\n});\n\nexport const metadataSchema = z.discriminatedUnion('type', [\n chapterHeadingSchema,\n verseSchema,\n paratextSchema,\n]);\n\nexport const codexSchema = z.object({\n cells: z.array(\n z.object({\n kind: z.number(),\n languageId: z.string(),\n value: z.string(),\n metadata: z\n .object({\n type: z.string(),\n })\n .passthrough()\n .nullable()\n .optional(),\n })\n ),\n});\n\n/**\n * Defines a parser for codex files.\n */\nexport class CodexParser {\n // TODO:\n // /**\n // * Whether to preserve markdown in the parsed content.\n // */\n // preserveMarkdown: boolean = true;\n\n private _noteCounter: number = 0;\n\n /**\n * Parses the specified codex content.\n *\n * @param codex The codex content to parse.\n * @returns The parse tree that was generated.\n */\n public parse(codex: string): ParseTree {\n const json = JSON.parse(codex);\n const data = codexSchema.parse(json);\n\n let root: ParseTree = {\n type: 'root',\n content: [],\n };\n\n let chapters: Map<number, Chapter> = new Map();\n let lastChapter: Chapter | null = null;\n\n function addChapterContent(\n chapterNumber: number,\n content: ChapterContent[]\n ) {\n let chapter = chapters.get(chapterNumber);\n if (!chapter) {\n chapter = {\n type: 'chapter',\n number: chapterNumber,\n content: [],\n footnotes: [],\n };\n lastChapter = chapter;\n chapters.set(chapterNumber, chapter);\n }\n\n chapter.content.push(...content);\n }\n\n function addReference(ref: VerseRef, content: Verse['content']) {\n addChapterContent(ref.chapter, [\n {\n type: 'verse',\n number: ref.verse,\n content: content,\n },\n ]);\n }\n\n function stripHTML(value: string) {\n return value.replace(/<[^>]*>/g, '');\n }\n\n function toHeading(content: string) {\n return {\n type: 'heading',\n content: [content],\n } satisfies ChapterContent;\n }\n\n for (let cell of data.cells) {\n if (cell.languageId === 'html' && cell.value) {\n if (!cell.metadata) continue; // ignore html cells without metadata\n const metadata = metadataSchema.parse(cell.metadata);\n if (metadata.type === 'text') {\n const reference = parseVerseReference(metadata.id);\n if (!reference)\n throw new Error('Could not find verse reference.');\n\n if (!root.id) root.id = reference.book; // set book id if not already set\n\n const content = stripHTML(cell.value);\n\n // add line breaks in heading if there are multiple lines\n const lines = content.split('\\n').reduce(\n (prev, current) => {\n if (prev.length === 0) return [current];\n else\n return [\n ...prev,\n {\n lineBreak: true,\n },\n current,\n ] satisfies Verse['content'];\n },\n [] as Verse['content']\n );\n\n addReference(reference, lines);\n } else if (metadata.type === 'paratext') {\n const reference = parseVerseReference(`${metadata.id}:1`); // chapter headings do not have a verse reference always default to 1\n if (reference) {\n const content = stripHTML(cell.value);\n\n // add line breaks in heading if there are multiple lines\n const lines = content\n .split('\\n')\n .reduce<ChapterContent[]>((prev, current) => {\n if (prev.length === 0)\n return [toHeading(current)];\n else\n return [\n ...prev,\n {\n type: 'line_break',\n },\n toHeading(current),\n ] satisfies ChapterContent[];\n }, []);\n\n lines.forEach((line) =>\n addChapterContent(reference.chapter, [line])\n );\n } else {\n // parse paratext that isn't a chapter reference as footnote\n const content = stripHTML(cell.value);\n if (!cell.metadata) {\n if (lastChapter) {\n (lastChapter as Chapter).footnotes.push({\n noteId: this._noteCounter++,\n text: content,\n caller: null,\n });\n }\n }\n }\n }\n }\n }\n\n for (let chapter of chapters.values()) {\n root.content.push(chapter);\n }\n\n return root;\n }\n}\n"],
|
|
5
|
+
"mappings": ";AAAA,SAAS,SAAS;AAQlB,SAAS,2BAAqC;AAEvC,aAAM,uBAAuB,EAAE,OAAO;AAAA,EACzC,MAAM,EAAE,QAAQ,iBAAiB;AAAA,EACjC,MAAM,EAAE,OAAO;AAAA,IACX,SAAS,EAAE,OAAO,OAAO,EAAE,IAAI;AAAA,EACnC,CAAC;AACL,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,MAAM,EAAE,QAAQ,MAAM;AAAA,EACtB,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,OAAO;AAAA,EACnC,MAAM,EAAE,QAAQ,UAAU;AAAA,EAC1B,IAAI,EAAE,OAAO;AACjB,CAAC;AAEM,aAAM,iBAAiB,EAAE,mBAAmB,QAAQ;AAAA,EACvD;AAAA,EACA;AAAA,EACA;AACJ,CAAC;AAEM,aAAM,cAAc,EAAE,OAAO;AAAA,EAChC,OAAO,EAAE;AAAA,IACL,EAAE,OAAO;AAAA,MACL,MAAM,EAAE,OAAO;AAAA,MACf,YAAY,EAAE,OAAO;AAAA,MACrB,OAAO,EAAE,OAAO;AAAA,MAChB,UAAU,EACL,OAAO;AAAA,QACJ,MAAM,EAAE,OAAO;AAAA,MACnB,CAAC,EACA,YAAY,EACZ,SAAS,EACT,SAAS;AAAA,IAClB,CAAC;AAAA,EACL;AACJ,CAAC;AAKM,aAAM,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOb,eAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQxB,MAAM,OAA0B;AACnC,UAAM,OAAO,KAAK,MAAM,KAAK;AAC7B,UAAM,OAAO,YAAY,MAAM,IAAI;AAEnC,QAAI,OAAkB;AAAA,MAClB,MAAM;AAAA,MACN,SAAS,CAAC;AAAA,IACd;AAEA,QAAI,WAAiC,oBAAI,IAAI;AAC7C,QAAI,cAA8B;AAElC,aAAS,kBACL,eACA,SACF;AACE,UAAI,UAAU,SAAS,IAAI,aAAa;AACxC,UAAI,CAAC,SAAS;AACV,kBAAU;AAAA,UACN,MAAM;AAAA,UACN,QAAQ;AAAA,UACR,SAAS,CAAC;AAAA,UACV,WAAW,CAAC;AAAA,QAChB;AACA,sBAAc;AACd,iBAAS,IAAI,eAAe,OAAO;AAAA,MACvC;AAEA,cAAQ,QAAQ,KAAK,GAAG,OAAO;AAAA,IACnC;AAEA,aAAS,aAAa,KAAe,SAA2B;AAC5D,wBAAkB,IAAI,SAAS;AAAA,QAC3B;AAAA,UACI,MAAM;AAAA,UACN,QAAQ,IAAI;AAAA,UACZ;AAAA,QACJ;AAAA,MACJ,CAAC;AAAA,IACL;AAEA,aAAS,UAAU,OAAe;AAC9B,aAAO,MAAM,QAAQ,YAAY,EAAE;AAAA,IACvC;AAEA,aAAS,UAAU,SAAiB;AAChC,aAAO;AAAA,QACH,MAAM;AAAA,QACN,SAAS,CAAC,OAAO;AAAA,MACrB;AAAA,IACJ;AAEA,aAAS,QAAQ,KAAK,OAAO;AACzB,UAAI,KAAK,eAAe,UAAU,KAAK,OAAO;AAC1C,YAAI,CAAC,KAAK,SAAU;AACpB,cAAM,WAAW,eAAe,MAAM,KAAK,QAAQ;AACnD,YAAI,SAAS,SAAS,QAAQ;AAC1B,gBAAM,YAAY,oBAAoB,SAAS,EAAE;AACjD,cAAI,CAAC;AACD,kBAAM,IAAI,MAAM,iCAAiC;AAErD,cAAI,CAAC,KAAK,GAAI,MAAK,KAAK,UAAU;AAElC,gBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,gBAAM,QAAQ,QAAQ,MAAM,IAAI,EAAE;AAAA,YAC9B,CAAC,MAAM,YAAY;AACf,kBAAI,KAAK,WAAW,EAAG,QAAO,CAAC,OAAO;AAAA;AAElC,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,WAAW;AAAA,kBACf;AAAA,kBACA;AAAA,gBACJ;AAAA,YACR;AAAA,YACA,CAAC;AAAA,UACL;AAEA,uBAAa,WAAW,KAAK;AAAA,QACjC,WAAW,SAAS,SAAS,YAAY;AACrC,gBAAM,YAAY,oBAAoB,GAAG,SAAS,EAAE,IAAI;AACxD,cAAI,WAAW;AACX,kBAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,kBAAM,QAAQ,QACT,MAAM,IAAI,EACV,OAAyB,CAAC,MAAM,YAAY;AACzC,kBAAI,KAAK,WAAW;AAChB,uBAAO,CAAC,UAAU,OAAO,CAAC;AAAA;AAE1B,uBAAO;AAAA,kBACH,GAAG;AAAA,kBACH;AAAA,oBACI,MAAM;AAAA,kBACV;AAAA,kBACA,UAAU,OAAO;AAAA,gBACrB;AAAA,YACR,GAAG,CAAC,CAAC;AAET,kBAAM;AAAA,cAAQ,CAAC,SACX,kBAAkB,UAAU,SAAS,CAAC,IAAI,CAAC;AAAA,YAC/C;AAAA,UACJ,OAAO;AAEH,kBAAM,UAAU,UAAU,KAAK,KAAK;AACpC,gBAAI,CAAC,KAAK,UAAU;AAChB,kBAAI,aAAa;AACb,gBAAC,YAAwB,UAAU,KAAK;AAAA,kBACpC,QAAQ,KAAK;AAAA,kBACb,MAAM;AAAA,kBACN,QAAQ;AAAA,gBACZ,CAAC;AAAA,cACL;AAAA,YACJ;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,aAAS,WAAW,SAAS,OAAO,GAAG;AACnC,WAAK,QAAQ,KAAK,OAAO;AAAA,IAC7B;AAEA,WAAO;AAAA,EACX;AACJ;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../parser/commentary-csv-parser.ts"],
|
|
4
|
-
"sourcesContent": ["import {\
|
|
5
|
-
"mappings": ";AAKA,SAAS,WAAW,2BAA2B;AAC/C,SAAS,aAAa;AASf,aAAM,oBAAoB;AAAA,EAC7B,MAAM,MAAmC;AAErC,UAAM,YAAY,KAAK,QAAQ,IAAI;AACnC,WAAO,KAAK,UAAU,YAAY,CAAC;AAEnC,UAAM,QAAQ,MAAgB,MAAM;AAAA,MAChC,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MASR,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,WAAW;AAAA,IACf,CAAC;AACD,WAAO,KAAK;AAAA,MACR,MAAM,KAAK,IAAI,CAAC,UAAoB;AAAA,QAChC,MAAM,KAAK,CAAC;AAAA,QACZ,SAAS,KAAK,CAAC;AAAA,QACf,OAAO,KAAK,CAAC;AAAA,QACb,cAAc,KAAK,CAAC;AAAA,MACxB,EAAE;AAAA,IACN;AAAA,EACJ;AAAA,EAEA,WAAW,MAAsC;AAC7C,QAAI,OAA4B;AAAA,MAC5B,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,IACZ;AAEA,QAAI,OAAkC;AACtC,QAAI,UAAwC;AAC5C,aAAS,QAAQ,MAAM;AACnB,UAAI,SAAS,KAAK,IAAI,GAAG;AACrB,cAAM,KAAK,UAAU,KAAK,IAAI;AAC9B,YAAI,CAAC,IAAI;AACL,gBAAM,IAAI,MAAM,mBAAmB,KAAK,IAAI;AAAA,QAChD;AAEA,eAAO,KAAK,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK;AAChD,YAAI,CAAC,MAAM;AACP,iBAAO;AAAA,YACH,MAAM;AAAA,YACN,MAAM;AAAA,YACN,cAAc;AAAA,YACd,UAAU,CAAC;AAAA,UACf;AACA,eAAK,MAAM,KAAK,IAAI;AAAA,QACxB;AAEA,YAAI,CAAC,KAAK,cAAc;AACpB,eAAK,eAAe,SAAS,KAAK,YAAY,IACxC,KAAK,eACL;AAAA,QACV;AAAA,MACJ,WAAW,SAAS,KAAK,OAAO,GAAG;AAC/B,cAAM,SAAS,SAAS,KAAK,OAAO;AACpC,YAAI,MAAM,MAAM,GAAG;AACf,gBAAM,IAAI,MAAM,6BAA6B,KAAK,OAAO;AAAA,QAC7D;AAEA,YAAI,CAAC,MAAM;
|
|
4
|
+
"sourcesContent": ["import {\n CommentaryBookNode,\n CommentaryChapterNode,\n CommentaryParseTree,\n} from './types';\nimport { getBookId, parseVerseReference } from '../utils';\nimport { parse } from 'papaparse';\n\nexport interface CsvLine {\n book: string;\n chapter: string;\n verse: string;\n commentaries: string;\n}\n\nexport class CommentaryCsvParser {\n parse(data: string): CommentaryParseTree {\n // remove header automatically\n const firstLine = data.indexOf('\\n');\n data = data.substring(firstLine + 1);\n\n const lines = parse<string[]>(data, {\n header: false,\n // transformHeader: (_, index) => {\n // const headers = ['book', 'chapter', 'verse', 'commentaries'];\n // if (index < headers.length) {\n // return headers[index];\n // } else {\n // return '';\n // }\n // },\n delimiter: ',',\n escapeChar: '\"',\n quoteChar: '\"',\n });\n return this.parseLines(\n lines.data.map((line: string[]) => ({\n book: line[0],\n chapter: line[1],\n verse: line[2],\n commentaries: line[3],\n }))\n );\n }\n\n parseLines(data: CsvLine[]): CommentaryParseTree {\n let tree: CommentaryParseTree = {\n type: 'commentary/root',\n books: [],\n };\n\n let book: CommentaryBookNode | null = null;\n let chapter: CommentaryChapterNode | null = null;\n for (let line of data) {\n if (hasValue(line.book)) {\n const id = getBookId(line.book);\n if (!id) {\n throw new Error('Invalid book: ' + line.book);\n }\n\n book = tree.books.find((b) => b.book === id) ?? null;\n if (!book) {\n book = {\n type: 'book',\n book: id,\n introduction: null,\n chapters: [],\n };\n tree.books.push(book);\n }\n\n if (!book.introduction) {\n book.introduction = hasValue(line.commentaries)\n ? line.commentaries\n : null;\n }\n } else if (hasValue(line.chapter)) {\n const number = parseInt(line.chapter);\n if (isNaN(number)) {\n throw new Error('Invalid chapter number: ' + line.chapter);\n }\n\n if (!book) {\n throw new Error('Chapter without book');\n }\n\n chapter = {\n type: 'chapter',\n number,\n introduction: hasValue(line.commentaries)\n ? line.commentaries\n : null,\n verses: [],\n };\n book.chapters.push(chapter);\n } else if (hasValue(line.verse)) {\n const ref = parseVerseReference(line.verse);\n if (ref) {\n if (ref.book === book?.book) {\n // @ts-ignore\n if (ref.chapter === chapter?.number) {\n chapter.verses.push({\n type: 'verse',\n number: ref.verse,\n content: [line.commentaries],\n });\n } else {\n chapter = {\n type: 'chapter',\n number: ref.chapter,\n introduction: null,\n verses: [\n {\n type: 'verse',\n number: ref.verse,\n content: [line.commentaries],\n },\n ],\n };\n\n book.chapters.push(chapter);\n }\n }\n }\n }\n }\n\n return tree;\n }\n}\n\nfunction hasValue(value: string | null): boolean {\n return (\n value !== null &&\n value !== undefined &&\n value !== '' &&\n value.trim() !== ''\n );\n}\n"],
|
|
5
|
+
"mappings": ";AAKA,SAAS,WAAW,2BAA2B;AAC/C,SAAS,aAAa;AASf,aAAM,oBAAoB;AAAA,EAC7B,MAAM,MAAmC;AAErC,UAAM,YAAY,KAAK,QAAQ,IAAI;AACnC,WAAO,KAAK,UAAU,YAAY,CAAC;AAEnC,UAAM,QAAQ,MAAgB,MAAM;AAAA,MAChC,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MASR,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,WAAW;AAAA,IACf,CAAC;AACD,WAAO,KAAK;AAAA,MACR,MAAM,KAAK,IAAI,CAAC,UAAoB;AAAA,QAChC,MAAM,KAAK,CAAC;AAAA,QACZ,SAAS,KAAK,CAAC;AAAA,QACf,OAAO,KAAK,CAAC;AAAA,QACb,cAAc,KAAK,CAAC;AAAA,MACxB,EAAE;AAAA,IACN;AAAA,EACJ;AAAA,EAEA,WAAW,MAAsC;AAC7C,QAAI,OAA4B;AAAA,MAC5B,MAAM;AAAA,MACN,OAAO,CAAC;AAAA,IACZ;AAEA,QAAI,OAAkC;AACtC,QAAI,UAAwC;AAC5C,aAAS,QAAQ,MAAM;AACnB,UAAI,SAAS,KAAK,IAAI,GAAG;AACrB,cAAM,KAAK,UAAU,KAAK,IAAI;AAC9B,YAAI,CAAC,IAAI;AACL,gBAAM,IAAI,MAAM,mBAAmB,KAAK,IAAI;AAAA,QAChD;AAEA,eAAO,KAAK,MAAM,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,KAAK;AAChD,YAAI,CAAC,MAAM;AACP,iBAAO;AAAA,YACH,MAAM;AAAA,YACN,MAAM;AAAA,YACN,cAAc;AAAA,YACd,UAAU,CAAC;AAAA,UACf;AACA,eAAK,MAAM,KAAK,IAAI;AAAA,QACxB;AAEA,YAAI,CAAC,KAAK,cAAc;AACpB,eAAK,eAAe,SAAS,KAAK,YAAY,IACxC,KAAK,eACL;AAAA,QACV;AAAA,MACJ,WAAW,SAAS,KAAK,OAAO,GAAG;AAC/B,cAAM,SAAS,SAAS,KAAK,OAAO;AACpC,YAAI,MAAM,MAAM,GAAG;AACf,gBAAM,IAAI,MAAM,6BAA6B,KAAK,OAAO;AAAA,QAC7D;AAEA,YAAI,CAAC,MAAM;AACP,gBAAM,IAAI,MAAM,sBAAsB;AAAA,QAC1C;AAEA,kBAAU;AAAA,UACN,MAAM;AAAA,UACN;AAAA,UACA,cAAc,SAAS,KAAK,YAAY,IAClC,KAAK,eACL;AAAA,UACN,QAAQ,CAAC;AAAA,QACb;AACA,aAAK,SAAS,KAAK,OAAO;AAAA,MAC9B,WAAW,SAAS,KAAK,KAAK,GAAG;AAC7B,cAAM,MAAM,oBAAoB,KAAK,KAAK;AAC1C,YAAI,KAAK;AACL,cAAI,IAAI,SAAS,MAAM,MAAM;AAEzB,gBAAI,IAAI,YAAY,SAAS,QAAQ;AACjC,sBAAQ,OAAO,KAAK;AAAA,gBAChB,MAAM;AAAA,gBACN,QAAQ,IAAI;AAAA,gBACZ,SAAS,CAAC,KAAK,YAAY;AAAA,cAC/B,CAAC;AAAA,YACL,OAAO;AACH,wBAAU;AAAA,gBACN,MAAM;AAAA,gBACN,QAAQ,IAAI;AAAA,gBACZ,cAAc;AAAA,gBACd,QAAQ;AAAA,kBACJ;AAAA,oBACI,MAAM;AAAA,oBACN,QAAQ,IAAI;AAAA,oBACZ,SAAS,CAAC,KAAK,YAAY;AAAA,kBAC/B;AAAA,gBACJ;AAAA,cACJ;AAEA,mBAAK,SAAS,KAAK,OAAO;AAAA,YAC9B;AAAA,UACJ;AAAA,QACJ;AAAA,MACJ;AAAA,IACJ;AAEA,WAAO;AAAA,EACX;AACJ;AAEA,SAAS,SAAS,OAA+B;AAC7C,SACI,UAAU,QACV,UAAU,UACV,UAAU,MACV,MAAM,KAAK,MAAM;AAEzB;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
import { getLogger } from "../log.js";
|
|
2
3
|
export class Rewindable {
|
|
3
4
|
buffer = [];
|
|
4
5
|
_bufferSize = 0;
|
|
@@ -36,19 +37,23 @@ export class Rewindable {
|
|
|
36
37
|
let result = this._iterator.next(...args);
|
|
37
38
|
this.buffer.unshift(result.value);
|
|
38
39
|
if (this.buffer.length > this._bufferSize) {
|
|
39
|
-
this.buffer.splice(
|
|
40
|
+
this.buffer.splice(
|
|
41
|
+
this._bufferSize,
|
|
42
|
+
this.buffer.length - this._bufferSize
|
|
43
|
+
);
|
|
40
44
|
}
|
|
41
45
|
return result;
|
|
42
46
|
}
|
|
43
47
|
}
|
|
44
48
|
export function* debug(label, iterator) {
|
|
49
|
+
const logger = getLogger();
|
|
45
50
|
for (let value of uncompletable(iterator)) {
|
|
46
51
|
if (value instanceof Element) {
|
|
47
|
-
|
|
52
|
+
logger.log(label, value.outerHTML);
|
|
48
53
|
} else if (value instanceof Node) {
|
|
49
|
-
|
|
54
|
+
logger.log(label, value.textContent);
|
|
50
55
|
} else {
|
|
51
|
-
|
|
56
|
+
logger.log(label, value);
|
|
52
57
|
}
|
|
53
58
|
yield value;
|
|
54
59
|
}
|
|
@@ -62,10 +67,16 @@ function* iterateNodes(node) {
|
|
|
62
67
|
}
|
|
63
68
|
}
|
|
64
69
|
export function parentChar(node) {
|
|
65
|
-
return matchingParent(
|
|
70
|
+
return matchingParent(
|
|
71
|
+
node,
|
|
72
|
+
(n) => n instanceof Element && n.nodeName === "char"
|
|
73
|
+
);
|
|
66
74
|
}
|
|
67
75
|
export function parentNote(node) {
|
|
68
|
-
return matchingParent(
|
|
76
|
+
return matchingParent(
|
|
77
|
+
node,
|
|
78
|
+
(n) => n instanceof Element && n.nodeName === "note"
|
|
79
|
+
);
|
|
69
80
|
}
|
|
70
81
|
export function matchingParent(node, filter) {
|
|
71
82
|
let parent = node.parentNode;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../parser/iterators.ts"],
|
|
4
|
-
"sourcesContent": ["\
|
|
5
|
-
"mappings": ";
|
|
4
|
+
"sourcesContent": ["import { getLogger } from '../log.js';\n\n/**\n * Defines an interface that represents an iterator that can be rewound.\n */\nexport interface RewindableIterator<T> extends IterableIterator<T> {\n /**\n * Rewinds the iterator by the specified number of elements.\n * @param number The number of elements to rewind.\n */\n rewind(number: number): void;\n}\n\n/**\n * Defines a class that implements an iterator that can be rewound.\n */\nexport class Rewindable<T> implements RewindableIterator<T> {\n buffer: T[] = [];\n\n private _bufferSize: number = 0;\n private _position: number = -1;\n private _iterator: IterableIterator<T>;\n\n constructor(iterator: IterableIterator<T>, bufferSize: number = 2) {\n this._iterator = iterator;\n this._bufferSize = bufferSize;\n\n if (this._iterator.return) {\n this.return = (value?: any) => {\n return this._iterator.return!(value);\n };\n }\n if (this._iterator.throw) {\n this.throw = (value?: any) => {\n return this._iterator.throw!(value);\n };\n }\n }\n\n rewind(number: number): void {\n this._position += number;\n }\n\n [Symbol.iterator](): IterableIterator<T> {\n return this;\n }\n\n next(...args: [] | [undefined]): IteratorResult<T, any> {\n if (this._position >= 0) {\n let item = this.buffer[this._position];\n this._position--;\n return {\n value: item,\n done: false,\n };\n }\n\n let result = this._iterator.next(...args);\n this.buffer.unshift(result.value);\n if (this.buffer.length > this._bufferSize) {\n this.buffer.splice(\n this._bufferSize,\n this.buffer.length - this._bufferSize\n );\n }\n return result;\n }\n\n return?(value?: any): IteratorResult<T, any>;\n throw?(e?: any): IteratorResult<T, any>;\n}\n\nexport function* debug(label: string, iterator: IterableIterator<any>) {\n const logger = getLogger();\n for (let value of uncompletable(iterator)) {\n if (value instanceof Element) {\n logger.log(label, value.outerHTML);\n } else if (value instanceof Node) {\n logger.log(label, value.textContent);\n } else {\n logger.log(label, value);\n }\n yield value;\n }\n}\n\n/**\n * Creates a new rewindable iterator from the given iterator.\n * @param iterator The iterator.\n * @param bufferSize The size of the buffer.\n */\nexport function rewindable<T>(\n iterator: IterableIterator<T>,\n bufferSize: number = 2\n): Rewindable<T> {\n return new Rewindable(iterator, bufferSize);\n}\n\nfunction* iterateNodes(node: Node) {\n for (let i = 0; i < node.childNodes.length; i++) {\n yield node.childNodes[i];\n }\n}\n\n/**\n * Gets the char element that is the parent of the given node.\n * @param node The node.\n */\nexport function parentChar(node: Node): Element | null {\n return matchingParent(\n node,\n (n) => n instanceof Element && n.nodeName === 'char'\n ) as Element | null;\n}\n\n/**\n * Gets the note element that is the parent of the given node.\n * @param node The node.\n */\nexport function parentNote(node: Node): Element | null {\n return matchingParent(\n node,\n (n) => n instanceof Element && n.nodeName === 'note'\n ) as Element | null;\n}\n\n/**\n * Gets the parent node that matches the given filter.\n * @param node The node to start the search from.\n * @param filter The filter that should be used.\n */\nexport function matchingParent(\n node: Node,\n filter: (node: Node) => boolean\n): Node | null {\n let parent = node.parentNode;\n while (parent) {\n if (filter(parent)) {\n return parent;\n }\n parent = parent.parentNode;\n }\n return null;\n}\n\n/**\n * Determines if the given node is a child of the parent node.\n * @param node The node to test.\n * @param parent The parent.\n * @returns\n */\nexport function isParent(node: Node, parent: Node): boolean {\n let parentNode = node.parentNode;\n while (parentNode) {\n if (parentNode === parent) {\n return true;\n }\n parentNode = parentNode.parentNode;\n }\n return false;\n}\n\n/**\n * Iterates through all of the nodes in the tree in a depth-first traversal.\n * @param node The node to start the traversal from.\n */\nexport function iterateAll(node: Node): RewindableIterator<Node> {\n return rewindable(_iterateAll(node));\n}\n\nexport function* iterateUntil<T>(\n iterator: IterableIterator<T>,\n predicate: (value: T) => boolean\n): IterableIterator<T> {\n for (let value of iterator) {\n if (predicate(value)) {\n return;\n }\n yield value;\n }\n}\n\nfunction* _iterateAll(node: Node): IterableIterator<Node> {\n for (let child of node.childNodes) {\n yield child;\n yield* iterateAll(child);\n }\n}\n\n/**\n * Iterates only the elements in the iterator.\n * @param iterator The iterator that should be used.\n */\nexport function* elements(\n iterator: IterableIterator<Node>\n): IterableIterator<Element> {\n for (let node of uncompletable(iterator)) {\n if (node instanceof Element) {\n yield node;\n }\n }\n}\n\n/**\n * Iterates only the children of the given node in the iterator.\n * @param iterator The iterator that should be used.\n * @param parent The parent node.\n */\nexport function* children(\n iterator: RewindableIterator<Node>,\n parent: Node\n): IterableIterator<Node> {\n for (let node of uncompletable(iterator)) {\n if (!isParent(node, parent)) {\n iterator.rewind(1);\n return;\n }\n\n yield node;\n }\n}\n\n/**\n * Wraps the given iterable in a generator that will prevent consumers from calling return() on the iterator.\n * @param iterable The iterable.\n */\nexport function* uncompletable<T>(\n iterable: IterableIterator<T>\n): IterableIterator<T> {\n while (true) {\n const { done, value } = iterable.next();\n if (done) {\n return;\n }\n yield value;\n }\n}\n\n/**\n * Converts the given iterable into an async iterable.\n * @param input The input iterable.\n */\nexport async function* toAsyncIterable<T>(\n input: Iterable<T>\n): AsyncIterable<T> {\n for (let item of input) {\n yield item;\n }\n}\n\n/**\n * Batches items from the given iterator.\n * @param input The input iterator.\n * @param batchSize The size of each batch.\n */\nexport function* batch<T>(\n input: Iterable<T>,\n batchSize: number\n): Iterable<T[]> {\n while (true) {\n let batch = [] as T[];\n for (let item of input) {\n batch.push(item);\n if (batch.length >= batchSize) {\n break;\n }\n }\n\n if (batch.length === 0) {\n return;\n }\n\n yield batch;\n }\n}\n"],
|
|
5
|
+
"mappings": ";AAAA,SAAS,iBAAiB;AAgBnB,aAAM,WAA+C;AAAA,EACxD,SAAc,CAAC;AAAA,EAEP,cAAsB;AAAA,EACtB,YAAoB;AAAA,EACpB;AAAA,EAER,YAAY,UAA+B,aAAqB,GAAG;AAC/D,SAAK,YAAY;AACjB,SAAK,cAAc;AAEnB,QAAI,KAAK,UAAU,QAAQ;AACvB,WAAK,SAAS,CAAC,UAAgB;AAC3B,eAAO,KAAK,UAAU,OAAQ,KAAK;AAAA,MACvC;AAAA,IACJ;AACA,QAAI,KAAK,UAAU,OAAO;AACtB,WAAK,QAAQ,CAAC,UAAgB;AAC1B,eAAO,KAAK,UAAU,MAAO,KAAK;AAAA,MACtC;AAAA,IACJ;AAAA,EACJ;AAAA,EAEA,OAAO,QAAsB;AACzB,SAAK,aAAa;AAAA,EACtB;AAAA,EAEA,CAAC,OAAO,QAAQ,IAAyB;AACrC,WAAO;AAAA,EACX;AAAA,EAEA,QAAQ,MAAgD;AACpD,QAAI,KAAK,aAAa,GAAG;AACrB,UAAI,OAAO,KAAK,OAAO,KAAK,SAAS;AACrC,WAAK;AACL,aAAO;AAAA,QACH,OAAO;AAAA,QACP,MAAM;AAAA,MACV;AAAA,IACJ;AAEA,QAAI,SAAS,KAAK,UAAU,KAAK,GAAG,IAAI;AACxC,SAAK,OAAO,QAAQ,OAAO,KAAK;AAChC,QAAI,KAAK,OAAO,SAAS,KAAK,aAAa;AACvC,WAAK,OAAO;AAAA,QACR,KAAK;AAAA,QACL,KAAK,OAAO,SAAS,KAAK;AAAA,MAC9B;AAAA,IACJ;AACA,WAAO;AAAA,EACX;AAIJ;AAEO,iBAAU,MAAM,OAAe,UAAiC;AACnE,QAAM,SAAS,UAAU;AACzB,WAAS,SAAS,cAAc,QAAQ,GAAG;AACvC,QAAI,iBAAiB,SAAS;AAC1B,aAAO,IAAI,OAAO,MAAM,SAAS;AAAA,IACrC,WAAW,iBAAiB,MAAM;AAC9B,aAAO,IAAI,OAAO,MAAM,WAAW;AAAA,IACvC,OAAO;AACH,aAAO,IAAI,OAAO,KAAK;AAAA,IAC3B;AACA,UAAM;AAAA,EACV;AACJ;AAOO,gBAAS,WACZ,UACA,aAAqB,GACR;AACb,SAAO,IAAI,WAAW,UAAU,UAAU;AAC9C;AAEA,UAAU,aAAa,MAAY;AAC/B,WAAS,IAAI,GAAG,IAAI,KAAK,WAAW,QAAQ,KAAK;AAC7C,UAAM,KAAK,WAAW,CAAC;AAAA,EAC3B;AACJ;AAMO,gBAAS,WAAW,MAA4B;AACnD,SAAO;AAAA,IACH;AAAA,IACA,CAAC,MAAM,aAAa,WAAW,EAAE,aAAa;AAAA,EAClD;AACJ;AAMO,gBAAS,WAAW,MAA4B;AACnD,SAAO;AAAA,IACH;AAAA,IACA,CAAC,MAAM,aAAa,WAAW,EAAE,aAAa;AAAA,EAClD;AACJ;AAOO,gBAAS,eACZ,MACA,QACW;AACX,MAAI,SAAS,KAAK;AAClB,SAAO,QAAQ;AACX,QAAI,OAAO,MAAM,GAAG;AAChB,aAAO;AAAA,IACX;AACA,aAAS,OAAO;AAAA,EACpB;AACA,SAAO;AACX;AAQO,gBAAS,SAAS,MAAY,QAAuB;AACxD,MAAI,aAAa,KAAK;AACtB,SAAO,YAAY;AACf,QAAI,eAAe,QAAQ;AACvB,aAAO;AAAA,IACX;AACA,iBAAa,WAAW;AAAA,EAC5B;AACA,SAAO;AACX;AAMO,gBAAS,WAAW,MAAsC;AAC7D,SAAO,WAAW,YAAY,IAAI,CAAC;AACvC;AAEO,iBAAU,aACb,UACA,WACmB;AACnB,WAAS,SAAS,UAAU;AACxB,QAAI,UAAU,KAAK,GAAG;AAClB;AAAA,IACJ;AACA,UAAM;AAAA,EACV;AACJ;AAEA,UAAU,YAAY,MAAoC;AACtD,WAAS,SAAS,KAAK,YAAY;AAC/B,UAAM;AACN,WAAO,WAAW,KAAK;AAAA,EAC3B;AACJ;AAMO,iBAAU,SACb,UACyB;AACzB,WAAS,QAAQ,cAAc,QAAQ,GAAG;AACtC,QAAI,gBAAgB,SAAS;AACzB,YAAM;AAAA,IACV;AAAA,EACJ;AACJ;AAOO,iBAAU,SACb,UACA,QACsB;AACtB,WAAS,QAAQ,cAAc,QAAQ,GAAG;AACtC,QAAI,CAAC,SAAS,MAAM,MAAM,GAAG;AACzB,eAAS,OAAO,CAAC;AACjB;AAAA,IACJ;AAEA,UAAM;AAAA,EACV;AACJ;AAMO,iBAAU,cACb,UACmB;AACnB,SAAO,MAAM;AACT,UAAM,EAAE,MAAM,MAAM,IAAI,SAAS,KAAK;AACtC,QAAI,MAAM;AACN;AAAA,IACJ;AACA,UAAM;AAAA,EACV;AACJ;AAMA,uBAAuB,gBACnB,OACgB;AAChB,WAAS,QAAQ,OAAO;AACpB,UAAM;AAAA,EACV;AACJ;AAOO,iBAAU,MACb,OACA,WACa;AACb,SAAO,MAAM;AACT,QAAIA,SAAQ,CAAC;AACb,aAAS,QAAQ,OAAO;AACpB,MAAAA,OAAM,KAAK,IAAI;AACf,UAAIA,OAAM,UAAU,WAAW;AAC3B;AAAA,MACJ;AAAA,IACJ;AAEA,QAAIA,OAAM,WAAW,GAAG;AACpB;AAAA,IACJ;AAEA,UAAMA;AAAA,EACV;AACJ;",
|
|
6
6
|
"names": ["batch"]
|
|
7
7
|
}
|