@storyteller-platform/align 0.1.26 → 0.1.28

package/dist/align/align.cjs

@@ -78,11 +78,16 @@ __export(align_exports, {
   concatTranscriptions: () => concatTranscriptions
 });
 module.exports = __toCommonJS(align_exports);
+var import_node_crypto = require("node:crypto");
+var import_node_fs = require("node:fs");
 var import_promises = require("node:fs/promises");
+var import_node_os = require("node:os");
 var import_node_path = require("node:path");
 var import_posix = require("node:path/posix");
+var import_shared = require("@readium/shared");
 var import_itertools = require("itertools");
 var import_memoize = __toESM(require("memoize"), 1);
+var import_yazl = require("yazl");
 var import_audiobook = require("@storyteller-platform/audiobook");
 var import_epub = require("@storyteller-platform/epub");
 var import_ghost_story = require("@storyteller-platform/ghost-story");
@@ -90,20 +95,24 @@ var import_ffmpeg = require("../common/ffmpeg.cjs");
 var import_parseDom = require("../markup/parseDom.cjs");
 var import_segmentation = require("../markup/segmentation.cjs");
 var import_transform = require("../markup/transform.cjs");
+var import_guidedNavigation = require("../readium/guidedNavigation.cjs");
 var import_getSentenceRanges = require("./getSentenceRanges.cjs");
 var import_interpolateSentenceRanges = require("./interpolateSentenceRanges.cjs");
 var import_search = require("./search.cjs");
 var import_slugify = require("./slugify.cjs");
 var import_textFragments = require("./textFragments.cjs");
 async function align(input, output, transcriptionsDir, audiobookDir, options) {
-  var _stack = [];
+  var _stack2 = [];
   try {
-    await (0, import_promises.mkdir)((0, import_posix.dirname)(output), { recursive: true });
-    await (0, import_promises.copyFile)(input, output);
+    const outFormat = options.outFormat ?? "epub";
+    if (outFormat === "epub") {
+      await (0, import_promises.mkdir)((0, import_posix.dirname)(output), { recursive: true });
+      await (0, import_promises.copyFile)(input, output);
+    }
     const audiobookFiles = await (0, import_promises.readdir)(audiobookDir).then(
       (filenames) => filenames.filter((f) => (0, import_audiobook.isAudioFile)(f)).map((f) => (0, import_node_path.join)(audiobookDir, f))
     );
-    const epub = __using(_stack, await import_epub.Epub.from(output));
+    const epub = __using(_stack2, await import_epub.Epub.from(outFormat === "epub" ? output : input));
     const transcriptions = await (0, import_promises.readdir)(transcriptionsDir).then(
       (filenames) => filenames.filter((f) => f.endsWith(".json")).map((f) => (0, import_node_path.join)(transcriptionsDir, f))
     ).then(
@@ -135,7 +144,57 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
       options.logger
     );
     const timing = await aligner.alignBook(options.onProgress);
-    await epub.saveAndClose();
+    if (outFormat === "epub") {
+      await epub.saveAndClose();
+    } else {
+      var _stack = [];
+      try {
+        const guidedNavigationDocuments = await (0, import_guidedNavigation.generateGuidedNavigationDocuments)(epub);
+        const manifest = (0, import_guidedNavigation.generateGuidedNavigationManifest)(
+          new import_shared.LocalizedString(
+            await epub.getTitle() ?? (0, import_posix.basename)(input, (0, import_node_path.extname)(input))
+          ),
+          guidedNavigationDocuments
+        );
+        const tmpArchivePath = (0, import_node_path.join)(
+          (0, import_node_os.tmpdir)(),
+          `storyteller-platform-epub-${(0, import_node_crypto.randomUUID)()}`
+        );
+        const { promise, resolve } = Promise.withResolvers();
+        const zipfile = new import_yazl.ZipFile();
+        const writeStream = (0, import_node_fs.createWriteStream)(tmpArchivePath);
+        writeStream.on("close", () => {
+          resolve();
+        });
+        const stack = __using(_stack, new AsyncDisposableStack(), true);
+        stack.defer(async () => {
+          writeStream.close();
+          await (0, import_promises.rm)(tmpArchivePath, { force: true });
+        });
+        zipfile.outputStream.pipe(writeStream);
+        zipfile.addBuffer(
+          Buffer.from(JSON.stringify(manifest.serialize())),
+          "manifest.json"
+        );
+        for (const doc of guidedNavigationDocuments) {
+          const selfLink = doc.links?.findWithRel("self");
+          if (!selfLink) continue;
+          zipfile.addBuffer(
+            Buffer.from(JSON.stringify(doc.serialize())),
+            selfLink.href
+          );
+        }
+        zipfile.end();
+        await promise;
+        await (0, import_promises.cp)(tmpArchivePath, output);
+        epub.discardAndClose();
+      } catch (_) {
+        var _error = _, _hasError = true;
+      } finally {
+        var _promise = __callDispose(_stack, _error, _hasError);
+        _promise && await _promise;
+      }
+    }
     if (options.reportsPath) {
       await (0, import_promises.mkdir)((0, import_node_path.dirname)(options.reportsPath), { recursive: true });
       await (0, import_promises.writeFile)(
@@ -147,10 +206,10 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
       );
     }
     return timing;
-  } catch (_) {
-    var _error = _, _hasError = true;
+  } catch (_2) {
+    var _error2 = _2, _hasError2 = true;
   } finally {
-    __callDispose(_stack, _error, _hasError);
+    __callDispose(_stack2, _error2, _hasError2);
   }
 }
 class Aligner {
@@ -187,6 +246,7 @@ class Aligner {
     const locale = this.languageOverride ?? await this.epub.getLanguage() ?? new Intl.Locale("en-US");
     const { chapter, sentenceRanges, wordRanges, xml } = alignedChapter;
     const sentences = await this.getChapterSentences(chapter.id);
+    let sentenceIdToBlockFragment = null;
     const sentenceIdToFragment = new Map(
       sentenceRanges.map((range) => [
         range.id,
@@ -207,34 +267,67 @@ class Aligner {
     );
     const wordRangeMap = new Map(wordRanges.map((w) => [w[0].sentenceId, w]));
     if (this.textRef === "text-fragment") {
-      const trie = new import_textFragments.TextFragmentTrie(
-        sentences.map((s) => s.text.replace("\n", " ")),
+      sentenceIdToBlockFragment = /* @__PURE__ */ new Map();
+      const blocks = [[]];
+      for (const [i, sentence] of (0, import_itertools.enumerate)(sentences)) {
+        const text = sentence.text;
+        blocks.at(-1)?.push(text);
+        if (text.includes("\n") && i < sentences.length - 1) {
+          blocks.push([]);
+        }
+      }
+      const blockFactory = new import_textFragments.TextFragmentFactory(
+        blocks.map((block) => block.join("")),
         locale
       );
-      for (const range of sentenceRanges) {
-        const sentence = sentences[range.id];
-        sentenceIdToFragment.set(
-          range.id,
-          trie.findMinimalFragment(
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            range.id
-          )
+      let sentenceRangeIndex = 0;
+      for (const [i, block] of (0, import_itertools.enumerate)(blocks)) {
+        sentenceIdToBlockFragment.set(
+          sentenceRangeIndex,
+          blockFactory.findMinimalFragment(i)
+        );
+        const sentenceFactory = new import_textFragments.TextFragmentFactory(
+          block.map((s) => s.replace("\n", "")),
+          locale
         );
+        const blockRanges = sentenceRanges.slice(
+          sentenceRangeIndex,
+          sentenceRangeIndex + block.length
+        );
+        for (const [j, range] of (0, import_itertools.enumerate)(blockRanges)) {
+          sentenceIdToFragment.set(
+            range.id,
+            sentenceFactory.findMinimalFragment(j)
+          );
+        }
         if (this.granularity === "word") {
-          const wordRanges2 = wordRangeMap.get(range.id);
-          const toFragment = wordIdToFragment.get(range.id);
-          const words = sentence.words.entries.filter((w) => w.text.match(/\S/));
-          const wordTrie = new import_textFragments.TextFragmentTrie(
-            words.map((w) => w.text.replace("\n", " ")),
-            locale
+          const wordFactory = new import_textFragments.TextFragmentFactory(
+            blockRanges.flatMap((range) => {
+              const sentence = sentences[range.id];
+              const wordRanges2 = wordRangeMap.get(range.id);
+              const toFragment = wordIdToFragment.get(range.id);
+              if (!wordRanges2 || !toFragment) return [];
+              const words = sentence.words.entries.filter(
+                (w) => w.text.match(/\S/)
+              );
+              return words.map((w) => w.text.replace("\n", ""));
+            })
           );
-          for (const wordRange of wordRanges2) {
-            toFragment.set(
-              wordRange.id,
-              wordTrie.findMinimalFragment(wordRange.id)
-            );
+          let wordRangeIndex = 0;
+          for (const range of blockRanges) {
+            const wordRanges2 = wordRangeMap.get(range.id);
+            const toFragment = wordIdToFragment.get(range.id);
+            if (!wordRanges2 || !toFragment) continue;
+            for (const [k, wordRange] of (0, import_itertools.enumerate)(wordRanges2)) {
+              toFragment.set(
+                wordRange.id,
+                wordFactory.findMinimalFragment(k + wordRangeIndex)
+              );
+            }
+            wordRangeIndex += wordRanges2.length;
           }
         }
+        sentenceRangeIndex += block.length;
       }
     }
     const audiofiles = Array.from(
@@ -274,6 +367,7 @@ class Aligner {
         this.granularity,
         sentenceRanges,
         wordRangeMap,
+        sentenceIdToBlockFragment,
         sentenceIdToFragment,
         wordIdToFragment
       ),
@@ -572,7 +666,23 @@ class Aligner {
     return this.timing;
   }
 }
-function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToFragment, wordIdToFragment) {
+function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToBlockFragment, sentenceIdToFragment, wordIdToFragment) {
+  const subSequences = sentenceIdToBlockFragment ? createTextRangeLargeSequences(
+    chapter,
+    granularity,
+    sentenceRanges,
+    wordRanges,
+    sentenceIdToBlockFragment,
+    sentenceIdToFragment,
+    wordIdToFragment
+  ) : createTextRangeSmallSequences(
+    chapter,
+    granularity,
+    sentenceRanges,
+    wordRanges,
+    sentenceIdToFragment,
+    wordIdToFragment
+  );
   return [
     import_epub.Epub.createXmlElement(
       "smil",
@@ -590,60 +700,89 @@ function createMediaOverlay(chapter, granularity, sentenceRanges, wordRanges, se
               "epub:textref": `../${chapter.href}`,
               "epub:type": "chapter"
             },
-            sentenceRanges.map((sentenceRange) => {
-              if (granularity === "sentence" || !wordRanges.has(sentenceRange.id)) {
-                return import_epub.Epub.createXmlElement(
-                  "par",
-                  {
-                    id: `${chapter.id}-s${sentenceRange.id}`
-                  },
-                  [
-                    import_epub.Epub.createXmlElement("text", {
-                      src: `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
-                    }),
-                    import_epub.Epub.createXmlElement("audio", {
-                      src: `../Audio/${(0, import_posix.basename)(sentenceRange.audiofile)}`,
-                      clipBegin: `${sentenceRange.start.toFixed(3)}s`,
-                      clipEnd: `${sentenceRange.end.toFixed(3)}s`
-                    })
-                  ]
-                );
-              }
-              const words = wordRanges.get(sentenceRange.id);
-              const wordToFragment = wordIdToFragment.get(sentenceRange.id);
-              return import_epub.Epub.createXmlElement(
-                "seq",
-                {
-                  id: `${chapter.id}-s${sentenceRange.id}`,
-                  "epub:type": "text-range-small",
-                  "epub:textref": `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
-                },
-                words.map(
-                  (word) => import_epub.Epub.createXmlElement(
-                    "par",
-                    {
-                      id: `${chapter.id}-s${sentenceRange.id}-w${word.id}`
-                    },
-                    [
-                      import_epub.Epub.createXmlElement("text", {
-                        src: `../${chapter.href}#${wordToFragment.get(word.id)}`
-                      }),
-                      import_epub.Epub.createXmlElement("audio", {
-                        src: `../Audio/${(0, import_posix.basename)(word.audiofile)}`,
-                        clipBegin: `${word.start.toFixed(3)}s`,
-                        clipEnd: `${word.end.toFixed(3)}s`
-                      })
-                    ]
-                  )
-                )
-              );
-            })
+            subSequences
           )
         ])
       ]
     )
   ];
 }
+function createTextRangeLargeSequences(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToBlockFragment, sentenceIdToFragment, wordIdToFragment) {
+  const blockStarts = sentenceIdToBlockFragment.entries().toArray().toSorted(([a], [b]) => a - b);
+  return blockStarts.map(([sentenceId, fragment], index) => {
+    const blockEnd = index === blockStarts.length - 1 ? sentenceRanges.length - 1 : (
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      blockStarts[index + 1][0]
+    );
+    const sentences = sentenceRanges.slice(sentenceId, blockEnd);
+    return import_epub.Epub.createXmlElement(
+      "seq",
+      {
+        id: `${chapter.id}-b${index}`,
+        "epub:type": "text-range-large",
+        "epub:textref": `../${chapter.href}#${fragment}`
+      },
+      createTextRangeSmallSequences(
+        chapter,
+        granularity,
+        sentences,
+        wordRanges,
+        sentenceIdToFragment,
+        wordIdToFragment
+      )
+    );
+  });
+}
+function createTextRangeSmallSequences(chapter, granularity, sentenceRanges, wordRanges, sentenceIdToFragment, wordIdToFragment) {
+  return sentenceRanges.map((sentenceRange) => {
+    if (granularity === "sentence" || !wordRanges.has(sentenceRange.id)) {
+      return import_epub.Epub.createXmlElement(
+        "par",
+        {
+          id: `${chapter.id}-s${sentenceRange.id}`
+        },
+        [
+          import_epub.Epub.createXmlElement("text", {
+            src: `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
+          }),
+          import_epub.Epub.createXmlElement("audio", {
+            src: `../Audio/${(0, import_posix.basename)(sentenceRange.audiofile)}`,
+            clipBegin: `${sentenceRange.start.toFixed(3)}s`,
+            clipEnd: `${sentenceRange.end.toFixed(3)}s`
+          })
+        ]
+      );
+    }
+    const words = wordRanges.get(sentenceRange.id);
+    const wordToFragment = wordIdToFragment.get(sentenceRange.id);
+    return import_epub.Epub.createXmlElement(
+      "seq",
+      {
+        id: `${chapter.id}-s${sentenceRange.id}`,
+        "epub:type": "text-range-small",
+        "epub:textref": `../${chapter.href}#${sentenceIdToFragment.get(sentenceRange.id)}`
+      },
+      words.map(
+        (word) => import_epub.Epub.createXmlElement(
+          "par",
+          {
+            id: `${chapter.id}-s${sentenceRange.id}-w${word.id}`
+          },
+          [
+            import_epub.Epub.createXmlElement("text", {
+              src: `../${chapter.href}#${wordToFragment.get(word.id)}`
+            }),
+            import_epub.Epub.createXmlElement("audio", {
+              src: `../Audio/${(0, import_posix.basename)(word.audiofile)}`,
+              clipBegin: `${word.start.toFixed(3)}s`,
+              clipEnd: `${word.end.toFixed(3)}s`
+            })
+          ]
+        )
+      )
+    );
+  });
+}
 function concatTranscriptions(transcriptions, audiofiles) {
   return transcriptions.reduce(
     (acc, transcription, index) => ({

package/dist/align/align.d.cts

@@ -41,6 +41,7 @@ interface AlignOptions {
     reportsPath?: string | null | undefined;
     granularity?: "sentence" | "word" | null | undefined;
     textRef?: "id-fragment" | "text-fragment" | null | undefined;
+    outFormat?: "epub" | "gnp" | null | undefined;
     primaryLocale?: Intl.Locale | null | undefined;
     logger?: Logger | null | undefined;
     onProgress?: ((progress: number) => void) | null | undefined;

package/dist/align/align.d.ts

@@ -41,6 +41,7 @@ interface AlignOptions {
     reportsPath?: string | null | undefined;
     granularity?: "sentence" | "word" | null | undefined;
     textRef?: "id-fragment" | "text-fragment" | null | undefined;
+    outFormat?: "epub" | "gnp" | null | undefined;
     primaryLocale?: Intl.Locale | null | undefined;
     logger?: Logger | null | undefined;
     onProgress?: ((progress: number) => void) | null | undefined;