@storyteller-platform/align 0.1.57 → 0.1.58

package/dist/align/align.cjs

@@ -81,16 +81,16 @@ module.exports = __toCommonJS(align_exports);
 var import_node_crypto = require("node:crypto");
 var import_node_fs = require("node:fs");
 var import_promises = require("node:fs/promises");
-var import_node_os = require("node:os");
 var import_node_path = require("node:path");
 var import_posix = require("node:path/posix");
 var import_shared = require("@readium/shared");
 var import_itertools = require("itertools");
 var import_memoize = __toESM(require("memoize"), 1);
 var import_yazl = require("yazl");
-var import_audiobook = require("@storyteller-platform/audiobook");
 var import_epub = require("@storyteller-platform/epub");
+var import_fs = require("@storyteller-platform/fs");
 var import_ghost_story = require("@storyteller-platform/ghost-story");
+var import_media_types = require("@storyteller-platform/media-types");
 var import_ffmpeg = require("../common/ffmpeg.cjs");
 var import_parseDom = require("../markup/parseDom.cjs");
 var import_segmentation = require("../markup/segmentation.cjs");
@@ -105,8 +105,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
   var _stack2 = [];
   try {
     const outFormat = options.outFormat ?? "epub";
-    const epubPath = outFormat === "epub" ? (0, import_node_path.join)(
-      (0, import_node_os.tmpdir)(),
+    const epubPath = outFormat === "epub" ? (0, import_fs.scratchPath)(
       `storyteller-platform-align-${(0, import_node_crypto.randomUUID)()}`,
       (0, import_node_path.basename)(output)
     ) : input;
@@ -124,7 +123,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
       await (0, import_promises.copyFile)(input, epubPath);
     }
     const audiobookFiles = await (0, import_promises.readdir)(audiobookDir).then(
-      (filenames) => filenames.filter((f) => (0, import_audiobook.isAudioFile)(f)).map((f) => (0, import_node_path.join)(audiobookDir, f))
+      (filenames) => filenames.filter((f) => import_media_types.MediaType.fromPath(f)?.kind === "audio").map((f) => (0, import_node_path.join)(audiobookDir, f))
     );
     const epub = __using(_stack2, await import_epub.Epub.from(epubPath));
     const transcriptions = await (0, import_promises.readdir)(transcriptionsDir).then(
@@ -172,8 +171,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
           ),
           guidedNavigationDocuments
         );
-        const tmpArchivePath = (0, import_node_path.join)(
-          (0, import_node_os.tmpdir)(),
+        const tmpArchivePath = (0, import_fs.scratchPath)(
           `storyteller-platform-epub-${(0, import_node_crypto.randomUUID)()}`
         );
         const { promise, resolve } = Promise.withResolvers();
@@ -235,6 +233,9 @@ class Aligner {
     this.languageOverride = languageOverride;
     this.logger = logger;
     this.transcription = concatTranscriptions(transcriptions, audiofiles);
+    audiofiles.forEach((audiofile, index) => {
+      this.audiofileTranscripts[audiofile] = transcriptions[index]?.transcript ?? "";
+    });
     this.getChapterSentences = (0, import_memoize.default)(this.getChapterSentences.bind(this));
     this.granularity = granularity ?? "sentence";
     this.textRef = textRef ?? "id-fragment";
@@ -246,6 +247,7 @@ class Aligner {
   granularity;
   textRef;
   audioFileDurations = {};
+  audiofileTranscripts = {};
   report = {
     chapters: [],
     unalignedChapters: [],
@@ -369,7 +371,7 @@ class Aligner {
         const duration = await (0, import_ffmpeg.getTrackDuration)(audiofile);
         this.totalDuration += duration;
         const audio = await (0, import_promises.readFile)(audiofile);
-        const mediaType = (0, import_audiobook.lookupAudioMime)(base) ?? void 0;
+        const mediaType = import_media_types.MediaType.fromPath(base)?.mime;
         await this.epub.addManifestItem(
           {
             id,
@@ -386,7 +388,7 @@ class Aligner {
       {
         id: mediaOverlayId,
         href: `MediaOverlays/${chapterStem}.smil`,
-        mediaType: "application/smil+xml"
+        mediaType: import_media_types.MediaType.SMIL.mime
       },
       createMediaOverlay(
         chapter,
@@ -414,7 +416,7 @@ class Aligner {
       value: import_epub.Epub.formatSmilDuration(chapterDuration)
     });
   }
-  addChapterReport(chapter, chapterSentences, sentenceRanges, startSentence, endSentence, mapping, transcriptionOffset, endTranscriptionOffset) {
+  addChapterReport(spineIndex, chapter, chapterSentences, sentenceRanges, startSentence, endSentence, mapping, transcriptionOffset, endTranscriptionOffset, score) {
     const audioFiles = sentenceRanges.reduce(
       (acc, range) => {
         const existing = acc.find(
@@ -438,6 +440,8 @@ class Aligner {
     const mappedEndTranscriptionOffset = cursor.map(endTranscriptionOffset);
     this.report.chapters.push({
       href: chapter.href,
+      score,
+      spineIndex,
       transcriptionOffset: mappedTranscriptionOffset,
       endTranscriptionOffset: mappedEndTranscriptionOffset,
       transcriptionContext: {
@@ -505,7 +509,7 @@ class Aligner {
       }
     }
   }
-  async alignChapter(chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline, mapping) {
+  async alignChapter(spineIndex, chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline, mapping) {
     const timing = (0, import_ghost_story.createTiming)();
     timing.start("read contents");
     const manifest = await this.epub.getManifest();
@@ -525,7 +529,8 @@ class Aligner {
       wordRanges,
       transcriptionOffset: endTranscriptionOffset,
       firstFoundSentence,
-      lastFoundSentence
+      lastFoundSentence,
+      score
     } = await (0, import_getSentenceRanges.getSentenceRanges)(
       transcriptionText,
       mappedTimeline,
@@ -544,7 +549,7 @@ class Aligner {
     import_epub.Epub.addLinkToXhtmlHead(chapterXml, {
       rel: "stylesheet",
       href: storytellerStylesheetUrl,
-      type: "text/css"
+      type: import_media_types.MediaType.CSS.mime
     });
     this.alignedChapters.push({
       chapter,
@@ -555,6 +560,7 @@ class Aligner {
       endOffset: endTranscriptionOffset
     });
     this.addChapterReport(
+      spineIndex,
       chapter,
       chapterSentences,
       sentenceRanges,
@@ -562,7 +568,8 @@ class Aligner {
       lastFoundSentence,
       mapping,
       transcriptionOffset,
-      endTranscriptionOffset
+      endTranscriptionOffset,
+      score
     );
     return {
       lastSentenceRange: sentenceRanges.at(-1) ?? null,
@@ -614,7 +621,8 @@ class Aligner {
       if (manifest[chapterId]?.properties?.includes("nav")) {
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "is-nav"
+          reason: "is-nav",
+          spineIndex: index
         });
         continue;
       }
@@ -629,7 +637,8 @@ class Aligner {
         this.logger?.info(`Chapter #${index} has no text; skipping`);
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "no-text"
+          reason: "no-text",
+          spineIndex: index
         });
         continue;
       }
@@ -640,7 +649,8 @@ class Aligner {
         );
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "too-short"
+          reason: "too-short",
+          spineIndex: index
         });
         continue;
       }
@@ -655,6 +665,9 @@ class Aligner {
         this.report.unalignedChapters.push({
           href: spineItem.href,
           reason: "not-found",
+          spineIndex: index,
+          chapterSentenceCount: chapterSentences.length,
+          score: 0,
           start: chapterSentences.slice(0, 3).map((s) => s.text).join("").slice(0, 80),
           end: chapterSentences.slice(-3).map((s) => s.text).join("").slice(-80)
         });
@@ -665,6 +678,7 @@ class Aligner {
         continue;
       }
       const result = await this.alignChapter(
+        index,
         chapterId,
         transcriptionText,
         Math.max(start, 0),
@@ -730,7 +744,12 @@ class Aligner {
     }
     for (const audiofile of this.audiofiles) {
       if (!this.report.audioFiles.some(({ filepath }) => filepath === audiofile)) {
-        this.report.unalignedAudioFiles.push({ filepath: audiofile });
+        const transcript = this.audiofileTranscripts[audiofile]?.trim();
+        this.report.unalignedAudioFiles.push({
+          filepath: audiofile,
+          duration: this.audioFileDurations[audiofile] ?? null,
+          ...transcript ? { transcription: { text: transcript.slice(0, 2e3) } } : {}
+        });
       }
     }
     await this.epub.addMetadata({
@@ -747,7 +766,7 @@ class Aligner {
       {
         id: "storyteller_readaloud_styles",
         href: "Styles/storyteller-readaloud.css",
-        mediaType: "text/css"
+        mediaType: import_media_types.MediaType.CSS.mime
       },
       `
 .-epub-media-overlay-active {

package/dist/align/align.d.cts

@@ -13,6 +13,8 @@ interface AudioFileContext {
 }
 interface ChapterReport {
     href: string;
+    score: number;
+    spineIndex: number;
     transcriptionOffset: number;
     endTranscriptionOffset: number;
     transcriptionContext: {
@@ -42,10 +44,14 @@ interface ChapterReport {
 type UnalignedChapterReason = "too-short" | "not-found" | "is-nav" | "no-text";
 interface UnalignedChapterReport {
     href: string;
+    spineIndex: number;
     reason: Exclude<UnalignedChapterReason, "not-found">;
 }
 interface UnalignedNotFoundChapterReport {
     href: string;
+    spineIndex: number;
+    chapterSentenceCount: number;
+    score: number;
     reason: "not-found";
     start: string;
     end: string;
@@ -61,6 +67,10 @@ interface AudioFileReport {
 }
 interface UnalignedAudioFileReport {
     filepath: string;
+    duration?: number | null;
+    transcription?: {
+        text: string;
+    };
 }
 interface Report {
     chapters: ChapterReport[];
@@ -90,6 +100,7 @@ declare class Aligner {
     private granularity;
     private textRef;
     private audioFileDurations;
+    private audiofileTranscripts;
     report: Report;
     constructor(epub: Epub, audiofiles: string[], transcriptions: Pick<RecognitionResult, "transcript" | "timeline">[], granularity: "sentence" | "word" | null | undefined, textRef: "id-fragment" | "text-fragment" | null | undefined, languageOverride?: (Intl.Locale | null) | undefined, logger?: (Logger | null) | undefined);
     private getChapterSentences;
@@ -107,4 +118,4 @@ declare class Aligner {
 }
 declare function concatTranscriptions(transcriptions: Pick<RecognitionResult, "transcript" | "timeline">[], audiofiles: string[]): StorytellerTranscription;
 
-export { type AlignOptions, Aligner, align, concatTranscriptions };
+export { type AlignOptions, Aligner, type AudioFileContext, type AudioFileReport, type ChapterReport, type Report, type UnalignedAudioFileReport, type UnalignedChapterReason, type UnalignedChapterReport, type UnalignedNotFoundChapterReport, align, concatTranscriptions };

package/dist/align/align.d.ts

@@ -13,6 +13,8 @@ interface AudioFileContext {
 }
 interface ChapterReport {
     href: string;
+    score: number;
+    spineIndex: number;
     transcriptionOffset: number;
     endTranscriptionOffset: number;
     transcriptionContext: {
@@ -42,10 +44,14 @@ interface ChapterReport {
 type UnalignedChapterReason = "too-short" | "not-found" | "is-nav" | "no-text";
 interface UnalignedChapterReport {
     href: string;
+    spineIndex: number;
     reason: Exclude<UnalignedChapterReason, "not-found">;
 }
 interface UnalignedNotFoundChapterReport {
     href: string;
+    spineIndex: number;
+    chapterSentenceCount: number;
+    score: number;
     reason: "not-found";
     start: string;
     end: string;
@@ -61,6 +67,10 @@ interface AudioFileReport {
 }
 interface UnalignedAudioFileReport {
     filepath: string;
+    duration?: number | null;
+    transcription?: {
+        text: string;
+    };
 }
 interface Report {
     chapters: ChapterReport[];
@@ -90,6 +100,7 @@ declare class Aligner {
     private granularity;
     private textRef;
     private audioFileDurations;
+    private audiofileTranscripts;
     report: Report;
     constructor(epub: Epub, audiofiles: string[], transcriptions: Pick<RecognitionResult, "transcript" | "timeline">[], granularity: "sentence" | "word" | null | undefined, textRef: "id-fragment" | "text-fragment" | null | undefined, languageOverride?: (Intl.Locale | null) | undefined, logger?: (Logger | null) | undefined);
     private getChapterSentences;
@@ -107,4 +118,4 @@ declare class Aligner {
 }
 declare function concatTranscriptions(transcriptions: Pick<RecognitionResult, "transcript" | "timeline">[], audiofiles: string[]): StorytellerTranscription;
 
-export { type AlignOptions, Aligner, align, concatTranscriptions };
+export { type AlignOptions, Aligner, type AudioFileContext, type AudioFileReport, type ChapterReport, type Report, type UnalignedAudioFileReport, type UnalignedChapterReason, type UnalignedChapterReport, type UnalignedNotFoundChapterReport, align, concatTranscriptions };

package/dist/align/align.js

@@ -13,7 +13,6 @@ import {
   rm,
   writeFile
 } from "node:fs/promises";
-import { tmpdir } from "node:os";
 import {
   basename as autoBasename,
   dirname as autoDirname,
@@ -26,14 +25,15 @@ import { LocalizedString } from "@readium/shared";
 import { enumerate, max } from "itertools";
 import memoize from "memoize";
 import { ZipFile } from "yazl";
-import { isAudioFile, lookupAudioMime } from "@storyteller-platform/audiobook";
 import {
   Epub
 } from "@storyteller-platform/epub";
+import { scratchPath } from "@storyteller-platform/fs";
 import {
   createAggregator,
   createTiming
 } from "@storyteller-platform/ghost-story";
+import { MediaType } from "@storyteller-platform/media-types";
 import { getTrackDuration } from "../common/ffmpeg.js";
 import { parseDom } from "../markup/parseDom.js";
 import { segmentChapter } from "../markup/segmentation.js";
@@ -60,8 +60,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
   var _stack2 = [];
   try {
     const outFormat = options.outFormat ?? "epub";
-    const epubPath = outFormat === "epub" ? autoJoin(
-      tmpdir(),
+    const epubPath = outFormat === "epub" ? scratchPath(
       `storyteller-platform-align-${randomUUID()}`,
       autoBasename(output)
     ) : input;
@@ -79,7 +78,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
       await copyFile(input, epubPath);
     }
     const audiobookFiles = await readdir(audiobookDir).then(
-      (filenames) => filenames.filter((f) => isAudioFile(f)).map((f) => autoJoin(audiobookDir, f))
+      (filenames) => filenames.filter((f) => MediaType.fromPath(f)?.kind === "audio").map((f) => autoJoin(audiobookDir, f))
     );
     const epub = __using(_stack2, await Epub.from(epubPath));
     const transcriptions = await readdir(transcriptionsDir).then(
@@ -127,8 +126,7 @@ async function align(input, output, transcriptionsDir, audiobookDir, options) {
           ),
           guidedNavigationDocuments
         );
-        const tmpArchivePath = autoJoin(
-          tmpdir(),
+        const tmpArchivePath = scratchPath(
           `storyteller-platform-epub-${randomUUID()}`
         );
         const { promise, resolve } = Promise.withResolvers();
@@ -190,6 +188,9 @@ class Aligner {
     this.languageOverride = languageOverride;
     this.logger = logger;
     this.transcription = concatTranscriptions(transcriptions, audiofiles);
+    audiofiles.forEach((audiofile, index) => {
+      this.audiofileTranscripts[audiofile] = transcriptions[index]?.transcript ?? "";
+    });
     this.getChapterSentences = memoize(this.getChapterSentences.bind(this));
     this.granularity = granularity ?? "sentence";
     this.textRef = textRef ?? "id-fragment";
@@ -201,6 +202,7 @@ class Aligner {
   granularity;
   textRef;
   audioFileDurations = {};
+  audiofileTranscripts = {};
   report = {
     chapters: [],
     unalignedChapters: [],
@@ -324,7 +326,7 @@ class Aligner {
         const duration = await getTrackDuration(audiofile);
         this.totalDuration += duration;
         const audio = await readFile(audiofile);
-        const mediaType = lookupAudioMime(base) ?? void 0;
+        const mediaType = MediaType.fromPath(base)?.mime;
         await this.epub.addManifestItem(
           {
             id,
@@ -341,7 +343,7 @@ class Aligner {
       {
         id: mediaOverlayId,
         href: `MediaOverlays/${chapterStem}.smil`,
-        mediaType: "application/smil+xml"
+        mediaType: MediaType.SMIL.mime
       },
       createMediaOverlay(
         chapter,
@@ -369,7 +371,7 @@ class Aligner {
       value: Epub.formatSmilDuration(chapterDuration)
     });
   }
-  addChapterReport(chapter, chapterSentences, sentenceRanges, startSentence, endSentence, mapping, transcriptionOffset, endTranscriptionOffset) {
+  addChapterReport(spineIndex, chapter, chapterSentences, sentenceRanges, startSentence, endSentence, mapping, transcriptionOffset, endTranscriptionOffset, score) {
     const audioFiles = sentenceRanges.reduce(
       (acc, range) => {
         const existing = acc.find(
@@ -393,6 +395,8 @@ class Aligner {
     const mappedEndTranscriptionOffset = cursor.map(endTranscriptionOffset);
     this.report.chapters.push({
       href: chapter.href,
+      score,
+      spineIndex,
       transcriptionOffset: mappedTranscriptionOffset,
       endTranscriptionOffset: mappedEndTranscriptionOffset,
       transcriptionContext: {
@@ -460,7 +464,7 @@ class Aligner {
       }
     }
   }
-  async alignChapter(chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline, mapping) {
+  async alignChapter(spineIndex, chapterId, transcriptionText, transcriptionOffset, transcriptionEndOffset, locale, mappedTimeline, mapping) {
     const timing = createTiming();
     timing.start("read contents");
     const manifest = await this.epub.getManifest();
@@ -480,7 +484,8 @@ class Aligner {
       wordRanges,
       transcriptionOffset: endTranscriptionOffset,
       firstFoundSentence,
-      lastFoundSentence
+      lastFoundSentence,
+      score
     } = await getSentenceRanges(
       transcriptionText,
       mappedTimeline,
@@ -499,7 +504,7 @@ class Aligner {
     Epub.addLinkToXhtmlHead(chapterXml, {
       rel: "stylesheet",
       href: storytellerStylesheetUrl,
-      type: "text/css"
+      type: MediaType.CSS.mime
     });
     this.alignedChapters.push({
       chapter,
@@ -510,6 +515,7 @@ class Aligner {
       endOffset: endTranscriptionOffset
     });
     this.addChapterReport(
+      spineIndex,
       chapter,
       chapterSentences,
       sentenceRanges,
@@ -517,7 +523,8 @@ class Aligner {
       lastFoundSentence,
       mapping,
       transcriptionOffset,
-      endTranscriptionOffset
+      endTranscriptionOffset,
+      score
     );
     return {
       lastSentenceRange: sentenceRanges.at(-1) ?? null,
@@ -569,7 +576,8 @@ class Aligner {
       if (manifest[chapterId]?.properties?.includes("nav")) {
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "is-nav"
+          reason: "is-nav",
+          spineIndex: index
         });
         continue;
       }
@@ -584,7 +592,8 @@ class Aligner {
         this.logger?.info(`Chapter #${index} has no text; skipping`);
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "no-text"
+          reason: "no-text",
+          spineIndex: index
         });
         continue;
       }
@@ -595,7 +604,8 @@ class Aligner {
         );
         this.report.unalignedChapters.push({
           href: spineItem.href,
-          reason: "too-short"
+          reason: "too-short",
+          spineIndex: index
         });
         continue;
       }
@@ -610,6 +620,9 @@ class Aligner {
         this.report.unalignedChapters.push({
           href: spineItem.href,
           reason: "not-found",
+          spineIndex: index,
+          chapterSentenceCount: chapterSentences.length,
+          score: 0,
           start: chapterSentences.slice(0, 3).map((s) => s.text).join("").slice(0, 80),
           end: chapterSentences.slice(-3).map((s) => s.text).join("").slice(-80)
         });
@@ -620,6 +633,7 @@ class Aligner {
         continue;
       }
       const result = await this.alignChapter(
+        index,
         chapterId,
         transcriptionText,
         Math.max(start, 0),
@@ -685,7 +699,12 @@ class Aligner {
     }
     for (const audiofile of this.audiofiles) {
       if (!this.report.audioFiles.some(({ filepath }) => filepath === audiofile)) {
-        this.report.unalignedAudioFiles.push({ filepath: audiofile });
+        const transcript = this.audiofileTranscripts[audiofile]?.trim();
+        this.report.unalignedAudioFiles.push({
+          filepath: audiofile,
+          duration: this.audioFileDurations[audiofile] ?? null,
+          ...transcript ? { transcription: { text: transcript.slice(0, 2e3) } } : {}
+        });
       }
     }
     await this.epub.addMetadata({
@@ -702,7 +721,7 @@ class Aligner {
       {
         id: "storyteller_readaloud_styles",
         href: "Styles/storyteller-readaloud.css",
-        mediaType: "text/css"
+        mediaType: MediaType.CSS.mime
       },
       `
 .-epub-media-overlay-active {