@storyteller-platform/align 0.1.56 → 0.1.58

package/dist/cli/bin.js

@@ -5,10 +5,10 @@ import {
 } from "../chunk-BIEQXUOY.js";
 import { randomUUID } from "node:crypto";
 import { rmSync } from "node:fs";
-import * as os from "node:os";
-import { basename, join } from "node:path";
+import { basename } from "node:path";
 import {
   constant,
+  flag,
   group,
   integer,
   merge,
@@ -23,8 +23,10 @@ import { run } from "@optique/run";
 import { path } from "@optique/run/valueparser";
 import { Presets, SingleBar } from "cli-progress";
 import { Epub } from "@storyteller-platform/epub";
+import { scratchPath } from "@storyteller-platform/fs";
 import packageJson from "../../package.json" with { type: "json" };
 import { align } from "../align/align.js";
+import { ctcAlign } from "../align/ctc/align.js";
 import { alignCommand, alignParser } from "../align/parse.js";
 import { createLogger } from "../common/logging.js";
 import {
@@ -34,10 +36,13 @@ import {
   loggingParser,
   removeNcxParser
 } from "../common/parse.js";
+import { emit } from "../emit/emit.js";
+import { emitCommand, emitParser } from "../emit/parse.js";
 import { markup } from "../markup/markup.js";
 import { markupCommand } from "../markup/parse.js";
 import { processCommand, processParser } from "../process/parse.js";
 import { processAudiobook } from "../process/processAudiobook.js";
+import { snapshotCtcAlignment } from "../snapshot/ctc/snapshot.js";
 import { snapshotCommand } from "../snapshot/parse.js";
 import { snapshotAlignment } from "../snapshot/snapshot.js";
 import { transcribeCommand, transcribeParser } from "../transcribe/parse.js";
@@ -50,9 +55,6 @@ const pipelineCommand = merge(
     processedAudio: optional(
       option("--processed-audio", path({ type: "directory" }))
     ),
-    transcriptions: optional(
-      option("--transcriptions", path({ type: "directory" }))
-    ),
     markedup: optional(
       option("--markedup", path({ type: "file", extensions: [".epub"] }))
     ),
@@ -67,7 +69,25 @@ const pipelineCommand = merge(
     output: option("--output", path({ type: "file", extensions: [".epub"] }))
   }),
   processParser,
-  group("Transcription", transcribeParser),
+  or(
+    merge(
+      object({
+        ctc: flag("--ctc", {
+          description: message`Use CTC emissions instead of transcriptions for alignment`
+        }),
+        emissions: optional(option("--emissions", path({ type: "directory" })))
+      }),
+      emitParser
+    ),
+    merge(
+      group("Transcription", transcribeParser),
+      object({
+        transcriptions: optional(
+          option("--transcriptions", path({ type: "directory" }))
+        )
+      })
+    )
+  ),
   autoUpgradeParser,
   removeNcxParser,
   granularityParser,
@@ -78,6 +98,7 @@ const pipelineCommand = merge(
 const parser = or(
   processCommand,
   transcribeCommand,
+  emitCommand,
   upgradeEpubCommand,
   markupCommand,
   alignCommand,
@@ -87,6 +108,13 @@ const parser = or(
 async function main() {
   var _stack2 = [];
   try {
+    process.on("SIGINT", () => {
+      try {
+        progressBar.stop();
+      } finally {
+        process.exit(130);
+      }
+    });
     const parsed = run(parser, {
       showChoices: true,
       showDefault: true,
@@ -110,6 +138,17 @@ async function main() {
         progressBar.start(100, 0);
       }
     }
+    function updateProgressBar(progress) {
+      if (parsed.action === "upgrade-epub") return;
+      if (parsed.noProgress || parsed.logLevel !== "silent") {
+        return;
+      }
+      const floored = Math.floor(progress * 100);
+      if (floored === Math.floor(progressBar.getProgress() * 100)) {
+        return;
+      }
+      progressBar.update(floored);
+    }
     resetProgressBar();
     process.on("SIGINT", () => {
       controller.abort();
@@ -131,11 +170,7 @@ async function main() {
           parallelism: parsed.parallelism,
           signal: controller.signal,
           logger,
-          ...!parsed.noProgress && parsed.logLevel === "silent" && {
-            onProgress: (progress) => {
-              progressBar.update(Math.floor(progress * 100));
-            }
-          }
+          onProgress: updateProgressBar
         });
         if (parsed.time) {
           timing.print("Process audiobook");
@@ -151,11 +186,7 @@ async function main() {
             ...parsed,
             signal: controller.signal,
             logger,
-            ...!parsed.noProgress && parsed.logLevel === "silent" && {
-              onProgress: (progress) => {
-                progressBar.update(Math.floor(progress * 100));
-              }
-            }
+            onProgress: updateProgressBar
           }
         );
         if (parsed.time) {
@@ -163,6 +194,13 @@ async function main() {
         }
         break;
       }
+      case "emit": {
+        await emit(parsed.input, parsed.output, {
+          ...parsed,
+          onProgress: updateProgressBar
+        });
+        break;
+      }
       case "upgrade-epub": {
         await upgradeEpub(parsed.input, parsed.output, {
           removeNcx: parsed.removeNcx
@@ -173,8 +211,7 @@ async function main() {
         let input = parsed.input;
         if (parsed.autoupgrade) {
           logger.info("Upgrading EPUB 2 to EPUB 3");
-          input = join(
-            os.tmpdir(),
+          input = scratchPath(
             `stalign-autoupgrade-${randomUUID()}`,
             basename(parsed.input)
           );
@@ -184,11 +221,7 @@ async function main() {
           primaryLocale: parsed.language ?? new Intl.Locale("en-US"),
           granularity: parsed.granularity,
           logger,
-          ...!parsed.noProgress && parsed.logLevel === "silent" && {
-            onProgress: (progress) => {
-              progressBar.update(Math.floor(progress * 100));
-            }
-          }
+          onProgress: updateProgressBar
         });
         if (parsed.time) {
           timing.print("Mark up EPUB");
@@ -199,34 +232,50 @@ async function main() {
         let input = parsed.epub;
         if (parsed.autoupgrade) {
           logger.info("Upgrading EPUB 2 to EPUB 3");
-          input = join(
-            os.tmpdir(),
+          input = scratchPath(
             `stalign-autoupgrade-${randomUUID()}`,
             basename(parsed.epub)
           );
           await upgradeEpub(parsed.epub, input, { removeNcx: parsed.removeNcx });
         }
-        const timing = await align(
-          input,
-          parsed.output,
-          parsed.transcriptions,
-          parsed.audiobook,
-          {
-            granularity: parsed.granularity,
-            textRef: parsed.textRef,
-            outFormat: parsed.outFormat,
-            primaryLocale: parsed.language,
-            reportsPath: parsed.reports,
-            logger,
-            ...!parsed.noProgress && parsed.logLevel === "silent" && {
-              onProgress: (progress) => {
-                progressBar.update(Math.floor(progress * 100));
-              }
+        if ("ctc" in parsed) {
+          const timing = await ctcAlign(
+            input,
+            parsed.output,
+            parsed.emissions,
+            parsed.audiobook,
+            {
+              granularity: parsed.granularity,
+              textRef: parsed.textRef,
+              outFormat: parsed.outFormat,
+              primaryLocale: parsed.language,
+              reportsPath: parsed.reports,
+              logger,
+              onProgress: updateProgressBar
             }
+          );
+          if (parsed.time) {
+            timing.print("Align EPUB and audiobook");
+          }
+        } else {
+          const timing = await align(
+            input,
+            parsed.output,
+            parsed.transcriptions,
+            parsed.audiobook,
+            {
+              granularity: parsed.granularity,
+              textRef: parsed.textRef,
+              outFormat: parsed.outFormat,
+              primaryLocale: parsed.language,
+              reportsPath: parsed.reports,
+              logger,
+              onProgress: updateProgressBar
+            }
+          );
+          if (parsed.time) {
+            timing.print("Align EPUB and audiobook");
           }
-        );
-        if (parsed.time) {
-          timing.print("Align EPUB and audiobook");
         }
         break;
       }
@@ -236,8 +285,7 @@ async function main() {
           let input = parsed.epub;
           if (parsed.autoupgrade) {
             logger.info("Upgrading EPUB 2 to EPUB 3");
-            input = join(
-              os.tmpdir(),
+            input = scratchPath(
               `stalign-autoupgrade-${randomUUID()}`,
               basename(parsed.epub)
             );
@@ -246,7 +294,7 @@ async function main() {
           const epub = __using(_stack, await Epub.from(input));
           const primaryLocale = parsed.language ?? await epub.getLanguage() ?? new Intl.Locale("en-US");
           epub.discardAndClose();
-          const processedAudio = parsed.processedAudio ?? join(os.tmpdir(), `stalign-processed-${randomUUID()}`);
+          const processedAudio = parsed.processedAudio ?? scratchPath(`stalign-processed-${randomUUID()}`);
           if (!parsed.processedAudio) {
             stack.defer(() => {
               rmSync(processedAudio, { recursive: true, force: true });
@@ -263,11 +311,7 @@ async function main() {
               maxLength: parsed.maxLength / 60,
               parallelism: parsed.parallelTranscodes,
               signal: controller.signal,
-              ...!parsed.noProgress && parsed.logLevel === "silent" && {
-                onProgress: (progress) => {
-                  progressBar.update(Math.floor(progress * 100));
-                }
-              }
+              onProgress: updateProgressBar
             }
           );
           resetProgressBar();
@@ -279,36 +323,56 @@ async function main() {
           }
           logger.info("Transcribing...");
           startProgressBar();
-          const transcriptions = parsed.transcriptions ?? join(os.tmpdir(), `stalign-transcriptions-${randomUUID()}`);
-          if (!parsed.transcriptions) {
-            stack.defer(() => {
-              rmSync(transcriptions, { recursive: true, force: true });
-            });
-          }
-          const transcribeTiming = await transcribe(
-            processedAudio,
-            transcriptions,
-            primaryLocale,
-            {
+          let emissions;
+          let transcriptions;
+          if ("ctc" in parsed) {
+            emissions = parsed.emissions ?? scratchPath(`stalign-emissions-${randomUUID()}`);
+            if (!parsed.emissions) {
+              stack.defer(() => {
+                rmSync(emissions, { recursive: true, force: true });
+              });
+            }
+            const transcribeTiming = await emit(processedAudio, emissions, {
               ...parsed,
-              parallelism: parsed.parallelTranscribes,
               signal: controller.signal,
               logger,
-              ...!parsed.noProgress && parsed.logLevel === "silent" && {
-                onProgress: (progress) => {
-                  progressBar.update(Math.floor(progress * 100));
-                }
+              onProgress: updateProgressBar
+            });
+            resetProgressBar();
+            logger.info(
+              `Generating emissions for audiobook complete, emissions saved to ${emissions}.`
+            );
+            if (parsed.time) {
+              transcribeTiming.print();
+            }
+          } else {
+            transcriptions = parsed.transcriptions ?? scratchPath(`stalign-transcriptions-${randomUUID()}`);
+            if (!parsed.transcriptions) {
+              stack.defer(() => {
+                rmSync(transcriptions, { recursive: true, force: true });
+              });
+            }
+            const transcribeTiming = await transcribe(
+              processedAudio,
+              transcriptions,
+              primaryLocale,
+              {
+                ...parsed,
+                parallelism: parsed.parallelTranscribes,
+                signal: controller.signal,
+                logger,
+                onProgress: updateProgressBar
               }
+            );
+            resetProgressBar();
+            logger.info(
+              `Transcribing audiobook complete, transcriptions saved to ${transcriptions}.`
+            );
+            if (parsed.time) {
+              transcribeTiming.print();
             }
-          );
-          resetProgressBar();
-          logger.info(
-            `Transcribing audiobook complete, transcriptions saved to ${transcriptions}.`
-          );
-          if (parsed.time) {
-            transcribeTiming.print();
           }
-          const markedup = parsed.textRef === "id-fragment" ? parsed.markedup ?? join(os.tmpdir(), `stalign-markedup-${randomUUID()}.epub`) : parsed.epub;
+          const markedup = parsed.textRef === "id-fragment" ? parsed.markedup ?? scratchPath(`stalign-markedup-${randomUUID()}.epub`) : parsed.epub;
           if (parsed.textRef === "id-fragment") {
             logger.info("Marking up EPUB...");
             startProgressBar();
@@ -321,11 +385,7 @@ async function main() {
               granularity: parsed.granularity,
               primaryLocale,
               logger,
-              ...!parsed.noProgress && parsed.logLevel === "silent" && {
-                onProgress: (progress) => {
-                  progressBar.update(Math.floor(progress * 100));
-                }
-              }
+              onProgress: updateProgressBar
             });
             resetProgressBar();
             logger.info(`Markup complete, marked up EPUB saved to ${markedup}.`);
@@ -337,29 +397,52 @@ async function main() {
           }
           logger.info("Aligning EPUB with audiobook...");
           startProgressBar();
-          const alignTiming = await align(
-            markedup,
-            parsed.output,
-            transcriptions,
-            processedAudio,
-            {
-              granularity: parsed.granularity,
-              textRef: parsed.textRef,
-              outFormat: parsed.outFormat,
-              primaryLocale,
-              reportsPath: parsed.reports,
-              logger,
-              ...!parsed.noProgress && parsed.logLevel === "silent" && {
-                onProgress: (progress) => {
-                  progressBar.update(Math.floor(progress * 100));
-                }
+          if ("ctc" in parsed) {
+            const timing = await ctcAlign(
+              markedup,
+              parsed.output,
+              emissions,
+              processedAudio,
+              {
+                granularity: parsed.granularity,
+                textRef: parsed.textRef,
+                outFormat: parsed.outFormat,
+                primaryLocale: parsed.language,
+                reportsPath: parsed.reports,
+                logger,
+                onProgress: updateProgressBar
               }
+            );
+            resetProgressBar();
+            logger.info(
+              `Alignment complete, aligned EPUB saved to ${parsed.output}`
+            );
+            if (parsed.time) {
+              timing.print("Align EPUB and audiobook");
+            }
+          } else {
+            const alignTiming = await align(
+              markedup,
+              parsed.output,
+              transcriptions,
+              processedAudio,
+              {
+                granularity: parsed.granularity,
+                textRef: parsed.textRef,
+                outFormat: parsed.outFormat,
+                primaryLocale,
+                reportsPath: parsed.reports,
+                logger,
+                onProgress: updateProgressBar
+              }
+            );
+            resetProgressBar();
+            logger.info(
+              `Alignment complete, aligned EPUB saved to ${parsed.output}`
+            );
+            if (parsed.time) {
+              alignTiming.print();
             }
-          );
-          resetProgressBar();
-          logger.info(`Alignment complete, aligned EPUB saved to ${parsed.output}`);
-          if (parsed.time) {
-            alignTiming.print();
           }
           break;
         } catch (_) {
@@ -369,7 +452,15 @@ async function main() {
         }
       }
       case "snapshot": {
-        await snapshotAlignment(parsed.epub, parsed.transcriptions, parsed.output);
+        if ("ctc" in parsed) {
+          await snapshotCtcAlignment(parsed.epub, parsed.emissions, parsed.output);
+        } else {
+          await snapshotAlignment(
+            parsed.epub,
+            parsed.transcriptions,
+            parsed.output
+          );
+        }
       }
     }
   } catch (_2) {

package/dist/common/ffmpeg.cjs

@@ -42,7 +42,8 @@ var import_promises = require("node:fs/promises");
 var import_node_path = require("node:path");
 var import_node_util = require("node:util");
 var import_memoize = __toESM(require("memoize"), 1);
-var import_mime = require("../process/mime.cjs");
+var import_audiobook = require("@storyteller-platform/audiobook");
+var import_media_types = require("@storyteller-platform/media-types");
 var import_shell = require("./shell.cjs");
 const execPromise = (0, import_node_util.promisify)(import_node_child_process.exec);
 async function execCmd(command, logger, signal) {
@@ -190,12 +191,15 @@ function commonFfmpegArguments(options) {
   const args = ["-vn"];
   if (codec) {
     args.push("-c:a", codec);
-    if (codec === "libopus") {
+    if (codec === import_audiobook.AUDIO_ENCODERS.opus) {
       args.push("-b:a", bitrate && /^\d+[kK]$/i.test(bitrate) ? bitrate : "32K");
-    } else if (codec === "libmp3lame" && bitrate) {
+    } else if (codec === import_audiobook.AUDIO_ENCODERS.mp3 && bitrate) {
       args.push("-b:a", bitrate);
     }
-  } else if ((0, import_mime.areSameType)(sourceExtension, destExtension) || destExtension == ".mp4") {
+  } else if ((0, import_audiobook.canCopyCodec)(
+    import_media_types.MediaType.fromExtension(sourceExtension),
+    import_media_types.MediaType.fromExtension(destExtension)
+  )) {
     args.push("-c:a", "copy");
   }
   args.push("-map", "0:a");
@@ -238,7 +242,8 @@ async function splitFile(input, output, start, end, encoding, signal, logger) {
   return true;
 }
 async function transcodeFile(input, output, encoding, signal, logger) {
-  if (!encoding?.codec && (0, import_mime.areSameType)(input, output)) {
+  const container = import_media_types.MediaType.fromPath(input);
+  if (!encoding?.codec && container !== null && container === import_media_types.MediaType.fromPath(output)) {
     logger?.info(
       `Input and output container and codec are the same, copying ${input} to output directory`
     );

package/dist/common/ffmpeg.js

@@ -4,7 +4,8 @@ import { copyFile } from "node:fs/promises";
 import { extname } from "node:path";
 import { promisify } from "node:util";
 import memoize from "memoize";
-import { areSameType } from "../process/mime.js";
+import { AUDIO_ENCODERS, canCopyCodec } from "@storyteller-platform/audiobook";
+import { MediaType } from "@storyteller-platform/media-types";
 import { quotePath } from "./shell.js";
 const execPromise = promisify(exec);
 async function execCmd(command, logger, signal) {
@@ -152,12 +153,15 @@ function commonFfmpegArguments(options) {
   const args = ["-vn"];
   if (codec) {
     args.push("-c:a", codec);
-    if (codec === "libopus") {
+    if (codec === AUDIO_ENCODERS.opus) {
       args.push("-b:a", bitrate && /^\d+[kK]$/i.test(bitrate) ? bitrate : "32K");
-    } else if (codec === "libmp3lame" && bitrate) {
+    } else if (codec === AUDIO_ENCODERS.mp3 && bitrate) {
       args.push("-b:a", bitrate);
     }
-  } else if (areSameType(sourceExtension, destExtension) || destExtension == ".mp4") {
+  } else if (canCopyCodec(
+    MediaType.fromExtension(sourceExtension),
+    MediaType.fromExtension(destExtension)
+  )) {
     args.push("-c:a", "copy");
   }
   args.push("-map", "0:a");
@@ -200,7 +204,8 @@ async function splitFile(input, output, start, end, encoding, signal, logger) {
   return true;
 }
 async function transcodeFile(input, output, encoding, signal, logger) {
-  if (!encoding?.codec && areSameType(input, output)) {
+  const container = MediaType.fromPath(input);
+  if (!encoding?.codec && container !== null && container === MediaType.fromPath(output)) {
     logger?.info(
       `Input and output container and codec are the same, copying ${input} to output directory`
     );

package/dist/common/logging.cjs

@@ -31,12 +31,44 @@ __export(logging_exports, {
   createLogger: () => createLogger
 });
 module.exports = __toCommonJS(logging_exports);
+var import_node_fs = require("node:fs");
+var import_node_path = require("node:path");
 var import_pino = __toESM(require("pino"), 1);
 var import_pino_pretty = __toESM(require("pino-pretty"), 1);
+function todayDateString() {
+  return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+}
+function getLogDir() {
+  const dataDir = process.env["STORYTELLER_DATA_DIR"];
+  if (!dataDir) return null;
+  return (0, import_node_path.join)(dataDir, "logs");
+}
 function createLogger(level = "info") {
+  const prettyStream = (0, import_pino_pretty.default)({
+    ignore: "pid,hostname",
+    translateTime: "SYS:standard"
+  });
+  const logDir = getLogDir();
+  if (!logDir) {
+    return (0, import_pino.default)({ level }, prettyStream);
+  }
+  try {
+    (0, import_node_fs.mkdirSync)(logDir, { recursive: true });
+  } catch {
+    return (0, import_pino.default)({ level }, prettyStream);
+  }
+  const filePath = (0, import_node_path.join)(logDir, `storyteller-${todayDateString()}.log`);
+  const fileStream = import_pino.default.destination({
+    dest: filePath,
+    mkdir: true,
+    sync: false
+  });
   return (0, import_pino.default)(
     { level },
-    (0, import_pino_pretty.default)({ ignore: "pid,hostname", translateTime: "SYS:standard" })
+    import_pino.default.multistream([
+      { stream: prettyStream, level },
+      { stream: fileStream, level }
+    ])
   );
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/common/logging.js

@@ -1,10 +1,42 @@
 import "../chunk-BIEQXUOY.js";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
 import pino from "pino";
 import PinoPretty from "pino-pretty";
+function todayDateString() {
+  return (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
+}
+function getLogDir() {
+  const dataDir = process.env["STORYTELLER_DATA_DIR"];
+  if (!dataDir) return null;
+  return join(dataDir, "logs");
+}
 function createLogger(level = "info") {
+  const prettyStream = PinoPretty({
+    ignore: "pid,hostname",
+    translateTime: "SYS:standard"
+  });
+  const logDir = getLogDir();
+  if (!logDir) {
+    return pino({ level }, prettyStream);
+  }
+  try {
+    mkdirSync(logDir, { recursive: true });
+  } catch {
+    return pino({ level }, prettyStream);
+  }
+  const filePath = join(logDir, `storyteller-${todayDateString()}.log`);
+  const fileStream = pino.destination({
+    dest: filePath,
+    mkdir: true,
+    sync: false
+  });
   return pino(
     { level },
-    PinoPretty({ ignore: "pid,hostname", translateTime: "SYS:standard" })
+    pino.multistream([
+      { stream: prettyStream, level },
+      { stream: fileStream, level }
+    ])
   );
 }
 export {