@storyteller-platform/ghost-story 0.1.5 → 0.1.7

package/dist/cli/bin.cjs

@@ -24,6 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var import_zli = require("@robingenz/zli");
 var import_cli_progress = require("cli-progress");
+var import_fs_extra = require("fs-extra");
 var import_zod = require("zod");
 var import_config = require("./config.cjs");
 var import_install = require("./install.cjs");
@@ -194,7 +195,16 @@ const transcribeCommand = (0, import_zli.defineCommand)({
       import_cli_progress.Presets.shades_classic
     );
     try {
-      const [inputPath] = args;
+      let [inputPath] = args;
+      const path = await import("node:path");
+      inputPath = path.resolve(process.cwd(), inputPath);
+      let outputPath = options.output ? path.resolve(process.cwd(), options.output) : void 0;
+      if (outputPath && !path.extname(outputPath)) {
+        outputPath = `${outputPath}.json`;
+      }
+      if (outputPath) {
+        (0, import_fs_extra.ensureDirSync)(path.dirname(outputPath));
+      }
       const result = await recognize(inputPath, {
         engine: options.engine,
         language: options.language,
@@ -219,14 +229,14 @@ const transcribeCommand = (0, import_zli.defineCommand)({
           }
         }
       });
-      if (!options.output) {
+      if (!outputPath) {
         console.log(JSON.stringify(result, null, 2));
         return;
       }
       const { writeFile } = await import("node:fs/promises");
-      await writeFile(options.output, JSON.stringify(result, null, 2));
+      await writeFile(outputPath, JSON.stringify(result, null, 2));
       console.log(`
-Transcription written to ${options.output}`);
+Transcription written to ${outputPath}`);
     } finally {
       if (!options.noProgress) {
         progressBar.stop();

package/dist/cli/bin.js

@@ -6,6 +6,7 @@ import {
   processConfig
 } from "@robingenz/zli";
 import { Presets, SingleBar } from "cli-progress";
+import { ensureDirSync } from "fs-extra";
 import { z } from "zod";
 import {
   BUILD_VARIANTS,
@@ -193,7 +194,16 @@ const transcribeCommand = defineCommand({
       Presets.shades_classic
     );
     try {
-      const [inputPath] = args;
+      let [inputPath] = args;
+      const path = await import("node:path");
+      inputPath = path.resolve(process.cwd(), inputPath);
+      let outputPath = options.output ? path.resolve(process.cwd(), options.output) : void 0;
+      if (outputPath && !path.extname(outputPath)) {
+        outputPath = `${outputPath}.json`;
+      }
+      if (outputPath) {
+        ensureDirSync(path.dirname(outputPath));
+      }
       const result = await recognize(inputPath, {
         engine: options.engine,
         language: options.language,
@@ -218,14 +228,14 @@ const transcribeCommand = defineCommand({
           }
         }
       });
-      if (!options.output) {
+      if (!outputPath) {
         console.log(JSON.stringify(result, null, 2));
         return;
       }
       const { writeFile } = await import("node:fs/promises");
-      await writeFile(options.output, JSON.stringify(result, null, 2));
+      await writeFile(outputPath, JSON.stringify(result, null, 2));
       console.log(`
-Transcription written to ${options.output}`);
+Transcription written to ${outputPath}`);
     } finally {
       if (!options.noProgress) {
         progressBar.stop();

package/dist/cli/config.cjs

@@ -35,6 +35,8 @@ __export(config_exports, {
   MODEL_SIZES: () => MODEL_SIZES,
   RECOGNITION_ENGINES: () => RECOGNITION_ENGINES,
   SILERO_VAD_VERSION: () => SILERO_VAD_VERSION,
+  WHISPER_CPP_PATCH_LEVEL: () => WHISPER_CPP_PATCH_LEVEL,
+  WHISPER_CPP_UPSTREAM_VERSION: () => WHISPER_CPP_UPSTREAM_VERSION,
   WHISPER_CPP_VERSION: () => WHISPER_CPP_VERSION,
   WHISPER_MODELS: () => WHISPER_MODELS,
   WHISPER_MODEL_VERSION: () => WHISPER_MODEL_VERSION,
@@ -72,7 +74,9 @@ var import_node_os = __toESM(require("node:os"), 1);
 var import_node_path = __toESM(require("node:path"), 1);
 var import_zod = __toESM(require("zod"), 1);
 var import_FileSystem = require("../utilities/FileSystem.cjs");
-const WHISPER_CPP_VERSION = "1.8.3";
+const WHISPER_CPP_UPSTREAM_VERSION = "1.8.3";
+const WHISPER_CPP_PATCH_LEVEL = 2;
+const WHISPER_CPP_VERSION = `${WHISPER_CPP_UPSTREAM_VERSION}-st.${WHISPER_CPP_PATCH_LEVEL}`;
 const WHISPER_MODEL_VERSION = "1.0.0";
 const SILERO_VAD_VERSION = "6.2.0";
 const GITLAB_PROJECT_PATH = "storyteller-platform/storyteller";
@@ -478,6 +482,8 @@ function writeConfig(config) {
   MODEL_SIZES,
   RECOGNITION_ENGINES,
   SILERO_VAD_VERSION,
+  WHISPER_CPP_PATCH_LEVEL,
+  WHISPER_CPP_UPSTREAM_VERSION,
   WHISPER_CPP_VERSION,
   WHISPER_MODELS,
   WHISPER_MODEL_VERSION,

package/dist/cli/config.d.cts

@@ -1,6 +1,8 @@
 import z from 'zod';
 
-declare const WHISPER_CPP_VERSION = "1.8.3";
+declare const WHISPER_CPP_UPSTREAM_VERSION = "1.8.3";
+declare const WHISPER_CPP_PATCH_LEVEL = 2;
+declare const WHISPER_CPP_VERSION = "1.8.3-st.2";
 declare const WHISPER_MODEL_VERSION = "1.0.0";
 declare const SILERO_VAD_VERSION = "6.2.0";
 declare const GITLAB_PROJECT_PATH = "storyteller-platform/storyteller";
@@ -73,4 +75,4 @@ type CLIConfig = z.infer<typeof cliConfigSchema>;
 declare function readConfig(): CLIConfig;
 declare function writeConfig(config: Partial<CLIConfig>): void;
 
-export { BUILD_VARIANTS, type BuildVariant, type CLIConfig, GITLAB_PROJECT_ID, GITLAB_PROJECT_PATH, GITLAB_WHIPSER_ML_ID, MODEL_SIZES, RECOGNITION_ENGINES, SILERO_VAD_VERSION, WHISPER_CPP_VERSION, WHISPER_MODELS, WHISPER_MODEL_VERSION, type WhisperModel, applyLegacyCpuFallback, cliConfigSchema, detectPlatform, getBinaryDownloadUrl, getCompatibleVariants, getConfiguredVariant, getCoremlModelDownloadUrl, getCoremlModelPath, getInstallDir, getInstalledVariant, getModelDir, getModelDownloadUrl, getModelPath, getVadExecutablePath, getVadModelDownloadUrl, getVadModelPath, getWhisperBaseDir, getWhisperExecutablePath, getWhisperServerExecutablePath, isValidModel, isValidVariant, isVariantCompatibleWithCurrentPlatform, needsCoremlModel, readConfig, resolveVariant, writeConfig };
+export { BUILD_VARIANTS, type BuildVariant, type CLIConfig, GITLAB_PROJECT_ID, GITLAB_PROJECT_PATH, GITLAB_WHIPSER_ML_ID, MODEL_SIZES, RECOGNITION_ENGINES, SILERO_VAD_VERSION, WHISPER_CPP_PATCH_LEVEL, WHISPER_CPP_UPSTREAM_VERSION, WHISPER_CPP_VERSION, WHISPER_MODELS, WHISPER_MODEL_VERSION, type WhisperModel, applyLegacyCpuFallback, cliConfigSchema, detectPlatform, getBinaryDownloadUrl, getCompatibleVariants, getConfiguredVariant, getCoremlModelDownloadUrl, getCoremlModelPath, getInstallDir, getInstalledVariant, getModelDir, getModelDownloadUrl, getModelPath, getVadExecutablePath, getVadModelDownloadUrl, getVadModelPath, getWhisperBaseDir, getWhisperExecutablePath, getWhisperServerExecutablePath, isValidModel, isValidVariant, isVariantCompatibleWithCurrentPlatform, needsCoremlModel, readConfig, resolveVariant, writeConfig };

package/dist/cli/config.d.ts

@@ -1,6 +1,8 @@
 import z from 'zod';
 
-declare const WHISPER_CPP_VERSION = "1.8.3";
+declare const WHISPER_CPP_UPSTREAM_VERSION = "1.8.3";
+declare const WHISPER_CPP_PATCH_LEVEL = 2;
+declare const WHISPER_CPP_VERSION = "1.8.3-st.2";
 declare const WHISPER_MODEL_VERSION = "1.0.0";
 declare const SILERO_VAD_VERSION = "6.2.0";
 declare const GITLAB_PROJECT_PATH = "storyteller-platform/storyteller";
@@ -73,4 +75,4 @@ type CLIConfig = z.infer<typeof cliConfigSchema>;
 declare function readConfig(): CLIConfig;
 declare function writeConfig(config: Partial<CLIConfig>): void;
 
-export { BUILD_VARIANTS, type BuildVariant, type CLIConfig, GITLAB_PROJECT_ID, GITLAB_PROJECT_PATH, GITLAB_WHIPSER_ML_ID, MODEL_SIZES, RECOGNITION_ENGINES, SILERO_VAD_VERSION, WHISPER_CPP_VERSION, WHISPER_MODELS, WHISPER_MODEL_VERSION, type WhisperModel, applyLegacyCpuFallback, cliConfigSchema, detectPlatform, getBinaryDownloadUrl, getCompatibleVariants, getConfiguredVariant, getCoremlModelDownloadUrl, getCoremlModelPath, getInstallDir, getInstalledVariant, getModelDir, getModelDownloadUrl, getModelPath, getVadExecutablePath, getVadModelDownloadUrl, getVadModelPath, getWhisperBaseDir, getWhisperExecutablePath, getWhisperServerExecutablePath, isValidModel, isValidVariant, isVariantCompatibleWithCurrentPlatform, needsCoremlModel, readConfig, resolveVariant, writeConfig };
+export { BUILD_VARIANTS, type BuildVariant, type CLIConfig, GITLAB_PROJECT_ID, GITLAB_PROJECT_PATH, GITLAB_WHIPSER_ML_ID, MODEL_SIZES, RECOGNITION_ENGINES, SILERO_VAD_VERSION, WHISPER_CPP_PATCH_LEVEL, WHISPER_CPP_UPSTREAM_VERSION, WHISPER_CPP_VERSION, WHISPER_MODELS, WHISPER_MODEL_VERSION, type WhisperModel, applyLegacyCpuFallback, cliConfigSchema, detectPlatform, getBinaryDownloadUrl, getCompatibleVariants, getConfiguredVariant, getCoremlModelDownloadUrl, getCoremlModelPath, getInstallDir, getInstalledVariant, getModelDir, getModelDownloadUrl, getModelPath, getVadExecutablePath, getVadModelDownloadUrl, getVadModelPath, getWhisperBaseDir, getWhisperExecutablePath, getWhisperServerExecutablePath, isValidModel, isValidVariant, isVariantCompatibleWithCurrentPlatform, needsCoremlModel, readConfig, resolveVariant, writeConfig };

package/dist/cli/config.js

@@ -4,7 +4,9 @@ import os from "node:os";
 import path from "node:path";
 import z from "zod";
 import { getAppDataDir } from "../utilities/FileSystem.js";
-const WHISPER_CPP_VERSION = "1.8.3";
+const WHISPER_CPP_UPSTREAM_VERSION = "1.8.3";
+const WHISPER_CPP_PATCH_LEVEL = 2;
+const WHISPER_CPP_VERSION = `${WHISPER_CPP_UPSTREAM_VERSION}-st.${WHISPER_CPP_PATCH_LEVEL}`;
 const WHISPER_MODEL_VERSION = "1.0.0";
 const SILERO_VAD_VERSION = "6.2.0";
 const GITLAB_PROJECT_PATH = "storyteller-platform/storyteller";
@@ -409,6 +411,8 @@ export {
   MODEL_SIZES,
   RECOGNITION_ENGINES,
   SILERO_VAD_VERSION,
+  WHISPER_CPP_PATCH_LEVEL,
+  WHISPER_CPP_UPSTREAM_VERSION,
   WHISPER_CPP_VERSION,
   WHISPER_MODELS,
   WHISPER_MODEL_VERSION,

package/dist/cli/install.cjs

@@ -43,6 +43,8 @@ var import_node_os = __toESM(require("node:os"), 1);
 var import_node_path = __toESM(require("node:path"), 1);
 var import_promises2 = require("node:stream/promises");
 var import_node_zlib = require("node:zlib");
+var import_chalk = __toESM(require("chalk"), 1);
+var import_cli_progress = require("cli-progress");
 var import_tar = require("tar");
 var import_config = require("./config.cjs");
 async function downloadFile(options) {
@@ -68,9 +70,27 @@ async function downloadFile(options) {
   }
   await (0, import_promises.mkdir)(import_node_path.default.dirname(destPath), { recursive: true });
   const fileStream = (0, import_node_fs.createWriteStream)(destPath);
+  const progressBar = new import_cli_progress.SingleBar(
+    {
+      etaBuffer: 2,
+      hideCursor: null,
+      noTTYOutput: !process.stderr.isTTY,
+      autopadding: true,
+      format: `${import_chalk.default.yellow("{bar}")} | {percentage}% | {current}/{sum} MB | ({speed} MB/s)`
+    },
+    import_cli_progress.Presets.shades_classic
+  );
   const reader = response.body.getReader();
   let downloaded = 0;
-  let lastPrinted = 0;
+  let lastTime = Date.now();
+  let lastDownloaded = 0;
+  if (printOutput) {
+    progressBar.start(totalSize, 0, {
+      current: "0 MB",
+      sum: `${(totalSize / 1024 / 1024).toFixed(2)} MB`,
+      speed: 0
+    });
+  }
   try {
     while (true) {
       const { done, value } = await reader.read();
@@ -81,18 +101,25 @@ async function downloadFile(options) {
       fileStream.write(value);
       onProgress == null ? void 0 : onProgress(downloaded, totalSize);
       if (printOutput && totalSize > 0) {
-        const shouldPrint = downloaded - lastPrinted > 10 * 1024 * 1024 || downloaded === totalSize;
+        const deltaTime = Date.now() - lastTime;
+        const shouldPrint = deltaTime > 1e3 || downloaded === totalSize;
         if (shouldPrint) {
-          const percent = (downloaded / totalSize * 100).toFixed(1);
-          console.log(
-            `  ${(downloaded / 1024 / 1024).toFixed(1)} MB / ${(totalSize / 1024 / 1024).toFixed(1)} MB (${percent}%)`
-          );
-          lastPrinted = downloaded;
+          const speed = (downloaded - lastDownloaded) / 1024 / 1024 / (deltaTime / 1e3);
+          lastTime = Date.now();
+          lastDownloaded = downloaded;
+          progressBar.update(downloaded, {
+            current: (downloaded / 1024 / 1024).toFixed(2),
+            sum: (totalSize / 1024 / 1024).toFixed(2),
+            speed: speed.toFixed(2)
+          });
         }
       }
     }
   } finally {
     fileStream.end();
+    if (printOutput) {
+      progressBar.stop();
+    }
   }
   await new Promise((resolve, reject) => {
     fileStream.on("finish", resolve);

package/dist/cli/install.js

@@ -9,6 +9,8 @@ import os from "node:os";
 import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import { createGunzip } from "node:zlib";
+import chalk from "chalk";
+import { Presets, SingleBar } from "cli-progress";
 import { extract } from "tar";
 import {
   MODEL_SIZES,
@@ -52,9 +54,27 @@ async function downloadFile(options) {
   }
   await mkdir(path.dirname(destPath), { recursive: true });
   const fileStream = createWriteStream(destPath);
+  const progressBar = new SingleBar(
+    {
+      etaBuffer: 2,
+      hideCursor: null,
+      noTTYOutput: !process.stderr.isTTY,
+      autopadding: true,
+      format: `${chalk.yellow("{bar}")} | {percentage}% | {current}/{sum} MB | ({speed} MB/s)`
+    },
+    Presets.shades_classic
+  );
   const reader = response.body.getReader();
   let downloaded = 0;
-  let lastPrinted = 0;
+  let lastTime = Date.now();
+  let lastDownloaded = 0;
+  if (printOutput) {
+    progressBar.start(totalSize, 0, {
+      current: "0 MB",
+      sum: `${(totalSize / 1024 / 1024).toFixed(2)} MB`,
+      speed: 0
+    });
+  }
   try {
     while (true) {
       const { done, value } = await reader.read();
@@ -65,18 +85,25 @@ async function downloadFile(options) {
       fileStream.write(value);
       onProgress == null ? void 0 : onProgress(downloaded, totalSize);
       if (printOutput && totalSize > 0) {
-        const shouldPrint = downloaded - lastPrinted > 10 * 1024 * 1024 || downloaded === totalSize;
+        const deltaTime = Date.now() - lastTime;
+        const shouldPrint = deltaTime > 1e3 || downloaded === totalSize;
         if (shouldPrint) {
-          const percent = (downloaded / totalSize * 100).toFixed(1);
-          console.log(
-            `  ${(downloaded / 1024 / 1024).toFixed(1)} MB / ${(totalSize / 1024 / 1024).toFixed(1)} MB (${percent}%)`
-          );
-          lastPrinted = downloaded;
+          const speed = (downloaded - lastDownloaded) / 1024 / 1024 / (deltaTime / 1e3);
+          lastTime = Date.now();
+          lastDownloaded = downloaded;
+          progressBar.update(downloaded, {
+            current: (downloaded / 1024 / 1024).toFixed(2),
+            sum: (totalSize / 1024 / 1024).toFixed(2),
+            speed: speed.toFixed(2)
+          });
         }
       }
     }
   } finally {
     fileStream.end();
+    if (printOutput) {
+      progressBar.stop();
+    }
   }
   await new Promise((resolve, reject) => {
     fileStream.on("finish", resolve);

package/dist/recognition/WhisperCppSTT.cjs

@@ -319,7 +319,7 @@ function buildTranscribeArgs(options) {
     "--print-progress",
     options.language ? ["--language", options.language.toLowerCase()] : null,
     options.flashAttention ? ["--flash-attn"] : null,
-    options.suppressNonSpeechTokens ? ["--suppress-nst", "--no-prints"] : null,
+    options.suppressNonSpeechTokens ? ["--suppress-nst"] : null,
     ["--processors", String(options.processors)],
     ["--threads", String(options.threads)]
   ];

package/dist/recognition/WhisperCppSTT.js

@@ -298,7 +298,7 @@ function buildTranscribeArgs(options) {
     "--print-progress",
     options.language ? ["--language", options.language.toLowerCase()] : null,
     options.flashAttention ? ["--flash-attn"] : null,
-    options.suppressNonSpeechTokens ? ["--suppress-nst", "--no-prints"] : null,
+    options.suppressNonSpeechTokens ? ["--suppress-nst"] : null,
     ["--processors", String(options.processors)],
     ["--threads", String(options.threads)]
   ];

package/dist/utilities/SpacelessScripts.cjs

@@ -0,0 +1,48 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var SpacelessScripts_exports = {};
+__export(SpacelessScripts_exports, {
+  spacelessScriptPattern: () => spacelessScriptPattern,
+  spacelessScripts: () => spacelessScripts,
+  startsWithSpacelessScript: () => startsWithSpacelessScript
+});
+module.exports = __toCommonJS(SpacelessScripts_exports);
+const spacelessScripts = [
+  { name: "thai", from: 3585, to: 3663 },
+  { name: "lao", from: 3713, to: 3807 },
+  { name: "tibetan", from: 3840, to: 4095 },
+  { name: "myanmar", from: 4096, to: 4255 },
+  { name: "khmer", from: 6016, to: 6143 },
+  { name: "hiragana", from: 12352, to: 12447 },
+  { name: "katakana", from: 12448, to: 12543 },
+  { name: "cjk-ext-a", from: 13312, to: 19903 },
+  { name: "cjk-unified", from: 19968, to: 40959 },
+  { name: "cjk-compat", from: 63744, to: 64255 }
+];
+const charClass = spacelessScripts.map((s) => `${String.fromCharCode(s.from)}-${String.fromCharCode(s.to)}`).join("");
+const spacelessScriptPattern = new RegExp(`[${charClass}]`);
+function startsWithSpacelessScript(text) {
+  return spacelessScriptPattern.test(text.charAt(0));
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  spacelessScriptPattern,
+  spacelessScripts,
+  startsWithSpacelessScript
+});

package/dist/utilities/SpacelessScripts.d.cts

@@ -0,0 +1,10 @@
+interface ScriptRange {
+    name: string;
+    from: number;
+    to: number;
+}
+declare const spacelessScripts: readonly ScriptRange[];
+declare const spacelessScriptPattern: RegExp;
+declare function startsWithSpacelessScript(text: string): boolean;
+
+export { spacelessScriptPattern, spacelessScripts, startsWithSpacelessScript };

package/dist/utilities/SpacelessScripts.d.ts

@@ -0,0 +1,10 @@
+interface ScriptRange {
+    name: string;
+    from: number;
+    to: number;
+}
+declare const spacelessScripts: readonly ScriptRange[];
+declare const spacelessScriptPattern: RegExp;
+declare function startsWithSpacelessScript(text: string): boolean;
+
+export { spacelessScriptPattern, spacelessScripts, startsWithSpacelessScript };

package/dist/utilities/SpacelessScripts.js

@@ -0,0 +1,22 @@
+const spacelessScripts = [
+  { name: "thai", from: 3585, to: 3663 },
+  { name: "lao", from: 3713, to: 3807 },
+  { name: "tibetan", from: 3840, to: 4095 },
+  { name: "myanmar", from: 4096, to: 4255 },
+  { name: "khmer", from: 6016, to: 6143 },
+  { name: "hiragana", from: 12352, to: 12447 },
+  { name: "katakana", from: 12448, to: 12543 },
+  { name: "cjk-ext-a", from: 13312, to: 19903 },
+  { name: "cjk-unified", from: 19968, to: 40959 },
+  { name: "cjk-compat", from: 63744, to: 64255 }
+];
+const charClass = spacelessScripts.map((s) => `${String.fromCharCode(s.from)}-${String.fromCharCode(s.to)}`).join("");
+const spacelessScriptPattern = new RegExp(`[${charClass}]`);
+function startsWithSpacelessScript(text) {
+  return spacelessScriptPattern.test(text.charAt(0));
+}
+export {
+  spacelessScriptPattern,
+  spacelessScripts,
+  startsWithSpacelessScript
+};

package/dist/utilities/Timeline.cjs

@@ -24,6 +24,7 @@ __export(Timeline_exports, {
   getUTF32Chars: () => getUTF32Chars
 });
 module.exports = __toCommonJS(Timeline_exports);
+var import_SpacelessScripts = require("./SpacelessScripts.cjs");
 function addWordTextOffsetsToTimelineInPlace(timeline, text) {
   const { utf16To32Mapping } = getUTF32Chars(text);
   let currentOffset = 0;
@@ -96,16 +97,22 @@ function getUTF32Chars(str) {
 function buildTranscriptFromTimeline(timeline) {
   const words = [];
   function collectWords(entries) {
-    for (const entry of entries) {
-      if (entry.type === "word") {
-        words.push(entry.text);
-      } else if (entry.timeline) {
+    for (let i = 0; i < entries.length; i++) {
+      const entry = entries[i];
+      if (entry.timeline) {
         collectWords(entry.timeline);
+        continue;
+      }
+      if (!(0, import_SpacelessScripts.startsWithSpacelessScript)(entry.text) && i !== entries.length - 1) {
+        words.push(entry.text);
+        words.push(" ");
+        continue;
       }
+      words.push(entry.text);
     }
   }
   collectWords(timeline);
-  return words.join(" ");
+  return words.join("");
 }
 function addTimeOffsetToTimeline(targetTimeline, timeOffset) {
   const newTimeline = structuredClone(targetTimeline);

package/dist/utilities/Timeline.js

@@ -1,3 +1,4 @@
+import { startsWithSpacelessScript } from "./SpacelessScripts.js";
 function addWordTextOffsetsToTimelineInPlace(timeline, text) {
   const { utf16To32Mapping } = getUTF32Chars(text);
   let currentOffset = 0;
@@ -70,16 +71,22 @@ function getUTF32Chars(str) {
 function buildTranscriptFromTimeline(timeline) {
   const words = [];
   function collectWords(entries) {
-    for (const entry of entries) {
-      if (entry.type === "word") {
-        words.push(entry.text);
-      } else if (entry.timeline) {
+    for (let i = 0; i < entries.length; i++) {
+      const entry = entries[i];
+      if (entry.timeline) {
         collectWords(entry.timeline);
+        continue;
+      }
+      if (!startsWithSpacelessScript(entry.text) && i !== entries.length - 1) {
+        words.push(entry.text);
+        words.push(" ");
+        continue;
       }
+      words.push(entry.text);
     }
   }
   collectWords(timeline);
-  return words.join(" ");
+  return words.join("");
 }
 function addTimeOffsetToTimeline(targetTimeline, timeOffset) {
   const newTimeline = structuredClone(targetTimeline);

package/dist/utilities/WhisperTimeline.cjs

@@ -28,6 +28,7 @@ __export(WhisperTimeline_exports, {
   scoreTimeline: () => scoreTimeline
 });
 module.exports = __toCommonJS(WhisperTimeline_exports);
+var import_SpacelessScripts = require("./SpacelessScripts.cjs");
 const WHISPER_SAMPLE_RATE = 16e3;
 function calculateWhisperSplits(durationSeconds, numProcessors, sampleRate = WHISPER_SAMPLE_RATE) {
   if (numProcessors <= 1) return [];
@@ -42,26 +43,107 @@ function calculateWhisperSplits(durationSeconds, numProcessors, sampleRate = WHI
   return splits;
 }
 const specialTokenPattern = /\[_.+\]|<\|[a-z_]+\|>/g;
+const REPLACEMENT_CHAR = "\uFFFD";
+function isSpecialToken(text) {
+  return text.startsWith("[_") || text.startsWith("<|");
+}
+function hasUtf8Corruption(text) {
+  return text.includes(REPLACEMENT_CHAR);
+}
+function buildAnchor(items, startIdx, options, maxItems = 5) {
+  let anchor = "";
+  let count = 0;
+  for (let j = startIdx; j < items.length && count < maxItems; j++) {
+    const item = items[j];
+    if (!item) break;
+    const text = options.getText(item);
+    if (options.shouldSkipInAnchor(item)) continue;
+    if (hasUtf8Corruption(text)) break;
+    anchor += text;
+    count++;
+  }
+  return anchor;
+}
+function forEachMergedUtf8Run(items, segmentText, options, emit) {
+  let segPos = 0;
+  let i = 0;
+  while (i < items.length) {
+    const item = items[i];
+    if (!item) break;
+    const text = options.getText(item);
+    const isSkippable = options.shouldSkipInAnchor(item);
+    if (isSkippable || !hasUtf8Corruption(text)) {
+      if (!isSkippable) {
+        segPos += text.length;
+      }
+      emit({
+        first: item,
+        last: item,
+        text,
+        probability: options.getProbability(item),
+        isMerged: false
+      });
+      i++;
+      continue;
+    }
+    const runStart = i;
+    let probability = 1;
+    while (i < items.length) {
+      const runItem = items[i];
+      if (!runItem) break;
+      const runText = options.getText(runItem);
+      const shouldStop = options.shouldSkipInAnchor(runItem) || !hasUtf8Corruption(runText);
+      if (shouldStop) break;
+      probability *= options.getProbability(runItem);
+      i++;
+    }
+    const first = items[runStart];
+    const last = items[i - 1];
+    if (!first || !last) continue;
+    const anchor = buildAnchor(items, i, options);
+    const anchorIdx = anchor.length > 0 ? segmentText.indexOf(anchor, segPos) : -1;
+    const runEndSegPos = anchorIdx >= 0 ? anchorIdx : segmentText.length;
+    const mergedText = segmentText.slice(segPos, runEndSegPos);
+    segPos = runEndSegPos;
+    emit({
+      first,
+      last,
+      text: mergedText,
+      probability,
+      isMerged: true
+    });
+  }
+}
 function parseWhisperCppOutput(transcription) {
   return transcription.map((segment) => {
-    var _a, _b;
     const words = [];
     let lastTokenEndMs = 0;
-    for (const token of segment.tokens) {
-      const cleanedText = token.text.replace(specialTokenPattern, "");
-      if (cleanedText.trim().length === 0) continue;
-      const offsetFrom = ((_a = token.offsets) == null ? void 0 : _a.from) ?? lastTokenEndMs;
-      const offsetTo = ((_b = token.offsets) == null ? void 0 : _b.to) ?? lastTokenEndMs;
-      if (token.offsets) {
-        lastTokenEndMs = token.offsets.to;
+    forEachMergedUtf8Run(
+      segment.tokens,
+      segment.text,
+      {
+        getText: (token) => token.text,
+        getProbability: (token) => token.p,
+        shouldSkipInAnchor: (token) => isSpecialToken(token.text)
+      },
+      (run) => {
+        var _a, _b;
+        const cleanedText = run.text.replace(specialTokenPattern, "");
+        if (cleanedText.trim().length === 0) return;
+        const fallbackOffset = run.isMerged ? 0 : lastTokenEndMs;
+        const offsetFrom = ((_a = run.first.offsets) == null ? void 0 : _a.from) ?? fallbackOffset;
+        const offsetTo = ((_b = run.last.offsets) == null ? void 0 : _b.to) ?? fallbackOffset;
+        if (run.isMerged || run.last.offsets) {
+          lastTokenEndMs = offsetTo;
+        }
+        words.push({
+          text: cleanedText,
+          start: offsetFrom / 1e3,
+          end: offsetTo / 1e3,
+          confidence: run.probability
+        });
       }
-      words.push({
-        text: cleanedText,
-        start: offsetFrom / 1e3,
-        end: offsetTo / 1e3,
-        confidence: token.p
-      });
-    }
+    );
     return {
       text: segment.text,
       segmentStart: segment.offsets.from / 1e3,
@@ -72,12 +154,25 @@ function parseWhisperCppOutput(transcription) {
 }
 function parseWhisperServerOutput(segments) {
   return segments.map((segment) => {
-    const words = (segment.words ?? []).map((word) => ({
-      text: word.word,
-      start: word.start,
-      end: word.end,
-      confidence: word.probability ?? 0
-    }));
+    const words = [];
+    forEachMergedUtf8Run(
+      segment.words ?? [],
+      segment.text,
+      {
+        getText: (word) => word.word,
+        getProbability: (word) => word.probability ?? 1,
+        shouldSkipInAnchor: () => false
+      },
+      (run) => {
+        const confidence = run.isMerged ? run.probability : run.first.probability ?? 0;
+        words.push({
+          text: run.text,
+          start: run.first.start,
+          end: run.last.end,
+          confidence
+        });
+      }
+    );
     return {
       text: segment.text,
       segmentStart: segment.start,
@@ -256,7 +351,8 @@ function extractCorrectedTimeline(segments, options = {}) {
         end: segmentEnd
       });
       const lastEntry = timeline[timeline.length - 1];
-      if (lastEntry && !word.text.startsWith(" ")) {
+      const isSubwordContinuation = !word.text.startsWith(" ") && !(0, import_SpacelessScripts.startsWithSpacelessScript)(trimmedText);
+      if (lastEntry && isSubwordContinuation) {
         lastEntry.text += trimmedText;
         if (lastEntry.confidence !== void 0) {
           lastEntry.confidence = Math.min(lastEntry.confidence, word.confidence);

package/dist/utilities/WhisperTimeline.js

@@ -1,3 +1,4 @@
+import { startsWithSpacelessScript } from "./SpacelessScripts.js";
 const WHISPER_SAMPLE_RATE = 16e3;
 function calculateWhisperSplits(durationSeconds, numProcessors, sampleRate = WHISPER_SAMPLE_RATE) {
   if (numProcessors <= 1) return [];
@@ -12,26 +13,107 @@ function calculateWhisperSplits(durationSeconds, numProcessors, sampleRate = WHI
   return splits;
 }
 const specialTokenPattern = /\[_.+\]|<\|[a-z_]+\|>/g;
+const REPLACEMENT_CHAR = "\uFFFD";
+function isSpecialToken(text) {
+  return text.startsWith("[_") || text.startsWith("<|");
+}
+function hasUtf8Corruption(text) {
+  return text.includes(REPLACEMENT_CHAR);
+}
+function buildAnchor(items, startIdx, options, maxItems = 5) {
+  let anchor = "";
+  let count = 0;
+  for (let j = startIdx; j < items.length && count < maxItems; j++) {
+    const item = items[j];
+    if (!item) break;
+    const text = options.getText(item);
+    if (options.shouldSkipInAnchor(item)) continue;
+    if (hasUtf8Corruption(text)) break;
+    anchor += text;
+    count++;
+  }
+  return anchor;
+}
+function forEachMergedUtf8Run(items, segmentText, options, emit) {
+  let segPos = 0;
+  let i = 0;
+  while (i < items.length) {
+    const item = items[i];
+    if (!item) break;
+    const text = options.getText(item);
+    const isSkippable = options.shouldSkipInAnchor(item);
+    if (isSkippable || !hasUtf8Corruption(text)) {
+      if (!isSkippable) {
+        segPos += text.length;
+      }
+      emit({
+        first: item,
+        last: item,
+        text,
+        probability: options.getProbability(item),
+        isMerged: false
+      });
+      i++;
+      continue;
+    }
+    const runStart = i;
+    let probability = 1;
+    while (i < items.length) {
+      const runItem = items[i];
+      if (!runItem) break;
+      const runText = options.getText(runItem);
+      const shouldStop = options.shouldSkipInAnchor(runItem) || !hasUtf8Corruption(runText);
+      if (shouldStop) break;
+      probability *= options.getProbability(runItem);
+      i++;
+    }
+    const first = items[runStart];
+    const last = items[i - 1];
+    if (!first || !last) continue;
+    const anchor = buildAnchor(items, i, options);
+    const anchorIdx = anchor.length > 0 ? segmentText.indexOf(anchor, segPos) : -1;
+    const runEndSegPos = anchorIdx >= 0 ? anchorIdx : segmentText.length;
+    const mergedText = segmentText.slice(segPos, runEndSegPos);
+    segPos = runEndSegPos;
+    emit({
+      first,
+      last,
+      text: mergedText,
+      probability,
+      isMerged: true
+    });
+  }
+}
 function parseWhisperCppOutput(transcription) {
   return transcription.map((segment) => {
-    var _a, _b;
     const words = [];
     let lastTokenEndMs = 0;
-    for (const token of segment.tokens) {
-      const cleanedText = token.text.replace(specialTokenPattern, "");
-      if (cleanedText.trim().length === 0) continue;
-      const offsetFrom = ((_a = token.offsets) == null ? void 0 : _a.from) ?? lastTokenEndMs;
-      const offsetTo = ((_b = token.offsets) == null ? void 0 : _b.to) ?? lastTokenEndMs;
-      if (token.offsets) {
-        lastTokenEndMs = token.offsets.to;
+    forEachMergedUtf8Run(
+      segment.tokens,
+      segment.text,
+      {
+        getText: (token) => token.text,
+        getProbability: (token) => token.p,
+        shouldSkipInAnchor: (token) => isSpecialToken(token.text)
+      },
+      (run) => {
+        var _a, _b;
+        const cleanedText = run.text.replace(specialTokenPattern, "");
+        if (cleanedText.trim().length === 0) return;
+        const fallbackOffset = run.isMerged ? 0 : lastTokenEndMs;
+        const offsetFrom = ((_a = run.first.offsets) == null ? void 0 : _a.from) ?? fallbackOffset;
+        const offsetTo = ((_b = run.last.offsets) == null ? void 0 : _b.to) ?? fallbackOffset;
+        if (run.isMerged || run.last.offsets) {
+          lastTokenEndMs = offsetTo;
+        }
+        words.push({
+          text: cleanedText,
+          start: offsetFrom / 1e3,
+          end: offsetTo / 1e3,
+          confidence: run.probability
+        });
       }
-      words.push({
-        text: cleanedText,
-        start: offsetFrom / 1e3,
-        end: offsetTo / 1e3,
-        confidence: token.p
-      });
-    }
+    );
     return {
       text: segment.text,
       segmentStart: segment.offsets.from / 1e3,
@@ -42,12 +124,25 @@ function parseWhisperCppOutput(transcription) {
 }
 function parseWhisperServerOutput(segments) {
   return segments.map((segment) => {
-    const words = (segment.words ?? []).map((word) => ({
-      text: word.word,
-      start: word.start,
-      end: word.end,
-      confidence: word.probability ?? 0
-    }));
+    const words = [];
+    forEachMergedUtf8Run(
+      segment.words ?? [],
+      segment.text,
+      {
+        getText: (word) => word.word,
+        getProbability: (word) => word.probability ?? 1,
+        shouldSkipInAnchor: () => false
+      },
+      (run) => {
+        const confidence = run.isMerged ? run.probability : run.first.probability ?? 0;
+        words.push({
+          text: run.text,
+          start: run.first.start,
+          end: run.last.end,
+          confidence
+        });
+      }
+    );
     return {
       text: segment.text,
       segmentStart: segment.start,
@@ -226,7 +321,8 @@ function extractCorrectedTimeline(segments, options = {}) {
         end: segmentEnd
       });
       const lastEntry = timeline[timeline.length - 1];
-      if (lastEntry && !word.text.startsWith(" ")) {
+      const isSubwordContinuation = !word.text.startsWith(" ") && !startsWithSpacelessScript(trimmedText);
+      if (lastEntry && isSubwordContinuation) {
         lastEntry.text += trimmedText;
         if (lastEntry.confidence !== void 0) {
           lastEntry.confidence = Math.min(lastEntry.confidence, word.confidence);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@storyteller-platform/ghost-story",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "An easy-to-use speech toolset. Fork of the original echogarden project.",
   "author": "Thomas F. K. Jorna",
   "license": "GPL-3.0",