@internetarchive/bookreader 5.0.0-12-fixsorting-7 → 5.0.0-120

package/src/plugins/tts/AbstractTTSEngine.js

@@ -1,12 +1,12 @@
 import PageChunkIterator from './PageChunkIterator.js';
+import { hasLocalStorage } from './utils.js';
 /** @typedef {import('./utils.js').ISO6391} ISO6391 */
 /** @typedef {import('./PageChunk.js')} PageChunk */
 
 /**
  * @export
  * @typedef {Object} TTSEngineOptions
- * @property {String} server
- * @property {String} bookPath
+ * @property {import('@/src/util/strings.js').StringWithVars} pageChunkUrl
  * @property {ISO6391} bookLanguage
  * @property {Function} onLoadingStart
  * @property {Function} onLoadingComplete
@@ -28,6 +28,7 @@ import PageChunkIterator from './PageChunkIterator.js';
  * @property {() => void} resume
  * @property {() => void} finish force the sound to 'finish'
  * @property {number => void} setPlaybackRate
+ * @property {SpeechSynthesisVoice => void} setVoice
  **/
 
 /** Handling bookreader's text-to-speech */
@@ -39,6 +40,7 @@ export default class AbstractTTSEngine {
   constructor(options) {
     this.playing = false;
     this.paused = false;
+    /** @type {TTSEngineOptions} */
     this.opts = options;
     /** @type {PageChunkIterator} */
     this._chunkIterator = null;
@@ -50,9 +52,7 @@ export default class AbstractTTSEngine {
     /** @type {SpeechSynthesisVoice} */
     this.voice = null;
     // Listen for voice changes (fired by subclasses)
-    this.events.on('voiceschanged', () => {
-      this.voice = AbstractTTSEngine.getBestBookVoice(this.getVoices(), this.opts.bookLanguage);
-    });
+    this.events.on('voiceschanged', this.updateBestVoice);
     this.events.trigger('voiceschanged');
   }
 
@@ -71,20 +71,23 @@ export default class AbstractTTSEngine {
   /** @abstract */
   init() { return null; }
 
+  updateBestVoice = () => {
+    this.voice = AbstractTTSEngine.getBestBookVoice(this.getVoices(), this.opts.bookLanguage);
+  }
+
   /**
    * @param {number} leafIndex
    * @param {number} numLeafs total number of leafs in the current book
    */
   start(leafIndex, numLeafs) {
     this.playing = true;
+    this.paused = false;
     this.opts.onLoadingStart();
 
     this._chunkIterator = new PageChunkIterator(numLeafs, leafIndex, {
-      server: this.opts.server,
-      bookPath: this.opts.bookPath,
-      pageBufferSize: 5,
+      pageChunkUrl: this.opts.pageChunkUrl,
+      pageBufferSize: 3,
     });
-
     this.step();
     this.events.trigger('start');
   }
@@ -92,6 +95,7 @@ export default class AbstractTTSEngine {
   stop() {
     if (this.activeSound) this.activeSound.stop();
     this.playing = false;
+    this.paused = true;
     this._chunkIterator = null;
     this.activeSound = null;
     this.events.trigger('stop');
@@ -124,13 +128,26 @@ export default class AbstractTTSEngine {
   }
 
   /** @public */
-  jumpBackward() {
-    Promise.all([
+  async jumpBackward() {
+    await Promise.all([
       this.activeSound.stop(),
       this._chunkIterator.decrement()
-        .then(() => this._chunkIterator.decrement())
-    ])
-      .then(() => this.step());
+        .then(() => this._chunkIterator.decrement()),
+    ]);
+    this.step();
+  }
+
+  /** @param {string} voiceURI */
+  setVoice(voiceURI) {
+    // if the user actively selects a voice, don't re-choose best voice anymore
+    // MS Edge fires voices changed randomly very often
+    this.events.off('voiceschanged', this.updateBestVoice);
+    this.voice = this.getVoices().find(voice => voice.voiceURI === voiceURI);
+    // if the current book has a language set, store the selected voice with the book language as a suffix
+    if (this.opts.bookLanguage && hasLocalStorage()) {
+      localStorage.setItem(`BRtts-voice-${this.opts.bookLanguage}`, this.voice.voiceURI);
+    }
+    if (this.activeSound) this.activeSound.setVoice(this.voice);
   }
 
   /** @param {number} newRate */
@@ -140,36 +157,33 @@ export default class AbstractTTSEngine {
   }
 
   /** @private */
-  step() {
-    this._chunkIterator.next()
-      .then(chunk => {
-        if (chunk == PageChunkIterator.AT_END) {
-          this.stop();
-          this.opts.onDone();
-          return;
-        }
-
-        this.opts.onLoadingStart();
-        const sound = this.createSound(chunk);
-        sound.chunk = chunk;
-        sound.rate = this.playbackRate;
-        sound.voice = this.voice;
-        sound.load(() => this.opts.onLoadingComplete());
-
-        this.opts.onLoadingComplete();
-        return this.opts.beforeChunkPlay(chunk).then(() => sound);
-      })
-      .then(sound => {
-        if (!this.playing) return;
-
-        const playPromise = this.playSound(sound)
-          .then(() => this.opts.afterChunkPlay(sound.chunk));
-        if (this.paused) this.pause();
-        return playPromise;
-      })
-      .then(() => {
-        if (this.playing) return this.step();
-      });
+  async step() {
+    const chunk = await  this._chunkIterator.next();
+    if (chunk == PageChunkIterator.AT_END) {
+      this.stop();
+      this.opts.onDone();
+      return;
+    }
+    this.opts.onLoadingStart();
+    const sound = this.createSound(chunk);
+    sound.chunk = chunk;
+    sound.rate = this.playbackRate;
+    sound.voice = this.voice;
+    sound.load(() => this.opts.onLoadingComplete());
+
+    this.opts.onLoadingComplete();
+
+    await this.opts.beforeChunkPlay(chunk);
+
+    if (!this.playing) return;
+
+    const playPromise = await this.playSound(sound)
+      .then(()=> this.opts.afterChunkPlay(sound.chunk));
+
+    if (this.paused) this.pause();
+    await playPromise;
+
+    if (this.playing) return this.step();
   }
 
   /**
@@ -190,8 +204,8 @@ export default class AbstractTTSEngine {
   }
 
   /** Convenience wrapper for {@see AbstractTTSEngine.getBestVoice} */
-  getBestVoice() {
-    return AbstractTTSEngine.getBestBookVoice(this.getVoices(), this.opts.bookLanguage);
+  getBestVoice(languageOverride) {
+    return AbstractTTSEngine.getBestBookVoice(this.getVoices(), languageOverride || this.opts.bookLanguage);
   }
 
   /**
@@ -212,10 +226,12 @@ export default class AbstractTTSEngine {
     // user languages that match the book language
     const matchingUserLangs = userLanguages.filter(lang => lang.startsWith(bookLanguage));
 
-    // Try to find voices that intersect these two sets
-    return AbstractTTSEngine.getMatchingVoice(matchingUserLangs, bookLangVoices) ||
+    // First try to find the last chosen voice from localStorage for the current book language
+    return AbstractTTSEngine.getMatchingStoredVoice(bookLangVoices, bookLanguage)
+        // Try to find voices that intersect these two sets
+        || AbstractTTSEngine.getMatchingVoice(matchingUserLangs, bookLangVoices)
         // no user languages match the books; let's return the best voice for the book language
-        (bookLangVoices.find(v => v.default) || bookLangVoices[0])
+        || (bookLangVoices.find(v => v.default) || bookLangVoices[0])
         // No voices match the book language? let's find a voice in the user's language
         // and ignore book lang
         || AbstractTTSEngine.getMatchingVoice(userLanguages, voices)
@@ -223,6 +239,19 @@ export default class AbstractTTSEngine {
         || (voices.find(v => v.default) || voices[0]);
   }
 
+  /**
+   * @private
+   * Get the voice last selected by the user for the book language from localStorage.
+   * Returns undefined if no voice is stored or found.
+   * @param {SpeechSynthesisVoice[]} voices  browser voices to choose from
+   * @param {ISO6391} bookLanguage  book language to look for
+   * @return {SpeechSynthesisVoice | undefined}
+   */
+  static getMatchingStoredVoice(voices, bookLanguage) {
+    const storedVoice = hasLocalStorage() && localStorage.getItem(`BRtts-voice-${bookLanguage}`);
+    return (storedVoice ? voices.find(v => v.voiceURI === storedVoice) : undefined);
+  }
+
   /**
      * @private
      * Get the best voice that matches one of the BCP47 languages (order by preference)

package/src/plugins/tts/FestivalTTSEngine.js

@@ -1,5 +1,6 @@
 import AbstractTTSEngine from './AbstractTTSEngine.js';
-import { sleep } from './utils.js';
+import { sleep } from '../../BookReader/utils.js';
+import { applyVariables } from '../../util/strings.js';
 /* global soundManager */
 import 'soundmanager2';
 import 'jquery.browser';
@@ -17,19 +18,21 @@ export default class FestivalTTSEngine extends AbstractTTSEngine {
     return typeof(soundManager) !== 'undefined' && soundManager.supported();
   }
 
-  /** @param {TTSEngineOptions} options */
+  /** @param {TTSEngineOptions | { festivalUrl: import('@/src/util/strings.js').StringWithVars }} options */
   constructor(options) {
     super(options);
     // $.browsers is sometimes undefined on some Android browsers :/
     // Likely related to when $.browser was moved to npm
     /** @type {'mp3' | 'ogg'} format of audio to get */
-    this.audioFormat = $.browser?.mozilla ? 'ogg' : 'mp3';
+    this.audioFormat = $.browser?.mozilla ? 'ogg' : 'mp3'; //eslint-disable-line no-jquery/no-browser
+    /** @type {import('@/src/util/strings.js').StringWithVars} */
+    this.festivalUrl = options.festivalUrl;
   }
 
   /** @override */
   getVoices() {
     return [
-      { default: true, lang: "en-US", localService: false, name: "Festival - English (US)", voiceURI: null }
+      { default: true, lang: "en-US", localService: false, name: "Festival - English (US)", voiceURI: null },
     ];
   }
 
@@ -45,7 +48,7 @@ export default class FestivalTTSEngine extends AbstractTTSEngine {
       url: '/bookreader/BookReader/soundmanager/swf',
       useHTML5Audio: true,
       //flash 8 version of swf is buggy when calling play() on a sound that is still loading
-      flashVersion: 9
+      flashVersion: 9,
     });
   }
 
@@ -68,19 +71,7 @@ export default class FestivalTTSEngine extends AbstractTTSEngine {
 
   /** @override */
   createSound(chunk) {
-    return new FestivalTTSSound(this.getSoundUrl(chunk.text));
-  }
-
-  /**
-   * @private
-   * Get URL for audio that says this text
-   * @param {String} dataString the thing to say
-   * @return {String} url
-   */
-  getSoundUrl(dataString) {
-    return 'https://' + this.opts.server + '/BookReader/BookReaderGetTTS.php?string='
-                  + encodeURIComponent(dataString)
-                  + '&format=.' + this.audioFormat;
+    return new FestivalTTSSound(applyVariables(this.festivalUrl, { text: chunk.text, format: this.audioFormat }));
   }
 
   /**
@@ -91,10 +82,10 @@ export default class FestivalTTSEngine extends AbstractTTSEngine {
    * See https://stackoverflow.com/questions/12206631/html5-audio-cant-play-through-javascript-unless-triggered-manually-once
    * @return {PromiseLike}
    */
-  iOSCaptureUserIntentHack() {
+  async iOSCaptureUserIntentHack() {
     const sound = soundManager.createSound({ url: SILENCE_1MS[this.audioFormat] });
-    return new Promise(res => sound.play({onfinish: res}))
-      .then(() => sound.destruct());
+    await new Promise(res => sound.play({onfinish: res}));
+    sound.destruct();
   }
 }
 
@@ -122,21 +113,20 @@ class FestivalTTSSound {
         if (this.rate != 1) this.sound.setPlaybackRate(this.rate);
         onload();
       },
-      onresume: () => {
-        sleep(25).then(() => {
-          if (this.rate != 1) this.sound.setPlaybackRate(this.rate);
-        });
-      }
+      onresume: async () => {
+        await sleep(25);
+        if (this.rate != 1) this.sound.setPlaybackRate(this.rate);
+      },
     });
     return this.sound.load();
   }
 
-  play() {
-    return new Promise(res => {
+  async play() {
+    await new Promise(res => {
       this._finishResolver = res;
       this.sound.play({ onfinish: res });
-    })
-      .then(() => this.sound.destruct());
+    });
+    this.sound.destruct();
   }
 
   /** @override */

package/src/plugins/tts/PageChunk.js

@@ -1,3 +1,10 @@
+import { applyVariables } from "../../util/strings.js";
+
+/** A chunk ends at the first sentence end after this many words */
+const MIN_WORDS_IN_CHUNK = 25;
+/** A chunk with no sentence end is cut at the first line end after this many words */
+const MAX_WORDS_IN_CHUNK = 50;
+
 /**
  * Class to manage a 'chunk' (approximately a paragraph) of text on a page.
  */
@@ -16,32 +23,61 @@ export default class PageChunk {
   }
 
   /**
-   * @param {string} server
-   * @param {string} bookPath
+   * @param {import('@/src/util/strings.js').StringWithVars} pageChunkUrl
    * @param {number} leafIndex
    * @return {Promise<PageChunk[]>}
    */
-  static fetch(server, bookPath, leafIndex) {
-    // jquery's ajax "PromiseLike" implementation is inconsistent with
-    // modern Promises, so convert it to a full promise (it doesn't forward
-    // a returned promise to the next handler in the chain, which kind of
-    // defeats the entire point of using promises to avoid "callback hell")
-    return new Promise((res, rej) => {
-      $.ajax({
-        type: 'GET',
-        url: `https://${server}/BookReader/BookReaderGetTextWrapper.php`,
-        dataType:'jsonp',
-        cache: true,
-        data: {
-          path: `${bookPath}_djvu.xml`,
-          page: leafIndex
-        },
-        error: rej,
-      })
-        .then(chunks => {
-          res(PageChunk._fromTextWrapperResponse(leafIndex, chunks));
-        });
+  static async fetch(pageChunkUrl, leafIndex) {
+    if (window.br.plugins.translate?.translationManager.active) {
+      const translateLayers = await window.br.plugins.translate.getTranslateLayers(leafIndex);
+      const paragraphs = Array.from(translateLayers[0].childNodes);
+
+      const pageChunks = [];
+      for (const [idx, item] of paragraphs.entries()) {
+        // Should not read paragraphs w/ header or footer roles
+        if (!item.classList.contains('ocr-role-header-footer')) {
+          const translatedChunk = new PageChunk(leafIndex, idx, item.textContent, []);
+          pageChunks.push(translatedChunk);
+        }
+      }
+      if (pageChunks.length === 0) {
+        const placeholder = new PageChunk(leafIndex, 0, "", []);
+        pageChunks.push(placeholder);
+      }
+      return pageChunks;
+    }
+
+    const ocrPage = await PageChunk._getTextSelectionOcrPage(leafIndex);
+    if (ocrPage) {
+      return PageChunk._fromTextWrapperResponse(leafIndex, PageChunk._chunkOcrPage(ocrPage));
+    }
+
+    const chunks = await $.ajax({
+      type: 'GET',
+      url: applyVariables(pageChunkUrl, { pageIndex: leafIndex }),
+      cache: true,
+      xhrFields: {
+        withCredentials: window.br.protected,
+      },
     });
+    return PageChunk._fromTextWrapperResponse(leafIndex, chunks);
+  }
+
+  /**
+   * The text selection plugin's OCR for the page, if it has any. It's cached
+   * and shared with the text layer, so using it avoids requesting the page's
+   * text a second time.
+   * @param {number} leafIndex
+   * @return {Promise<Element | undefined>}
+   */
+  static async _getTextSelectionOcrPage(leafIndex) {
+    const textSelection = window.br.plugins.textSelection;
+    if (!textSelection?.options.enabled) return undefined;
+    try {
+      return await textSelection.getPageText(leafIndex);
+    } catch (e) {
+      return undefined;
+    }
   }
 
   /**
@@ -58,6 +94,61 @@ export default class PageChunk {
     });
   }
 
+  /**
+   * Split a page of djvu xml OCR into roughly paragraph-sized chunks, in the
+   * same shape as BookReaderGetTextWrapper.php's response: each chunk is its
+   * text followed by a box per line (or part of a line) it covers. Running
+   * headers/footers are skipped.
+   * @param {Element} ocrPage djvu xml `OBJECT` element
+   * @return {Array<[String, ...DJVURect[]]>}
+   */
+  static _chunkOcrPage(ocrPage) {
+    /** @type {Array<[String, ...DJVURect[]]>} */
+    const chunks = [];
+    /** @type {string[]} */
+    let words = [];
+    /** @type {DJVURect[]} */
+    let rects = [];
+    /** @type {DJVURect | null} bounds of the current line's words in this chunk */
+    let lineRect = null;
+
+    const endChunk = () => {
+      if (lineRect) rects.push(lineRect);
+      chunks.push([words.join(' '), ...rects]);
+      words = [];
+      rects = [];
+      lineRect = null;
+    };
+
+    for (const line of ocrPage.querySelectorAll('LINE')) {
+      if (line.closest('PARAGRAPH')?.getAttribute('x-role') === 'header-footer') continue;
+
+      for (const word of line.querySelectorAll('WORD')) {
+        const [left, bottom, right, top] = (word.getAttribute('coords') ?? '').split(',').map(parseFloat);
+        const text = word.textContent.trim();
+        // Same words the text layer drops as unpositionable
+        if (!text || isNaN(top) || (left === 0 && top === 0)) continue;
+
+        lineRect = lineRect ? [
+          Math.min(lineRect[0], left),
+          Math.max(lineRect[1], bottom),
+          Math.max(lineRect[2], right),
+          Math.min(lineRect[3], top),
+        ] : [left, bottom, right, top];
+        words.push(text);
+
+        if (text.endsWith('.') && words.length > MIN_WORDS_IN_CHUNK) endChunk();
+      }
+
+      if (lineRect) rects.push(lineRect);
+      lineRect = null;
+      if (words.length > MAX_WORDS_IN_CHUNK) endChunk();
+    }
+
+    if (words.length) endChunk();
+    return chunks;
+  }
+
   /**
    * @private
    * Sometimes the first rectangle will be ridiculously wide/tall. Find those and fix them
@@ -97,7 +188,10 @@ export default class PageChunk {
    * @return {string}
    */
   static _removeDanglingHyphens(text) {
-    return text.replace(/-\s+/g, '');
+    // Some books mis-OCR a dangling hyphen as a ¬ (mathematical not sign) . Since in math
+    // the not sign should not appear followed by a space, we think we can safely assume
+    // this should be replaced.
+    return text.replace(/[-¬]\s+/g, '');
   }
 }
 

package/src/plugins/tts/PageChunkIterator.js

@@ -53,22 +53,18 @@ export default class PageChunkIterator {
    * in the correct order.
    * @return {PromiseLike<"__PageChunkIterator.AT_END__" | PageChunk>}
    */
-  _nextUncontrolled() {
+  async  _nextUncontrolled() {
     if (this._cursor.page == this.pageCount) {
       return Promise.resolve(PageChunkIterator.AT_END);
     }
-
     this._recenterBuffer(this._cursor.page);
-
-    return this._fetchPageChunks(this._cursor.page)
-      .then(chunks => {
-        if (this._cursor.chunk == chunks.length) {
-          this._cursor.page++;
-          this._cursor.chunk = 0;
-          return this._nextUncontrolled();
-        }
-        return chunks[this._cursor.chunk++];
-      });
+    const chunks = await this._fetchPageChunks(this._cursor.page);
+    if (this._cursor.chunk == chunks.length) {
+      this._cursor.page++;
+      this._cursor.chunk = 0;
+      return this._nextUncontrolled();
+    }
+    return chunks[this._cursor.chunk++];
   }
 
   /**
@@ -142,7 +138,7 @@ export default class PageChunkIterator {
    * @param {number} index
    */
   _fetchPageChunksDirect(index) {
-    return PageChunk.fetch(this.opts.server, this.opts.bookPath, index);
+    return PageChunk.fetch(this.opts.pageChunkUrl, index);
   }
 }
 
@@ -150,14 +146,12 @@ PageChunkIterator.AT_END = "__PageChunkIterator.AT_END__";
 
 /** @type {PageChunkIteratorOptions} */
 const DEFAULT_OPTS = {
-  server: null,
-  bookPath: null,
+  pageChunkUrl: null,
   pageBufferSize: 2,
 };
 
 /**
  * @typedef {Object} PageChunkIteratorOptions
- * @property {string} server
- * @property {string} bookPath
+ * @property {import('@/src/util/strings.js').StringWithVars} pageChunkUrl
  * @property {number} [pageBufferSize] number of pages to buffer before/after the current page
  */