@sc-voice/tools 2.17.0 → 2.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "2.17.0",
+  "version": "2.19.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [

package/src/defines.mjs

@@ -8,5 +8,7 @@ export const DBG = {
   DEEPL_MOCK_XLT: 0, // use mock translation
   DEEPL_TEST_API: 0, // test with live DeepL API ($$$)
   DEEPL_XLT: 0, // test live translation
+  L7C_FETCH_LEGACY: 0,
+  L7C_FETCH_LEGACY_SC: 0, // ignore test cache and use SC 
   WORD_MAP_TRANFORMER: 0,
 };

package/src/text/legacy-doc.mjs

@@ -37,20 +37,33 @@ export class LegacyDoc {
     return true;
   }
 
-  static async fetchLegacy(opts = {}) {
-    const msg = 'L7c.fetch:';
-    const dbg = DBG.FETCH_LEGACY;
+  static legacyUrl(opts={}) {
     let {
-      endPoint = 'https://suttacentral.net/api/suttas',
+      endPoint = 'https://staging.suttacentral.net/api/suttas',
       sutta_uid,
       lang,
       author,
+    } = opts;
+
+    return [endPoint, sutta_uid, `${author}?lang=${lang}`].join('/');
+  }
+
+  static async fetchLegacy(opts = {}) {
+    const msg = 'L7c.fetchLegacy:';
+    const dbg = DBG.L7C_FETCH_LEGACY;
+    let {
       maxBuffer = 10 * 1024 * 1024,
+      cache,
     } = opts;
-    let url = [endPoint, sutta_uid, `${author}?lang=${lang}`].join(
-      '/',
-    );
-    let res = await fetch(url);
+    let url = LegacyDoc.legacyUrl(opts);
+    let res;
+    if (cache) {
+      res = cache(url);
+      dbg && console.log(msg, '[1]cached', res.ok);
+    } else {
+      res = await fetch(url, {maxBuffer});
+      dbg && console.log(msg, '[2]scapi', res.ok);
+    }
     if (!res.ok) {
       throw new Error(`${msg} {res.status} ${url}`);
     }

package/src/text/tfidf-space.mjs

@@ -52,9 +52,11 @@ export class TfidfSpace {
 
   static removeNonWords(s) {
     const RE_RESERVED = /[_-]/g; // allowed in bow words
+    const RE_LQUOTE = /[“‘«]/g;
     const RE_PUNCT = /[.,:;$"'“”‘’!?«»\[\]]/g;
     const RE_SPACE = /\s+/g;
     return TfidfSpace.removeHtml(s)
+      .replace(RE_LQUOTE, '__LQUOTE ')
       .replace(RE_PUNCT, '')
       .replace(RE_SPACE, ' ')
       .trim();
@@ -107,7 +109,7 @@ export class TfidfSpace {
       // Bag-of-words maps word to wordCount(word,doc)
       throw new Error(`${msg} bow?`);
     }
-    let nWords = Object.values(bow).reduce((a,v)=>a+v);
+    let nWords = Object.values(bow).reduce((a, v) => a + v);
     let docInfo = { id, bow, nWords };
     corpus.wordDocCount.increment(bow.oneHot());
     corpus.addDocument(id, docInfo);
@@ -118,7 +120,7 @@ export class TfidfSpace {
   addDocument(id, doc) {
     let { corpus } = this;
     let { bow, words } = this.countWords(doc);
-    
+
     return this.addCorpusDocument(id, bow, words.length);
   }
 
@@ -138,7 +140,7 @@ export class TfidfSpace {
 
     // More efficient implementation of tf * idf
     let words = Object.keys(bow);
-    let nWords = words.reduce((a,w)=>a+bow[w],0);
+    let nWords = words.reduce((a, w) => a + bow[w], 0);
 
     let vTfIdf = words.reduce((a, word) => {
       let wd = bow[word] || 0;
@@ -157,7 +159,8 @@ export class TfidfSpace {
     return vTfIdf;
   }
 
-  tfidf(text) { // TfIdf of words in text w/r to corpus
+  tfidf(text) {
+    // TfIdf of words in text w/r to corpus
     let { bow } = this.countWords(text);
     return this.tfidfOfBow(bow);
   }

package/src/text/word-vector.mjs

@@ -20,12 +20,12 @@ export class WordVector extends Object {
     return this.$length;
   }
 
-  toString(opts={}) {
-    let { order='value', precision=2 } = opts;
+  toString(opts = {}) {
+    let { order = 'value', precision = 2 } = opts;
     let entries = Object.entries(this);
     switch (order) {
       case 'key':
-        entries.sort((a,b)=>{
+        entries.sort((a, b) => {
           let [ka] = a;
           let [kb] = b;
           return ka.localeCompare(kb);
@@ -33,16 +33,16 @@ export class WordVector extends Object {
         break;
       case 'value':
       default:
-        entries.sort((a,b)=>{
+        entries.sort((a, b) => {
           let [ka, va] = a;
           let [kb, vb] = b;
-          return (vb-va) || ka.localeCompare(kb);
+          return vb - va || ka.localeCompare(kb);
         });
         break;
     }
     let sv = entries.reduce((a, e) => {
       let [k, v] = e;
-      let vf = v.toFixed(precision).replace(/\.0*$/,'');
+      let vf = v.toFixed(precision).replace(/\.0*$/, '');
       a.push(`${k}:${vf}`);
       return a;
     }, []);