@sc-voice/tools 2.19.0 → 2.22.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "2.19.0",
+  "version": "2.22.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [

package/src/text/legacy-doc.mjs

@@ -39,7 +39,7 @@ export class LegacyDoc {
 
   static legacyUrl(opts={}) {
     let {
-      endPoint = 'https://staging.suttacentral.net/api/suttas',
+      endPoint = 'https://suttacentral.net/api/suttas',
       sutta_uid,
       lang,
       author,
@@ -65,7 +65,7 @@ export class LegacyDoc {
       dbg && console.log(msg, '[2]scapi', res.ok);
     }
     if (!res.ok) {
-      throw new Error(`${msg} {res.status} ${url}`);
+      throw new Error(`${msg} ${res.status} ${url}`);
     }
     let json = await res.json();
     let { translation } = json;

package/src/text/tfidf-space.mjs

@@ -15,6 +15,7 @@ export class TfidfSpace {
       idfWeight = GOLDEN_FUDGE, // IDF dampening
       idfFunction = TfidfSpace.idfTunable,
       normalizeText,
+      leftQuoteToken,
     } = opts;
     if (lang == null) {
       throw new Error(`${msg} lang?`);
@@ -31,7 +32,7 @@ export class TfidfSpace {
           throw new Error(`${msg} normalizeText?`);
       }
     }
-    Object.defineProperty(this, 'normalizeText', {
+    Object.defineProperty(this, '_normalizeText', {
       value: normalizeText,
     });
     Object.defineProperty(this, 'idfFunction', {
@@ -43,6 +44,7 @@ export class TfidfSpace {
       lang,
       corpus,
       idfWeight,
+      leftQuoteToken,
     });
   }
 
@@ -50,23 +52,26 @@ export class TfidfSpace {
     return s.replace(/<[^>]*>/gi, '');
   }
 
-  static removeNonWords(s) {
+  static removeNonWords(s, opts={}) {
     const RE_RESERVED = /[_-]/g; // allowed in bow words
     const RE_LQUOTE = /[“‘«]/g;
     const RE_PUNCT = /[.,:;$"'“”‘’!?«»\[\]]/g;
     const RE_SPACE = /\s+/g;
+    let {
+      leftQuoteToken = '', // TBD: is this useful?
+    } = opts;
     return TfidfSpace.removeHtml(s)
-      .replace(RE_LQUOTE, '__LQUOTE ')
+      .replace(RE_LQUOTE, leftQuoteToken)
       .replace(RE_PUNCT, '')
       .replace(RE_SPACE, ' ')
       .trim();
   }
 
-  static normalizeEN(s) {
-    return TfidfSpace.removeNonWords(s.toLowerCase());
+  static normalizeEN(s, opts={}) {
+    return TfidfSpace.removeNonWords(s.toLowerCase(), opts);
   }
 
-  static normalizeFR(s) {
+  static normalizeFR(s, opts={}) {
     let sAbbr = s
       .toLowerCase()
       .replace(/\bd[’']/gi, 'de ')
@@ -76,7 +81,7 @@ export class TfidfSpace {
       .replace(/\bm[’']/gi, 'm_')
       .replace(/\bn[’']/gi, 'n_')
       .replace(/\bc[’']/gi, 'c_');
-    return TfidfSpace.removeNonWords(sAbbr);
+    return TfidfSpace.removeNonWords(sAbbr, opts);
   }
 
   static idfStandard(nDocs, wdc, idfWeight) {
@@ -165,6 +170,10 @@ export class TfidfSpace {
     return this.tfidfOfBow(bow);
   }
 
+  normalizeText(str) {
+    return this._normalizeText(str, this);
+  }
+
   countWords(str) {
     const msg = 'w7e.countWords:';
     if (str == null) {

package/src/text/word-vector.mjs

@@ -21,7 +21,13 @@ export class WordVector extends Object {
   }
 
   toString(opts = {}) {
-    let { order = 'value', precision = 2 } = opts;
+    const msg = 'w10r.toString:';
+    let { order = 'value', minValue, precision = 2 } = opts;
+
+    if (minValue == null) {
+      minValue = Math.pow(10, -precision) / 2;
+    }
+
     let entries = Object.entries(this);
     switch (order) {
       case 'key':
@@ -42,8 +48,12 @@ export class WordVector extends Object {
     }
     let sv = entries.reduce((a, e) => {
       let [k, v] = e;
-      let vf = v.toFixed(precision).replace(/\.0*$/, '');
-      a.push(`${k}:${vf}`);
+      if (minValue <= v) {
+        let vf = v.toFixed(precision)
+          .replace(/\.0*$/, '')
+          .replace(/0\./,'.');
+        a.push(`${k}:${vf}`);
+      }
       return a;
     }, []);
     return sv.join(',');