@sc-voice/tools 2.16.0 → 2.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "2.16.0",
+  "version": "2.18.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [

package/src/text/tfidf-space.mjs

@@ -46,11 +46,20 @@ export class TfidfSpace {
     });
   }
 
+  static removeHtml(s) {
+    return s.replace(/<[^>]*>/gi, '');
+  }
+
   static removeNonWords(s) {
     const RE_RESERVED = /[_-]/g; // allowed in bow words
-    const RE_PUNCT = /[.,:;$"'“”‘’!?«»]/g;
+    const RE_LQUOTE = /[“‘«]/g;
+    const RE_PUNCT = /[.,:;$"'“”‘’!?«»\[\]]/g;
     const RE_SPACE = /\s+/g;
-    return s.replace(RE_PUNCT, '').replace(RE_SPACE, ' ').trim();
+    return TfidfSpace.removeHtml(s)
+      .replace(RE_LQUOTE, '__LQUOTE ')
+      .replace(RE_PUNCT, '')
+      .replace(RE_SPACE, ' ')
+      .trim();
   }
 
   static normalizeEN(s) {
@@ -100,7 +109,7 @@ export class TfidfSpace {
       // Bag-of-words maps word to wordCount(word,doc)
       throw new Error(`${msg} bow?`);
     }
-    let nWords = Object.values(bow).reduce((a,v)=>a+v);
+    let nWords = Object.values(bow).reduce((a, v) => a + v);
     let docInfo = { id, bow, nWords };
     corpus.wordDocCount.increment(bow.oneHot());
     corpus.addDocument(id, docInfo);
@@ -111,7 +120,7 @@ export class TfidfSpace {
   addDocument(id, doc) {
     let { corpus } = this;
     let { bow, words } = this.countWords(doc);
-    
+
     return this.addCorpusDocument(id, bow, words.length);
   }
 
@@ -131,7 +140,7 @@ export class TfidfSpace {
 
     // More efficient implementation of tf * idf
     let words = Object.keys(bow);
-    let nWords = words.reduce((a,w)=>a+bow[w],0);
+    let nWords = words.reduce((a, w) => a + bow[w], 0);
 
     let vTfIdf = words.reduce((a, word) => {
       let wd = bow[word] || 0;
@@ -150,7 +159,8 @@ export class TfidfSpace {
     return vTfIdf;
   }
 
-  tfidf(text) { // TfIdf of words in text w/r to corpus
+  tfidf(text) {
+    // TfIdf of words in text w/r to corpus
     let { bow } = this.countWords(text);
     return this.tfidfOfBow(bow);
   }

package/src/text/word-vector.mjs

@@ -20,12 +20,12 @@ export class WordVector extends Object {
     return this.$length;
   }
 
-  toString(opts={}) {
-    let { order='value', precision=2 } = opts;
+  toString(opts = {}) {
+    let { order = 'value', precision = 2 } = opts;
     let entries = Object.entries(this);
     switch (order) {
       case 'key':
-        entries.sort((a,b)=>{
+        entries.sort((a, b) => {
           let [ka] = a;
           let [kb] = b;
           return ka.localeCompare(kb);
@@ -33,16 +33,16 @@ export class WordVector extends Object {
         break;
       case 'value':
       default:
-        entries.sort((a,b)=>{
+        entries.sort((a, b) => {
           let [ka, va] = a;
           let [kb, vb] = b;
-          return (vb-va) || ka.localeCompare(kb);
+          return vb - va || ka.localeCompare(kb);
         });
         break;
     }
     let sv = entries.reduce((a, e) => {
       let [k, v] = e;
-      let vf = v.toFixed(precision).replace(/\.0*$/,'');
+      let vf = v.toFixed(precision).replace(/\.0*$/, '');
       a.push(`${k}:${vf}`);
       return a;
     }, []);