@sc-voice/tools 2.0.0 → 2.2.0

package/README.md

@@ -1,12 +1,38 @@
-# merkle-json
+# @sc-voice/tools
+Javascript libary for SC-Voice applications
+
+* Math 
+* Text
+* Graph
+
+## Math.Fraction
+```
+let f = new Fraction(9, 240, 'segments');
+console.log(f.value); // 0.375
+console.log(f.numerator, f.denominator); // 9 240
+console.log(f.n, f.d); // 9 240
+console.log(f.toString()); // 1/2 segments
+console.log(Fraction.gcd(9, 240)); // 3
+console.log(f.difference); // -150
+console.log(f.remainder); // 9
+console.log(f.percent); // '4%'
+console.log(f.add(new Fraction(1,80)); // new Fraction(4,80)
+```
+
+## Text.WordSpace
+Used for text similarity comparison, WordSpace creates and compares
+Vectors of words weighted by normalized frequency of occurrence.
+Weights and scores are normalized to the interval [0..1].
+(NOTE: Odd how bag-of-words works better here than fastText?)
+
+## Text.MerkleJson
 Computing the hash of JSON objects can be tricky because JSON.stringify()
 does not have a guaranteed string representation of a Javascript object.
 Specifically, the following are equivalent and valid outputs of JSON.stringify():
 
 ```js
 var json = "{size:{w:100,h:200}}";
-var json = "{size:{h:100,w:200}}";
-```
+var json = "{size:{h:100,w:200}}"; ```
 
 MerkleJson guarantees a unique hash code for any Javascript object.
 In addition, MerkleJson is efficient in that it only recalculates 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "2.0.0",
+  "version": "2.2.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [

package/src/text/logger.mjs

@@ -41,24 +41,51 @@ export class LogEntry {
   }
 }
 
+const LEVEL_DEBUG = { id: 'D', priority: -1 };
+const LEVEL_INFO = { id: 'I', priority: 0 };
+const LEVEL_WARN = { id: 'W', priority: 1 };
+const LEVEL_ERROR = { id: 'E', priority: 2 };
+const LEVEL_LOG = { id: 'L', priority: 3 };
+
 export class Logger {
   constructor(opts = {}) {
-    let { sink = console, msBase = Date.now() } = opts;
+    let {
+      sink = console,
+      msBase = Date.now(),
+      logLevel = Logger.LEVEL_WARN,
+    } = opts;
     Object.assign(this, {
       history: [],
       sink,
       msBase,
+      logLevel,
     });
   }
 
+  static get LEVEL_DEBUG() {
+    return LEVEL_DEBUG;
+  }
+  static get LEVEL_INFO() {
+    return LEVEL_INFO;
+  }
+  static get LEVEL_WARN() {
+    return LEVEL_WARN;
+  }
+  static get LEVEL_ERROR() {
+    return LEVEL_ERROR;
+  }
+  static get LEVEL_LOG() {
+    return LEVEL_LOG;
+  }
+
   addEntry(level, args, fSink) {
     const msg = 'l4r.addEntry;';
     const dbg = DBG.L4R_ADD_ENTRY;
-    let { history, sink, msBase } = this;
+    let { logLevel, history, sink, msBase } = this;
     let ms = Date.now() - msBase;
     let entry = LogEntry.fromArgs(level, args, ms);
     history.push(entry);
-    if (sink) {
+    if (sink && level.priority >= logLevel.priority) {
       dbg && console.log(msg, 'sink');
       fSink?.apply(sink, args);
     }
@@ -66,22 +93,22 @@ export class Logger {
   }
 
   debug(...args) {
-    return this.addEntry('D', args, this.sink?.debug);
+    return this.addEntry(LEVEL_DEBUG, args, this.sink?.debug);
   }
 
   info(...args) {
-    return this.addEntry('I', args, this.sink?.info);
-  }
-
-  log(...args) {
-    return this.addEntry('L', args, this.sink?.log);
+    return this.addEntry(LEVEL_INFO, args, this.sink?.info);
   }
 
   warn(...args) {
-    return this.addEntry('W', args, this.sink?.warn);
+    return this.addEntry(LEVEL_WARN, args, this.sink?.warn);
   }
 
   error(...args) {
-    return this.addEntry('E', args, this.sink?.error);
+    return this.addEntry(LEVEL_ERROR, args, this.sink?.error);
+  }
+
+  log(...args) {
+    return this.addEntry(LEVEL_INFO, args, this.sink?.log);
   }
 }

package/src/text/word-space.mjs

@@ -1,5 +1,9 @@
 import { DBG } from '../defines.mjs';
 
+// The golden ratio is pretty.
+// 1.6180339887498948482045868343656381177203091798057628621354;
+const GOLDEN_FUDGE = 1.618033988749895;
+
 class Vector extends Object {
   constructor(props) {
     super();
@@ -49,6 +53,17 @@ class Vector extends Object {
     }, new Vector(this));
   }
 
+  increment(vec2) {
+    let keys = Object.keys(vec2);
+    return keys.reduce((a, k) => {
+      let v2 = vec2[k];
+      if (v2) {
+        a[k] = (a[k] || 0) + v2;
+      }
+      return a;
+    }, this);
+  }
+
   dot(vec2) {
     const msg = 'V4r.dot:';
     if (vec2 == null) {
@@ -165,6 +180,9 @@ export class WordSpace {
   constructor(opts = {}) {
     let {
       lang, // 2-letter code: fr, en, es, pt
+      corpusBow = new Vector(), // corpus bag of words
+      corpusSize = 0, // number of retrieval units (docs, segments, etc.)
+      idfWeight = GOLDEN_FUDGE, // IDF dampening
       minWord = 4, // minimum word length
       normalize,
       normalizeVector = WordSpace.normalizeVector,
@@ -189,6 +207,9 @@ export class WordSpace {
 
     Object.assign(this, {
       lang,
+      corpusBow,
+      corpusSize,
+      idfWeight,
       minWord,
       normalizeVector,
       reWordMap,
@@ -197,6 +218,12 @@ export class WordSpace {
     });
   }
 
+  static createTfIdf() {
+    let minWord = 1;
+    let normalizeVector = (v) => v;
+    return new WordSpace({ minWord, normalizeVector });
+  }
+
   static get WordMapTransformer() {
     return WordMapTransformer;
   }
@@ -206,8 +233,7 @@ export class WordSpace {
   }
 
   // Golden Ratio fudge factor scales a count of 1 to ~0.8
-  // 1.6180339887498948482045868343656381177203091798057628621354
-  static normalizeVector(v, scale = 1.618033988749895) {
+  static normalizeVector(v, scale = GOLDEN_FUDGE) {
     let vNew = new Vector(v);
     Object.entries(v).forEach((e) => {
       let [key, value] = e;
@@ -217,8 +243,83 @@ export class WordSpace {
     return vNew;
   }
 
+  addDocument(doc) {
+    let { corpusBow } = this;
+    this.corpusSize += 1;
+    let { bow } = this.countWords(doc, 1); // one-hot
+    corpusBow.increment(bow);
+
+    return this;
+  }
+
+  inverseDocumentFrequency(word, idfWeight) {
+    return this.idf(word, idfWeight);
+  }
+
+  idf(word, idfWeight = this.idfWeight) {
+    const msg = 'w7e.idf:';
+    let { corpusBow, corpusSize } = this;
+    let wCount = corpusBow[word] || 0;
+    // Map to [0:ignore..1:important]
+    return corpusSize
+      ? 1 - Math.exp(((wCount - corpusSize) / wCount) * idfWeight)
+      : 1;
+  }
+
+  termFrequency(word, document) {
+    return this.tf(word, document);
+  }
+
+  tf(word, doc) {
+    let { bow, words } = this.countWords(doc);
+    let count = bow[word] || 0;
+    return count ? count / words.length : 0;
+  }
+
+  tfidf(doc) {
+    const msg = 'w7e.tfidf:';
+    let { corpusBow, corpusSize, idfWeight } = this;
+
+    // More efficient implementation of tf * idf
+    let { bow, words } = this.countWords(doc);
+    let nWords = words.length;
+
+    let vTfIdf = words.reduce((a, word) => {
+      let wd = bow[word] || 0;
+      let tf = wd ? wd / nWords : 0;
+      let wc = corpusBow[word] || 0;
+      let idf = corpusSize
+        ? 1 - Math.exp(((wc - corpusSize) / wc) * idfWeight)
+        : 1;
+      let tfidf = tf * idf;
+      if (tfidf) {
+        a[word] = tfidf;
+      }
+      return a;
+    }, new Vector());
+
+    return vTfIdf;
+  }
+
+  countWords(str, maxCount) {
+    const msg = 'w7e.countWords:';
+    if (str == null) {
+      throw new Error(`${msg} str?`);
+    }
+    let dbg = 0;
+    let sNorm = this.transformText(str);
+    let words = sNorm.split(' ');
+    let bow = words.reduce((a, w) => {
+      let count = (a[w] || 0) + 1;
+      a[w] = maxCount ? Math.min(maxCount, count) : count;
+      return a;
+    }, new Vector());
+
+    return { bow, words };
+  }
+
   string2Vector(str, scale = 1) {
-    const msg = 'W7e.string2Vector:';
+    const msg = 'w7e.string2Vector:';
     if (str == null) {
       throw new Error(`${msg} str?`);
     }