@sc-voice/tools 3.30.0 → 3.31.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "3.30.0",
+  "version": "3.31.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [

package/src/defines.mjs

@@ -15,5 +15,6 @@ export const DBG = {
   DEEPL_XLT: 0, // test live translation
   L7C_FETCH_LEGACY: 0,
   L7C_FETCH_LEGACY_SC: 0, // ignore test cache and use SC
+  W7E_BOW_OF_TEXT: 0,
   WORD_MAP_TRANFORMER: 0,
 };

package/src/math/interval.mjs

@@ -2,14 +2,14 @@ import { Unicode } from '../text/unicode.mjs';
 const { EMPTY_SET, INFINITY } = Unicode;
 import { ColorConsole } from '../text/color-console.mjs';
 const { cc } = ColorConsole;
-import { DBG } from '../defines.mjs';
 import util from 'node:util';
+import { DBG } from '../defines.mjs';
 
 const MINUS_INFINITY = `-${INFINITY}`;
 const PLUS_INFINITY = `+${INFINITY}`;
 
 export class Interval {
-  static styleText = (text)=>text; 
+  static styleText; // (text) => text
   static collapseDegenerate = false;
 
   constructor(a, b, opts = {}) {
@@ -69,7 +69,7 @@ export class Interval {
     return INFINITY;
   }
 
-  get size(){
+  get size() {
     return this.hi - this.lo;
   }
 

package/src/text/tfidf-space.mjs

@@ -1,6 +1,8 @@
 import { DBG } from '../defines.mjs';
 import { Corpus } from './corpus.mjs';
 import { WordVector } from './word-vector.mjs';
+import { ColorConsole } from './color-console.mjs';
+const { cc } = ColorConsole;
 
 // The golden ratio is pretty.
 // 1.6180339887498948482045868343656381177203091798057628621354;
@@ -48,6 +50,22 @@ export class TfidfSpace {
     });
   }
 
+  // Create wordWeight function that weighs the first words
+  // of a document more than the remainder
+  static wordWeightFromPrefix(prefixLength, prefixBias=0.5) {
+    const msg = 't8e.wordWeightFromPrefix';
+
+    let wordWeight = (w,i,nWords) => {
+      const nWeighted = Math.min(nWords, prefixLength);
+      const nUnweighted = nWords - nWeighted;
+      const wf = nUnweighted ? prefixBias : 1;
+      return i < nWeighted 
+        ? wf * nWords / nWeighted 
+        : (1 - wf) * nWords / nUnweighted;
+      }
+    return wordWeight;
+  }
+
   static removeHtml(s) {
     return s.replace(/<[^>]*>/gi, '');
   }
@@ -190,16 +208,22 @@ export class TfidfSpace {
     return { bow, words };
   }
 
-  bowOfText(text) {
+  bowOfText(text, opts={}) {
     const msg = 'w7e.bowOfText:';
+    let dbg = DBG.W7E_BOW_OF_TEXT;
     if (text == null) {
       throw new Error(`${msg} text?`);
     }
-    let dbg = 0;
+    let {
+      wordWeight = (word,i,n) => 1,
+    } = opts;
     let sNorm = this.normalizeText(text);
     let words = sNorm.split(' ');
-    let bow = words.reduce((a, w) => {
-      a[w] = (a[w] || 0) + 1;
+    let nWords = words.length;
+    let bow = words.reduce((a, word, i) => {
+      let ww = wordWeight(word, i, nWords);
+      a[word] = (a[word] || 0) + ww;
+      dbg && cc.fyi1(msg+0.1, {i, word, ww, sum:a[word]});
       return a;
     }, new WordVector());