@sc-voice/tools 1.1.0 → 1.2.0

package/index.mjs

@@ -5,9 +5,6 @@ export const ScvMath = {
   Fraction,
 };
 
-import { 
-  Aligner, Alignment, AlignmentStatus 
-} from './src/text/aligner.mjs';
 import { BilaraPath } from './src/text/bilara-path.mjs';
 import { EbtDoc } from './src/text/ebt-doc.mjs';
 import { LegacyDoc } from './src/text/legacy-doc.mjs';
@@ -17,9 +14,6 @@ import { Unicode } from './src/text/unicode.mjs';
 import { WordSpace } from './src/text/word-space.mjs';
 
 export const Text = {
-  Aligner,
-  Alignment,
-  AlignmentStatus,
   BilaraPath,
   EbtDoc,
   LegacyDoc,
@@ -29,3 +23,24 @@ export const Text = {
   WordSpace,
 };
 
+import { default as Sankey } from './src/graph/sankey.mjs';
+export const Graph = {
+  Sankey, 
+}
+
+import { 
+  Aligner, Alignment, AlignmentStatus 
+} from './src/translate/aligner.mjs';
+import { DpdTransformer } from './src/translate/dpd-transformer.mjs';
+import { MockDeepL } from './src/translate/mock-deepl.mjs';
+import { DeepLAdapter } from './src/translate/deepl-adapter.mjs';
+import { QuoteParser } from './src/translate/quote-parser.mjs';
+export const Translate = {
+  Aligner,
+  Alignment,
+  AlignmentStatus,
+  DeepLAdapter,
+  DpdTransformer,
+  MockDeepL,
+  QuoteParser,
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sc-voice/tools",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Utilities for SC-Voice",
   "main": "index.mjs",
   "files": [
@@ -31,6 +31,7 @@
   "homepage": "https://github.com/sc-voice/tools#readme",
   "devDependencies": {
     "@biomejs/biome": "1.9.4",
+    "deepl-node": "^1.15.0",
     "eslint": "^9.17.0",
     "mocha": "^11.0.1",
     "should": "^13.2.3"

package/src/defines.mjs

@@ -1,9 +1,14 @@
 
 export const DBG = {
-  APPLY_WORD_MAP: 0,
   ALIGN_ALL: 0,
   ALIGN_LINE: 0,
   ML_DOC_VECTORS: 0, // 'mn8:3.4',
   MN8_MOHAN: 0,
+  DEEPL_ADAPTER: 0,
+  DEEPL_MOCK: 0,  // use mock-deepl
+  DEEPL_MOCK_XLT: 0, // use mock translation
+  DEEPL_TEST_API: 0, // test with live DeepL API ($$$)
+  DEEPL_XLT: 0,  // test live translation
+  WORD_MAP_TRANFORMER: 0,
 };
 

package/src/graph/sankey.mjs

@@ -0,0 +1,56 @@
+import { DBG } from '../defines.mjs';
+
+export default class Sankey {
+  constructor(opts = {}) {
+    let { links = [] } = opts;
+
+    Object.assign(this, {
+      links: [...links],
+    });
+
+    let nodeMap = {};
+    Object.defineProperty(this, 'nodeMap', { value: nodeMap });
+
+    // organize links by source/target
+    let linkMap = {};
+    Object.defineProperty(this, 'linkMap', { value: linkMap });
+    this.links.forEach((link) => {
+      let { source, target } = link;
+      let key = `${source}|${target}`;
+      linkMap[key] = link;
+      nodeMap[source] = (nodeMap[source] || 0) + 1;
+      nodeMap[target] = (nodeMap[target] || 0) + 1;
+    });
+  }
+
+  get nodes() {
+    let { nodeMap } = this;
+    let nodes = Object.keys(nodeMap).map((n) => ({ id: n }));
+    return nodes;
+  }
+
+  addLink({ source, target, value = 0 }) {
+    const msg = 'Sankey.addLink()';
+    const dbg = DBG.SANKEY;
+    let { links, linkMap, nodeMap } = this;
+    if (source == null) {
+      throw new Error(`${msg} source? [${source}]`);
+    }
+    nodeMap[source] = (nodeMap[source] || 0) + 1;
+    if (target == null) {
+      throw new Error(`${msg} target? [${target}]`);
+    }
+    if (Number.isNaN(value)) {
+      throw new Error(`${msg} value? [${value}]`);
+    }
+    nodeMap[target] = (nodeMap[target] || 0) + 1;
+
+    let key = `${source}|${target}`;
+    let link = linkMap[key];
+    if (link == null) {
+      link = linkMap[key] = { source, target, value: 0 };
+      links.push(link);
+    }
+    link.value += value;
+  }
+}

package/src/text/ebt-doc.mjs

@@ -3,7 +3,11 @@ import { SuttaCentralId } from './sutta-central-id.mjs';
 let privateCtor = false;
 
 const INHERITED_KEYS = [
-  'lang', 'author', 'author_uid', 'wordSpace'
+  'lang',
+  'author',
+  'author_uid',
+  'wordSpace',
+  'footer',
 ];
 
 const HDR_KEY = '__header__';
@@ -27,7 +31,7 @@ export class EbtDoc {
 
   static create(opts = {}) {
     const msg = 'E4c.create:';
-    let { suid, bilaraPath, segMap = {}, parent = {} } = opts;
+    let { segMap = {}, parent = {}, suid, bilaraPath } = opts;
     if (segMap == null) {
       throw new Error(`${msg} segMap?`);
     }

package/src/text/legacy-doc.mjs

@@ -1,3 +1,5 @@
+import { DBG } from '../defines.mjs';
+
 let privateCtor = false;
 
 const HTML_FILTER = (() => {
@@ -35,13 +37,38 @@ export class LegacyDoc {
     return true;
   }
 
-  static create(rawDoc) {
+  static async fetchLegacy(opts = {}) {
+    const msg = 'L7c.fetch:';
+    const dbg = DBG.FETCH_LEGACY;
+    let {
+      endPoint = 'https://suttacentral.net/api/suttas',
+      sutta_uid,
+      lang,
+      author,
+      maxBuffer = 10 * 1024 * 1024,
+    } = opts;
+    let url = [endPoint, sutta_uid, `${author}?lang=${lang}`].join(
+      '/',
+    );
+    let res = await fetch(url);
+    if (!res.ok) {
+      throw new Error(`${msg} {res.status} ${url}`);
+    }
+    let json = await res.json();
+    let { translation } = json;
+    return LegacyDoc.create(translation);
+  }
+
+  static create(translation) {
     const msg = 'LegacyDoc.create:';
     if (typeof legacy === 'string') {
       legacy = JSON.parse(legacy);
     }
 
-    let { uid, lang, title, author, author_uid, text } = rawDoc;
+    let { uid, lang, title, author, author_uid, text } = translation;
+    if (typeof text === 'string') {
+      text = text.split('\n');
+    }
 
     let para;
     let lines = text.filter((line) => !HTML_FILTER.test(line));

package/src/text/word-space.mjs

@@ -63,7 +63,7 @@ class Vector extends Object {
     }, 0);
   }
 
-  intersect(vec2) {
+  intersect(vec2 = {}) {
     let keys = Object.keys(this);
     return keys.reduce((a, k) => {
       let v1 = this[k];
@@ -89,26 +89,21 @@ class Vector extends Object {
   }
 } // Vector
 
-export class WordSpace {
-  constructor(opts = {}) {
-    let {
-      lang, // 2-letter code: fr, en, es, pt
-      minWord = 4, // minimum word length
-      normalize,
-      normalizeVector = WordSpace.normalizeVector,
-      wordMap = {}, // word replacement map
-      reWordMap,
-    } = opts;
-
-    wordMap = Object.keys(wordMap).reduce((a, w) => {
+export class WordMapTransformer {
+  // DEPRECATED
+  constructor(oWordMap = {}, opts = {}) {
+    let wordMap = Object.keys(oWordMap).reduce((a, w) => {
       let wLow = w.toLowerCase();
-      a[wLow] = wordMap[w].toLowerCase();
+      a[wLow] = oWordMap[w].toLowerCase();
       return a;
     }, {});
+
+    let { lang = 'en', normalize } = opts;
+
     if (!normalize) {
       switch (lang) {
         case 'fr':
-          normalize = WordSpace.normalizeFR;
+          normalize = WordMapTransformer.normalizeFR;
           break;
         default:
           normalize = (s) => s;
@@ -116,27 +111,8 @@ export class WordSpace {
       }
     }
 
-    Object.assign(this, {
-      lang,
-      minWord,
-      normalize,
-      normalizeVector,
-      reWordMap,
-      wordMap,
-    });
-  }
-
-  static compileWordMap(wordMap) {
-    return (
-      wordMap &&
-      Object.keys(wordMap).map((pat) => {
-        let rep = wordMap[pat];
-        return {
-          re: new RegExp(pat, 'iugm'),
-          rep,
-        };
-      })
-    );
+    this.wordMap = wordMap;
+    this.normalize = normalize;
   }
 
   static normalizeFR(s) {
@@ -150,34 +126,92 @@ export class WordSpace {
       .trim();
   }
 
-  applyWordMap(text) {
-    const msg = 'W7e.applyWordMap:';
-    const dbg = DBG.APPLY_WORD_MAP;
-    let { wordMap, reWordMap } = this;
+  #compileWordMap() {
+    let { wordMap } = this;
+    return Object.keys(wordMap).map((pat) => {
+      let rep = wordMap[pat];
+      return {
+        re: new RegExp(pat, 'iugm'),
+        rep,
+      };
+    });
+  }
+
+  transform(text) {
+    const msg = 'W16r.transform:';
+    const dbg = DBG.WORD_MAP_TRANSFORMER;
+    let { wordMap, reWordMap, normalize } = this;
     if (reWordMap == null) {
-      reWordMap = WordSpace.compileWordMap(wordMap);
+      reWordMap = this.#compileWordMap();
       this.reWordMap = reWordMap;
     }
     dbg && console.log(msg, { text });
-    let rslt = text;
+    let textMapped = text;
     for (let i = 0; i < reWordMap.length; i++) {
       let { re, rep } = reWordMap[i];
-      rslt = rslt.replaceAll(re, rep);
-      dbg && console.log(msg, { i, rslt, re });
+      textMapped = textMapped.replaceAll(re, rep);
+      dbg && console.log(msg, { i, textMapped, re });
     }
+    let rslt = normalize(textMapped)
+      .toLowerCase()
+      .trim()
+      .replace(/[-]/g, ' ')
+      .replace(/[.,_:;"'“”‘’!?]/g, '');
     return rslt;
   }
+}
+
+export class WordSpace {
+  constructor(opts = {}) {
+    let {
+      lang, // 2-letter code: fr, en, es, pt
+      minWord = 4, // minimum word length
+      normalize,
+      normalizeVector = WordSpace.normalizeVector,
+      transformText,
+      transformer,
+      reWordMap,
+    } = opts;
+
+    if (transformer == null) {
+      let wordMap = opts.wordMap;
+      transformer = new WordMapTransformer(wordMap, {
+        lang,
+        normalize,
+      });
+      if (transformText == null) {
+        transformText = (text) => transformer.transform(text);
+      }
+    }
+    Object.defineProperty(this, 'transformText', {
+      value: transformText,
+    });
+
+    Object.assign(this, {
+      lang,
+      minWord,
+      normalizeVector,
+      reWordMap,
+      transformer,
+      wordMap: opts.wordMap, // DEPRECATED
+    });
+  }
+
+  static get WordMapTransformer() {
+    return WordMapTransformer;
+  }
 
   static get Vector() {
     return Vector;
   }
 
-  static normalizeVector(v) {
-    let tau = 0.618034; // Golden ratio
+  // Golden Ratio fudge factor scales a count of 1 to ~0.8
+  // 1.6180339887498948482045868343656381177203091798057628621354
+  static normalizeVector(v, scale=1.618033988749895) {
     let vNew = new Vector(v);
     Object.entries(v).forEach((e) => {
       let [key, value] = e;
-      vNew[key] = 1 - Math.exp(-value / tau);
+      vNew[key] = 1 - Math.exp(-value * scale);
     });
 
     return vNew;
@@ -189,13 +223,8 @@ export class WordSpace {
       throw new Error(`${msg} str?`);
     }
     let dbg = 0;
-    let { normalize, normalizeVector, minWord, wordMap } = this;
-    let sWordMap = this.applyWordMap(str);
-    let sNorm = normalize(sWordMap)
-      .toLowerCase()
-      .trim()
-      .replace(/[-]/g, ' ')
-      .replace(/[.,_:;"'“”‘’!?]/g, '');
+    let { normalize, normalizeVector, minWord } = this;
+    let sNorm = this.transformText(str);
     let words = sNorm.split(' ');
     let v = words.reduce((a, w) => {
       if (w.length >= minWord) {

package/src/{text → translate}/aligner.mjs

@@ -1,10 +1,13 @@
 import { DBG } from '../defines.mjs';
 import { Fraction } from '../math/fraction.mjs';
-import { EbtDoc } from './ebt-doc.mjs';
-import { LegacyDoc } from './legacy-doc.mjs';
-import { SuttaCentralId } from './sutta-central-id.mjs';
-import { Unicode } from './unicode.mjs';
-import { WordSpace } from './word-space.mjs';
+import { EbtDoc } from '../text/ebt-doc.mjs';
+import { LegacyDoc } from '../text/legacy-doc.mjs';
+import { SuttaCentralId } from '../text/sutta-central-id.mjs';
+import { Unicode } from '../text/unicode.mjs';
+import {
+  WordMapTransformer,
+  WordSpace,
+} from '../text/word-space.mjs';
 
 const STATE_OK = 'ok';
 const STATE_WARN = 'warn';
@@ -33,13 +36,72 @@ const {
 
 let alignmentCtor = false;
 
+class PaliTransformer {
+  constructor(transformer) {
+    let { wordMap } = transformer;
+    this.transformer = transformer;
+
+    let reList;
+    let entries = Object.entries(wordMap);
+    reList = entries.reduce((a, e) => {
+      let [legacyText, paliText] = e;
+      if (paliText) {
+        a.set(paliText, new RegExp(`\\b${paliText}`, 'gi'));
+      }
+      return a;
+    }, new Map());
+    this.reList = reList;
+  }
+
+  get wordMap() {
+    return this.transformer.wordMap;
+  }
+
+  transform(text) {
+    const msg = 'P14r.transform';
+    const dbg = DBG.PALI_TRANSFORMER;
+    let { transformer } = this;
+    dbg && console.log(msg, text);
+    return transformer.transform(text);
+  }
+
+  normalize(text) {
+    const msg = 'P14r.normalize';
+    const dbg = DBG.PALI_TRANSFORMER;
+    let { transformer } = this;
+    dbg && console.log(msg, text);
+    return transformer.normalize(text);
+  }
+}
+
+export class DpdTransformer {
+  constructor(opts = {}) {
+    const msg = 'D12r.ctor:';
+    let { dictionary } = opts;
+    if (dictionary == null) {
+      throw new Error(`${msg} dictionary?`);
+    }
+
+    this.dictionary = dictionary;
+  }
+
+  transform(text) {
+    return text;
+  }
+
+  normalize(text) {
+    return text;
+  }
+}
+
 export class Aligner {
   constructor(opts = {}) {
-    const msg = 'Aligner.ctor:';
+    const msg = 'A5r.ctor:';
     let {
-      alignPali = true,
+      alignMethod = 'alignPali',
       authorAligned, // author of segment aligned document
       authorLegacy, // author of legacy document
+      dbgScid,
       groupDecay = 0.5, // group exponential decay
       groupSize = 1, // comparison group size
       lang, // 2-letter ISO language (en, fr, es, pt)
@@ -52,16 +114,26 @@ export class Aligner {
       wordSpace,
     } = opts;
     if (wordSpace == null) {
-      wordSpace = new WordSpace({ lang, minWord, normalizeVector });
+      wordSpace = new WordSpace({
+        lang,
+        minWord,
+        normalizeVector,
+      });
+    }
+    if (alignMethod === 'alignPali') {
+      wordSpace.transformer = new PaliTransformer(
+        wordSpace.transformer,
+      );
     }
     if (lang == null) {
       lang = wordSpace.lang;
     }
 
     Object.assign(this, {
-      alignPali,
+      alignMethod,
       authorAligned,
       authorLegacy,
+      dbgScid,
       groupSize,
       groupDecay,
       lang,
@@ -97,6 +169,7 @@ export class Aligner {
     const msg = 'A7t.createAlignment:';
     const dbg = DBG.CREATE_ALIGNMENT;
     let {
+      dbgScid = this.dbgScid,
       legacyDoc,
       mlDoc,
       minScore = this.minScore,
@@ -112,7 +185,8 @@ export class Aligner {
       throw new Error(`${msg} mlDoc?`);
     }
 
-    let nLines = legacyDoc.lines.length;
+    let { author, author_uid, lines, footer } = legacyDoc;
+    let nLines = lines.length;
     let lineCursor = new Fraction(0, nLines, 'lines');
     let scids = Object.keys(mlDoc.segMap);
     let nSegs = scids.length;
@@ -128,18 +202,25 @@ export class Aligner {
       throw new Error(`${msg} minScanSize? ${minScanSize} `);
     }
 
-    let { sutta_uid:suid, docAuthor, bilaraPaths } = mlDoc;
-    let { author, author_uid } = legacyDoc;
-    let bilaraPath = bilaraPaths.reduce((a,p)=>{
+    let { sutta_uid: suid, docAuthor, bilaraPaths } = mlDoc;
+    let bilaraPath = bilaraPaths.reduce((a, p) => {
       if (p.includes(docAuthor)) {
         a = p.replaceAll(docAuthor, author_uid);
       }
       return a;
     });
-    let docOpts = { suid, lang, author, author_uid, bilaraPath };
+    let docOpts = {
+      suid,
+      lang,
+      author,
+      author_uid,
+      bilaraPath,
+      footer,
+    };
 
     const optsAlignment = {
       aligner: this,
+      dbgScid,
       ebtDoc: EbtDoc.create(docOpts),
       legacyDoc,
       lineCursor,
@@ -162,14 +243,14 @@ export class Aligner {
   mlDocVectors(mld) {
     const msg = 'Aligner.mlDocVectors';
     const dbg = DBG.ML_DOC_VECTORS;
-    let { alignPali, groupDecay, groupSize, wordSpace } = this;
-    let { wordMap } = wordSpace;
+    let { alignMethod, groupDecay, groupSize, wordSpace } = this;
+    let { wordMap } = wordSpace.transformer;
     let { segMap, lang } = mld;
     let segs = Object.entries(segMap);
     let iLastSeg = segs.length - 1;
     let reList;
 
-    if (alignPali) {
+    if (alignMethod === 'alignPali') {
       let entries = Object.entries(wordMap);
       reList = entries.reduce((a, e) => {
         let [legacyText, paliText] = e;
@@ -184,26 +265,34 @@ export class Aligner {
     let segGroup = [];
     for (let i = segs.length; i-- > 0; ) {
       let [scid, seg] = segs[i];
+      let vGroup = new WordSpace.Vector();
+
       let { pli } = seg;
       let segData = seg[lang] || '';
-      let vGroup = new WordSpace.Vector();
-      if (alignPali) {
-        // for aligning Pali, we add all Pali words that
-        // occur in the Pali for a segment to the
-        // vector input text
-        let pliWords = [];
-        reList.forEach((re, paliText, map) => {
-          let nMatch = pli.match(re)?.length || 0;
-          if (nMatch) {
-            for (let i = 0; i < nMatch; i++) {
-              pliWords.push(paliText);
+      switch (alignMethod) {
+        case 'alignPali':
+          {
+            // for aligning Pali, we add all Pali words that
+            // occur in the Pali for a segment to the
+            // vector input text
+            let pliWords = [];
+            reList.forEach((re, paliText, map) => {
+              let nMatch = pli.match(re)?.length || 0;
+              if (nMatch) {
+                for (let i = 0; i < nMatch; i++) {
+                  pliWords.push(paliText);
+                }
+              }
+            });
+            if (pliWords.length) {
+              segData += ' ' + pliWords.join(' ');
+              dbg === scid &&
+                console.log(msg, 'segData', scid, segData);
             }
           }
-        });
-        if (pliWords.length) {
-          segData += ' ' + pliWords.join(' ');
-          dbg === scid && console.log(msg, 'segData', scid, segData);
-        }
+          break;
+        case 'DPD':
+          break;
       }
       segGroup.unshift(segData);
       if (segGroup.length > groupSize) {
@@ -267,7 +356,7 @@ export class Alignment {
     if (typeof opts !== 'object') {
       throw new Error(`${msg} opts?`);
     }
-    let { dbgScid } = opts;
+    let { dbgScid = this.dbgScid } = opts;
     // biome-ignore format:
     let { ebtDoc, legacyDoc, lineCursor, maxScanSize, minScanSize,
       minScore, mlDoc, scids, segCursor, vMLDoc, wordSpace,
@@ -281,6 +370,7 @@ export class Alignment {
     for (let i = 0; scanning(i); i++) {
       let scid = scids[segCursor.numerator + i];
       if (scid == null) {
+        console.log(error, '[1]scid?', segCursor.toString());
         break;
       }
       let vSeg = vMLDoc[scid];
@@ -407,7 +497,7 @@ export class Alignment {
       aligner, ebtDoc, legacyDoc, lineCursor, maxScanSize, minScanSize,
       mlDoc, scidsExp, segCursor, vMLDoc,
     } = this;
-    let { lang, alignPali, wordSpace } = aligner;
+    let { lang, alignMethod, wordSpace } = aligner;
     let { segMap } = mlDoc;
     let scids = Object.keys(segMap);
     scids.sort(SuttaCentralId.compareLow);
@@ -417,7 +507,6 @@ export class Alignment {
 
     while (lineCursor.difference < 0) {
       let line = lines[lineCursor.numerator];
-      dbg > 1 && console.log(msg, lineCursor.toString(), line);
       let curScid = scids[segCursor.numerator];
       let dbgScid = scidsExp?.[lineCursor.numerator];
       let r = this.alignLine(line, { dbgScid });
@@ -425,12 +514,12 @@ export class Alignment {
       // biome-ignore format:
       if (r == null) {
         let { vSeg, vLegacy, intersection } = this.status;
-        dbg && console.log( msg, 'UNMATCHED', 
+        dbg && console.log(msg, 'UNMATCHED', 
           lineCursor.toString(),
           segCursor.toString(),
           { curScid, line, minScanSize, maxScanSize, vSeg, vLegacy, intersection },
         );
-        throw new Error(`${msg} unmatched`);
+        return null;
       }
     }