@sc-voice/tools 1.1.0 → 1.3.0

package/src/{text → translate}/aligner.mjs

@@ -1,10 +1,13 @@
 import { DBG } from '../defines.mjs';
 import { Fraction } from '../math/fraction.mjs';
-import { EbtDoc } from './ebt-doc.mjs';
-import { LegacyDoc } from './legacy-doc.mjs';
-import { SuttaCentralId } from './sutta-central-id.mjs';
-import { Unicode } from './unicode.mjs';
-import { WordSpace } from './word-space.mjs';
+import { EbtDoc } from '../text/ebt-doc.mjs';
+import { LegacyDoc } from '../text/legacy-doc.mjs';
+import { SuttaCentralId } from '../text/sutta-central-id.mjs';
+import { Unicode } from '../text/unicode.mjs';
+import {
+  WordMapTransformer,
+  WordSpace,
+} from '../text/word-space.mjs';
 
 const STATE_OK = 'ok';
 const STATE_WARN = 'warn';
@@ -33,13 +36,72 @@ const {
 
 let alignmentCtor = false;
 
+class PaliTransformer {
+  constructor(transformer) {
+    let { wordMap } = transformer;
+    this.transformer = transformer;
+
+    let reList;
+    let entries = Object.entries(wordMap);
+    reList = entries.reduce((a, e) => {
+      let [legacyText, paliText] = e;
+      if (paliText) {
+        a.set(paliText, new RegExp(`\\b${paliText}`, 'gi'));
+      }
+      return a;
+    }, new Map());
+    this.reList = reList;
+  }
+
+  get wordMap() {
+    return this.transformer.wordMap;
+  }
+
+  transform(text) {
+    const msg = 'P14r.transform';
+    const dbg = DBG.PALI_TRANSFORMER;
+    let { transformer } = this;
+    dbg && console.log(msg, text);
+    return transformer.transform(text);
+  }
+
+  normalize(text) {
+    const msg = 'P14r.normalize';
+    const dbg = DBG.PALI_TRANSFORMER;
+    let { transformer } = this;
+    dbg && console.log(msg, text);
+    return transformer.normalize(text);
+  }
+}
+
+export class DpdTransformer {
+  constructor(opts = {}) {
+    const msg = 'D12r.ctor:';
+    let { dictionary } = opts;
+    if (dictionary == null) {
+      throw new Error(`${msg} dictionary?`);
+    }
+
+    this.dictionary = dictionary;
+  }
+
+  transform(text) {
+    return text;
+  }
+
+  normalize(text) {
+    return text;
+  }
+}
+
 export class Aligner {
   constructor(opts = {}) {
-    const msg = 'Aligner.ctor:';
+    const msg = 'A5r.ctor:';
     let {
-      alignPali = true,
+      alignMethod = 'alignPali',
       authorAligned, // author of segment aligned document
       authorLegacy, // author of legacy document
+      dbgScid,
       groupDecay = 0.5, // group exponential decay
       groupSize = 1, // comparison group size
       lang, // 2-letter ISO language (en, fr, es, pt)
@@ -52,16 +114,26 @@ export class Aligner {
       wordSpace,
     } = opts;
     if (wordSpace == null) {
-      wordSpace = new WordSpace({ lang, minWord, normalizeVector });
+      wordSpace = new WordSpace({
+        lang,
+        minWord,
+        normalizeVector,
+      });
+    }
+    if (alignMethod === 'alignPali') {
+      wordSpace.transformer = new PaliTransformer(
+        wordSpace.transformer,
+      );
     }
     if (lang == null) {
       lang = wordSpace.lang;
     }
 
     Object.assign(this, {
-      alignPali,
+      alignMethod,
       authorAligned,
       authorLegacy,
+      dbgScid,
       groupSize,
       groupDecay,
       lang,
@@ -97,6 +169,7 @@ export class Aligner {
     const msg = 'A7t.createAlignment:';
     const dbg = DBG.CREATE_ALIGNMENT;
     let {
+      dbgScid = this.dbgScid,
       legacyDoc,
       mlDoc,
       minScore = this.minScore,
@@ -112,7 +185,8 @@ export class Aligner {
       throw new Error(`${msg} mlDoc?`);
     }
 
-    let nLines = legacyDoc.lines.length;
+    let { author, author_uid, lines, footer } = legacyDoc;
+    let nLines = lines.length;
     let lineCursor = new Fraction(0, nLines, 'lines');
     let scids = Object.keys(mlDoc.segMap);
     let nSegs = scids.length;
@@ -128,18 +202,25 @@ export class Aligner {
       throw new Error(`${msg} minScanSize? ${minScanSize} `);
     }
 
-    let { sutta_uid:suid, docAuthor, bilaraPaths } = mlDoc;
-    let { author, author_uid } = legacyDoc;
-    let bilaraPath = bilaraPaths.reduce((a,p)=>{
+    let { sutta_uid: suid, docAuthor, bilaraPaths } = mlDoc;
+    let bilaraPath = bilaraPaths.reduce((a, p) => {
       if (p.includes(docAuthor)) {
         a = p.replaceAll(docAuthor, author_uid);
       }
       return a;
     });
-    let docOpts = { suid, lang, author, author_uid, bilaraPath };
+    let docOpts = {
+      suid,
+      lang,
+      author,
+      author_uid,
+      bilaraPath,
+      footer,
+    };
 
     const optsAlignment = {
       aligner: this,
+      dbgScid,
       ebtDoc: EbtDoc.create(docOpts),
       legacyDoc,
       lineCursor,
@@ -162,14 +243,14 @@ export class Aligner {
   mlDocVectors(mld) {
     const msg = 'Aligner.mlDocVectors';
     const dbg = DBG.ML_DOC_VECTORS;
-    let { alignPali, groupDecay, groupSize, wordSpace } = this;
-    let { wordMap } = wordSpace;
+    let { alignMethod, groupDecay, groupSize, wordSpace } = this;
+    let { wordMap } = wordSpace.transformer;
     let { segMap, lang } = mld;
     let segs = Object.entries(segMap);
     let iLastSeg = segs.length - 1;
     let reList;
 
-    if (alignPali) {
+    if (alignMethod === 'alignPali') {
       let entries = Object.entries(wordMap);
       reList = entries.reduce((a, e) => {
         let [legacyText, paliText] = e;
@@ -184,26 +265,34 @@ export class Aligner {
     let segGroup = [];
     for (let i = segs.length; i-- > 0; ) {
       let [scid, seg] = segs[i];
+      let vGroup = new WordSpace.Vector();
+
       let { pli } = seg;
       let segData = seg[lang] || '';
-      let vGroup = new WordSpace.Vector();
-      if (alignPali) {
-        // for aligning Pali, we add all Pali words that
-        // occur in the Pali for a segment to the
-        // vector input text
-        let pliWords = [];
-        reList.forEach((re, paliText, map) => {
-          let nMatch = pli.match(re)?.length || 0;
-          if (nMatch) {
-            for (let i = 0; i < nMatch; i++) {
-              pliWords.push(paliText);
+      switch (alignMethod) {
+        case 'alignPali':
+          {
+            // for aligning Pali, we add all Pali words that
+            // occur in the Pali for a segment to the
+            // vector input text
+            let pliWords = [];
+            reList.forEach((re, paliText, map) => {
+              let nMatch = pli.match(re)?.length || 0;
+              if (nMatch) {
+                for (let i = 0; i < nMatch; i++) {
+                  pliWords.push(paliText);
+                }
+              }
+            });
+            if (pliWords.length) {
+              segData += ' ' + pliWords.join(' ');
+              dbg === scid &&
+                console.log(msg, 'segData', scid, segData);
             }
           }
-        });
-        if (pliWords.length) {
-          segData += ' ' + pliWords.join(' ');
-          dbg === scid && console.log(msg, 'segData', scid, segData);
-        }
+          break;
+        case 'DPD':
+          break;
       }
       segGroup.unshift(segData);
       if (segGroup.length > groupSize) {
@@ -267,7 +356,7 @@ export class Alignment {
     if (typeof opts !== 'object') {
       throw new Error(`${msg} opts?`);
     }
-    let { dbgScid } = opts;
+    let { dbgScid = this.dbgScid } = opts;
     // biome-ignore format:
     let { ebtDoc, legacyDoc, lineCursor, maxScanSize, minScanSize,
       minScore, mlDoc, scids, segCursor, vMLDoc, wordSpace,
@@ -281,6 +370,7 @@ export class Alignment {
     for (let i = 0; scanning(i); i++) {
       let scid = scids[segCursor.numerator + i];
       if (scid == null) {
+        console.log(error, '[1]scid?', segCursor.toString());
         break;
       }
       let vSeg = vMLDoc[scid];
@@ -407,7 +497,7 @@ export class Alignment {
       aligner, ebtDoc, legacyDoc, lineCursor, maxScanSize, minScanSize,
       mlDoc, scidsExp, segCursor, vMLDoc,
     } = this;
-    let { lang, alignPali, wordSpace } = aligner;
+    let { lang, alignMethod, wordSpace } = aligner;
     let { segMap } = mlDoc;
     let scids = Object.keys(segMap);
     scids.sort(SuttaCentralId.compareLow);
@@ -417,7 +507,6 @@ export class Alignment {
 
     while (lineCursor.difference < 0) {
       let line = lines[lineCursor.numerator];
-      dbg > 1 && console.log(msg, lineCursor.toString(), line);
       let curScid = scids[segCursor.numerator];
       let dbgScid = scidsExp?.[lineCursor.numerator];
       let r = this.alignLine(line, { dbgScid });
@@ -425,12 +514,12 @@ export class Alignment {
       // biome-ignore format:
       if (r == null) {
         let { vSeg, vLegacy, intersection } = this.status;
-        dbg && console.log( msg, 'UNMATCHED', 
+        dbg && console.log(msg, 'UNMATCHED', 
           lineCursor.toString(),
           segCursor.toString(),
           { curScid, line, minScanSize, maxScanSize, vSeg, vLegacy, intersection },
         );
-        throw new Error(`${msg} unmatched`);
+        return null;
       }
     }
 

package/src/translate/deepl-adapter.mjs

@@ -0,0 +1,353 @@
+import * as deepl from 'deepl-node';
+import { DBG } from '../defines.mjs';
+import { MockDeepL } from './mock-deepl.mjs';
+
+const EMPTY_TEXT = '911911911';
+const TRANSLATE_OPTS = {
+  tag_handling: 'xml',
+  formality: 'more',
+};
+const DST_AUTHOR = 'no-author';
+
+let mockApi = DBG.MOCK_DEEPL;
+
+export class DeepLAdapter {
+  #authKey;
+
+  constructor(opts = {}) {
+    let {
+      authKey,
+      glossary,
+      glossaryName,
+      initialized,
+      sourceLang, // deepl lang
+      targetLang, // deepl lang
+      translateOpts,
+      translator,
+      dstLang,
+      dstLang2, // bilara-data lang
+      srcLang2, // bilara-data lang
+      srcLang,
+    } = DeepLAdapter.srcDstLangs(opts);
+
+    let emsg = 'use DeepLAdapter.create()';
+    let check = 1;
+    if (null == authKey) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == dstLang2) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == glossaryName) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == initialized) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == sourceLang) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == targetLang) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == srcLang2) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == translateOpts) throw new Error(`${emsg} ${check}`);
+    check++;
+    if (null == translator) throw new Error(`${emsg} ${check}`);
+    check++;
+
+    this.#authKey = authKey;
+
+    Object.assign(this, {
+      dstLang,
+      dstLang2,
+      glossary,
+      glossaryName,
+      initialized,
+      srcLang,
+      srcLang2,
+      sourceLang,
+      targetLang,
+      translateOpts: JSON.parse(JSON.stringify(translateOpts)),
+      translator,
+    });
+  }
+
+  static srcDstLangs(opts = {}) {
+    let { srcLang = 'en', dstLang = 'pt-pt' } = opts;
+    srcLang = srcLang.toLowerCase();
+    let srcLang2 = srcLang.split('-')[0];
+    dstLang = dstLang.toLowerCase();
+    let dstLang2 = dstLang.split('-')[0];
+
+    return Object.assign({}, opts, {
+      srcLang,
+      srcLang2,
+      dstLang,
+      dstLang2,
+    });
+  }
+
+  static deeplLang(lang) {
+    switch (lang) {
+      case 'pt':
+        return 'pt-PT';
+      default:
+        return lang;
+    }
+  }
+
+  static glossaryName(opts = {}) {
+    const msg = 'D10r.glossaryName()';
+    const dbg = DBG.GLOSSARY;
+    let { dstAuthor = DST_AUTHOR } = opts;
+    let {
+      dstLang,
+      dstLang2, // bilara-data lang
+      srcLang2, // bilara-data lang
+      srcLang,
+    } = DeepLAdapter.srcDstLangs(opts);
+    let name =
+      `D10r_${srcLang2}_${dstLang2}_${dstAuthor}`.toLowerCase();
+    dbg && console.log(msg, name);
+    return name;
+  }
+
+  static async create(opts = {}) {
+    const msg = 'D10r.create()';
+    const dbg = DBG.GLOSSARY;
+    let {
+      authKey,
+      srcLang,
+      srcLang2,
+      dstLang,
+      dstLang2,
+      dstAuthor = DST_AUTHOR,
+      sourceLang,
+      targetLang,
+      translateOpts = TRANSLATE_OPTS,
+      updateGlossary = false,
+      translator,
+    } = DeepLAdapter.srcDstLangs(opts);
+    dbg && console.log(msg, '[1]opts', opts);
+    if (authKey == null) {
+      throw new Error(`${msg} authKey?`);
+    }
+    sourceLang = sourceLang || DeepLAdapter.deeplLang(srcLang);
+    targetLang = targetLang || DeepLAdapter.deeplLang(dstLang);
+    if (translator == null) {
+      dbg && console.log(msg, '[2]new deepl.Translator()');
+      let deeplOpts = {};
+      translator = mockApi
+        ? new MockDeepL.Translator(authKey)
+        : new deepl.Translator(authKey);
+    }
+
+    let glossaryName = DeepLAdapter.glossaryName({
+      srcLang,
+      dstLang,
+      dstAuthor,
+    });
+    let glossaries = await translator.listGlossaries();
+    let glossary = glossaries.reduce((a, g) => {
+      return g.name === glossaryName ? g : a;
+    }, null);
+    if (updateGlossary) {
+      console.warn(msg, '[3]updateGlossary', glossaryName);
+      dbg && console.log(msg, '[4]uploadGlossary');
+      glossary = await DeepLAdapter.uploadGlossary({
+        srcLang,
+        dstLang,
+        dstAuthor,
+        translator,
+        glossaries,
+      });
+    }
+    if (glossary) {
+      let { glossaryId, name } = glossary;
+      dbg &&
+        console.warn(
+          msg,
+          '[5]using glossary',
+          name,
+          glossaryId && glossaryId.substring(0, 8),
+        );
+    } else {
+      let dbg = DBG.GLOSSARY;
+      dbg && console.log(msg, '[6]no glossary');
+    }
+    translateOpts = translateOpts
+      ? JSON.parse(JSON.stringify(translateOpts))
+      : TRANSLATE_OPTS;
+    if (glossary) {
+      translateOpts.glossary = glossary;
+    }
+    let initialized = true;
+
+    let ctorOpts = {
+      authKey,
+      dstLang,
+      dstLang2,
+      glossary,
+      glossaryName,
+      initialized,
+      srcLang,
+      srcLang2,
+      sourceLang,
+      targetLang,
+      translateOpts,
+      translator,
+    };
+    dbg &&
+      console.log(msg, '[7]ctor', {
+        sourceLang,
+        targetLang,
+        glossaryName,
+      });
+    return new DeepLAdapter(ctorOpts);
+  }
+
+  static setMockApi(value) {
+    mockApi = value;
+  }
+
+  static asGlossaryEntries(strObj) {
+    const msg = 'd12r.asToGlossaryEntries:';
+    let dbg = DBG.KVG_TO_GLOSSARY_ENTRIES;
+
+    if (strObj instanceof deepl.GlossaryEntries) {
+      return strObj;
+    }
+    let nEntries = 0;
+    let entries;
+
+    if (typeof strObj === 'string') {
+      // assume kvg string
+      entries = strObj.split('\n').reduce((a, kv) => {
+        let [key, value] = kv.split(/\|/);
+        if (key && !value) {
+          throw new Error(`${msg} [1]no value for key:${key}`);
+        } else if (!key && value) {
+          throw new Error(`${msg} [2]no key for value:${value}`);
+        } else if (!key && !value) {
+          // ignore
+        } else {
+          key = key.trim();
+          value = value.trim();
+          a[key] = value;
+          dbg > 1 && console.log(msg, '[3]', { key, value });
+          nEntries++;
+        }
+        return a;
+      }, []);
+    } else if (typeof strObj === 'object') {
+      entries = strObj;
+    } else {
+      throw new Error(`${msg} string or object?`);
+    }
+
+    return new deepl.GlossaryEntries({ entries });
+  }
+
+  static async uploadGlossary(opts = {}) {
+    const msg = 'D10r.uploadGlossary()';
+    const dbg = DBG.GLOSSARY;
+    const dbgv = DBG.VERBOSE && dbg;
+    let {
+      srcLang,
+      srcLang2,
+      dstLang,
+      dstLang2,
+      dstAuthor,
+      translator,
+      glossaries,
+      glossaryEntries,
+    } = DeepLAdapter.srcDstLangs(opts);
+    if (glossaryEntries == null) {
+      throw new Error(`${msg} glossaryEntries?`);
+    }
+    let nEntries = Object.keys(glossaryEntries).length;
+    let glossaryName = DeepLAdapter.glossaryName({
+      srcLang,
+      dstLang,
+      dstAuthor,
+    });
+    let glossary;
+
+    if (glossaries == null) {
+      glossaries = await translator.listGlossaries();
+    }
+    for (let i = 0; i < glossaries.length; i++) {
+      let g = glossaries[i];
+      if (g.name === glossaryName) {
+        dbg && console.log(msg, '[1]deleting', g.glossaryId);
+        await translator.deleteGlossary(g.glossaryId);
+      }
+    }
+
+    let sourceLang = DeepLAdapter.deeplLang(srcLang);
+    let targetLang = DeepLAdapter.deeplLang(dstLang);
+    glossary = await translator.createGlossary(
+      glossaryName,
+      sourceLang,
+      targetLang,
+      glossaryEntries,
+    );
+    let { glossaryId } = glossary;
+    dbg &&
+      console.log(msg, '[6]createGlossary', {
+        fName,
+        glossaryName,
+        sourceLang,
+        targetLang,
+        glossaryId,
+        nEntries,
+      });
+
+    return glossary;
+  }
+
+  async deleteGlossary(id) {
+    const msg = 'd12r.deleteGlossary:';
+    let { translator } = this;
+    dbg && console.log(msg, '[1]deleting', id);
+    await translator.deleteGlossary(id);
+    dbg>1 && console.log(msg, '[2]deleted', id);
+  }
+
+  async listGlossaries() {
+    let { translator } = this;
+
+    let glossaries = await translator.listGlossaries();
+    return glossaries;
+  }
+
+  async translate(texts) {
+    const msg = 'D10r.translate()';
+    const dbg = DBG.DEEPL_XLT;
+    const dbgv = dbg && DBG.VERBOSE;
+    let { translator, srcLang, dstLang, translateOpts } = this;
+
+    let sourceLang = DeepLAdapter.deeplLang(srcLang);
+    let targetLang = DeepLAdapter.deeplLang(dstLang);
+    texts = texts.map((t) => t || EMPTY_TEXT);
+    dbgv && console.log(msg, '[1]translateOpts', translateOpts);
+    let results = await translator.translateText(
+      texts,
+      sourceLang,
+      targetLang,
+      translateOpts,
+    );
+    if (dbg) {
+      results.forEach((result, i) => {
+        console.log(
+          msg,
+          `\n[${i}<] `,
+          `${texts[i]}$`,
+          `\n[${i}>] `,
+          `${results[i]?.text}$`,
+        );
+      });
+    }
+    results = results.map((r) =>
+      r.text === EMPTY_TEXT ? '' : r.text,
+    );
+
+    return results;
+  }
+} // DeepLAdapter

package/src/translate/dpd-transformer.mjs

@@ -0,0 +1,17 @@
+export class DpdTransformer {
+  constructor(opts = {}) {
+    const msg = 'D14r.ctor:';
+    let { dictionary } = opts;
+    if (dictionary == null) {
+      throw new Error(`${msg} dictionary?`);
+    }
+
+    Object.assign(this, {
+      dictionary,
+    });
+  }
+
+  transform(text) {
+    return text;
+  }
+}