calliope-ts 0.0.4 → 0.1.0

package/src/parser.ts

@@ -1,501 +0,0 @@
-// parser.ts — Syntactic dependency parser powered by UDPipe (English GUM model),
-// producing a ClsDocument with a full dependency graph and a phrase‑structure node
-// tree in the Universal Dependencies format used in McAleese's Calliope.
-//
-// HISTORY: this module previously ran a staged FinNLP pipeline
-// (en-norm → lexed → en-pos → en-parse) with hand-written tag/dep correction
-// layers (tagfix.ts / depfix.ts) to patch en-parse's systematic errors. It now
-// delegates tokenisation, POS tagging, and dependency parsing to UDPipe via the
-// `udpipe-node` package (a pure-WASM build — no native binary, no subprocess),
-// which is a far more accurate parser. UDPipe's output maps cleanly onto the
-// existing data model:
-//   • XPOS column is Penn Treebank  → ClsWord.lexicalClass (unchanged downstream)
-//   • DEPREL column is Universal Dependencies → ClsDependency.dependentType
-// The correction layers are therefore no longer applied on this path.
-
-import { createUDPipe } from 'udpipe-node/wasm';
-import type { UDSentence, UDWord } from 'udpipe-node';
-import { correctUDPipePos } from './calliope/postag.js';
-import { pennTagOf } from './calliope/udpos.js';
-import {
-  ClsDocument,
-  ClsSentence,
-  ClsWord,
-  ClsDependency,
-  ClsNode,
-} from './types.js';
-
-// ── UDPipe instance (lazy singleton) ────────────────────────────────
-// The "./wasm" entry point pre-initialises the WASM runtime via top-level await,
-// so by the time this module is imported the engine is ready and construction /
-// parsing are fully synchronous — `parseDocument` keeps its synchronous contract.
-let _nlp: ReturnType<typeof createUDPipe> | null = null;
-function nlp(): ReturnType<typeof createUDPipe> {
-  // CALLIOPE_UDPIPE_MODEL lets us swap the UDPipe model (EWT / GUM / LinES /
-  // ParTUT) for auditing — different treebanks tag XPOS quite differently, so the
-  // model choice materially affects the parse the phonological pipeline consumes.
-  // Unset → the bundled GUM model.
-  const modelPath = process.env.CALLIOPE_UDPIPE_MODEL || undefined;
-  return (_nlp ??= createUDPipe({ defaultInputMode: 'presegmented', modelPath }));
-}
-
-// ── POS / punctuation classification (unchanged) ─────────────────────
-
-const CONTENT_POS = new Set([
-  'NN', 'NNS', 'NNP', 'NNPS',
-  'JJ', 'JJR', 'JJS',
-  'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ',
-  'RB', 'RBR', 'RBS',
-  'CD',                         // cardinal numbers (content‑like)
-]);
-
-/** Punctuation POS tags that should not be syllabified. */
-const PUNCT_TAGS = new Set([
-  ',', '.', ':', ';', '!', '?',
-  '-LRB-', '-RRB-', '``', "''",
-  '--', '...', '"', "'",
-  '(', ')', '[', ']', '{', '}',
-]);
-
-export function isPunctuation(tag: string): boolean {
-  return PUNCT_TAGS.has(tag);
-}
-
-/**
- * Quotation-mark tags. Quotes are tokens (never syllabified) but NOT prosodic
- * breaks: a quoted word inside a clause is read in one breath — no intonational
- * boundary, no caesura.
- */
-const QUOTE_TAGS = new Set(['``', "''", '"', "'"]);
-
-export function isQuoteTag(tag: string): boolean {
-  return QUOTE_TAGS.has(tag);
-}
-
-function isContentWord(tag: string): boolean {
-  return CONTENT_POS.has(tag);
-}
-
-/**
- * Lowercase the first alphabetic character of every line. Kept available but
- * NOT called by default — empirically net-negative with UDPipe (see parseDocument).
- */
-export function lowerLineInitials(text: string): string {
-  return text
-    .split('\n')
-    .map((line) => line.replace(/[A-Za-z]/, (c) => c.toLowerCase()))
-    .join('\n');
-}
-
-// Archaic / Early-Modern English forms the UD model (trained on modern text)
-// systematically mis-tags. This is a closed lexicon of forms that are virtually
-// never modern words, so an unconditional retag is safe domain adaptation for a
-// verse tool (it replaces the role the old en-pos correction layer played for
-// these tokens). Surface forms are matched lowercased, sans apostrophes.
-const ARCHAIC_POS: Record<string, string> = {
-  thy: 'PRP$', thine: 'PRP$',
-  thee: 'PRP', thou: 'PRP', ye: 'PRP',
-  hath: 'VBZ', doth: 'VBZ', saith: 'VBZ',
-  hast: 'VBP', dost: 'VBP', wilt: 'MD',
-  shalt: 'MD', canst: 'MD', wouldst: 'MD', shouldst: 'MD', couldst: 'MD',
-  hadst: 'VBD', didst: 'VBD', wast: 'VBD', wert: 'VBD',
-};
-
-// ── Dash handling (unchanged) ────────────────────────────────────────
-
-/**
- * Standalone en/em/figure/bar/minus dashes (or a run of 2+ hyphens) are prosodic
- * breaks (a dash caesura), not stress-bearing tokens. Re-tagged to the Penn dash
- * class ':' so they drop out of syllabification & scansion and mark a pause.
- */
-const DASH_GLYPH_RE = /^(?:[‒–—―−]+|-{2,})$/;
-function isDashGlyph(word: string): boolean {
-  return DASH_GLYPH_RE.test(word);
-}
-
-const DASH_CLASS = '‒–—―−';
-const DASH_GLYPHS_RE = new RegExp(`[${DASH_CLASS}]`, 'g');
-const DASH_PAREN_RE = new RegExp(`([${DASH_CLASS}])([^${DASH_CLASS}]*?[.!?][^${DASH_CLASS}]*?)([${DASH_CLASS}])`, 'g');
-
-/**
- * Normalize dash *usages* to colon-class clause-breaks BEFORE parsing. A dash is
- * an ι (intonational-unit) boundary — a stronger pause than a comma. We fold
- * every dash usage into a canonical glyph, neutralise sentence-final punctuation
- * inside a dash-delimited parenthetical (so the line stays one sentence), then
- * rewrite the dashes to a colon-class break (which prosodic.ts reads as an ι
- * boundary). Unspaced hyphen compounds ("torch-flames") are left intact.
- */
-function normalizeDashesToClauseBreaks(text: string): string {
-  text = text.replace(/(^|\s)-+(?=\s|$)/g, '$1–');
-  text = text.replace(/-{2,}/g, '–');
-  text = text.replace(DASH_PAREN_RE, (_m, a, inner, b) => a + inner.replace(/[.!?]+/g, ',') + b);
-  text = text.replace(DASH_GLYPHS_RE, ' : ');
-  text = text.replace(/(?:\s*:\s*){2,}/g, ' : ')
-             .replace(/\s+:/g, ' :')
-             .replace(/:(\S)/g, ': $1')
-             .replace(/^\s*:\s*/, '')
-             .replace(/\s{2,}/g, ' ')
-             .trim();
-  return text;
-}
-
-// ── Clitic / contraction re‑merge (UDPipe-specific) ──────────────────
-// UDPipe tokenises contractions and elisions on the apostrophe boundary, e.g.
-//   it's   → it + 's            don't → do + n't        we'll → we + 'll
-//   th'expense → th' + expense  'Tis  → ' + Tis         fix'd → fix + 'd
-// For scansion a contraction must be ONE orthographic word (one syllable count,
-// one stress domain). We re-merge using UDPipe's SpaceAfter flag (which marks
-// tokens that were contiguous in the source) plus the apostrophe shape:
-//   • a LEFT clitic  (apostrophe-initial, or n't) merges into the previous word,
-//     EXCEPT the possessive 's (XPOS=POS), which stays split (as it always has);
-//   • a RIGHT proclitic (a short apostrophe-final piece like "th'", or a bare
-//     leading apostrophe before an aphaeresis like 'tis/'twas) merges into the
-//     next word.
-
-const APOS = /['’]/;
-const LEFT_CLITIC_RE = /^['’]([a-z]+)?$|^n['’]?t$/i;       // 's 've 'll 'd 're 'm n't
-const RIGHT_PROCLITIC_RE = /^[a-z]{1,3}['’]$/i;            // th' o' d' ne'
-const APHAERESIS = new Set(['tis', 'twas', 'twere', 'twill', 'twould', 'gainst', 'neath', 'tween', 'twixt', 'til', 'cause', 'em', 'round', 'bout']);
-
-interface Cluster {
-  tokens: UDWord[];
-  repr: UDWord;       // the token that carries the syntactic role / POS
-}
-
-/** Group UDPipe words into orthographic clusters, re-merging clitics. */
-function clusterWords(uds: UDWord[]): { clusters: Cluster[]; idToCluster: Map<number, number> } {
-  const idToCluster = new Map<number, number>();
-  const clusters: Cluster[] = [];
-
-  for (let i = 0; i < uds.length; i++) {
-    const w = uds[i];
-    const prev = uds[i - 1];
-    const contiguous = prev ? prev.spaceAfter === false : false;
-    const isLeftClitic =
-      contiguous && w.xpos !== 'POS' && LEFT_CLITIC_RE.test(w.form);
-
-    if (isLeftClitic && clusters.length > 0) {
-      clusters[clusters.length - 1].tokens.push(w);
-      idToCluster.set(w.id, clusters.length - 1);
-      continue;
-    }
-    clusters.push({ tokens: [w], repr: w });
-    idToCluster.set(w.id, clusters.length - 1);
-  }
-
-  // Right-merge pass: a cluster that is a lone proclitic (th') or a bare leading
-  // apostrophe before an aphaeresis ('tis) folds into the following cluster.
-  const merged: Cluster[] = [];
-  for (let c = 0; c < clusters.length; c++) {
-    const cl = clusters[c];
-    const next = clusters[c + 1];
-    const onlyTok = cl.tokens.length === 1 ? cl.tokens[0] : null;
-    const contiguous = onlyTok ? onlyTok.spaceAfter === false : false;
-    const nextWord = next?.repr;
-
-    const isProclitic =
-      !!onlyTok && contiguous && !!nextWord &&
-      (RIGHT_PROCLITIC_RE.test(onlyTok.form) ||
-        (/^['’]$/.test(onlyTok.form) && APHAERESIS.has(nextWord.form.toLowerCase())));
-
-    if (isProclitic && next) {
-      next.tokens.unshift(onlyTok!);                 // prepend proclitic
-      for (const t of cl.tokens) idToCluster.set(t.id, merged.length); // re-point to next cluster's eventual index
-      // The next cluster will be pushed next iteration; fix its index mapping then.
-      // Mark by leaving cl out (skip pushing it).
-      // Re-point all of next's tokens to current merged length too:
-      continue;
-    }
-    merged.push(cl);
-  }
-
-  // Rebuild idToCluster cleanly against the merged list (indices shifted by right-merges).
-  idToCluster.clear();
-  for (let c = 0; c < merged.length; c++) {
-    for (const t of merged[c].tokens) idToCluster.set(t.id, c);
-    // representative = first token that is neither a left-clitic nor a proclitic
-    merged[c].repr =
-      merged[c].tokens.find(
-        (t) => !(t.xpos !== 'POS' && LEFT_CLITIC_RE.test(t.form)) && !RIGHT_PROCLITIC_RE.test(t.form) && !/^['’]$/.test(t.form),
-      ) ?? merged[c].tokens[0];
-  }
-
-  return { clusters: merged, idToCluster };
-}
-
-// Dependency labels are passed through to `ClsDependency.dependentType` as RAW
-// Universal Dependencies relations (obl, nsubj:pass, compound, nmod:poss, …) —
-// they are deliberately NOT folded into the old Stanford names. The canonical
-// normaliser `calliope/deps.ts` maps every UD relation onto the engine's Scenario
-// label space (canonicalRel), so new UD tags are accommodated there, not hidden
-// here.
-
-// ── Public API ───────────────────────────────────────────────────────
-
-export function parseDocument(text: string): ClsDocument {
-  // Normalise curly/typographic apostrophes to straight ' so contractions and
-  // elisions tokenise identically regardless of glyph.
-  text = text.replace(/[‘’ʼ′]/g, "'");
-  // Collapse runs of sentence-final punctuation (ellipsis, "!!") to a single mark.
-  text = text.replace(/([.!?])\1+/g, '$1');
-  // Dashes → colon-class clause-breaks (see helper above).
-  text = normalizeDashesToClauseBreaks(text);
-
-  // NOTE on line-initial caps: lowering the first letter of each line before
-  // tagging (the role the old `normalizeCaps` played) was tested and is NET
-  // NEGATIVE with UDPipe — it recovers cases like "Nap"/"Gap" (UH→NN) but a
-  // line-initial capital often HELPS UDPipe's parse (e.g. "Through Eden took…"
-  // parses "Eden" as nsubj when capitalised, obl when lowercased), so it changes
-  // more scansions than it fixes. Left disabled; see lowerLineInitials() below.
-
-  const udSentences: UDSentence[] = nlp().parse(text, { inputMode: 'presegmented' });
-
-  const sentences: ClsSentence[] = [];
-  let absoluteOffset = 0;
-
-  udSentences.forEach((ud, si) => {
-    const { clusters, idToCluster } = clusterWords(ud.words);
-
-    // ---- 1. Build ClsWord array ----
-    const words: ClsWord[] = clusters.map((cl, i) => {
-      // Preserve the ORIGINAL case of the surface form (UDPipe keeps it); only
-      // lowercase a private lookup key for the archaic-lexicon / dash checks.
-      // Lowercasing `word` itself lost every proper-noun capital ("pakistan",
-      // "marcel proust") in the display and projection; downstream stress/name
-      // lookups all lowercase internally, so case in `word` is display-only.
-      const surfaceRaw = cl.tokens.map((t) => t.form).join('');
-      const surface = surfaceRaw.toLowerCase();
-      // Penn tag: use the raw XPOS when it already is Penn (EWT/GUM), else derive
-      // it from UPOS+FEATS (LinES/ParTUT emit non-Penn XPOS the pipeline can't read).
-      const rawTag = pennTagOf(cl.repr);
-      const archaic = ARCHAIC_POS[surface.replace(/['’]/g, '')];
-      const tag = isDashGlyph(surfaceRaw) ? ':' : (archaic ?? rawTag);
-      return {
-        index: i + 1,
-        lexicalClass: tag,
-        lexicalDetails: cl.repr.feats,
-        lexicalPlural: tag === 'NNS' || tag === 'NNPS',
-        position: '',
-        word: surfaceRaw,
-        absoluteIndex: absoluteOffset + i,
-        isContent: isContentWord(tag),
-        syllables: [],
-        phraseStress: 0,
-        dependency: undefined,
-        node: undefined,
-      };
-    });
-
-    // Sentence-initial de-capitalisation (mirrors en-norm.normalizeCaps in the
-    // pre-UD path): lower the first letter of the sentence's first word UNLESS it
-    // is a proper noun, so "The"→"the" and "I"→"i" (an orthographic capital forced
-    // by line position carries no lexical signal) while mid-line proper nouns
-    // ("Marcel Proust", "Pakistan") keep their caps for display/projection.
-    for (const w of words) {
-      if (isPunctuation(w.lexicalClass)) continue;
-      if (!/^(NNP|NNPS)$/.test(w.lexicalClass) && /^[A-Z]/.test(w.word)) {
-        w.displayWord = w.word;      // keep the original surface for reports/phonopoetics
-        w.word = w.word[0].toLowerCase() + w.word.slice(1);
-      }
-      break;
-    }
-
-    // ---- 2. Build ClsDependency array ----
-    const dependencies: ClsDependency[] = [];
-    clusters.forEach((cl, depIdx) => {
-      const r = cl.repr;
-      let govIdx: number | undefined;
-      if (r.head === 0) {
-        govIdx = undefined;                            // attaches to root
-      } else {
-        const g = idToCluster.get(r.head);
-        // If the representative's head fell inside its own cluster (e.g. a copula
-        // clitic), follow that clitic's head out of the cluster.
-        if (g === depIdx) {
-          const external = cl.tokens
-            .map((t) => idToCluster.get(t.head))
-            .find((gi) => gi !== undefined && gi !== depIdx);
-          govIdx = external;
-        } else {
-          govIdx = g;
-        }
-      }
-
-      const depWord = words[depIdx];
-      const govWord = govIdx !== undefined ? words[govIdx] : null;
-
-      dependencies.push({
-        index: depIdx + 1,
-        governorIndex: govIdx !== undefined ? govIdx + 1 : 0,
-        dependentIndex: depIdx + 1,
-        dependentType: govIdx === undefined ? 'root' : r.deprel,
-        governorName: govWord ? govWord.word : 'ROOT',
-        dependentName: depWord.word,
-        governor: govWord as unknown as ClsWord,
-        dependent: depWord,
-      });
-    });
-
-    // Ensure a ROOT dependency exists.
-    if (!dependencies.some((d) => d.governorIndex === 0) && words.length > 0) {
-      dependencies.push({
-        index: 0,
-        governorIndex: 0,
-        dependentIndex: 1,
-        dependentType: 'root',
-        governorName: 'ROOT',
-        dependentName: words[0].word,
-        governor: null as unknown as ClsWord,
-        dependent: words[0],
-      });
-    }
-
-    // Back‑reference: each word stores the dependency edge where it is dependent.
-    words.forEach((w) => {
-      w.dependency = dependencies.find((d) => d.dependent === w);
-    });
-
-    // UDPipe XPOS correction (the role en-pos + tagfix.ts played pre-UD): fix the
-    // systematic mis-tags UDPipe makes on terse, decontextualised verse via
-    // en-lexicon cross-check.  Runs HERE (in the parser, after deps are attached)
-    // so direct `parseDocument` consumers — and every engine — see corrected tags;
-    // rule (3) needs the dependency back-references just set above.
-    correctUDPipePos({ index: si + 1, nodes: null, dependencies, words, xml: '' });
-
-    // ---- 3. Build phrase‑structure node tree from the dependency graph ----
-    const rootNode = buildDepNodeTree(words, dependencies);
-    const wordNodeMap = new Map<number, ClsNode>();
-    collectWordNodes(rootNode, wordNodeMap);
-    words.forEach((w) => {
-      w.node = wordNodeMap.get(w.index);
-    });
-
-    sentences.push({
-      index: si + 1,
-      nodes: rootNode,
-      dependencies,
-      words,
-      xml: '',
-    });
-
-    absoluteOffset += words.length;
-  });
-
-  return { sentences, xml: '' };
-}
-
-// ── Dependency → constituency projection ─────────────────────────────
-// phonological.ts groups clitic groups into phonological phrases by finding the
-// smallest phrase node containing them, so it needs a properly nested, position-
-// ordered constituency tree. We synthesise one by projection: each head plus its
-// dependent subtrees forms a phrase, labelled by the head's POS family.
-
-function phraseType(tag: string): string {
-  if (/^(NN|NNS|NNP|NNPS|PRP|PRP\$|DT|CD|WP|WDT|EX)$/.test(tag)) return 'NP';
-  if (/^(VB|VBD|VBG|VBN|VBP|VBZ|MD)$/.test(tag)) return 'VP';
-  if (/^(IN|TO)$/.test(tag)) return 'PP';
-  if (/^(JJ|JJR|JJS)$/.test(tag)) return 'ADJP';
-  if (/^(RB|RBR|RBS|WRB)$/.test(tag)) return 'ADVP';
-  return 'XP';
-}
-
-function buildDepNodeTree(words: ClsWord[], deps: ClsDependency[]): ClsNode {
-  const sq: ClsNode = { index: '1', nodeName: 'SQ', parent: null, contains: [] };
-  if (words.length === 0) return sq;
-
-  // children[g] = list of dependent word-indices (1-based) governed by g (1-based);
-  // roots are governed by 0.
-  const children = new Map<number, number[]>();
-  for (const d of deps) {
-    if (d.dependentIndex < 1 || d.dependentIndex > words.length) continue;
-    const g = d.governorIndex;
-    if (!children.has(g)) children.set(g, []);
-    children.get(g)!.push(d.dependentIndex);
-  }
-
-  const build = (wordIdx: number, parent: ClsNode): ClsNode => {
-    const word = words[wordIdx - 1];
-    const kids = (children.get(wordIdx) ?? []).filter((k) => k !== wordIdx);
-    if (kids.length === 0) {
-      const leaf = createWordLeaf(word);
-      leaf.parent = parent;
-      return leaf;
-    }
-    const node: ClsNode = {
-      index: `ph_${wordIdx}`,
-      nodeName: phraseType(word.lexicalClass),
-      parent,
-      contains: [],
-    };
-    // Order head + dependents by surface position for a projective tree.
-    const ordered = [...kids, wordIdx].sort((a, b) => a - b);
-    for (const idx of ordered) {
-      if (idx === wordIdx) {
-        const leaf = createWordLeaf(word);
-        leaf.parent = node;
-        node.contains.push(leaf);
-      } else {
-        node.contains.push(build(idx, node));
-      }
-    }
-    return node;
-  };
-
-  const roots = (children.get(0) ?? []).sort((a, b) => a - b);
-  if (roots.length === 0) {
-    // No explicit root: attach all words as leaves under SQ.
-    for (const w of words) {
-      const leaf = createWordLeaf(w);
-      leaf.parent = sq;
-      sq.contains.push(leaf);
-    }
-    return sq;
-  }
-  for (const r of roots) {
-    const child = build(r, sq);
-    sq.contains.push(child);
-  }
-
-  // Attach any orphan words (rare) directly under SQ.
-  const attached = new Set<number>();
-  collectAttachedWordIndices(sq, attached);
-  for (const w of words) {
-    if (!attached.has(w.index)) {
-      const leaf = createWordLeaf(w);
-      leaf.parent = sq;
-      sq.contains.push(leaf);
-    }
-  }
-  return sq;
-}
-
-// ── Leaf / traversal helpers (unchanged) ─────────────────────────────
-
-function createWordLeaf(word: ClsWord): ClsNode {
-  return {
-    index: `w${word.index}`,
-    nodeName: word.index.toString(),
-    parent: null,
-    contains: [word],
-  };
-}
-
-function collectWordNodes(node: ClsNode, map: Map<number, ClsNode>): void {
-  for (const child of node.contains) {
-    if (child instanceof Object && 'word' in (child as any)) {
-      map.set((child as ClsWord).index, node);
-    } else if (child instanceof Object && 'index' in (child as any)) {
-      collectWordNodes(child as ClsNode, map);
-    }
-  }
-}
-
-function collectAttachedWordIndices(node: ClsNode, set: Set<number>): void {
-  for (const child of node.contains) {
-    if (child instanceof Object && 'word' in (child as any)) {
-      set.add((child as ClsWord).index);
-    } else if (child instanceof Object && 'index' in (child as any)) {
-      collectAttachedWordIndices(child as ClsNode, set);
-    }
-  }
-}