calliope-ts 0.0.2 → 0.0.3

package/dist/parser.js

@@ -1,234 +1,34 @@
-// parser.ts — Syntactic dependency parser powered by FinNLP, producing
-// ClsDocument with full dependency graph and phrase‑structure node tree
-// that matches the Universal Dependencies format as used in McAleese’s Calliope.
-import * as Lexed from 'lexed';
-import * as EnPos from 'en-pos';
-import * as EnParse from 'en-parse';
-import * as EnNorm from 'en-norm';
-import { correctTags } from './tagfix.js';
-import { applyDepFixes } from './depfix.js';
-const CONTRACTION_MAP = {
-    "we'll": { host: 'we', clitic: 'will' },
-    "we've": { host: 'we', clitic: 'have' },
-    "we're": { host: 'we', clitic: 'are' },
-    "we'd": { host: 'we', clitic: 'would' },
-    "i'll": { host: 'i', clitic: 'will' },
-    "i've": { host: 'i', clitic: 'have' },
-    "i'm": { host: 'i', clitic: 'am' },
-    "i'd": { host: 'i', clitic: 'would' },
-    "you'll": { host: 'you', clitic: 'will' },
-    "you've": { host: 'you', clitic: 'have' },
-    "you're": { host: 'you', clitic: 'are' },
-    "you'd": { host: 'you', clitic: 'would' },
-    "he'll": { host: 'he', clitic: 'will' },
-    "he's": { host: 'he', clitic: 'is' },
-    "he'd": { host: 'he', clitic: 'would' },
-    "she'll": { host: 'she', clitic: 'will' },
-    "she's": { host: 'she', clitic: 'is' },
-    "she'd": { host: 'she', clitic: 'would' },
-    "it'll": { host: 'it', clitic: 'will' },
-    "it'd": { host: 'it', clitic: 'would' },
-    "they'll": { host: 'they', clitic: 'will' },
-    "they've": { host: 'they', clitic: 'have' },
-    "they're": { host: 'they', clitic: 'are' },
-    "they'd": { host: 'they', clitic: 'would' },
-    "that's": { host: 'that', clitic: 'is' },
-    "that'll": { host: 'that', clitic: 'will' },
-    "this'll": { host: 'this', clitic: 'will' },
-    "it's": { host: 'it', clitic: 'is' },
-    "who'll": { host: 'who', clitic: 'will' },
-    "who's": { host: 'who', clitic: 'is' },
-    "who'd": { host: 'who', clitic: 'would' },
-    "who've": { host: 'who', clitic: 'have' },
-    "what's": { host: 'what', clitic: 'is' },
-    "there's": { host: 'there', clitic: 'is' },
-    "here's": { host: 'here', clitic: 'is' },
-    "where's": { host: 'where', clitic: 'is' },
-    "when's": { host: 'when', clitic: 'is' },
-    "how's": { host: 'how', clitic: 'is' },
-    "why's": { host: 'why', clitic: 'is' },
-    "one's": { host: 'one', clitic: 'is' },
-    "let's": { host: 'let', clitic: 'us' },
-    "y'all": { host: 'you', clitic: 'all' },
-    "don't": { host: 'do', clitic: 'not' },
-    "can't": { host: 'can', clitic: 'not' },
-    "won't": { host: 'will', clitic: 'not' },
-    "shouldn't": { host: 'should', clitic: 'not' },
-    "couldn't": { host: 'could', clitic: 'not' },
-    "wouldn't": { host: 'would', clitic: 'not' },
-    "isn't": { host: 'is', clitic: 'not' },
-    "aren't": { host: 'are', clitic: 'not' },
-    "wasn't": { host: 'was', clitic: 'not' },
-    "weren't": { host: 'were', clitic: 'not' },
-    "haven't": { host: 'have', clitic: 'not' },
-    "hasn't": { host: 'has', clitic: 'not' },
-    "hadn't": { host: 'had', clitic: 'not' },
-    "didn't": { host: 'did', clitic: 'not' },
-    "doesn't": { host: 'does', clitic: 'not' },
-    "ain't": { host: 'am', clitic: 'not' },
-    "might've": { host: 'might', clitic: 'have' },
-    "would've": { host: 'would', clitic: 'have' },
-    "should've": { host: 'should', clitic: 'have' },
-    "could've": { host: 'could', clitic: 'have' },
-    "must've": { host: 'must', clitic: 'have' },
-};
-/** Archaic poetic "-'d" preterite: any -'d form that is not a genuine pronoun/wh
- *  contraction (those live in CONTRACTION_MAP and are checked first). */
-const ARCHAIC_D_RE = /^[a-z]{2,}'d$/;
-/**
- * Tokenise the raw (un‑normalised) text into word‑like segments,
- * marking which are contracted forms.
- */
-function tokenizeRawText(text) {
-    const re = /\b[a-zA-Z]+(?:'[a-zA-Z]+)?\b/g;
-    const segments = [];
-    let match;
-    while ((match = re.exec(text)) !== null) {
-        const lower = match[0].toLowerCase();
-        const isContraction = lower in CONTRACTION_MAP;
-        segments.push({
-            text: lower,
-            isContraction,
-            isArchaicD: !isContraction && ARCHAIC_D_RE.test(lower),
-        });
-    }
-    return segments;
-}
-/**
- * Re‑merge contraction pairs in a sentence's ClsWord array.
- *
- * Segments from the raw text are walked in parallel with the sentence's
- * tokens. Non‑contraction segments consume 1 token; contraction segments
- * consume 2 tokens (host + clitic), which are merged into a single ClsWord
- * that preserves the host's properties and the original contracted form.
- *
- * Returns the updated word array.
- */
-function mergeContractionsInSentence(words, segments, startSegmentIdx) {
-    const merged = [];
-    let tokenIdx = 0;
-    let segIdx = startSegmentIdx;
-    while (tokenIdx < words.length && segIdx < segments.length) {
-        // Punctuation tokens have NO raw-text segment (tokenizeRawText matches only
-        // letter sequences), so they must not consume a segment.  Otherwise a
-        // sentence ending in "!"/"." over-advances segIdx and misaligns every later
-        // sentence — dropping a pronoun that precedes a contraction ("No more! He
-        // won't…" lost "He" and mis-tagged the contraction PRP).
-        if (isPunctuation(words[tokenIdx].lexicalClass)) {
-            merged.push(words[tokenIdx]);
-            tokenIdx++;
-            continue;
-        }
-        const seg = segments[segIdx];
-        // Archaic poetic preterite ("fix'd", "lov'd"): en-norm expands the -'d into a
-        // separate "would"/"had" token, splitting one syllable into two words AND
-        // shifting every later token off its raw segment.  Re-merge host + modal back
-        // into the apostrophized form — but ONLY when the spurious modal is actually
-        // there (conditional, so a hand-typed "fix'd" that survived intact is safe).
-        if (seg.isArchaicD) {
-            const next = tokenIdx + 1 < words.length ? words[tokenIdx + 1].word.toLowerCase() : '';
-            if (next === 'would' || next === 'had') {
-                merged.push({ ...words[tokenIdx], word: seg.text });
-                tokenIdx += 2;
-                segIdx++;
-                continue;
-            }
-            merged.push(words[tokenIdx]);
-            tokenIdx++;
-            segIdx++;
-            continue;
-        }
-        if (seg.isContraction) {
-            if (tokenIdx + 1 >= words.length) {
-                merged.push(words[tokenIdx]);
-                tokenIdx++;
-                segIdx++;
-                continue;
-            }
-            const hostWord = words[tokenIdx];
-            const cliticWord = words[tokenIdx + 1];
-            // Keep the host as the merged word, update its text to the contracted form.
-            const mergedWord = {
-                ...hostWord,
-                word: seg.text,
-            };
-            merged.push(mergedWord);
-            tokenIdx += 2;
-            segIdx++;
-        }
-        else {
-            merged.push(words[tokenIdx]);
-            tokenIdx++;
-            segIdx++;
-        }
-    }
-    // Append any remaining words that exceeded segment count.
-    while (tokenIdx < words.length) {
-        merged.push(words[tokenIdx]);
-        tokenIdx++;
-    }
-    return { words: merged, consumedSegments: segIdx - startSegmentIdx };
-}
-function mergeHyphenatedWords(words) {
-    const merged = [];
-    let i = 0;
-    while (i < words.length) {
-        if (i + 2 < words.length &&
-            words[i + 1].word === '-' &&
-            !isPunctuation(words[i].lexicalClass) &&
-            !isPunctuation(words[i + 2].lexicalClass)) {
-            const combined = words[i].word + '-' + words[i + 2].word;
-            const mergedWord = {
-                ...words[i],
-                word: combined,
-                lexicalClass: words[i + 2].lexicalClass.startsWith('N') ? words[i + 2].lexicalClass : words[i].lexicalClass,
-                isContent: words[i].isContent || words[i + 2].isContent,
-            };
-            merged.push(mergedWord);
-            i += 3;
-        }
-        else {
-            merged.push(words[i]);
-            i++;
-        }
-    }
-    return merged;
-}
-// ─── Mappings: FinNLP → Antelope‑compatible labels ──────────────────
-// FinNLP (aka Stanford Dependency Types) versus Antelope NLP dependency (aka Universal Dependencies) type equivalencies.
-const FIN_TO_ANTELOPE_LABEL = {
-    AUX: 'aux',
-    AUXPASS: 'auxpass',
-    NSUBJ: 'nsubj',
-    NSUBJPASS: 'nsubjpass',
-    DOBJ: 'dobj',
-    IOBJ: 'iobj',
-    OBL: 'pobj', // OBL (oblique) ≈ pobj in Antelope
-    DET: 'det',
-    CASE: 'prep', // CASE marks the preposition; matched to UD prep
-    CC: 'cc',
-    COMPMARK: 'mark',
-    ADVMARK: 'mark',
-    NOMD: 'poss', // nominal modifier ≈ possessive relation
-    AMOD: 'amod',
-    ADVMOD: 'advmod',
-    ADVCL: 'advcl',
-    XCOMP: 'xcomp',
-    CCOMP: 'ccomp',
-    ACL: 'acl',
-    VPRT: 'prt',
-    NUMDMOD: 'nummod',
-    EXPL: 'expl',
-    DISCOURSE: 'discourse',
-    PUNCT: 'punct',
-    INTERJ: 'intj',
-    EXT: 'dep', // extension – best mapped to generic 'dep'
-    DEP: 'dep',
-    ROOT: 'root',
-};
-function toAntelopeLabel(finLabel) {
-    return FIN_TO_ANTELOPE_LABEL[finLabel] ?? finLabel.toLowerCase();
+// parser.ts — Syntactic dependency parser powered by UDPipe (English GUM model),
+// producing a ClsDocument with a full dependency graph and a phrase‑structure node
+// tree in the Universal Dependencies format used in McAleese's Calliope.
+//
+// HISTORY: this module previously ran a staged FinNLP pipeline
+// (en-norm → lexed → en-pos → en-parse) with hand-written tag/dep correction
+// layers (tagfix.ts / depfix.ts) to patch en-parse's systematic errors. It now
+// delegates tokenisation, POS tagging, and dependency parsing to UDPipe via the
+// `udpipe-node` package (a pure-WASM build — no native binary, no subprocess),
+// which is a far more accurate parser. UDPipe's output maps cleanly onto the
+// existing data model:
+//   • XPOS column is Penn Treebank  → ClsWord.lexicalClass (unchanged downstream)
+//   • DEPREL column is Universal Dependencies → ClsDependency.dependentType
+// The correction layers are therefore no longer applied on this path.
+import { createUDPipe } from 'udpipe-node/wasm';
+import { correctUDPipePos } from './calliope/postag.js';
+import { pennTagOf } from './calliope/udpos.js';
+// ── UDPipe instance (lazy singleton) ────────────────────────────────
+// The "./wasm" entry point pre-initialises the WASM runtime via top-level await,
+// so by the time this module is imported the engine is ready and construction /
+// parsing are fully synchronous — `parseDocument` keeps its synchronous contract.
+let _nlp = null;
+function nlp() {
+    // CALLIOPE_UDPIPE_MODEL lets us swap the UDPipe model (EWT / GUM / LinES /
+    // ParTUT) for auditing — different treebanks tag XPOS quite differently, so the
+    // model choice materially affects the parse the phonological pipeline consumes.
+    // Unset → the bundled GUM model.
+    const modelPath = process.env.CALLIOPE_UDPIPE_MODEL || undefined;
+    return (_nlp ??= createUDPipe({ defaultInputMode: 'presegmented', modelPath }));
 }
+// ── POS / punctuation classification (unchanged) ─────────────────────
 const CONTENT_POS = new Set([
     'NN', 'NNS', 'NNP', 'NNPS',
     'JJ', 'JJR', 'JJS',
@@ -241,266 +41,253 @@ const PUNCT_TAGS = new Set([
     ',', '.', ':', ';', '!', '?',
     '-LRB-', '-RRB-', '``', "''",
     '--', '...', '"', "'",
-    '(', ')', '[', ']', '{', '}', // FinNLP emits literal bracket tags, not -LRB-/-RRB-
+    '(', ')', '[', ']', '{', '}',
 ]);
 export function isPunctuation(tag) {
     return PUNCT_TAGS.has(tag);
 }
 /**
- * Quotation-mark tags.  Quotes are tokens (never syllabified) but NOT prosodic
- * breaks: a quoted word inside a clause ('call them "wisdom" teeth') is read in
- * one breath — no intonational boundary, no caesura.  Treating quotes as IU
- * boundaries fragmented such lines into 3-4 IUs and flipped their meter.
+ * Quotation-mark tags. Quotes are tokens (never syllabified) but NOT prosodic
+ * breaks: a quoted word inside a clause is read in one breath — no intonational
+ * boundary, no caesura.
  */
 const QUOTE_TAGS = new Set(['``', "''", '"', "'"]);
 export function isQuoteTag(tag) {
     return QUOTE_TAGS.has(tag);
 }
+function isContentWord(tag) {
+    return CONTENT_POS.has(tag);
+}
 /**
- * Typographic dashes that FinNLP mis-tags as content words.  A standalone en-dash
- * "–", em-dash "—", horizontal bar "―", minus sign "−" or a run of 2+ hyphens is a
- * prosodic break (a dash caesura), NOT a stress-bearing token — but FinNLP's POS
- * model labels the bare glyph `NNP` (proper noun), so it flowed through the
- * pipeline, received a syllable, and even attracted a strong metrical beat
- * ("crunch – a guilt" scanned the dash as 's').  We re-tag any such glyph to the
- * Penn dash/colon class ':' (already an IU/caesura boundary) at parse time, so the
- * dash drops out of syllabification & scansion and instead marks a pause.
- * A *single* hyphen-minus is deliberately excluded — it joins hyphenated compounds
- * ("torch-flames") handled by mergeHyphenatedWords.
+ * Lowercase the first alphabetic character of every line. Kept available but
+ * NOT called by default — empirically net-negative with UDPipe (see parseDocument).
+ */
+export function lowerLineInitials(text) {
+    return text
+        .split('\n')
+        .map((line) => line.replace(/[A-Za-z]/, (c) => c.toLowerCase()))
+        .join('\n');
+}
+// Archaic / Early-Modern English forms the UD model (trained on modern text)
+// systematically mis-tags. This is a closed lexicon of forms that are virtually
+// never modern words, so an unconditional retag is safe domain adaptation for a
+// verse tool (it replaces the role the old en-pos correction layer played for
+// these tokens). Surface forms are matched lowercased, sans apostrophes.
+const ARCHAIC_POS = {
+    thy: 'PRP$', thine: 'PRP$',
+    thee: 'PRP', thou: 'PRP', ye: 'PRP',
+    hath: 'VBZ', doth: 'VBZ', saith: 'VBZ',
+    hast: 'VBP', dost: 'VBP', wilt: 'MD',
+    shalt: 'MD', canst: 'MD', wouldst: 'MD', shouldst: 'MD', couldst: 'MD',
+    hadst: 'VBD', didst: 'VBD', wast: 'VBD', wert: 'VBD',
+};
+// ── Dash handling (unchanged) ────────────────────────────────────────
+/**
+ * Standalone en/em/figure/bar/minus dashes (or a run of 2+ hyphens) are prosodic
+ * breaks (a dash caesura), not stress-bearing tokens. Re-tagged to the Penn dash
+ * class ':' so they drop out of syllabification & scansion and mark a pause.
  */
 const DASH_GLYPH_RE = /^(?:[‒–—―−]+|-{2,})$/;
 function isDashGlyph(word) {
     return DASH_GLYPH_RE.test(word);
 }
-const DASH_CLASS = '‒–—―−'; // figure / en / em / bar / minus
+const DASH_CLASS = '‒–—―−';
 const DASH_GLYPHS_RE = new RegExp(`[${DASH_CLASS}]`, 'g');
 const DASH_PAREN_RE = new RegExp(`([${DASH_CLASS}])([^${DASH_CLASS}]*?[.!?][^${DASH_CLASS}]*?)([${DASH_CLASS}])`, 'g');
 /**
- * Normalize dash *usages* to comma clause-breaks BEFORE parsing.  In verse a dash
- * is a comma-like prosodic break — not a word, not a sentence end — but FinNLP
- * mis-handles it two ways: it glues a SPACE-flanked hyphen-minus into the
- * neighbouring word ("I still carry - Oh" → token "carry-Oh", then OOV), and it
- * tags a bare en/em-dash as a proper noun (NNP) that pollutes the dependency tree.
- * Worse, a parenthetical aside set off by dashes often contains sentence-final
- * punctuation ("– Oh, Petersburg! –") that splits the line into separate
- * sentences and severs the main clause's dependencies (here, carry↔address).
- *
- * So we (1) fold every dash usage — em/en/figure/bar/minus, a 2+ hyphen run, or a
- * space-flanked single hyphen — into a canonical dash glyph (leaving unspaced
- * hyphen compounds like "torch-flames" intact for `mergeHyphenatedWords`);
- * (2) neutralize sentence-final punctuation INSIDE a dash-delimited parenthetical
- * so the line stays one sentence; (3) rewrite the dashes to commas, which FinNLP
- * parses cleanly and which are the same prosodic break (a comma is an IU boundary
- * → caesura).  The verbatim original (dashes and all) is preserved by the caller
- * for the reading-view projection; only the parser's working copy is normalized.
+ * Normalize dash *usages* to colon-class clause-breaks BEFORE parsing. A dash is
+ * an ι (intonational-unit) boundary — a stronger pause than a comma. We fold
+ * every dash usage into a canonical glyph, neutralise sentence-final punctuation
+ * inside a dash-delimited parenthetical (so the line stays one sentence), then
+ * rewrite the dashes to a colon-class break (which prosodic.ts reads as an ι
+ * boundary). Unspaced hyphen compounds ("torch-flames") are left intact.
  */
 function normalizeDashesToClauseBreaks(text) {
-    // (1) space-flanked single/multi hyphen-minus, and any 2+ hyphen run → en-dash
     text = text.replace(/(^|\s)-+(?=\s|$)/g, '$1–');
     text = text.replace(/-{2,}/g, '–');
-    // (2) neutralize sentence-final punctuation between paired dashes (keeps it one sentence)
     text = text.replace(DASH_PAREN_RE, (_m, a, inner, b) => a + inner.replace(/[.!?]+/g, ',') + b);
-    // (3) dash glyphs → comma clause-break
-    text = text.replace(DASH_GLYPHS_RE, ',');
-    // tidy: collapse comma runs, no space before a comma, one space after, no leading comma
-    text = text.replace(/(?:\s*,\s*){2,}/g, ', ')
-        .replace(/\s+,/g, ',')
-        .replace(/,(\S)/g, ', $1')
-        .replace(/^\s*,\s*/, '');
+    text = text.replace(DASH_GLYPHS_RE, ' : ');
+    text = text.replace(/(?:\s*:\s*){2,}/g, ' : ')
+        .replace(/\s+:/g, ' :')
+        .replace(/:(\S)/g, ': $1')
+        .replace(/^\s*:\s*/, '')
+        .replace(/\s{2,}/g, ' ')
+        .trim();
     return text;
 }
-function isContentWord(tag) {
-    return CONTENT_POS.has(tag);
+// ── Clitic / contraction re‑merge (UDPipe-specific) ──────────────────
+// UDPipe tokenises contractions and elisions on the apostrophe boundary, e.g.
+//   it's   → it + 's            don't → do + n't        we'll → we + 'll
+//   th'expense → th' + expense  'Tis  → ' + Tis         fix'd → fix + 'd
+// For scansion a contraction must be ONE orthographic word (one syllable count,
+// one stress domain). We re-merge using UDPipe's SpaceAfter flag (which marks
+// tokens that were contiguous in the source) plus the apostrophe shape:
+//   • a LEFT clitic  (apostrophe-initial, or n't) merges into the previous word,
+//     EXCEPT the possessive 's (XPOS=POS), which stays split (as it always has);
+//   • a RIGHT proclitic (a short apostrophe-final piece like "th'", or a bare
+//     leading apostrophe before an aphaeresis like 'tis/'twas) merges into the
+//     next word.
+const APOS = /['’]/;
+const LEFT_CLITIC_RE = /^['’]([a-z]+)?$|^n['’]?t$/i; // 's 've 'll 'd 're 'm n't
+const RIGHT_PROCLITIC_RE = /^[a-z]{1,3}['’]$/i; // th' o' d' ne'
+const APHAERESIS = new Set(['tis', 'twas', 'twere', 'twill', 'twould', 'gainst', 'neath', 'tween', 'twixt', 'til', 'cause', 'em', 'round', 'bout']);
+/** Group UDPipe words into orthographic clusters, re-merging clitics. */
+function clusterWords(uds) {
+    const idToCluster = new Map();
+    const clusters = [];
+    for (let i = 0; i < uds.length; i++) {
+        const w = uds[i];
+        const prev = uds[i - 1];
+        const contiguous = prev ? prev.spaceAfter === false : false;
+        const isLeftClitic = contiguous && w.xpos !== 'POS' && LEFT_CLITIC_RE.test(w.form);
+        if (isLeftClitic && clusters.length > 0) {
+            clusters[clusters.length - 1].tokens.push(w);
+            idToCluster.set(w.id, clusters.length - 1);
+            continue;
+        }
+        clusters.push({ tokens: [w], repr: w });
+        idToCluster.set(w.id, clusters.length - 1);
+    }
+    // Right-merge pass: a cluster that is a lone proclitic (th') or a bare leading
+    // apostrophe before an aphaeresis ('tis) folds into the following cluster.
+    const merged = [];
+    for (let c = 0; c < clusters.length; c++) {
+        const cl = clusters[c];
+        const next = clusters[c + 1];
+        const onlyTok = cl.tokens.length === 1 ? cl.tokens[0] : null;
+        const contiguous = onlyTok ? onlyTok.spaceAfter === false : false;
+        const nextWord = next?.repr;
+        const isProclitic = !!onlyTok && contiguous && !!nextWord &&
+            (RIGHT_PROCLITIC_RE.test(onlyTok.form) ||
+                (/^['’]$/.test(onlyTok.form) && APHAERESIS.has(nextWord.form.toLowerCase())));
+        if (isProclitic && next) {
+            next.tokens.unshift(onlyTok); // prepend proclitic
+            for (const t of cl.tokens)
+                idToCluster.set(t.id, merged.length); // re-point to next cluster's eventual index
+            // The next cluster will be pushed next iteration; fix its index mapping then.
+            // Mark by leaving cl out (skip pushing it).
+            // Re-point all of next's tokens to current merged length too:
+            continue;
+        }
+        merged.push(cl);
+    }
+    // Rebuild idToCluster cleanly against the merged list (indices shifted by right-merges).
+    idToCluster.clear();
+    for (let c = 0; c < merged.length; c++) {
+        for (const t of merged[c].tokens)
+            idToCluster.set(t.id, c);
+        // representative = first token that is neither a left-clitic nor a proclitic
+        merged[c].repr =
+            merged[c].tokens.find((t) => !(t.xpos !== 'POS' && LEFT_CLITIC_RE.test(t.form)) && !RIGHT_PROCLITIC_RE.test(t.form) && !/^['’]$/.test(t.form)) ?? merged[c].tokens[0];
+    }
+    return { clusters: merged, idToCluster };
 }
-// ─── Public API ───────────────────────────────────────────────────
-/**
- * Parse a multi‑sentence text string and return a ClsDocument whose
- * internal structure mirrors the Antelope NLP (aka Universal Dependency type)
- * output from McAleese’s original Calliope implementation.
- */
+// Dependency labels are passed through to `ClsDependency.dependentType` as RAW
+// Universal Dependencies relations (obl, nsubj:pass, compound, nmod:poss, …) —
+// they are deliberately NOT folded into the old Stanford names. The canonical
+// normaliser `calliope/deps.ts` maps every UD relation onto the engine's Scenario
+// label space (canonicalRel), so new UD tags are accommodated there, not hidden
+// here.
+// ── Public API ───────────────────────────────────────────────────────
 export function parseDocument(text) {
-    // Collapse runs of sentence-final punctuation (ellipsis "...", "!!", "??")
-    // to a single mark BEFORE tokenisation.  FinNLP otherwise glues the surplus
-    // marks onto the preceding word ("springtime..." → token "springtime.." → OOV,
-    // mis-tagged JJ, mis-syllabified, and re-phrased), which made two lines that
-    // differ only in trailing punctuation scan differently.  This is metrically
-    // harmless (punctuation bears no syllable); the verbatim original is preserved
-    // by the caller and used for the reading projection.
+    // Normalise curly/typographic apostrophes to straight ' so contractions and
+    // elisions tokenise identically regardless of glyph.
+    text = text.replace(/[‘’ʼ′]/g, "'");
+    // Collapse runs of sentence-final punctuation (ellipsis, "!!") to a single mark.
     text = text.replace(/([.!?])\1+/g, '$1');
-    // Dashes → comma clause-breaks (fixes "carry-Oh" gluing AND the parenthetical
-    // sentence-split that severs main-clause dependencies).  See the helper above.
+    // Dashes → colon-class clause-breaks (see helper above).
     text = normalizeDashesToClauseBreaks(text);
-    // Pre‑scan the raw text for contraction positions before FinNLP
-    // normalises them away.
-    const rawSegments = tokenizeRawText(text);
-    // Run the FinNLP pipeline STAGED rather than via `Fin.Run`, so the tag-
-    // correction layer (tagfix.ts) sits between en-pos and en-parse: corrected
-    // tags repair the tagging AND the dependency tree built from it.  The
-    // stages below mirror finnlp's own Run() exactly (en-norm → lexed →
-    // en-pos → en-parse); lemmas are skipped (unused downstream).
-    const intercepted = EnNorm.resolveContractions(EnNorm.replaceConfusables(text));
-    const lexer = new Lexed.Lexed(intercepted).lexer();
-    const runner = { raw: text, intercepted, sentences: [] };
-    for (let li = 0; li < lexer.sentences.length; li++) {
-        const tokens = EnNorm.normalizeCaps(lexer.tokens[li]);
-        const tagging = new EnPos.Tag(tokens).initial().smooth();
-        const tags = correctTags(tokens, tagging.tags);
-        const depsTree = EnParse.tree(tags, tokens)[0];
-        runner.sentences.push({
-            sentence: lexer.sentences[li],
-            tokens, tags, lemmas: [],
-            depsTree,
-            // Post-parse dependency repair (depfix.ts): systematic en-parse
-            // attachment errors (noun-compound double-objects, dangling DT).
-            deps: applyDepFixes(tokens, tags, EnParse.toArray(depsTree)),
-            confidence: 0,
-        });
-    }
+    // NOTE on line-initial caps: lowering the first letter of each line before
+    // tagging (the role the old `normalizeCaps` played) was tested and is NET
+    // NEGATIVE with UDPipe — it recovers cases like "Nap"/"Gap" (UH→NN) but a
+    // line-initial capital often HELPS UDPipe's parse (e.g. "Through Eden took…"
+    // parses "Eden" as nsubj when capitalised, obl when lowercased), so it changes
+    // more scansions than it fixes. Left disabled; see lowerLineInitials() below.
+    const udSentences = nlp().parse(text, { inputMode: 'presegmented' });
     const sentences = [];
     let absoluteOffset = 0;
-    let segmentIdx = 0;
-    for (let si = 0; si < runner.sentences.length; si++) {
-        const s = runner.sentences[si];
-        const rawTokens = s.tokens;
-        const rawTags = s.tags;
-        const rawDeps = s.deps;
+    udSentences.forEach((ud, si) => {
+        const { clusters, idToCluster } = clusterWords(ud.words);
         // ---- 1. Build ClsWord array ----
-        const wordsPre = rawTokens.map((word, i) => {
-            // Re-tag a mis-tagged dash glyph (FinNLP labels "–"/"—" as NNP) to the Penn
-            // dash class ':' so it acts as a caesura/IU boundary, not a stressable word.
-            const tag = isDashGlyph(word) ? ':' : rawTags[i];
-            return ({
-                index: i + 1, // 1‑based, matching Antelope
+        const words = clusters.map((cl, i) => {
+            // Preserve the ORIGINAL case of the surface form (UDPipe keeps it); only
+            // lowercase a private lookup key for the archaic-lexicon / dash checks.
+            // Lowercasing `word` itself lost every proper-noun capital ("pakistan",
+            // "marcel proust") in the display and projection; downstream stress/name
+            // lookups all lowercase internally, so case in `word` is display-only.
+            const surfaceRaw = cl.tokens.map((t) => t.form).join('');
+            const surface = surfaceRaw.toLowerCase();
+            // Penn tag: use the raw XPOS when it already is Penn (EWT/GUM), else derive
+            // it from UPOS+FEATS (LinES/ParTUT emit non-Penn XPOS the pipeline can't read).
+            const rawTag = pennTagOf(cl.repr);
+            const archaic = ARCHAIC_POS[surface.replace(/['’]/g, '')];
+            const tag = isDashGlyph(surfaceRaw) ? ':' : (archaic ?? rawTag);
+            return {
+                index: i + 1,
                 lexicalClass: tag,
-                lexicalDetails: '',
+                lexicalDetails: cl.repr.feats,
                 lexicalPlural: tag === 'NNS' || tag === 'NNPS',
                 position: '',
-                word,
+                word: surfaceRaw,
                 absoluteIndex: absoluteOffset + i,
                 isContent: isContentWord(tag),
-                syllables: [], // filled later by stress module
+                syllables: [],
                 phraseStress: 0,
-                dependency: undefined, // patched below
-                node: undefined, // patched below
-            });
-        });
-        // ---- 1a. Re‑merge contraction pairs ----
-        const { words: contractedWords, consumedSegments } = mergeContractionsInSentence(wordsPre, rawSegments, segmentIdx);
-        segmentIdx += consumedSegments;
-        // ---- 1b. Re‑merge hyphenated words ----
-        const words = mergeHyphenatedWords(contractedWords);
-        // Re‑index words after merging (1‑based).
-        words.forEach((w, i) => {
-            w.index = i + 1;
+                dependency: undefined,
+                node: undefined,
+            };
         });
-        // ---- 2. Build ClsDependency array ----
-        const dependencies = [];
-        // Build contraction merge map: wordsPre idx → contractedWords idx.
-        // MUST replay mergeContractionsInSentence exactly: punctuation tokens have no
-        // raw segment (consume none), and an archaic -'d merges only when the
-        // spurious would/had token follows.
-        const contractionMap = new Map();
-        let pi = 0;
-        let qi = 0;
-        let segOff = segmentIdx - consumedSegments;
-        while (pi < wordsPre.length && segOff < segmentIdx) {
-            if (isPunctuation(wordsPre[pi].lexicalClass)) {
-                contractionMap.set(pi, qi);
-                pi++;
-                qi++;
+        // Sentence-initial de-capitalisation (mirrors en-norm.normalizeCaps in the
+        // pre-UD path): lower the first letter of the sentence's first word UNLESS it
+        // is a proper noun, so "The"→"the" and "I"→"i" (an orthographic capital forced
+        // by line position carries no lexical signal) while mid-line proper nouns
+        // ("Marcel Proust", "Pakistan") keep their caps for display/projection.
+        for (const w of words) {
+            if (isPunctuation(w.lexicalClass))
                 continue;
+            if (!/^(NNP|NNPS)$/.test(w.lexicalClass) && /^[A-Z]/.test(w.word)) {
+                w.displayWord = w.word; // keep the original surface for reports/phonopoetics
+                w.word = w.word[0].toLowerCase() + w.word.slice(1);
             }
-            const seg = rawSegments[segOff];
-            const archaicMerge = seg.isArchaicD
-                && pi + 1 < wordsPre.length
-                && ['would', 'had'].includes(wordsPre[pi + 1].word.toLowerCase());
-            if ((seg.isContraction && pi + 1 < wordsPre.length) || archaicMerge) {
-                contractionMap.set(pi, qi);
-                contractionMap.set(pi + 1, qi);
-                pi += 2;
-                qi += 1;
-                segOff++;
-            }
-            else {
-                contractionMap.set(pi, qi);
-                pi++;
-                qi++;
-                segOff++;
-            }
-        }
-        while (pi < wordsPre.length) {
-            contractionMap.set(pi, qi);
-            pi++;
-            qi++;
+            break;
         }
-        // Build hyphen merge map: contractedWords idx → words idx
-        const hyphenMap = new Map();
-        let ci = 0;
-        let wi = 0;
-        while (ci < contractedWords.length) {
-            if (ci + 2 < contractedWords.length &&
-                contractedWords[ci + 1].word === '-' &&
-                !isPunctuation(contractedWords[ci].lexicalClass) &&
-                !isPunctuation(contractedWords[ci + 2].lexicalClass)) {
-                hyphenMap.set(ci, wi);
-                hyphenMap.set(ci + 1, wi);
-                hyphenMap.set(ci + 2, wi);
-                ci += 3;
-                wi++;
+        // ---- 2. Build ClsDependency array ----
+        const dependencies = [];
+        clusters.forEach((cl, depIdx) => {
+            const r = cl.repr;
+            let govIdx;
+            if (r.head === 0) {
+                govIdx = undefined; // attaches to root
             }
             else {
-                hyphenMap.set(ci, wi);
-                ci++;
-                wi++;
+                const g = idToCluster.get(r.head);
+                // If the representative's head fell inside its own cluster (e.g. a copula
+                // clitic), follow that clitic's head out of the cluster.
+                if (g === depIdx) {
+                    const external = cl.tokens
+                        .map((t) => idToCluster.get(t.head))
+                        .find((gi) => gi !== undefined && gi !== depIdx);
+                    govIdx = external;
+                }
+                else {
+                    govIdx = g;
+                }
             }
-        }
-        // Compose: wordsPre idx → words idx
-        const mergeMap2 = new Map();
-        for (const [preIdx, cIdx] of contractionMap) {
-            const wIdx = hyphenMap.get(cIdx);
-            if (wIdx !== undefined)
-                mergeMap2.set(preIdx, wIdx);
-        }
-        // Build dependencies from rawDeps, remapping governor and dependent indices.
-        for (let i = 0; i < rawDeps.length; i++) {
-            const dep = rawDeps[i];
-            const govPreIdx = dep.parent; // 0‑based, -1 for root
-            const depPreIdx = i;
-            const govPostIdx = govPreIdx >= 0 ? mergeMap2.get(govPreIdx) : undefined;
-            const depPostIdx = mergeMap2.get(depPreIdx);
-            if (depPostIdx === undefined)
-                continue;
-            if (govPreIdx >= 0 && govPostIdx === depPostIdx)
-                continue; // self-loop from merged pair
-            // If this dependent is the clitic half of a contraction, skip
-            // (its dependency info is already captured through the host mapping).
-            // Check if this is the second token of a contraction:
-            const isCliticHalf = i > 0 &&
-                mergeMap2.get(i) === mergeMap2.get(i - 1);
-            if (isCliticHalf)
-                continue;
-            const govWord = govPostIdx !== undefined && govPostIdx >= 0 ? words[govPostIdx] : undefined;
-            const depWord = words[depPostIdx];
-            // If governor was a clitic half that got merged into host, re-point to host
-            const actualGovWord = govWord || (govPreIdx >= 0 && govPreIdx < wordsPre.length
-                ? words[mergeMap2.get(govPreIdx)]
-                : undefined);
+            const depWord = words[depIdx];
+            const govWord = govIdx !== undefined ? words[govIdx] : null;
             dependencies.push({
-                index: depPostIdx + 1,
-                governorIndex: govPostIdx !== undefined ? govPostIdx + 1 : 0,
-                dependentIndex: depPostIdx + 1,
-                dependentType: toAntelopeLabel(dep.label),
-                governorName: (govPostIdx !== undefined && govPostIdx >= 0 && words[govPostIdx])
-                    ? words[govPostIdx].word : 'ROOT',
+                index: depIdx + 1,
+                governorIndex: govIdx !== undefined ? govIdx + 1 : 0,
+                dependentIndex: depIdx + 1,
+                dependentType: govIdx === undefined ? 'root' : r.deprel,
+                governorName: govWord ? govWord.word : 'ROOT',
                 dependentName: depWord.word,
-                governor: (govPostIdx !== undefined && govPostIdx >= 0 && words[govPostIdx])
-                    ? words[govPostIdx] : null,
+                governor: govWord,
                 dependent: depWord,
             });
-        }
-        // Ensure ROOT dependency exists.
-        const hasRoot = dependencies.some(d => d.governorIndex === 0);
-        if (!hasRoot && words.length > 0) {
+        });
+        // Ensure a ROOT dependency exists.
+        if (!dependencies.some((d) => d.governorIndex === 0) && words.length > 0) {
             dependencies.push({
                 index: 0,
                 governorIndex: 0,
@@ -512,16 +299,21 @@ export function parseDocument(text) {
                 dependent: words[0],
             });
         }
-        // Back‑reference: each word stores the dependency edge where it is the dependent.
-        words.forEach(w => {
-            w.dependency = dependencies.find(d => d.dependent === w);
+        // Back‑reference: each word stores the dependency edge where it is dependent.
+        words.forEach((w) => {
+            w.dependency = dependencies.find((d) => d.dependent === w);
         });
-        // ---- 3. Build phrase‑structure node tree from FinNLP's depsTree ----
-        const rootNode = buildNodeTree(s.depsTree, words);
-        // Attach each word’s corresponding leaf node (if any).
+        // UDPipe XPOS correction (the role en-pos + tagfix.ts played pre-UD): fix the
+        // systematic mis-tags UDPipe makes on terse, decontextualised verse via
+        // en-lexicon cross-check.  Runs HERE (in the parser, after deps are attached)
+        // so direct `parseDocument` consumers — and every engine — see corrected tags;
+        // rule (3) needs the dependency back-references just set above.
+        correctUDPipePos({ index: si + 1, nodes: null, dependencies, words, xml: '' });
+        // ---- 3. Build phrase‑structure node tree from the dependency graph ----
+        const rootNode = buildDepNodeTree(words, dependencies);
         const wordNodeMap = new Map();
         collectWordNodes(rootNode, wordNodeMap);
-        words.forEach(w => {
+        words.forEach((w) => {
             w.node = wordNodeMap.get(w.index);
         });
         sentences.push({
@@ -532,145 +324,111 @@ export function parseDocument(text) {
             xml: '',
         });
         absoluteOffset += words.length;
-    }
+    });
     return { sentences, xml: '' };
 }
-// ─── Node‑tree construction ──────────────────────────────────────
-/** Sentinel used for empty / unparsable trees. */
-const EMPTY_NODE = {
-    index: '0',
-    nodeName: 'EMPTY',
-    parent: null,
-    contains: [],
-};
-/**
- * Recursively convert a FinNLP NodeInterface tree into a ClsNode tree
- * that mirrors Antelope’s phrase‑structure output.
- *
- * The root of the FinNLP tree is always wrapped in an SQ node.
- */
-function buildNodeTree(finRoot, words) {
-    // Guard: missing or empty tree
-    if (!finRoot || !finRoot.tokens || finRoot.tokens.length === 0) {
-        // Create a minimal SQ node containing all words as direct leaves.
-        const sq = {
-            index: '1',
-            nodeName: 'SQ',
-            parent: null,
-            contains: words.map(w => createWordLeaf(w)),
-        };
+// ── Dependency → constituency projection ─────────────────────────────
+// phonological.ts groups clitic groups into phonological phrases by finding the
+// smallest phrase node containing them, so it needs a properly nested, position-
+// ordered constituency tree. We synthesise one by projection: each head plus its
+// dependent subtrees forms a phrase, labelled by the head's POS family.
+function phraseType(tag) {
+    if (/^(NN|NNS|NNP|NNPS|PRP|PRP\$|DT|CD|WP|WDT|EX)$/.test(tag))
+        return 'NP';
+    if (/^(VB|VBD|VBG|VBN|VBP|VBZ|MD)$/.test(tag))
+        return 'VP';
+    if (/^(IN|TO)$/.test(tag))
+        return 'PP';
+    if (/^(JJ|JJR|JJS)$/.test(tag))
+        return 'ADJP';
+    if (/^(RB|RBR|RBS|WRB)$/.test(tag))
+        return 'ADVP';
+    return 'XP';
+}
+function buildDepNodeTree(words, deps) {
+    const sq = { index: '1', nodeName: 'SQ', parent: null, contains: [] };
+    if (words.length === 0)
         return sq;
+    // children[g] = list of dependent word-indices (1-based) governed by g (1-based);
+    // roots are governed by 0.
+    const children = new Map();
+    for (const d of deps) {
+        if (d.dependentIndex < 1 || d.dependentIndex > words.length)
+            continue;
+        const g = d.governorIndex;
+        if (!children.has(g))
+            children.set(g, []);
+        children.get(g).push(d.dependentIndex);
     }
-    // The top‑level SQ node (Antelope style)
-    const sqNode = {
-        index: '1',
-        nodeName: 'SQ',
-        parent: null,
-        contains: [],
-    };
-    // Convert the root FinNLP node and attach it under SQ.
-    const convertedRoot = convertFinNode(finRoot, words, sqNode);
-    if (convertedRoot) {
-        convertedRoot.parent = sqNode;
-        sqNode.contains.push(convertedRoot);
-    }
-    // Ensure every word is represented somewhere in the tree.
-    // Words not yet attached (e.g., punctuation at the edges) are added
-    // directly under SQ.
-    const attachedIndices = new Set();
-    collectAttachedWordIndices(sqNode, attachedIndices);
-    for (const w of words) {
-        if (!attachedIndices.has(w.index)) {
-            const leaf = createWordLeaf(w);
-            leaf.parent = sqNode;
-            sqNode.contains.push(leaf);
+    const build = (wordIdx, parent) => {
+        const word = words[wordIdx - 1];
+        const kids = (children.get(wordIdx) ?? []).filter((k) => k !== wordIdx);
+        if (kids.length === 0) {
+            const leaf = createWordLeaf(word);
+            leaf.parent = parent;
+            return leaf;
         }
-    }
-    return sqNode;
-}
-/**
- * Convert a single FinNodeInterface (sub‑tree) into a ClsNode.
- */
-function convertFinNode(finNode, words, parentNode) {
-    // Determine whether this is a leaf (single‑word) node.
-    const isLeaf = (!finNode.left || finNode.left.length === 0) &&
-        (!finNode.right || finNode.right.length === 0);
-    if (isLeaf && finNode.tokens.length === 1) {
-        // Single‑word leaf → reference the ClsWord
-        const wordIdx = finNode.index[0]; // 0‑based
-        const word = words[wordIdx];
-        if (!word) {
-            // Fallback: create a text leaf
-            return {
-                index: `leaf_${wordIdx}`,
-                nodeName: finNode.tokens[0],
-                parent: parentNode,
-                contains: [],
-            };
+        const node = {
+            index: `ph_${wordIdx}`,
+            nodeName: phraseType(word.lexicalClass),
+            parent,
+            contains: [],
+        };
+        // Order head + dependents by surface position for a projective tree.
+        const ordered = [...kids, wordIdx].sort((a, b) => a - b);
+        for (const idx of ordered) {
+            if (idx === wordIdx) {
+                const leaf = createWordLeaf(word);
+                leaf.parent = node;
+                node.contains.push(leaf);
+            }
+            else {
+                node.contains.push(build(idx, node));
+            }
         }
-        return createWordLeaf(word);
-    }
-    // Phrase node – use the FinNLP type as label (NP, VP, PP, etc.)
-    const phraseType = finNode.type && finNode.type !== 'ROOT'
-        ? finNode.type
-        : 'XP';
-    const phraseNode = {
-        index: `ph_${finNode.index[0]}_${finNode.index[1]}`,
-        nodeName: phraseType,
-        parent: parentNode,
-        contains: [],
+        return node;
     };
-    // Process left children (pre‑head dependents)
-    if (finNode.left && finNode.left.length > 0) {
-        for (const leftChild of finNode.left) {
-            const childNode = convertFinNode(leftChild, words, phraseNode);
-            if (childNode) {
-                childNode.parent = phraseNode;
-                phraseNode.contains.push(childNode);
-            }
+    const roots = (children.get(0) ?? []).sort((a, b) => a - b);
+    if (roots.length === 0) {
+        // No explicit root: attach all words as leaves under SQ.
+        for (const w of words) {
+            const leaf = createWordLeaf(w);
+            leaf.parent = sq;
+            sq.contains.push(leaf);
         }
+        return sq;
     }
-    // The head token(s) of this node
-    for (let i = finNode.index[0]; i <= finNode.index[1]; i++) {
-        const word = words[i];
-        if (word) {
-            const leaf = createWordLeaf(word);
-            leaf.parent = phraseNode;
-            phraseNode.contains.push(leaf);
-        }
+    for (const r of roots) {
+        const child = build(r, sq);
+        sq.contains.push(child);
     }
-    // Process right children (post‑head dependents)
-    if (finNode.right && finNode.right.length > 0) {
-        for (const rightChild of finNode.right) {
-            const childNode = convertFinNode(rightChild, words, phraseNode);
-            if (childNode) {
-                childNode.parent = phraseNode;
-                phraseNode.contains.push(childNode);
-            }
+    // Attach any orphan words (rare) directly under SQ.
+    const attached = new Set();
+    collectAttachedWordIndices(sq, attached);
+    for (const w of words) {
+        if (!attached.has(w.index)) {
+            const leaf = createWordLeaf(w);
+            leaf.parent = sq;
+            sq.contains.push(leaf);
         }
     }
-    return phraseNode;
+    return sq;
 }
-// ─── Leaf‑node helpers ────────────────────────────────────────────
+// ── Leaf / traversal helpers (unchanged) ─────────────────────────────
 function createWordLeaf(word) {
     return {
         index: `w${word.index}`,
-        nodeName: word.index.toString(), // Antelope style: the word’s 1‑based index as string
+        nodeName: word.index.toString(),
         parent: null,
         contains: [word],
     };
 }
-// ─── Tree traversal helpers ───────────────────────────────────────
 function collectWordNodes(node, map) {
     for (const child of node.contains) {
         if (child instanceof Object && 'word' in child) {
-            // child is a ClsWord
-            const w = child;
-            // The leaf is the current node (since word leaves contain the word directly)
-            map.set(w.index, node);
+            map.set(child.index, node);
         }
         else if (child instanceof Object && 'index' in child) {
-            // child is a ClsNode
             collectWordNodes(child, map);
         }
     }
@@ -678,8 +436,7 @@ function collectWordNodes(node, map) {
 function collectAttachedWordIndices(node, set) {
     for (const child of node.contains) {
         if (child instanceof Object && 'word' in child) {
-            const w = child;
-            set.add(w.index);
+            set.add(child.index);
         }
         else if (child instanceof Object && 'index' in child) {
             collectAttachedWordIndices(child, set);